From 7409f1575235719bb40de14b6a695fe481a96a4b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 5 Mar 2021 13:59:29 -0500 Subject: [PATCH 001/378] update all grafana queries that were using autogen to use default --- salt/grafana/dashboards/eval/eval.json | 4 ++-- salt/grafana/dashboards/manager/manager.json | 2 +- salt/grafana/dashboards/managersearch/managersearch.json | 2 +- salt/grafana/dashboards/sensor_nodes/sensor.json | 4 ++-- salt/grafana/dashboards/standalone/standalone.json | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/grafana/dashboards/eval/eval.json b/salt/grafana/dashboards/eval/eval.json index cc4298bb2..1b67267a5 100644 --- a/salt/grafana/dashboards/eval/eval.json +++ b/salt/grafana/dashboards/eval/eval.json @@ -352,7 +352,7 @@ ], "measurement": "zeekcaptureloss", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ @@ -2176,7 +2176,7 @@ ], "measurement": "docker_container_mem", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/manager/manager.json b/salt/grafana/dashboards/manager/manager.json index 2ce913155..a53438dd6 100644 --- a/salt/grafana/dashboards/manager/manager.json +++ b/salt/grafana/dashboards/manager/manager.json @@ -1647,7 +1647,7 @@ ], "measurement": "influxsize", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/managersearch/managersearch.json b/salt/grafana/dashboards/managersearch/managersearch.json index 15bf3cc73..1b71a9e84 100644 --- a/salt/grafana/dashboards/managersearch/managersearch.json +++ b/salt/grafana/dashboards/managersearch/managersearch.json @@ -1631,7 +1631,7 @@ ], "measurement": "influxsize", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/sensor_nodes/sensor.json b/salt/grafana/dashboards/sensor_nodes/sensor.json index ea0a6a63b..f9dbc9f9c 100644 --- a/salt/grafana/dashboards/sensor_nodes/sensor.json +++ b/salt/grafana/dashboards/sensor_nodes/sensor.json @@ -351,7 +351,7 @@ ], "measurement": "zeekcaptureloss", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ @@ -2866,7 +2866,7 @@ ], "measurement": "healthcheck", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 60a5c6c6c..d745f13c8 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -4486,7 +4486,7 @@ ], "measurement": "zeekcaptureloss", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ @@ -5107,7 +5107,7 @@ ], "measurement": "influxsize", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ From 1f9e5ca3ccf1818dee7c6203e408f4c00ddf39d8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 9 Mar 2021 10:31:59 -0500 Subject: [PATCH 002/378] install influxdb python module add test retention policies --- salt/influxdb/init.sls | 32 ++++++++++++++++++++++++++++++++ setup/so-functions | 3 ++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index aace4e827..871b0a16a 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -57,6 +57,31 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +set_autogen_retention_policy: + influxdb_retention_policy.present: + - name: autogen + - database: telegraf + - duration: 1h + - replication: 1 + - default: True + - ssl: True + - unsafeSsl: True + - require: + - docker_container: so-influxdb + +set_so_long_term_retention_policy: + influxdb_retention_policy.present: + - name: so_long_term + - database: telegraf + - duration: 2h + - replication: 1 + - default: False + - ssl: True + - unsafeSsl: True + - require: + - docker_container: so-influxdb + + {% endif %} {% else %} @@ -66,3 +91,10 @@ append_so-influxdb_so-status.conf: - name: {{sls}}_state_not_allowed {% endif %} + +#influxdb: +# retention_policies: +# autogen: +# duration: 1h +# so_long_term: +# duration: 2h \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index 21602f320..5890f4726 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2022,6 +2022,7 @@ saltify() { set_progress_str 7 'Installing salt-master' retry 50 10 "apt-get -y install salt-master=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1 ;; *) # Copy down the gpg keys and install them from the manager @@ -2041,7 +2042,7 @@ saltify() { retry 50 10 "apt-get -y install salt-minion=3002.5+ds-1 salt-common=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 if [[ $OSVER != 'xenial' ]]; then - retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-influxdb" >> "$setup_log" 2>&1 || exit 1 else retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1 fi From c1e4c4cb3049382ef8c695a78d019a623f363612 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 9 Mar 2021 11:50:27 -0500 Subject: [PATCH 003/378] fix pip and python-influxdb install --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 391ac3558..9ea165548 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1917,7 +1917,8 @@ saltify() { reserve_group_ids >> "$setup_log" 2>&1 if [[ ! $is_iso ]]; then logCmd "yum -y install epel-release" - logCmd "yum -y install sqlite argon2 curl mariadb-devel" + logCmd "yum -y install sqlite argon2 curl mariadb-devel python3-pip" + retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1 fi # Download Ubuntu Keys in case manager updates = 1 mkdir -p /opt/so/gpg >> "$setup_log" 2>&1 @@ -2032,7 +2033,6 @@ saltify() { set_progress_str 7 'Installing salt-master' retry 50 10 "apt-get -y install salt-master=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 - retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1 ;; *) # Copy down the gpg keys and install them from the manager @@ -2054,7 +2054,7 @@ saltify() { if [[ $OSVER != 'xenial' ]]; then retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-influxdb" >> "$setup_log" 2>&1 || exit 1 else - retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-influxdb" >> "$setup_log" 2>&1 || exit 1 fi fi } From 75012cdcbabb66980a551ab2bca4ca2cde709fb5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 10 Mar 2021 15:20:11 -0500 Subject: [PATCH 004/378] create rps and cqs --- salt/common/init.sls | 3 ++ salt/influxdb/defaults.yaml | 33 ++++++++++++++++++ salt/influxdb/init.sls | 68 ++++++++++++++++++++++++++----------- 3 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 salt/influxdb/defaults.yaml diff --git a/salt/common/init.sls b/salt/common/init.sls index 16cba3c1a..eb7a413e2 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -105,6 +105,8 @@ commonpkgs: - python3-m2crypto - python3-mysqldb - git + - patch + heldpackages: pkg.installed: - pkgs: @@ -140,6 +142,7 @@ commonpkgs: - lvm2 - openssl - git + - patch heldpackages: pkg.installed: diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml new file mode 100644 index 000000000..57193e6fa --- /dev/null +++ b/salt/influxdb/defaults.yaml @@ -0,0 +1,33 @@ +influxdb: + retention_policies: + autogen: + default: True + duration: 1h + so_long_term: + default: False + duration: 30d + downsample: + so_long_term: + resolution: 30m + measurements: + - cpu + - disk + - diskio + - docker_container_cpu + - docker_container_mem + - docker_container_net + - elasticsearch_indices + - elasticsearch_jvm + - esteps + - healthcheck + - influxsize + - mem + - net + - pcapage + - processes + - redisqueue + - stenodrop + - suridrop + - system + - zeekcaptureloss + - zeekdrop \ No newline at end of file diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 871b0a16a..b20e6f64b 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -5,6 +5,8 @@ {% set MANAGER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{% import_yaml 'influxdb/defaults.yaml' as default_settings %} +{% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} @@ -57,30 +59,31 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb -set_autogen_retention_policy: +{% for rp in influxdb.retention_policies.keys() %} +{{rp}}_retention_policy: influxdb_retention_policy.present: - - name: autogen + - name: {{rp}} - database: telegraf - - duration: 1h + - duration: {{influxdb.retention_policies[rp].duration}} - replication: 1 - - default: True + - default: {{influxdb.retention_policies[rp].get('default', 'False')}} - ssl: True - - unsafeSsl: True - require: - docker_container: so-influxdb +{% endfor %} -set_so_long_term_retention_policy: - influxdb_retention_policy.present: - - name: so_long_term +{% for dest_rp in influxdb.downsample.keys() %} + {% for measurement in influxdb.downsample[dest_rp].measurements %} +so_downsample_{{measurement}}_cq: + influxdb_continuous_query.present: + - name: so_downsample_{{measurement}}_cq - database: telegraf - - duration: 2h - - replication: 1 - - default: False + - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) - ssl: True - - unsafeSsl: True - require: - docker_container: so-influxdb - + {% endfor %} +{% endfor %} {% endif %} @@ -92,9 +95,36 @@ set_so_long_term_retention_policy: {% endif %} -#influxdb: -# retention_policies: -# autogen: -# duration: 1h -# so_long_term: -# duration: 2h \ No newline at end of file +influxdb: + retention_policies: + autogen: + default: True + duration: 1h + so_long_term: + default: False + duration: 2h + downsample: + so_long_term: + resolution: 30m + measurements: + - cpu + - disk + - diskio + - docker_container_cpu + - docker_container_mem + - docker_container_net + - elasticsearch_indices + - elasticsearch_jvm + - esteps + - healthcheck + - influxsize + - mem + - net + - pcapage + - processes + - redisqueue + - stenodrop + - suridrop + - system + - zeekcaptureloss + - zeekdrop \ No newline at end of file From 8fc1656939468b69d6fb2dd5599d1762ed920ae6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 09:24:57 -0500 Subject: [PATCH 005/378] fix timeouts / retries in ssl state --- salt/ssl/init.sls | 110 +++++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 44 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 8d6c65bea..4646b1d07 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -68,8 +68,9 @@ removeesp12dir: - x509: /etc/pki/influxdb.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the talking to influxdb /etc/pki/influxdb.crt: @@ -86,8 +87,9 @@ removeesp12dir: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/influxdb.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 influxkeyperms: file.managed: @@ -111,8 +113,9 @@ influxkeyperms: - x509: /etc/pki/redis.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/redis.crt: x509.certificate_managed: @@ -153,8 +156,9 @@ rediskeyperms: - x509: /etc/pki/filebeat.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Request a cert and drop it where it needs to go to be distributed /etc/pki/filebeat.crt: @@ -175,8 +179,9 @@ rediskeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/filebeat.key -topk8 -out /etc/pki/filebeat.p8 -nocrypt" - onchanges: @@ -232,8 +237,9 @@ fbcrtlink: - x509: /etc/pki/registry.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the docker registry /etc/pki/registry.crt: @@ -250,8 +256,9 @@ fbcrtlink: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/registry.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 regkeyperms: file.managed: @@ -273,8 +280,9 @@ regkeyperms: - x509: /etc/pki/minio.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for minio /etc/pki/minio.crt: @@ -291,8 +299,9 @@ regkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/minio.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 miniokeyperms: file.managed: @@ -332,8 +341,9 @@ miniokeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: @@ -366,8 +376,9 @@ elasticp12perms: - x509: /etc/pki/managerssl.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the reverse proxy /etc/pki/managerssl.crt: @@ -385,8 +396,9 @@ elasticp12perms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 msslkeyperms: file.managed: @@ -409,8 +421,9 @@ msslkeyperms: - x509: /etc/pki/fleet.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/fleet.crt: x509.certificate_managed: @@ -425,8 +438,9 @@ msslkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 fleetkeyperms: file.managed: @@ -456,8 +470,9 @@ fbcertdir: - x509: /opt/so/conf/filebeat/etc/pki/filebeat.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Request a cert and drop it where it needs to go to be distributed /opt/so/conf/filebeat/etc/pki/filebeat.crt: @@ -478,8 +493,9 @@ fbcertdir: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /opt/so/conf/filebeat/etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Convert the key to pkcs#8 so logstash will work correctly. filebeatpkcs: @@ -520,8 +536,9 @@ chownfilebeatp8: - x509: /etc/pki/managerssl.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the reverse proxy /etc/pki/managerssl.crt: @@ -539,8 +556,9 @@ chownfilebeatp8: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 msslkeyperms: file.managed: @@ -563,8 +581,9 @@ msslkeyperms: - x509: /etc/pki/fleet.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/fleet.crt: x509.certificate_managed: @@ -579,8 +598,9 @@ msslkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 fleetkeyperms: file.managed: @@ -606,8 +626,9 @@ fleetkeyperms: - x509: /etc/pki/elasticsearch.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/elasticsearch.crt: x509.certificate_managed: @@ -623,8 +644,9 @@ fleetkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: From 6dba2879c52e65828598f609e0f409aa35c49300 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 09:25:44 -0500 Subject: [PATCH 006/378] change so_long_term rp to 6h for tetsing --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 57193e6fa..9647b70ea 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -5,7 +5,7 @@ influxdb: duration: 1h so_long_term: default: False - duration: 30d + duration: 6h downsample: so_long_term: resolution: 30m From 3385ba2ca2da7590eae56f513e241adc3fe7360c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 09:35:54 -0500 Subject: [PATCH 007/378] verify ssl --- salt/influxdb/init.sls | 42 +++++++----------------------------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index b20e6f64b..f55e255b7 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -68,6 +68,9 @@ append_so-influxdb_so-status.conf: - replication: 1 - default: {{influxdb.retention_policies[rp].get('default', 'False')}} - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb {% endfor %} @@ -80,6 +83,9 @@ so_downsample_{{measurement}}_cq: - database: telegraf - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb {% endfor %} @@ -93,38 +99,4 @@ so_downsample_{{measurement}}_cq: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} - -influxdb: - retention_policies: - autogen: - default: True - duration: 1h - so_long_term: - default: False - duration: 2h - downsample: - so_long_term: - resolution: 30m - measurements: - - cpu - - disk - - diskio - - docker_container_cpu - - docker_container_mem - - docker_container_net - - elasticsearch_indices - - elasticsearch_jvm - - esteps - - healthcheck - - influxsize - - mem - - net - - pcapage - - processes - - redisqueue - - stenodrop - - suridrop - - system - - zeekcaptureloss - - zeekdrop \ No newline at end of file +{% endif %} \ No newline at end of file From 3b74d987c1a1486dc5e720b93814d592ebc35c62 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 10:49:15 -0500 Subject: [PATCH 008/378] fix retry in ca state. add subjectAltName to influxdb.crt --- salt/ca/init.sls | 5 +++-- salt/ssl/init.sls | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/ca/init.sls b/salt/ca/init.sls index 0d35c10c1..485b01d3e 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -43,8 +43,9 @@ pki_private_key: - require: - file: /etc/pki - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 x509_pem_entries: module.run: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 4646b1d07..85854bab4 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -79,6 +79,7 @@ removeesp12dir: - signing_policy: influxdb - public_key: /etc/pki/influxdb.key - CN: {{ manager }} + - subjectAltName: DNS:{{ HOSTNAME }} - days_remaining: 0 - days_valid: 820 - backup: True From c4da57603087b0c6ab9f7bbad5147c439aabc471 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 12:20:32 -0500 Subject: [PATCH 009/378] ensure the presence of the telegraf database --- salt/influxdb/init.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index f55e255b7..608561857 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -59,6 +59,17 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +telegraf_database: + influxdb_database.present: + - name: telegraf + - database: telegraf + - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} + - require: + - docker_container: so-influxdb + {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: influxdb_retention_policy.present: @@ -73,6 +84,7 @@ append_so-influxdb_so-status.conf: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb + - influxdb_database: telegraf_database {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} @@ -88,6 +100,7 @@ so_downsample_{{measurement}}_cq: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb + - influxdb_database: telegraf_database {% endfor %} {% endfor %} From 3a8aea0de69ceb26ec4e115c00dd465001a16aae Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 09:11:48 -0400 Subject: [PATCH 010/378] removing domainstats and freqserver from so-image-common --- salt/common/tools/sbin/so-image-common | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 402ae97f3..a3b9f62e9 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -47,13 +47,11 @@ container_list() { TRUSTED_CONTAINERS=( "so-acng" "so-curator" - "so-domainstats" "so-elastalert" "so-elasticsearch" "so-filebeat" "so-fleet" "so-fleet-launcher" - "so-freqserver" "so-grafana" "so-idstools" "so-influxdb" From 53528d486c0a2793cf45ccda7ee2950bb2f29e4e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 09:44:56 -0400 Subject: [PATCH 011/378] remove minio --- salt/common/tools/sbin/so-image-common | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index a3b9f62e9..be5a327f0 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -58,7 +58,6 @@ container_list() { "so-kibana" "so-kratos" "so-logstash" - "so-minio" "so-mysql" "so-nginx" "so-pcaptools" From 11c3f14b42d64b5581304e383f81b96b6f9b6836 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 10:35:20 -0400 Subject: [PATCH 012/378] end patch files with newline --- salt/salt/files/influxdb_continuous_query.py.patch | 2 +- salt/salt/files/influxdb_retention_policy.py.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/files/influxdb_continuous_query.py.patch b/salt/salt/files/influxdb_continuous_query.py.patch index 049b04782..a596983ff 100644 --- a/salt/salt/files/influxdb_continuous_query.py.patch +++ b/salt/salt/files/influxdb_continuous_query.py.patch @@ -1,4 +1,4 @@ 60c60 < database, name, query, resample_time, coverage_period --- -> database, name, query, resample_time, coverage_period, **client_args \ No newline at end of file +> database, name, query, resample_time, coverage_period, **client_args diff --git a/salt/salt/files/influxdb_retention_policy.py.patch b/salt/salt/files/influxdb_retention_policy.py.patch index b16de431e..79ca3e536 100644 --- a/salt/salt/files/influxdb_retention_policy.py.patch +++ b/salt/salt/files/influxdb_retention_policy.py.patch @@ -1,4 +1,4 @@ 38c38 < hours = int(duration.split("h")) --- -> hours = int(duration.split("h")[0]) \ No newline at end of file +> hours = int(duration.split("h")[0]) From d75fafb19cddfadeeff95be574b6d492b42d145b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 17:30:27 -0400 Subject: [PATCH 013/378] add support for shard_duration to influxdb module and influxdb_retention_policy state - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- .../files/influxdb_retention_policy.py.patch | 12 ++++++++++++ salt/salt/files/influxmod.py.patch | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 salt/salt/files/influxmod.py.patch diff --git a/salt/salt/files/influxdb_retention_policy.py.patch b/salt/salt/files/influxdb_retention_policy.py.patch index 79ca3e536..7180f8934 100644 --- a/salt/salt/files/influxdb_retention_policy.py.patch +++ b/salt/salt/files/influxdb_retention_policy.py.patch @@ -2,3 +2,15 @@ < hours = int(duration.split("h")) --- > hours = int(duration.split("h")[0]) +52c52 +< def present(name, database, duration="7d", replication=1, default=False, **client_args): +--- +> def present(name, database, duration="7d", replication=1, default=False, shard_duration="1d", **client_args): +77c77 +< database, name, duration, replication, default, **client_args +--- +> database, name, duration, replication, shard_duration, default, **client_args +119c119 +< database, name, duration, replication, default, **client_args +--- +> database, name, duration, replication, shard_duration, default, **client_args diff --git a/salt/salt/files/influxmod.py.patch b/salt/salt/files/influxmod.py.patch new file mode 100644 index 000000000..b653265cb --- /dev/null +++ b/salt/salt/files/influxmod.py.patch @@ -0,0 +1,16 @@ +427c427 +< database, name, duration, replication, default=False, **client_args +--- +> database, name, duration, replication, shard_duration, default=False, **client_args +462c462 +< client.create_retention_policy(name, duration, replication, database, default) +--- +> client.create_retention_policy(name, duration, replication, database, default, shard_duration) +468c468 +< database, name, duration, replication, default=False, **client_args +--- +> database, name, duration, replication, shard_duration, default=False, **client_args +504c504 +< client.alter_retention_policy(name, database, duration, replication, default) +--- +> client.alter_retention_policy(name, database, duration, replication, default, shard_duration) From cd0ab5c70909b3234d59a04a5c347aacbf51ec06 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 25 Mar 2021 08:50:32 -0400 Subject: [PATCH 014/378] add support for shard_duration to influxdb module and influxdb_retention_policy state - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- .../files/{influxmod.py.patch => influxdbmod.py.patch} | 0 salt/salt/minion.sls | 7 ++++++- 2 files changed, 6 insertions(+), 1 deletion(-) rename salt/salt/files/{influxmod.py.patch => influxdbmod.py.patch} (100%) diff --git a/salt/salt/files/influxmod.py.patch b/salt/salt/files/influxdbmod.py.patch similarity index 100% rename from salt/salt/files/influxmod.py.patch rename to salt/salt/files/influxdbmod.py.patch diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index fa5c0659e..47cd179ac 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -53,4 +53,9 @@ influxdb_continuous_query.present_patch: influxdb_retention_policy.present_patch: file.patch: - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py - - source: salt://salt/files/influxdb_retention_policy.py.patch \ No newline at end of file + - source: salt://salt/files/influxdb_retention_policy.py.patch + +influxdbmod.py_shard_duration_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py + - source: salt://salt/files/influxdbmod.py.patch \ No newline at end of file From 889e624a8cbf3736d2f45b57b2a92cb7934dac73 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 25 Mar 2021 09:33:10 -0400 Subject: [PATCH 015/378] add shard_duration to state and defaults - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- salt/influxdb/defaults.yaml | 6 ++++-- salt/influxdb/init.sls | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 9647b70ea..88faaedba 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,10 +2,12 @@ influxdb: retention_policies: autogen: default: True - duration: 1h + duration: 1d + shard_duration: 1h so_long_term: default: False - duration: 6h + duration: 7d + shard_duration: 1d downsample: so_long_term: resolution: 30m diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 88f03042e..59673c93b 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -79,6 +79,7 @@ telegraf_database: - name: {{rp}} - database: telegraf - duration: {{influxdb.retention_policies[rp].duration}} + - shard_duration: {{influxdb.retention_policies[rp].shard_duration}} - replication: 1 - default: {{influxdb.retention_policies[rp].get('default', 'False')}} - ssl: True From e0f4abaa093bb9617ed77b64eeb6eab219bd6b98 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Mar 2021 10:36:56 -0400 Subject: [PATCH 016/378] try to do it with just 1 cq, modify defaults for testing --- salt/influxdb/defaults.yaml | 4 ++-- salt/influxdb/init.sls | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 88faaedba..3de7edd56 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,11 +2,11 @@ influxdb: retention_policies: autogen: default: True - duration: 1d + duration: 1h shard_duration: 1h so_long_term: default: False - duration: 7d + duration: 2d shard_duration: 1d downsample: so_long_term: diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 59673c93b..b3f3e5c9c 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -92,13 +92,14 @@ telegraf_database: - file: influxdb_retention_policy.present_patch {% endfor %} +{# {% for dest_rp in influxdb.downsample.keys() %} {% for measurement in influxdb.downsample[dest_rp].measurements %} so_downsample_{{measurement}}_cq: influxdb_continuous_query.present: - name: so_downsample_{{measurement}}_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}), * - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] @@ -109,6 +110,22 @@ so_downsample_{{measurement}}_cq: - file: influxdb_continuous_query.present_patch {% endfor %} {% endfor %} +#} + +so_downsample_cq: + influxdb_continuous_query.present: + - name: so_downsample_cq + - database: telegraf + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} + - require: + - docker_container: so-influxdb + - influxdb_database: telegraf_database + - file: influxdb_continuous_query.present_patch + {% endif %} From d1150f150fb174f6aaec9e13c5916ada0ae76e81 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Mar 2021 10:59:18 -0400 Subject: [PATCH 017/378] loop through the rps --- salt/influxdb/init.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index b3f3e5c9c..1b21612b3 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -112,6 +112,7 @@ so_downsample_{{measurement}}_cq: {% endfor %} #} +{% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq @@ -125,7 +126,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - +{% endfor %} {% endif %} From 30c6d4756a76687b9cbf95d9c9ff5085c538bb66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Mar 2021 09:38:37 -0400 Subject: [PATCH 018/378] change default long term resolution to 5m --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 3de7edd56..927cd7b52 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -10,7 +10,7 @@ influxdb: shard_duration: 1d downsample: so_long_term: - resolution: 30m + resolution: 5m measurements: - cpu - disk From 8e55e0b9944e6d46032b8fa2e00998a3efd9ded3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Mar 2021 13:36:52 -0400 Subject: [PATCH 019/378] start graphing data from so_long_term --- .../dashboards/standalone/standalone.new.json | 7841 +++++++++++++++++ 1 file changed, 7841 insertions(+) create mode 100644 salt/grafana/dashboards/standalone/standalone.new.json diff --git a/salt/grafana/dashboards/standalone/standalone.new.json b/salt/grafana/dashboards/standalone/standalone.new.json new file mode 100644 index 000000000..41c4d3541 --- /dev/null +++ b/salt/grafana/dashboards/standalone/standalone.new.json @@ -0,0 +1,7841 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "This Dashboard provides a general overview of Standalone Mode", + "editable": true, + "gnetId": 2381, + "graphTooltip": 0, + "id": 6, + "iteration": 1617106562807, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 60 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Usage", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* -1 + 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "groupBy": [], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_idle" + ], + "type": "field" + }, + { + "params": [ + "*-1+100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 39, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "uptime" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "{{ SERVERNAME }} - System Uptime", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=", + "value": "so-kibana" + }, + { + "condition": "AND", + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=", + "value": "so-kibana" + }, + { + "condition": "AND", + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Kibana CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "suridrop", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Suricata Packet Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 0 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stenodrop", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - PCAP Packet Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Used", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] + }, + { + "groupBy": [], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Disk Used(/)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 5 + }, + "hiddenSeries": false, + "id": 74, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Used", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] + }, + { + "groupBy": [], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Disk Used(/nsm)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 5 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - InfluxDB CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Logstash CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 5 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Suri CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 5 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Steno CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Interrupt": "#70DBED", + "Nice": "#629E51", + "SoftIRQ": "#EA6460", + "System": "#BF1B00", + "User": "#1F78C1", + "Wait": "#F2C96D", + "cpu.mean": "#629E51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "System", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_system" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "User", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_user" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Nice", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_nice" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Interrupt", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_irq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Wait", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "SoftIRQ", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_softirq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "Percent(%)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "InBound": "#629E51", + "OutBound": "#5195CE", + "net.non_negative_derivative": "super-light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 10 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "net", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "alias": "Outbound", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "net", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_sent" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Management Traffic", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 10 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Suri Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 10 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Steno Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "#cpu": "green", + "1 Minute Average": "#EAB839", + "15 Minute Average": "#BF1B00", + "5 Minute Average": "#E0752D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "#cpu", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "#cpu", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "n_cpus" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "1 Minute Average", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "5 Minute Average", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load5" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "15 Minute Average", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load15" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Load Average", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "InBound": "#629E51", + "OutBound": "#5195CE", + "net.non_negative_derivative": "light-orange" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 15 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "net", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MONINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MONINT }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Monitor Traffic", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 15 + }, + "id": 37, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "healthcheck", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "zeek_restart" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Zeek Restarts via Healthcheck", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 2, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": null, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 259200 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 432000 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 15 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Oldest Pcap", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "pcapage", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "seconds" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "pcapage", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_seconds" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - PCAP Retention", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Blocked": "#BF1B00", + "Running": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 7, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Blocked", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "blocked" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Running", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "processes", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "running" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Sleep", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "processes", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "sleeping" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Processes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 20 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "alias": "Outbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Logstash Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 20 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 20 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Threads", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "processes", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_threads" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Total Threads", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 25 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "alias": "Outbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - InfluxDB Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 25 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "zeekcaptureloss", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "loss" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek Capture Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 25 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "zeekdrop", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek Packet Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - IO Wait", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 30 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] + }, + { + "alias": "Outbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Proxy Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 30 + }, + "hiddenSeries": false, + "id": 69, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "influxsize", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "kbytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "influxsize", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_kbytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - InfluxDB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "deckbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 30 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "redisqueue", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "unparsed" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "redisqueue", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_unparsed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Redis Queue", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Read", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "read_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Write", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "write_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Disk I/O", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 35 + }, + "hiddenSeries": false, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "EPS", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "esteps", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "eps" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "esteps", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_eps" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Estimated EPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "EPS", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 35 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Redis Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 35 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "/ 16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Redis CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Buffered": "#6ED0E0", + "Cached": "#F9934E", + "Free": "#629E51", + "Used": "#58140C" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "decimals": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Used", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Buffered", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffered" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Cached", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "cached" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Free", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "free" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 40 + }, + "id": 33, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "docs_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": "", + "title": "{{ SERVERNAME }} - ES Documents", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "store_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_store_size_in_bytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES Store Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 40 + }, + "hiddenSeries": false, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "threads_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_threads_count" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES Thread Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 40 + }, + "hiddenSeries": false, + "id": 63, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "fielddata_memory_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_fielddata_memory_size_in_bytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES Fielddata Cache Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 67, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Total", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Used", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Memory(Used)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10s", + "value": "10s" + }, + "error": null, + "hide": 0, + "label": null, + "name": "Interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_Interval" + }, + { + "selected": true, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "10s, 1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Standalone Mode - {{ SERVERNAME }} Overview", + "uid": "so_overview", + "version": 1 + } \ No newline at end of file From 4f3b3a787c5cf9348518280193263b69aacf314b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Apr 2021 10:18:01 -0400 Subject: [PATCH 020/378] change defaults for testing, remove measurements list since cq uses wildcard now - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- salt/influxdb/defaults.yaml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 927cd7b52..ce5624be2 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,34 +2,12 @@ influxdb: retention_policies: autogen: default: True - duration: 1h + duration: 2d shard_duration: 1h so_long_term: default: False - duration: 2d + duration: 7d shard_duration: 1d downsample: so_long_term: - resolution: 5m - measurements: - - cpu - - disk - - diskio - - docker_container_cpu - - docker_container_mem - - docker_container_net - - elasticsearch_indices - - elasticsearch_jvm - - esteps - - healthcheck - - influxsize - - mem - - net - - pcapage - - processes - - redisqueue - - stenodrop - - suridrop - - system - - zeekcaptureloss - - zeekdrop \ No newline at end of file + resolution: 5m \ No newline at end of file From 5cda35db0aa0d8d9b6f2ded8a9a4ef959550ab58 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Apr 2021 10:45:54 -0400 Subject: [PATCH 021/378] change defaults for testing - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index ce5624be2..03ac4c3c6 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,7 +2,7 @@ influxdb: retention_policies: autogen: default: True - duration: 2d + duration: 1d shard_duration: 1h so_long_term: default: False From 8eef574342f27c46d581331114cd2d4dbfd5d027 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 08:28:01 -0400 Subject: [PATCH 022/378] install python3-influxdb and create requires --- salt/influxdb/init.sls | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 1b21612b3..67342f826 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -13,6 +13,10 @@ include: - salt.minion +python3-influxdb: + pkg.installed: + - name: python3-influxdb + # Influx DB influxconfdir: file.directory: @@ -62,6 +66,7 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb + telegraf_database: influxdb_database.present: - name: telegraf @@ -72,6 +77,7 @@ telegraf_database: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb + - pkg: python3-influxdb {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: @@ -90,28 +96,9 @@ telegraf_database: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_retention_policy.present_patch + - pkg: python3-influxdb {% endfor %} -{# -{% for dest_rp in influxdb.downsample.keys() %} - {% for measurement in influxdb.downsample[dest_rp].measurements %} -so_downsample_{{measurement}}_cq: - influxdb_continuous_query.present: - - name: so_downsample_{{measurement}}_cq - - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}), * - - ssl: True - - verify_ssl: /etc/pki/ca.crt - - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] - - influxdb_host: {{ MANAGER }} - - require: - - docker_container: so-influxdb - - influxdb_database: telegraf_database - - file: influxdb_continuous_query.present_patch - {% endfor %} -{% endfor %} -#} - {% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: @@ -126,6 +113,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch + - pkg: python3-influxdb {% endfor %} {% endif %} From d792c65ce326e246ae603fcf3b82dd162da4ac60 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 09:25:25 -0400 Subject: [PATCH 023/378] change how influx is patch and python3-influxdb is installed --- salt/influxdb/init.sls | 11 ++++------- salt/salt/minion.sls | 20 +------------------- salt/salt/python3-influxdb.sls | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 26 deletions(-) create mode 100644 salt/salt/python3-influxdb.sls diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 67342f826..55f350099 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -12,10 +12,7 @@ include: - salt.minion - -python3-influxdb: - pkg.installed: - - name: python3-influxdb + - salt.python3-influxdb # Influx DB influxconfdir: @@ -77,7 +74,7 @@ telegraf_database: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb - - pkg: python3-influxdb + - sls: salt.python3-influxdb {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: @@ -96,7 +93,7 @@ telegraf_database: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_retention_policy.present_patch - - pkg: python3-influxdb + - sls: salt.python3-influxdb {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} @@ -113,7 +110,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - - pkg: python3-influxdb + - sls: python3-influxdb {% endfor %} {% endif %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 2ac66c81a..41f9bfaf2 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -67,22 +67,4 @@ salt_minion_service: patch_pkg: pkg.installed: - - name: patch - - -#https://github.com/saltstack/salt/issues/59766 -influxdb_continuous_query.present_patch: - file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_continuous_query.py - - source: salt://salt/files/influxdb_continuous_query.py.patch - -#https://github.com/saltstack/salt/issues/59761 -influxdb_retention_policy.present_patch: - file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py - - source: salt://salt/files/influxdb_retention_policy.py.patch - -influxdbmod.py_shard_duration_patch: - file.patch: - - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py - - source: salt://salt/files/influxdbmod.py.patch \ No newline at end of file + - name: patch \ No newline at end of file diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls new file mode 100644 index 000000000..0f8cad42e --- /dev/null +++ b/salt/salt/python3-influxdb.sls @@ -0,0 +1,23 @@ +python3_influxdb: + pkg.installed: + - name: python3-influxdb + +#https://github.com/saltstack/salt/issues/59766 +influxdb_continuous_query.present_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_continuous_query.py + - source: salt://salt/files/influxdb_continuous_query.py.patch + - pkg: python3_influxdb + +#https://github.com/saltstack/salt/issues/59761 +influxdb_retention_policy.present_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py + - source: salt://salt/files/influxdb_retention_policy.py.patch + - pkg: python3_influxdb + +influxdbmod.py_shard_duration_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py + - source: salt://salt/files/influxdbmod.py.patch + - pkg: python3_influxdb \ No newline at end of file From af86a9dac01fe511677d72718f1ac30675a79fe1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 09:52:22 -0400 Subject: [PATCH 024/378] handle different paths for salt states/modules based on os --- salt/salt/map.jinja | 4 ++++ salt/salt/python3-influxdb.sls | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5d6d980be..489a41dd2 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -4,9 +4,13 @@ {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?', python_shell=True) %} + {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} + {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 0f8cad42e..f0536e36f 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,3 +1,6 @@ +{% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} +{% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} + python3_influxdb: pkg.installed: - name: python3-influxdb @@ -5,19 +8,19 @@ python3_influxdb: #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_continuous_query.py + - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - pkg: python3_influxdb #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py + - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - pkg: python3_influxdb influxdbmod.py_shard_duration_patch: file.patch: - - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py + - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - pkg: python3_influxdb \ No newline at end of file From b53017ee87603f1d15c441bb4913a65b2989fd1e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 10:44:46 -0400 Subject: [PATCH 025/378] ensure salt python modules are installed and patched during soup --- salt/common/tools/sbin/soup | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2a1ddab1c..2d9e0a146 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -729,6 +729,13 @@ echo "" echo "Starting Salt Master service." systemctl start salt-master +sleep 10 + +echo "" +echo "Ensuring python modules for Salt are installed and patched." +salt-call state.apply python3-influxdb -l info queue=True +echo "" + # Only regenerate osquery packages if Fleet is enabled FLEET_MANAGER=$(lookup_pillar fleet_manager) FLEET_NODE=$(lookup_pillar fleet_node) From 0a01d7b0412e14b9d16bb9e317bc5437504cc4a4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 11:14:13 -0400 Subject: [PATCH 026/378] fix var --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 97e61e6e2..febdd4a6c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -167,7 +167,7 @@ gpg_rpm_import() { if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then local RPMKEYSLOC="../salt/repo/client/files/centos/keys" else - local RPMKEYSLOC="$UPDATEDIR/salt/repo/client/files/centos/keys" + local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/centos/keys" fi RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') From 1f654d44440fd1431d911bda96c67ea1db1ba663 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 12:04:58 -0400 Subject: [PATCH 027/378] fix the state apply --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2d9e0a146..05c56008a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -733,7 +733,7 @@ sleep 10 echo "" echo "Ensuring python modules for Salt are installed and patched." -salt-call state.apply python3-influxdb -l info queue=True +salt-call state.apply salt.python3-influxdb -l info queue=True echo "" # Only regenerate osquery packages if Fleet is enabled From 17c95723ec6670b94fb1f03518ffa2b11a35a03e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 16:44:40 -0400 Subject: [PATCH 028/378] update package name --- salt/salt/python3-influxdb.sls | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index f0536e36f..41989816f 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,26 +1,26 @@ {% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} -python3_influxdb: +securityonion_python3_influxdb: pkg.installed: - - name: python3-influxdb + - name: securityonion-python3-influxdb #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - - pkg: python3_influxdb + - pkg: securityonion_python3_influxdb #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - - pkg: python3_influxdb + - pkg: securityonion_python3_influxdb influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - - pkg: python3_influxdb \ No newline at end of file + - pkg: securityonion_python3_influxdb \ No newline at end of file From c9298137b52c1466f087246de153d553d36d6d04 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 17:08:35 -0400 Subject: [PATCH 029/378] adding docker-ce.repo to delete list --- salt/repo/client/map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/repo/client/map.jinja b/salt/repo/client/map.jinja index ccfa1eae2..dedb96783 100644 --- a/salt/repo/client/map.jinja +++ b/salt/repo/client/map.jinja @@ -10,6 +10,7 @@ 'CentOS-Sources.repo', 'CentOS-Vault.repo', 'CentOS-x86_64-kernel.repo', + 'docker-ce.repo', 'epel.repo', 'epel-testing.repo', 'saltstack.repo', From a7247e98121f8dc1ee13760efc4403be3ae8a83b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 17:17:49 -0400 Subject: [PATCH 030/378] update package name --- salt/salt/map.jinja | 2 ++ salt/salt/python3-influxdb.sls | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 489a41dd2..fb6cadb1b 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -6,11 +6,13 @@ {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?', python_shell=True) %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} + {% set PYTHON3INFLUX= 'python3-influxdb' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} + {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 41989816f..36347cb95 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,26 +1,27 @@ {% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} +{% from "salt/map.jinja" import PYTHON3INFLUX with context %} -securityonion_python3_influxdb: +python3_influxdb: pkg.installed: - - name: securityonion-python3-influxdb + - name: {{ PYTHON3INFLUX }} #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - - pkg: securityonion_python3_influxdb + - pkg: python3_influxdb #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - - pkg: securityonion_python3_influxdb + - pkg: python3_influxdb influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - - pkg: securityonion_python3_influxdb \ No newline at end of file + - pkg: python3_influxdb \ No newline at end of file From 5cedf98f555d79c1b80276ebb31ce125f5073a66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:05:52 -0400 Subject: [PATCH 031/378] change path --- salt/salt/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index fb6cadb1b..67ab4a594 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -10,8 +10,8 @@ {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} - {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} - {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} + {% set SALT_STATE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% endif %} From 21077ef26e09c2f1a0578d24e13ee23d01db05d1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:09:11 -0400 Subject: [PATCH 032/378] undo path change --- salt/salt/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 67ab4a594..fb6cadb1b 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -10,8 +10,8 @@ {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} - {% set SALT_STATE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/states' %} - {% set SALT_MODULE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/modules' %} + {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% endif %} From f5ddb084b6f850f23652432a85fe60254f3169c3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:22:44 -0400 Subject: [PATCH 033/378] create salt.helper-packages state, use it to install the patch package --- salt/common/init.sls | 2 -- salt/salt/helper-packages.sls | 3 +++ salt/salt/minion.sls | 1 + salt/salt/python3-influxdb.sls | 15 ++++++++++++--- 4 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 salt/salt/helper-packages.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index 6d2a9aea7..cb6039756 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -95,7 +95,6 @@ commonpkgs: - python3-mysqldb - python3-packaging - git - - patch heldpackages: pkg.installed: @@ -135,7 +134,6 @@ commonpkgs: - lvm2 - openssl - git - - patch heldpackages: pkg.installed: diff --git a/salt/salt/helper-packages.sls b/salt/salt/helper-packages.sls new file mode 100644 index 000000000..93ad76a22 --- /dev/null +++ b/salt/salt/helper-packages.sls @@ -0,0 +1,3 @@ +patch_package: + pkg.installed: + - name: patch \ No newline at end of file diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 41f9bfaf2..0dc8760cb 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -7,6 +7,7 @@ include: - salt + - salt.helper_packages - systemd.reload {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 36347cb95..b939c3380 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -2,6 +2,9 @@ {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} {% from "salt/map.jinja" import PYTHON3INFLUX with context %} +include: + - salt.helper-packages + python3_influxdb: pkg.installed: - name: {{ PYTHON3INFLUX }} @@ -11,17 +14,23 @@ influxdb_continuous_query.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - - pkg: python3_influxdb + - require: + - pkg: python3_influxdb + - pkg: patch_package #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - - pkg: python3_influxdb + - require: + - pkg: python3_influxdb + - pkg: patch_package influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - - pkg: python3_influxdb \ No newline at end of file + - require: + - pkg: python3_influxdb + - pkg: patch_package \ No newline at end of file From 1fbf77d09067ca71c5b913d948cf714bf60904e9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:53:00 -0400 Subject: [PATCH 034/378] fix state name --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 0dc8760cb..2f4c3d2ac 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -7,7 +7,7 @@ include: - salt - - salt.helper_packages + - salt.helper-packages - systemd.reload {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} From 28982e0e0b9fa2cec0f8bd4f1ea734f99551da5a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 19:22:07 -0400 Subject: [PATCH 035/378] fix requirement --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 55f350099..8248c6228 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -110,7 +110,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - - sls: python3-influxdb + - sls: salt.python3-influxdb {% endfor %} {% endif %} From 8dd9564171b145126cfa4d02539c5ce395380def Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 28 Apr 2021 10:17:37 -0400 Subject: [PATCH 036/378] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 58e1f4fa9..678d59d4f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.50 +2.3.60 From 14557983e1714cb14d685b8168a9e7d2be47a54a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 28 Apr 2021 10:24:39 -0400 Subject: [PATCH 037/378] Update raid.sh --- salt/telegraf/scripts/raid.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index 03e309c38..f237e8021 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -16,6 +16,8 @@ # along with this program. If not, see . THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) +RAIDLOG=/var/log/raid/status.log +RAIDSTATUS=$(cat /var/log/raid/status.log) if [ ! "$THEGREP" ]; then From 7a21c44727f2b5b28cbf393040f5ebc1da183349 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 28 Apr 2021 13:27:11 -0400 Subject: [PATCH 038/378] Remove unused and incorrectly formatted osraid metric --- salt/common/tools/sbin/so-raid-status | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status index 11909e012..e7da94878 100755 --- a/salt/common/tools/sbin/so-raid-status +++ b/salt/common/tools/sbin/so-raid-status @@ -17,18 +17,6 @@ . /usr/sbin/so-common -#check_boss_raid() { -# BOSSBIN=/opt/boss/mvcli -# BOSSRC=$($BOSSBIN info -o vd | grep functional) -# -# if [[ $BOSSRC ]]; then -# # Raid is good -# BOSSRAID=0 -# else -# BOSSRAID=1 -# fi -#} - check_lsi_raid() { # For use for LSI on Ubuntu #MEGA=/opt/MegaRAID/MegeCli/MegaCli64 @@ -66,13 +54,11 @@ mkdir -p /opt/so/log/raid {%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %} #check_boss_raid check_software_raid -#echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log -echo "osraid=1 nsmraid=$SWRAID" > /opt/so/log/raid/status.log +echo "nsmraid=$SWRAID" > /opt/so/log/raid/status.log {%- elif grains['sosmodel'] in ['SOS1000F', 'SOS1000', 'SOSSN7200', 'SOS10K', 'SOS4000'] %} #check_boss_raid check_lsi_raid -#echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log -echo "osraid=1 nsmraid=$LSIRAID" > /opt/so/log/raid/status.log +echo "nsmraid=$LSIRAID" > /opt/so/log/raid/status.log {%- else %} exit 0 {%- endif %} From 5584c4f1aee056a0d8946bb74776157a1f71a859 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 28 Apr 2021 18:01:33 -0400 Subject: [PATCH 039/378] define and install the dependencies for python36-influxdb --- salt/salt/map.jinja | 1 + salt/salt/python3-influxdb.sls | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 21fcb9a58..a4289a6b8 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -15,6 +15,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} + {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index b939c3380..424f06493 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,10 +1,15 @@ {% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} {% from "salt/map.jinja" import PYTHON3INFLUX with context %} +{% from "salt/map.jinja" import PYTHON3INFLUXDEPS with context %} include: - salt.helper-packages +python3_influxdb_dependencies: + pkg.installed: + - pkgs: {{ PYTHON3INFLUXDEPS }} + python3_influxdb: pkg.installed: - name: {{ PYTHON3INFLUX }} From 454b541a2e02e8eddc8182726059ea60eb1eea79 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 28 Apr 2021 18:04:14 -0400 Subject: [PATCH 040/378] merge with dev, change version so test box doesnt try to upgrade to 2.3.60 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 678d59d4f..58e1f4fa9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.60 +2.3.50 From f8e01d5d53d83b3cf3221037abd2f18532c8985a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 06:43:05 -0400 Subject: [PATCH 041/378] let the state retry incase influxdb isnt fully up yet --- salt/influxdb/init.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 8248c6228..22e2c9856 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -75,6 +75,10 @@ telegraf_database: - require: - docker_container: so-influxdb - sls: salt.python3-influxdb + - timeout: 30 + - retry: + attempts: 5 + interval: 30 {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: From 907dbe638886e37947714e74d935270a6f9dde84 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 08:47:33 -0400 Subject: [PATCH 042/378] for for influx to be up so the reliant states dont fail --- salt/influxdb/init.sls | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 22e2c9856..609263bb5 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -63,6 +63,16 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +wait_for_influxdb: + http.query: + - name: 'https://{{MANAGER}}:8086/query?q=SHOW+DATABASES' + - ssl: True + - verify_ssl: False + - status: 200 + - timeout: 30 + - retry: + attempts: 5 + interval: 60 telegraf_database: influxdb_database.present: @@ -75,10 +85,7 @@ telegraf_database: - require: - docker_container: so-influxdb - sls: salt.python3-influxdb - - timeout: 30 - - retry: - attempts: 5 - interval: 30 + - http: wait_for_influxdb {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: From 93ee96b1cd224176f1e5e107cd3839e186b9a4ac Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 10:19:42 -0400 Subject: [PATCH 043/378] Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry --- setup/so-setup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index d7dd3b660..2160929cf 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -918,10 +918,11 @@ success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}') if [[ $success != 0 ]]; then SO_ERROR=1; fi # Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox -if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then +# Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry +if grep -q -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then SO_ERROR=1 - grep --color=never "ERROR" "$setup_log" > "$error_log" + grep --color=never "ERROR" "$setup_log" | grep -qvE "Status .* was not found" > "$error_log" fi if [[ -n $SO_ERROR ]]; then From 02b621bd2cec52d65c4924974ccbebd4e80b4f87 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 29 Apr 2021 11:29:07 -0400 Subject: [PATCH 044/378] Add Kratos to Backups --- salt/common/tools/sbin/so-config-backup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-config-backup b/salt/common/tools/sbin/so-config-backup index c8e504b4a..32be845ae 100755 --- a/salt/common/tools/sbin/so-config-backup +++ b/salt/common/tools/sbin/so-config-backup @@ -35,6 +35,7 @@ if [ ! -f $BACKUPFILE ]; then {%- endfor %} tar -rf $BACKUPFILE /etc/pki tar -rf $BACKUPFILE /etc/salt + tar -rf $BACKUPFILE /opt/so/conf/kratos fi From 76d735ff43eb7f55b59d4bffe6fa6541a5cb556c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 29 Apr 2021 11:49:20 -0400 Subject: [PATCH 045/378] Add ID Fix to nodeID --- salt/sensoroni/files/sensoroni.json | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 378d42373..1a6e6fc8c 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -15,6 +15,7 @@ "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", "agent": { + "nodeId": "{{ grains.host | lower }}", "role": "{{ grains.role }}", "description": "{{ DESCRIPTION }}", "address": "{{ ADDRESS }}", From 44ad8ce8881ff66c5b2a5f0f8ace6b231cf9d2e7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 29 Apr 2021 12:08:07 -0400 Subject: [PATCH 046/378] Switch to the ES-included community_id plugin --- salt/elasticsearch/files/ingest/filterlog | 2 +- salt/elasticsearch/files/ingest/sysmon | 4 ++-- salt/elasticsearch/files/ingest/zeek.common | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/files/ingest/filterlog b/salt/elasticsearch/files/ingest/filterlog index cb914f4a3..3e885fe54 100644 --- a/salt/elasticsearch/files/ingest/filterlog +++ b/salt/elasticsearch/files/ingest/filterlog @@ -51,7 +51,7 @@ }, { "set": { "field": "_index", "value": "so-firewall", "override": true } }, { "set": { "if": "ctx.network?.transport_id == '0'", "field": "network.transport", "value": "icmp", "override": true } }, - {"community_id": { "if": "ctx.network?.transport != null", "field":["source.ip","source.port","destination.ip","destination.port","network.transport"],"target_field":"network.community_id"}}, + {"community_id": {} }, { "set": { "field": "module", "value": "pfsense", "override": true } }, { "set": { "field": "dataset", "value": "firewall", "override": true } }, { "remove": { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"], "ignore_failure": true } } diff --git a/salt/elasticsearch/files/ingest/sysmon b/salt/elasticsearch/files/ingest/sysmon index 599899488..e4db4bcb4 100644 --- a/salt/elasticsearch/files/ingest/sysmon +++ b/salt/elasticsearch/files/ingest/sysmon @@ -1,7 +1,6 @@ { "description" : "sysmon", "processors" : [ - {"community_id": {"if": "ctx.winlog.event_data?.Protocol != null", "field":["winlog.event_data.SourceIp","winlog.event_data.SourcePort","winlog.event_data.DestinationIp","winlog.event_data.DestinationPort","winlog.event_data.Protocol"],"target_field":"network.community_id"}}, { "set": { "field": "event.code", "value": "{{winlog.event_id}}", "override": true } }, { "set": { "field": "event.module", "value": "sysmon", "override": true } }, { "set": { "if": "ctx.winlog?.computer_name != null", "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true } }, @@ -64,6 +63,7 @@ { "rename": { "field": "winlog.event_data.SourceIp", "target_field": "source.ip", "ignore_missing": true } }, { "rename": { "field": "winlog.event_data.SourcePort", "target_field": "source.port", "ignore_missing": true } }, { "rename": { "field": "winlog.event_data.targetFilename", "target_field": "file.target", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } } + { "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } }, + { "community_id": {} } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common index 563f5956b..e7b898c6f 100644 --- a/salt/elasticsearch/files/ingest/zeek.common +++ b/salt/elasticsearch/files/ingest/zeek.common @@ -8,11 +8,11 @@ { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, - {"community_id": {"if": "ctx.network?.transport != null", "field":["message2.id.orig_h","message2.id.orig_p","message2.id.resp_h","message2.id.resp_p","network.transport"],"target_field":"network.community_id"}}, { "rename": { "field": "message2.id.orig_h", "target_field": "source.ip", "ignore_missing": true } }, { "rename": { "field": "message2.id.orig_p", "target_field": "source.port", "ignore_missing": true } }, { "rename": { "field": "message2.id.resp_h", "target_field": "destination.ip", "ignore_missing": true } }, { "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } }, + { "community_id": {} }, { "set": { "if": "ctx.source?.ip != null", "field": "client.ip", "value": "{{source.ip}}" } }, { "set": { "if": "ctx.source?.port != null", "field": "client.port", "value": "{{source.port}}" } }, { "set": { "if": "ctx.destination?.ip != null", "field": "server.ip", "value": "{{destination.ip}}" } }, From 450a01784befcb723409be8e293905f134c4d1c8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:22:31 -0400 Subject: [PATCH 047/378] support installing via pip for ubuntu --- salt/salt/map.jinja | 7 +++++-- salt/salt/python3-influxdb.sls | 10 +++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index a4289a6b8..8a3c26bb8 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -7,7 +7,9 @@ {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} - {% set PYTHON3INFLUX= 'python3-influxdb' %} + {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} + {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] + {% set PYTHONINSTALLER = 'pip' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} @@ -15,7 +17,8 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} - {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} + {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] + {% set PYTHONINSTALLER = 'pkg' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 424f06493..67fde3be2 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -2,16 +2,19 @@ {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} {% from "salt/map.jinja" import PYTHON3INFLUX with context %} {% from "salt/map.jinja" import PYTHON3INFLUXDEPS with context %} +{% from "salt/map.jinja" import PYTHONINSTALLER with context %} include: - salt.helper-packages +{# python3_influxdb_dependencies: - pkg.installed: + {{PYTHONINSTALLER}}.installed: - pkgs: {{ PYTHON3INFLUXDEPS }} +#} python3_influxdb: - pkg.installed: + {{PYTHONINSTALLER}}.installed: - name: {{ PYTHON3INFLUX }} #https://github.com/saltstack/salt/issues/59766 @@ -38,4 +41,5 @@ influxdbmod.py_shard_duration_patch: - source: salt://salt/files/influxdbmod.py.patch - require: - pkg: python3_influxdb - - pkg: patch_package \ No newline at end of file + - pkg: patch_package + - reload_modules: True \ No newline at end of file From e7660d68cb50827b9ed6901b16c4adcb0d7661bc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:25:29 -0400 Subject: [PATCH 048/378] add %} --- salt/salt/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 8a3c26bb8..9cf9331f5 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -8,7 +8,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} - {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] + {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} {% else %} {% set SPLITCHAR = '-' %} From 888d637b67e6444edb012c57d8a3531b36d537ef Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:26:24 -0400 Subject: [PATCH 049/378] add %} --- salt/salt/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 9cf9331f5..9b84b72e2 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -17,7 +17,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} - {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] + {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} {% endif %} From 689a01423fea65468f4b592052c66bdf7eab37c7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:28:31 -0400 Subject: [PATCH 050/378] fix deps --- salt/salt/python3-influxdb.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 67fde3be2..3cad07144 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -23,7 +23,7 @@ influxdb_continuous_query.present_patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - require: - - pkg: python3_influxdb + - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package #https://github.com/saltstack/salt/issues/59761 @@ -32,7 +32,7 @@ influxdb_retention_policy.present_patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - require: - - pkg: python3_influxdb + - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package influxdbmod.py_shard_duration_patch: @@ -40,6 +40,6 @@ influxdbmod.py_shard_duration_patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - require: - - pkg: python3_influxdb + - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package - reload_modules: True \ No newline at end of file From d475e50bef82df40fb59ce15dc087fe088abc314 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:49:15 -0400 Subject: [PATCH 051/378] add deps for ubuntu --- salt/salt/map.jinja | 2 +- salt/salt/python3-influxdb.sls | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 9b84b72e2..5b211e969 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -8,7 +8,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} - {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] %} + {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} {% else %} {% set SPLITCHAR = '-' %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 3cad07144..f65a0c6b4 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -7,11 +7,9 @@ include: - salt.helper-packages -{# python3_influxdb_dependencies: {{PYTHONINSTALLER}}.installed: - pkgs: {{ PYTHON3INFLUXDEPS }} -#} python3_influxdb: {{PYTHONINSTALLER}}.installed: From b6561fd8e2f20a00a9a9c42212c66ceb4ad414ae Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 29 Apr 2021 15:35:22 -0400 Subject: [PATCH 052/378] Update defaultRoute with new path structure --- salt/kibana/etc/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/etc/config.json b/salt/kibana/etc/config.json index 9631e0343..7593a0c27 100644 --- a/salt/kibana/etc/config.json +++ b/salt/kibana/etc/config.json @@ -1,7 +1,7 @@ { "attributes": { "defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29", - "defaultRoute":"/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645", + "defaultRoute":"/kibana/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645", "discover:sampleSize":"100", "dashboard:defaultDarkTheme":true, "theme:darkMode":true, From dda07af4d45a38ae290fc9277e806a0be176525e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 29 Apr 2021 17:44:15 -0400 Subject: [PATCH 053/378] Update Kibana config defaults --- salt/kibana/etc/config.json | 10 ---------- salt/kibana/files/saved_objects.ndjson | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 salt/kibana/etc/config.json diff --git a/salt/kibana/etc/config.json b/salt/kibana/etc/config.json deleted file mode 100644 index 7593a0c27..000000000 --- a/salt/kibana/etc/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ "attributes": - { - "defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29", - "defaultRoute":"/kibana/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645", - "discover:sampleSize":"100", - "dashboard:defaultDarkTheme":true, - "theme:darkMode":true, - "timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}" - } -} diff --git a/salt/kibana/files/saved_objects.ndjson b/salt/kibana/files/saved_objects.ndjson index fee868127..863209d48 100644 --- a/salt/kibana/files/saved_objects.ndjson +++ b/salt/kibana/files/saved_objects.ndjson @@ -460,7 +460,7 @@ {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\",\"time_zone\":\"America/New_York\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":48,\"x\":0,\"y\":0,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":96,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":8,\"y\":20,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":28,\"y\":20,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":0,\"y\":72,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":48,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":24,\"x\":16,\"y\":72,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":28,\"y\":8,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":120,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"event_type\",\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"}]","timeRestore":false,"title":"z16.04 - Sysmon - Logs","version":1},"id":"6d189680-6d62-11e7-8ddb-e71eb260f4a3","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"8cfdeff0-6d6b-11e7-ad64-15aa071374a6","name":"panel_1","type":"visualization"},{"id":"0eb1fd80-6d70-11e7-b09b-f57b22df6524","name":"panel_2","type":"visualization"},{"id":"3072c750-6d71-11e7-b09b-f57b22df6524","name":"panel_3","type":"visualization"},{"id":"7bc74b40-6d71-11e7-b09b-f57b22df6524","name":"panel_4","type":"visualization"},{"id":"13ed0810-6d72-11e7-b09b-f57b22df6524","name":"panel_5","type":"visualization"},{"id":"3b6c92c0-6d72-11e7-b09b-f57b22df6524","name":"panel_6","type":"visualization"},{"id":"e09f6010-6d72-11e7-b09b-f57b22df6524","name":"panel_7","type":"visualization"},{"id":"29611940-6d75-11e7-b09b-f57b22df6524","name":"panel_8","type":"visualization"},{"id":"6b70b840-6d75-11e7-b09b-f57b22df6524","name":"panel_9","type":"visualization"},{"id":"248c1d20-6d6b-11e7-ad64-15aa071374a6","name":"panel_10","type":"search"},{"id":"AWDHHk1sxQT5EBNmq43Y","name":"panel_11","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQyLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} -{"attributes":{"buildNum":33984,"dashboard:defaultDarkTheme":true,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":"100","theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"id":"7.11.2","migrationVersion":{"config":"7.9.0"},"references":[],"type":"config","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ1LDRd"} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.12.1","id":"7.12.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":56,\"x\":0,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":8,\"y\":32,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":20,\"y\":32,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":56,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"columns\":[\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"uid\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":32,\"y\":32,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":40,\"h\":24,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"}]","timeRestore":false,"title":"z16.04 - Bro - Modbus","version":1},"id":"70c005f0-3583-11e7-a588-05992195c551","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"0d168a30-363f-11e7-a6f7-4f44d7bf1c33","name":"panel_1","type":"visualization"},{"id":"20eabd60-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_2","type":"visualization"},{"id":"3c65f500-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_3","type":"visualization"},{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"panel_4","type":"search"},{"id":"178209e0-6e1b-11e7-b553-7f80727663c1","name":"panel_5","type":"visualization"},{"id":"AWDG_9KpxQT5EBNmq4Oo","name":"panel_6","type":"visualization"},{"id":"453f8b90-4a58-11e8-9b0a-f1d33346f773","name":"panel_7","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ4LDRd"} From e32ca284c52c86aee3274bcf0940f6c6d26920c7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 30 Apr 2021 11:10:31 -0400 Subject: [PATCH 054/378] ensure proper version of python3-influxdb is installed prior to running the states that require it --- salt/influxdb/init.sls | 6 ++++++ salt/salt/python3-influxdb.sls | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 609263bb5..cda095a31 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -7,6 +7,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% import_yaml 'influxdb/defaults.yaml' as default_settings %} {% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} +{% from 'salt/map.jinja' import PYTHON3INFLUX with context %} +{% set PYTHONINFLUXVERSION = PYTHON3INFLUX.split("==")[1] | trim %} +{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) | trim %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} @@ -63,6 +66,8 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +# We have to make sure the influxdb module is the right version prior to state run since reload_modules is bugged +{% if PYTHONINFLUXVERSIONINSTALLED == PYTHONINFLUXVERSION %} wait_for_influxdb: http.query: - name: 'https://{{MANAGER}}:8086/query?q=SHOW+DATABASES' @@ -124,6 +129,7 @@ so_downsample_cq: - sls: salt.python3-influxdb {% endfor %} +{% endif %} {% endif %} {% else %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index f65a0c6b4..acadf2161 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -33,11 +33,12 @@ influxdb_retention_policy.present_patch: - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package +# We should be able to set reload_modules: True in this state in order to tell salt to reload its python modules due to us possibly installing +# and possibly modifying modules in this state. This is bugged according to https://github.com/saltstack/salt/issues/24925 influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - require: - {{PYTHONINSTALLER}}: python3_influxdb - - pkg: patch_package - - reload_modules: True \ No newline at end of file + - pkg: patch_package \ No newline at end of file From 038cadeae892c6efdb824cb6274eaee8bf4b4924 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 30 Apr 2021 12:31:57 -0400 Subject: [PATCH 055/378] change version to 2.3.60 to prepare to push to dev --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 58e1f4fa9..678d59d4f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.50 +2.3.60 From d6b9154a88368e710cd5683036b3ec76db4d6dfd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 30 Apr 2021 14:48:51 -0400 Subject: [PATCH 056/378] change how version to be installed is defined to work with centos --- salt/influxdb/init.sls | 4 ++-- salt/salt/map.jinja | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index cda095a31..c6120d292 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -8,8 +8,8 @@ {% import_yaml 'influxdb/defaults.yaml' as default_settings %} {% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} {% from 'salt/map.jinja' import PYTHON3INFLUX with context %} -{% set PYTHONINFLUXVERSION = PYTHON3INFLUX.split("==")[1] | trim %} -{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) | trim %} +{% from 'salt/map.jinja' import PYTHONINFLUXVERSION with context %} +{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5b211e969..b0b9ffb2a 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -7,7 +7,8 @@ {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} - {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} + {% set PYTHONINFLUXVERSION = '5.3.1' %} + {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION %} {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} {% else %} @@ -16,6 +17,7 @@ {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} + {% set PYTHONINFLUXVERSION = '5.3.1' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} From b80dd1ef3eb0f34df9a1f5642f08ba8da36abe02 Mon Sep 17 00:00:00 2001 From: Pete Date: Fri, 30 Apr 2021 20:21:50 +0000 Subject: [PATCH 057/378] fix 5-second sleep using wait here instead of sleep tries to wait until pid 5 exits and generates the error > /usr/sbin/so-playbook-reset: line 25: wait: pid 5 is not a child of this shell --- salt/common/tools/sbin/so-playbook-reset | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-playbook-reset b/salt/common/tools/sbin/so-playbook-reset index 9113fd2b8..ca1d3f57f 100755 --- a/salt/common/tools/sbin/so-playbook-reset +++ b/salt/common/tools/sbin/so-playbook-reset @@ -22,5 +22,5 @@ salt-call state.apply playbook.db_init,playbook,playbook.automation_user_create /usr/sbin/so-soctopus-restart echo "Importing Plays - this will take some time...." -wait 5 -/usr/sbin/so-playbook-ruleupdate \ No newline at end of file +sleep 5 +/usr/sbin/so-playbook-ruleupdate From 19dd9b97d29256247d24ecc9520ace3128116ab2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 3 May 2021 09:40:53 -0400 Subject: [PATCH 058/378] Don't ask for node description on eval and import installs --- setup/so-setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index d7dd3b660..f85cba628 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -291,7 +291,7 @@ if ! [[ -f $install_opt_file ]]; then collect_hostname fi - whiptail_node_description + [[ ! ( $is_eval || $is_import ) ]] && whiptail_node_description if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init_whiptail @@ -960,4 +960,4 @@ fi install_cleanup >> "$setup_log" 2>&1 -if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit; fi \ No newline at end of file +if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit; fi From 619402cc671bde3c99b4473d11d8effd37ddb752 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 3 May 2021 17:03:30 +0000 Subject: [PATCH 059/378] Add event_data to common template so elastalert/playbook event_data fields can be indexed and searchable --- salt/elasticsearch/templates/so/so-common-template.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index c1f0a6755..8adbdcf13 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -228,7 +228,11 @@ "event":{ "type":"object", "dynamic": true - }, + }, + "event_data":{ + "type":"object", + "dynamic": true + }, "file":{ "type":"object", "dynamic": true From 12306368cfe6125869d32319ef471d76856c2aee Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 08:37:52 -0400 Subject: [PATCH 060/378] add post upgrade function for 2.3.60 soup to migrate influxdb data --- salt/common/tools/sbin/soup | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b3484446f..395ebef36 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -284,6 +284,7 @@ postupgrade_changes() { [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2 [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30 [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40 + [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60 } post_rc1_to_2.3.21() { @@ -304,6 +305,15 @@ post_2.3.30_to_2.3.40() { POSTVERSION=2.3.40 } +post_2.3.5X_to_2.3.60() { + # These are post upgrade changes that need to take place after all the new states have been applied. + echo "" + echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes." + docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$SOUP_LOG" 2>&1 & + + POSTVERSION=2.3.60 +} + rc1_to_rc2() { From 9b065155f46bd609ccf24e9b818a0219ed00d728 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 08:56:13 -0400 Subject: [PATCH 061/378] log iunfluxdb migration to new log --- salt/common/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 395ebef36..b1e326c73 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -24,6 +24,7 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) DEFAULT_SALT_DIR=/opt/so/saltstack/default BATCHSIZE=5 SOUP_LOG=/root/soup.log +INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup add_common() { @@ -309,7 +310,7 @@ post_2.3.5X_to_2.3.60() { # These are post upgrade changes that need to take place after all the new states have been applied. echo "" echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes." - docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$SOUP_LOG" 2>&1 & + docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & POSTVERSION=2.3.60 } From f186a3dde923ce29fe9eccd7a7cd0e451d4c65b4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 09:30:38 -0400 Subject: [PATCH 062/378] make sure user sees influxdb migration message by requiring enter to be pressed to continue --- salt/common/tools/sbin/soup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b1e326c73..b08271e1f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -309,7 +309,9 @@ post_2.3.30_to_2.3.40() { post_2.3.5X_to_2.3.60() { # These are post upgrade changes that need to take place after all the new states have been applied. echo "" - echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes." + echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." + echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" + read -p "Press enter to continue." docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & POSTVERSION=2.3.60 From e2f52765e4ae48e851ddad7675df6239906c43c5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 09:34:42 -0400 Subject: [PATCH 063/378] add newline --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b08271e1f..de968caea 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -312,6 +312,7 @@ post_2.3.5X_to_2.3.60() { echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." + echo "" docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & POSTVERSION=2.3.60 From a0f183916237be25782ea7c1ed8534cb84a052e4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 09:59:16 -0400 Subject: [PATCH 064/378] run in background --- salt/common/tools/sbin/soup | 2 +- salt/influxdb/defaults.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index de968caea..4c82e48c7 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -313,7 +313,7 @@ post_2.3.5X_to_2.3.60() { echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." echo "" - docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & + docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 POSTVERSION=2.3.60 } diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 03ac4c3c6..54e175024 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,12 +2,12 @@ influxdb: retention_policies: autogen: default: True - duration: 1d - shard_duration: 1h + duration: 30d + shard_duration: 1d so_long_term: default: False - duration: 7d - shard_duration: 1d + duration: 0s + shard_duration: 7d downsample: so_long_term: resolution: 5m \ No newline at end of file From 2d91e509faecf7f32bb22b808e2c5817a3248bb7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 10:34:13 -0400 Subject: [PATCH 065/378] update wording --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 4c82e48c7..0e6f84a63 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -309,7 +309,7 @@ post_2.3.30_to_2.3.40() { post_2.3.5X_to_2.3.60() { # These are post upgrade changes that need to take place after all the new states have been applied. echo "" - echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." + echo "Migration of InfluxDB data will begin in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." echo "" From 3dcaa1f6fb8494d08935ceecec14d8e7972b0c32 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 10:37:25 -0400 Subject: [PATCH 066/378] more logging for influxdb migration --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0e6f84a63..07a9804b2 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -313,7 +313,7 @@ post_2.3.5X_to_2.3.60() { echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." echo "" - docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 + docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*" && echo "InfluxDB migration complete." || echo "InfluxDB migration failed."' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 POSTVERSION=2.3.60 } From 0622c77a7f51dda93ddce0bbf2f12b2b8cbb925f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 May 2021 10:50:13 -0400 Subject: [PATCH 067/378] Add filebeat modules --- salt/filebeat/modules/activemq.yml.disabled | 19 ++ salt/filebeat/modules/apache.yml.disabled | 19 ++ salt/filebeat/modules/auditd.yml.disabled | 10 + salt/filebeat/modules/aws.yml.disabled | 255 ++++++++++++++++++ salt/filebeat/modules/azure.yml.disabled | 45 ++++ salt/filebeat/modules/barracuda.yml.disabled | 41 +++ salt/filebeat/modules/bluecoat.yml.disabled | 22 ++ salt/filebeat/modules/cef.yml.disabled | 17 ++ salt/filebeat/modules/checkpoint.yml.disabled | 24 ++ salt/filebeat/modules/cisco.yml.disabled | 142 ++++++++++ salt/filebeat/modules/coredns.yml.disabled | 11 + .../filebeat/modules/crowdstrike.yml.disabled | 11 + salt/filebeat/modules/cyberark.yml.disabled | 22 ++ salt/filebeat/modules/cylance.yml.disabled | 22 ++ .../modules/elasticsearch.yml.disabled | 35 +++ salt/filebeat/modules/envoyproxy.yml.disabled | 11 + salt/filebeat/modules/f5.yml.disabled | 41 +++ salt/filebeat/modules/fortinet.yml.disabled | 83 ++++++ salt/filebeat/modules/gcp.yml.disabled | 76 ++++++ .../modules/google_workspace.yml.disabled | 53 ++++ .../filebeat/modules/googlecloud.yml.disabled | 58 ++++ salt/filebeat/modules/gsuite.yml.disabled | 53 ++++ salt/filebeat/modules/haproxy.yml.disabled | 14 + salt/filebeat/modules/ibmmq.yml.disabled | 11 + salt/filebeat/modules/icinga.yml.disabled | 27 ++ salt/filebeat/modules/iis.yml.disabled | 20 ++ salt/filebeat/modules/imperva.yml.disabled | 22 ++ salt/filebeat/modules/infoblox.yml.disabled | 22 ++ salt/filebeat/modules/iptables.yml.disabled | 13 + salt/filebeat/modules/juniper.yml.disabled | 54 ++++ salt/filebeat/modules/kafka.yml.disabled | 15 ++ salt/filebeat/modules/kibana.yml.disabled | 19 ++ salt/filebeat/modules/logstash.yml.disabled | 18 ++ salt/filebeat/modules/microsoft.yml.disabled | 49 ++++ salt/filebeat/modules/misp.yml.disabled | 17 ++ salt/filebeat/modules/mongodb.yml.disabled | 11 + salt/filebeat/modules/mssql.yml.disabled | 11 + salt/filebeat/modules/mysql.yml.disabled | 19 ++ .../modules/mysqlenterprise.yml.disabled | 14 + salt/filebeat/modules/nats.yml.disabled | 11 + salt/filebeat/modules/netflow.yml.disabled | 14 + salt/filebeat/modules/netscout.yml.disabled | 22 ++ salt/filebeat/modules/nginx.yml.disabled | 27 ++ salt/filebeat/modules/o365.yml.disabled | 48 ++++ salt/filebeat/modules/okta.yml.disabled | 10 + salt/filebeat/modules/oracle.yml.disabled | 13 + salt/filebeat/modules/osquery.yml.disabled | 15 ++ salt/filebeat/modules/panw.yml.disabled | 22 ++ salt/filebeat/modules/pensando.yml.disabled | 13 + salt/filebeat/modules/postgresql.yml.disabled | 11 + salt/filebeat/modules/proofpoint.yml.disabled | 22 ++ salt/filebeat/modules/rabbitmq.yml.disabled | 11 + salt/filebeat/modules/radware.yml.disabled | 22 ++ salt/filebeat/modules/redis.yml.disabled | 21 ++ salt/filebeat/modules/santa.yml.disabled | 9 + salt/filebeat/modules/snort.yml.disabled | 22 ++ salt/filebeat/modules/snyk.yml.disabled | 112 ++++++++ salt/filebeat/modules/sonicwall.yml.disabled | 22 ++ salt/filebeat/modules/sophos.yml.disabled | 46 ++++ salt/filebeat/modules/squid.yml.disabled | 22 ++ salt/filebeat/modules/suricata.yml.disabled | 11 + salt/filebeat/modules/system.yml.disabled | 19 ++ .../filebeat/modules/threatintel.yml.disabled | 105 ++++++++ salt/filebeat/modules/tomcat.yml.disabled | 22 ++ salt/filebeat/modules/traefik.yml.disabled | 11 + salt/filebeat/modules/zeek.yml.disabled | 84 ++++++ salt/filebeat/modules/zoom.yml.disabled | 22 ++ salt/filebeat/modules/zscaler.yml.disabled | 22 ++ 68 files changed, 2237 insertions(+) create mode 100644 salt/filebeat/modules/activemq.yml.disabled create mode 100644 salt/filebeat/modules/apache.yml.disabled create mode 100644 salt/filebeat/modules/auditd.yml.disabled create mode 100644 salt/filebeat/modules/aws.yml.disabled create mode 100644 salt/filebeat/modules/azure.yml.disabled create mode 100644 salt/filebeat/modules/barracuda.yml.disabled create mode 100644 salt/filebeat/modules/bluecoat.yml.disabled create mode 100644 salt/filebeat/modules/cef.yml.disabled create mode 100644 salt/filebeat/modules/checkpoint.yml.disabled create mode 100644 salt/filebeat/modules/cisco.yml.disabled create mode 100644 salt/filebeat/modules/coredns.yml.disabled create mode 100644 salt/filebeat/modules/crowdstrike.yml.disabled create mode 100644 salt/filebeat/modules/cyberark.yml.disabled create mode 100644 salt/filebeat/modules/cylance.yml.disabled create mode 100644 salt/filebeat/modules/elasticsearch.yml.disabled create mode 100644 salt/filebeat/modules/envoyproxy.yml.disabled create mode 100644 salt/filebeat/modules/f5.yml.disabled create mode 100644 salt/filebeat/modules/fortinet.yml.disabled create mode 100644 salt/filebeat/modules/gcp.yml.disabled create mode 100644 salt/filebeat/modules/google_workspace.yml.disabled create mode 100644 salt/filebeat/modules/googlecloud.yml.disabled create mode 100644 salt/filebeat/modules/gsuite.yml.disabled create mode 100644 salt/filebeat/modules/haproxy.yml.disabled create mode 100644 salt/filebeat/modules/ibmmq.yml.disabled create mode 100644 salt/filebeat/modules/icinga.yml.disabled create mode 100644 salt/filebeat/modules/iis.yml.disabled create mode 100644 salt/filebeat/modules/imperva.yml.disabled create mode 100644 salt/filebeat/modules/infoblox.yml.disabled create mode 100644 salt/filebeat/modules/iptables.yml.disabled create mode 100644 salt/filebeat/modules/juniper.yml.disabled create mode 100644 salt/filebeat/modules/kafka.yml.disabled create mode 100644 salt/filebeat/modules/kibana.yml.disabled create mode 100644 salt/filebeat/modules/logstash.yml.disabled create mode 100644 salt/filebeat/modules/microsoft.yml.disabled create mode 100644 salt/filebeat/modules/misp.yml.disabled create mode 100644 salt/filebeat/modules/mongodb.yml.disabled create mode 100644 salt/filebeat/modules/mssql.yml.disabled create mode 100644 salt/filebeat/modules/mysql.yml.disabled create mode 100644 salt/filebeat/modules/mysqlenterprise.yml.disabled create mode 100644 salt/filebeat/modules/nats.yml.disabled create mode 100644 salt/filebeat/modules/netflow.yml.disabled create mode 100644 salt/filebeat/modules/netscout.yml.disabled create mode 100644 salt/filebeat/modules/nginx.yml.disabled create mode 100644 salt/filebeat/modules/o365.yml.disabled create mode 100644 salt/filebeat/modules/okta.yml.disabled create mode 100644 salt/filebeat/modules/oracle.yml.disabled create mode 100644 salt/filebeat/modules/osquery.yml.disabled create mode 100644 salt/filebeat/modules/panw.yml.disabled create mode 100644 salt/filebeat/modules/pensando.yml.disabled create mode 100644 salt/filebeat/modules/postgresql.yml.disabled create mode 100644 salt/filebeat/modules/proofpoint.yml.disabled create mode 100644 salt/filebeat/modules/rabbitmq.yml.disabled create mode 100644 salt/filebeat/modules/radware.yml.disabled create mode 100644 salt/filebeat/modules/redis.yml.disabled create mode 100644 salt/filebeat/modules/santa.yml.disabled create mode 100644 salt/filebeat/modules/snort.yml.disabled create mode 100644 salt/filebeat/modules/snyk.yml.disabled create mode 100644 salt/filebeat/modules/sonicwall.yml.disabled create mode 100644 salt/filebeat/modules/sophos.yml.disabled create mode 100644 salt/filebeat/modules/squid.yml.disabled create mode 100644 salt/filebeat/modules/suricata.yml.disabled create mode 100644 salt/filebeat/modules/system.yml.disabled create mode 100644 salt/filebeat/modules/threatintel.yml.disabled create mode 100644 salt/filebeat/modules/tomcat.yml.disabled create mode 100644 salt/filebeat/modules/traefik.yml.disabled create mode 100644 salt/filebeat/modules/zeek.yml.disabled create mode 100644 salt/filebeat/modules/zoom.yml.disabled create mode 100644 salt/filebeat/modules/zscaler.yml.disabled diff --git a/salt/filebeat/modules/activemq.yml.disabled b/salt/filebeat/modules/activemq.yml.disabled new file mode 100644 index 000000000..43536ecbc --- /dev/null +++ b/salt/filebeat/modules/activemq.yml.disabled @@ -0,0 +1,19 @@ +# Module: activemq +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-activemq.html + +- module: activemq + # Audit logs + audit: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Application logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/apache.yml.disabled b/salt/filebeat/modules/apache.yml.disabled new file mode 100644 index 000000000..b923dd581 --- /dev/null +++ b/salt/filebeat/modules/apache.yml.disabled @@ -0,0 +1,19 @@ +# Module: apache +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-apache.html + +- module: apache + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/auditd.yml.disabled b/salt/filebeat/modules/auditd.yml.disabled new file mode 100644 index 000000000..76296ec85 --- /dev/null +++ b/salt/filebeat/modules/auditd.yml.disabled @@ -0,0 +1,10 @@ +# Module: auditd +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-auditd.html + +- module: auditd + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/aws.yml.disabled b/salt/filebeat/modules/aws.yml.disabled new file mode 100644 index 000000000..904bd976c --- /dev/null +++ b/salt/filebeat/modules/aws.yml.disabled @@ -0,0 +1,255 @@ +# Module: aws +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-aws.html + +- module: aws + cloudtrail: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + cloudwatch: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + ec2: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + elb: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + s3access: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + vpcflow: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 diff --git a/salt/filebeat/modules/azure.yml.disabled b/salt/filebeat/modules/azure.yml.disabled new file mode 100644 index 000000000..3b2bc1ecf --- /dev/null +++ b/salt/filebeat/modules/azure.yml.disabled @@ -0,0 +1,45 @@ +# Module: azure +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-azure.html + +- module: azure + # All logs + activitylogs: + enabled: true + var: + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + eventhub: "insights-operational-logs" + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + consumer_group: "$Default" + # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string + connection_string: "" + # the name of the storage account the state/offsets will be stored and updated + storage_account: "" + # the storage account key, this key will be used to authorize access to data in your storage account + storage_account_key: "" + + platformlogs: + enabled: false + # var: + # eventhub: "" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + + auditlogs: + enabled: false + # var: + # eventhub: "insights-logs-auditlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + signinlogs: + enabled: false + # var: + # eventhub: "insights-logs-signinlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" diff --git a/salt/filebeat/modules/barracuda.yml.disabled b/salt/filebeat/modules/barracuda.yml.disabled new file mode 100644 index 000000000..99ff85036 --- /dev/null +++ b/salt/filebeat/modules/barracuda.yml.disabled @@ -0,0 +1,41 @@ +# Module: barracuda +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-barracuda.html + +- module: barracuda + waf: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9503 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + spamfirewall: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9524 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/bluecoat.yml.disabled b/salt/filebeat/modules/bluecoat.yml.disabled new file mode 100644 index 000000000..6550c8eed --- /dev/null +++ b/salt/filebeat/modules/bluecoat.yml.disabled @@ -0,0 +1,22 @@ +# Module: bluecoat +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-bluecoat.html + +- module: bluecoat + director: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9505 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/cef.yml.disabled b/salt/filebeat/modules/cef.yml.disabled new file mode 100644 index 000000000..2de22edcc --- /dev/null +++ b/salt/filebeat/modules/cef.yml.disabled @@ -0,0 +1,17 @@ +# Module: cef +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cef.html + +- module: cef + log: + enabled: true + var: + syslog_host: localhost + syslog_port: 9003 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] diff --git a/salt/filebeat/modules/checkpoint.yml.disabled b/salt/filebeat/modules/checkpoint.yml.disabled new file mode 100644 index 000000000..9d34b8d72 --- /dev/null +++ b/salt/filebeat/modules/checkpoint.yml.disabled @@ -0,0 +1,24 @@ +# Module: checkpoint +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-checkpoint.html + +- module: checkpoint + firewall: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: syslog + + # The interface to listen to UDP based syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The UDP port to listen for syslog traffic. Defaults to 9001. + #var.syslog_port: 9001 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] diff --git a/salt/filebeat/modules/cisco.yml.disabled b/salt/filebeat/modules/cisco.yml.disabled new file mode 100644 index 000000000..9e4658045 --- /dev/null +++ b/salt/filebeat/modules/cisco.yml.disabled @@ -0,0 +1,142 @@ +# Module: cisco +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cisco.html + +- module: cisco + asa: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: syslog + + # The interface to listen to UDP based syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The UDP port to listen for syslog traffic. Defaults to 9001. + #var.syslog_port: 9001 + + # Set the log level from 1 (alerts only) to 7 (include all messages). + # Messages with a log level higher than the specified will be dropped. + # See https://www.cisco.com/c/en/us/td/docs/security/asa/syslog/b_syslog/syslogs-sev-level.html + #var.log_level: 7 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] + + ftd: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: syslog + + # The interface to listen to UDP based syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The UDP port to listen for syslog traffic. Defaults to 9003. + #var.syslog_port: 9003 + + # Set the log level from 1 (alerts only) to 7 (include all messages). + # Messages with a log level higher than the specified will be dropped. + # See https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide/syslogs-sev-level.html + #var.log_level: 7 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] + + ios: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: syslog + + # The interface to listen to UDP based syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The UDP port to listen for syslog traffic. Defaults to 9002. + #var.syslog_port: 9002 + + # Set custom paths for the log files when using file input. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + nexus: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9506 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + meraki: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9525 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + umbrella: + enabled: true + + #var.input: aws-s3 + # AWS SQS queue url + #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue + # Access ID to authenticate with the S3 input + #var.access_key_id: 123456 + # Access key to authenticate with the S3 input + #var.secret_access_key: PASSWORD + # The duration that the received messages are hidden from ReceiveMessage request + #var.visibility_timeout: 300s + # Maximum duration before AWS API request will be interrupted + #var.api_timeout: 120s + + amp: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + + # The API URL + #var.url: https://api.amp.cisco.com/v1/events + # The client ID used as a username for the API requests. + #var.client_id: + # The API key related to the client ID. + #var.api_key: + # How far to look back the first time the module is started. Expects an amount of hours. + #var.first_interval: 24h + # Overriding the default request timeout, optional. + #var.request_timeout: 60s diff --git a/salt/filebeat/modules/coredns.yml.disabled b/salt/filebeat/modules/coredns.yml.disabled new file mode 100644 index 000000000..46e9e55c1 --- /dev/null +++ b/salt/filebeat/modules/coredns.yml.disabled @@ -0,0 +1,11 @@ +# Module: coredns +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-coredns.html + +- module: coredns + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/crowdstrike.yml.disabled b/salt/filebeat/modules/crowdstrike.yml.disabled new file mode 100644 index 000000000..8d2c8531d --- /dev/null +++ b/salt/filebeat/modules/crowdstrike.yml.disabled @@ -0,0 +1,11 @@ +# Module: crowdstrike +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-crowdstrike.html + +- module: crowdstrike + + falcon: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/cyberark.yml.disabled b/salt/filebeat/modules/cyberark.yml.disabled new file mode 100644 index 000000000..e97955adf --- /dev/null +++ b/salt/filebeat/modules/cyberark.yml.disabled @@ -0,0 +1,22 @@ +# Module: cyberark +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cyberark.html + +- module: cyberark + corepas: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9527 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/cylance.yml.disabled b/salt/filebeat/modules/cylance.yml.disabled new file mode 100644 index 000000000..342d654d2 --- /dev/null +++ b/salt/filebeat/modules/cylance.yml.disabled @@ -0,0 +1,22 @@ +# Module: cylance +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cylance.html + +- module: cylance + protect: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9508 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/elasticsearch.yml.disabled b/salt/filebeat/modules/elasticsearch.yml.disabled new file mode 100644 index 000000000..e6074c05e --- /dev/null +++ b/salt/filebeat/modules/elasticsearch.yml.disabled @@ -0,0 +1,35 @@ +# Module: elasticsearch +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-elasticsearch.html + +- module: elasticsearch + # Server log + server: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + gc: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + audit: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + slowlog: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + deprecation: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/envoyproxy.yml.disabled b/salt/filebeat/modules/envoyproxy.yml.disabled new file mode 100644 index 000000000..543b17be5 --- /dev/null +++ b/salt/filebeat/modules/envoyproxy.yml.disabled @@ -0,0 +1,11 @@ +# Module: envoyproxy +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-envoyproxy.html + +- module: envoyproxy + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/f5.yml.disabled b/salt/filebeat/modules/f5.yml.disabled new file mode 100644 index 000000000..959842174 --- /dev/null +++ b/salt/filebeat/modules/f5.yml.disabled @@ -0,0 +1,41 @@ +# Module: f5 +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-f5.html + +- module: f5 + bigipapm: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9504 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + bigipafm: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9528 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/fortinet.yml.disabled b/salt/filebeat/modules/fortinet.yml.disabled new file mode 100644 index 000000000..281b7d788 --- /dev/null +++ b/salt/filebeat/modules/fortinet.yml.disabled @@ -0,0 +1,83 @@ +# Module: fortinet +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-fortinet.html + +- module: fortinet + firewall: + enabled: true + + # Set which input to use between tcp, udp (default) or file. + #var.input: udp + + # The interface to listen to syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The port to listen for syslog traffic. Defaults to 9004. + #var.syslog_port: 9004 + + # Set internal interfaces. used to override parsed network.direction + # based on a tagged interface. Both internal and external interfaces must be + # set to leverage this functionality. + #var.internal_interfaces: [ "LAN" ] + + # Set external interfaces. used to override parsed network.direction + # based on a tagged interface. Both internal and external interfaces must be + # set to leverage this functionality. + #var.external_interfaces: [ "WAN" ] + + clientendpoint: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9510 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + fortimail: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9529 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + fortimanager: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9530 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/gcp.yml.disabled b/salt/filebeat/modules/gcp.yml.disabled new file mode 100644 index 000000000..a09d0fe36 --- /dev/null +++ b/salt/filebeat/modules/gcp.yml.disabled @@ -0,0 +1,76 @@ +# Module: gcp +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-gcp.html + +- module: gcp + vpcflow: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be + # configured to use this topic as a sink for VPC flow logs. + var.topic: gcp-vpc-flowlogs + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + # Set internal networks. This is used to classify network.direction based + # off of what networks are considered "internal" either base off of a CIDR + # block or named network conditions. If this is not specified, then traffic + # direction is determined by whether it is between source and destination + # instance information rather than IP. + # + # For a full list of network conditions see: + # https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network + #var.internal_networks: [ "private" ] + + firewall: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-firewall + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-firewall-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + # Set internal networks. This is used to classify network.direction based + # off of what networks are considered "internal" either base off of a CIDR + # block or named network conditions. If this is not specified, then traffic + # is taken from the direction data in the rule_details event payload. + # + # For a full list of network conditions see: + # https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network + #var.internal_networks: [ "private" ] + + audit: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-audit + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-audit + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/salt/filebeat/modules/google_workspace.yml.disabled b/salt/filebeat/modules/google_workspace.yml.disabled new file mode 100644 index 000000000..6d364af98 --- /dev/null +++ b/salt/filebeat/modules/google_workspace.yml.disabled @@ -0,0 +1,53 @@ +# Module: google_workspace +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-google_workspace.html + +- module: google_workspace + saml: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + user_accounts: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + login: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + admin: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + drive: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + groups: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + diff --git a/salt/filebeat/modules/googlecloud.yml.disabled b/salt/filebeat/modules/googlecloud.yml.disabled new file mode 100644 index 000000000..9a28dc036 --- /dev/null +++ b/salt/filebeat/modules/googlecloud.yml.disabled @@ -0,0 +1,58 @@ +# Module: googlecloud +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-googlecloud.html + +# googlecloud module is deprecated, please use gcp instead +- module: gcp + vpcflow: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be + # configured to use this topic as a sink for VPC flow logs. + var.topic: gcp-vpc-flowlogs + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + firewall: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-firewall + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-firewall-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + audit: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-audit + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-audit + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/salt/filebeat/modules/gsuite.yml.disabled b/salt/filebeat/modules/gsuite.yml.disabled new file mode 100644 index 000000000..6aec3b65d --- /dev/null +++ b/salt/filebeat/modules/gsuite.yml.disabled @@ -0,0 +1,53 @@ +# Module: gsuite +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-gsuite.html + +# Gsuite module is deprecated and will be removed in future releases. Please use Google Workspace module instead. +- module: gsuite + saml: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + user_accounts: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + login: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + admin: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + drive: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + groups: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h diff --git a/salt/filebeat/modules/haproxy.yml.disabled b/salt/filebeat/modules/haproxy.yml.disabled new file mode 100644 index 000000000..b2615dbb8 --- /dev/null +++ b/salt/filebeat/modules/haproxy.yml.disabled @@ -0,0 +1,14 @@ +# Module: haproxy +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-haproxy.html + +- module: haproxy + # All logs + log: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/ibmmq.yml.disabled b/salt/filebeat/modules/ibmmq.yml.disabled new file mode 100644 index 000000000..bfaf3792d --- /dev/null +++ b/salt/filebeat/modules/ibmmq.yml.disabled @@ -0,0 +1,11 @@ +# Module: ibmmq +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-ibmmq.html + +- module: ibmmq + # All logs + errorlog: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/icinga.yml.disabled b/salt/filebeat/modules/icinga.yml.disabled new file mode 100644 index 000000000..a7c3ac6e1 --- /dev/null +++ b/salt/filebeat/modules/icinga.yml.disabled @@ -0,0 +1,27 @@ +# Module: icinga +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-icinga.html + +- module: icinga + # Main logs + main: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Debug logs + debug: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Startup logs + startup: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/iis.yml.disabled b/salt/filebeat/modules/iis.yml.disabled new file mode 100644 index 000000000..44c200ba1 --- /dev/null +++ b/salt/filebeat/modules/iis.yml.disabled @@ -0,0 +1,20 @@ +# Module: iis +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-iis.html + +- module: iis + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + \ No newline at end of file diff --git a/salt/filebeat/modules/imperva.yml.disabled b/salt/filebeat/modules/imperva.yml.disabled new file mode 100644 index 000000000..8e53deaa6 --- /dev/null +++ b/salt/filebeat/modules/imperva.yml.disabled @@ -0,0 +1,22 @@ +# Module: imperva +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-imperva.html + +- module: imperva + securesphere: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9511 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/infoblox.yml.disabled b/salt/filebeat/modules/infoblox.yml.disabled new file mode 100644 index 000000000..9e82f8340 --- /dev/null +++ b/salt/filebeat/modules/infoblox.yml.disabled @@ -0,0 +1,22 @@ +# Module: infoblox +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-infoblox.html + +- module: infoblox + nios: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9512 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/iptables.yml.disabled b/salt/filebeat/modules/iptables.yml.disabled new file mode 100644 index 000000000..1147e14dd --- /dev/null +++ b/salt/filebeat/modules/iptables.yml.disabled @@ -0,0 +1,13 @@ +# Module: iptables +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-iptables.html + +- module: iptables + log: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/juniper.yml.disabled b/salt/filebeat/modules/juniper.yml.disabled new file mode 100644 index 000000000..71112679d --- /dev/null +++ b/salt/filebeat/modules/juniper.yml.disabled @@ -0,0 +1,54 @@ +# Module: juniper +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-juniper.html + +- module: juniper + junos: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9513 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + netscreen: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9523 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + srx: + enabled: true + + # Set which input to use between tcp, udp (default) or file. + #var.input: udp + + # The interface to listen to syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The port to listen for syslog traffic. Defaults to 9006. + #var.syslog_port: 9006 diff --git a/salt/filebeat/modules/kafka.yml.disabled b/salt/filebeat/modules/kafka.yml.disabled new file mode 100644 index 000000000..23362c8a1 --- /dev/null +++ b/salt/filebeat/modules/kafka.yml.disabled @@ -0,0 +1,15 @@ +# Module: kafka +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-kafka.html + +- module: kafka + # All logs + log: + enabled: true + + # Set custom paths for Kafka. If left empty, + # Filebeat will look under /opt. + #var.kafka_home: + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/kibana.yml.disabled b/salt/filebeat/modules/kibana.yml.disabled new file mode 100644 index 000000000..a4956c4b6 --- /dev/null +++ b/salt/filebeat/modules/kibana.yml.disabled @@ -0,0 +1,19 @@ +# Module: kibana +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-kibana.html + +- module: kibana + # Server logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Audit logs + audit: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/logstash.yml.disabled b/salt/filebeat/modules/logstash.yml.disabled new file mode 100644 index 000000000..f14229409 --- /dev/null +++ b/salt/filebeat/modules/logstash.yml.disabled @@ -0,0 +1,18 @@ +# Module: logstash +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-logstash.html + +- module: logstash + # logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Slow logs + slowlog: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/microsoft.yml.disabled b/salt/filebeat/modules/microsoft.yml.disabled new file mode 100644 index 000000000..b0a1b10c6 --- /dev/null +++ b/salt/filebeat/modules/microsoft.yml.disabled @@ -0,0 +1,49 @@ +# Module: microsoft +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-microsoft.html + +- module: microsoft + # ATP configuration + defender_atp: + enabled: true + # How often the API should be polled + #var.interval: 5m + + # Oauth Client ID + #var.oauth2.client.id: "" + + # Oauth Client Secret + #var.oauth2.client.secret: "" + + # Oauth Token URL, should include the tenant ID + #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" + m365_defender: + enabled: true + # How often the API should be polled + #var.interval: 5m + + # Oauth Client ID + #var.oauth2.client.id: "" + + # Oauth Client Secret + #var.oauth2.client.secret: "" + + # Oauth Token URL, should include the tenant ID + #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" + dhcp: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9515 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/misp.yml.disabled b/salt/filebeat/modules/misp.yml.disabled new file mode 100644 index 000000000..9a489fa0f --- /dev/null +++ b/salt/filebeat/modules/misp.yml.disabled @@ -0,0 +1,17 @@ +# Module: misp +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-misp.html + +- module: misp + threat: + enabled: true + # API key to access MISP + #var.api_key + + # Array object in MISP response + #var.http_request_body.limit: 1000 + + # URL of the MISP REST API + #var.url + + # You can also pass SSL options. For example: + #var.ssl.verification_mode: none diff --git a/salt/filebeat/modules/mongodb.yml.disabled b/salt/filebeat/modules/mongodb.yml.disabled new file mode 100644 index 000000000..266d2e4e8 --- /dev/null +++ b/salt/filebeat/modules/mongodb.yml.disabled @@ -0,0 +1,11 @@ +# Module: mongodb +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mongodb.html + +- module: mongodb + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/mssql.yml.disabled b/salt/filebeat/modules/mssql.yml.disabled new file mode 100644 index 000000000..bfe4c6e64 --- /dev/null +++ b/salt/filebeat/modules/mssql.yml.disabled @@ -0,0 +1,11 @@ +# Module: mssql +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mssql.html + +- module: mssql + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] diff --git a/salt/filebeat/modules/mysql.yml.disabled b/salt/filebeat/modules/mysql.yml.disabled new file mode 100644 index 000000000..e6be4045b --- /dev/null +++ b/salt/filebeat/modules/mysql.yml.disabled @@ -0,0 +1,19 @@ +# Module: mysql +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mysql.html + +- module: mysql + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Slow logs + slowlog: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/mysqlenterprise.yml.disabled b/salt/filebeat/modules/mysqlenterprise.yml.disabled new file mode 100644 index 000000000..37e10d0eb --- /dev/null +++ b/salt/filebeat/modules/mysqlenterprise.yml.disabled @@ -0,0 +1,14 @@ +# Module: mysqlenterprise +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mysqlenterprise.html + +- module: mysqlenterprise + audit: + enabled: true + + # Sets the input type. Currently only supports file + #var.input: file + + # Set paths for the log files when file input is used. + # Should only be used together with file input + # var.paths: + # - /home/user/mysqlauditlogs/audit.*.log diff --git a/salt/filebeat/modules/nats.yml.disabled b/salt/filebeat/modules/nats.yml.disabled new file mode 100644 index 000000000..65e44962d --- /dev/null +++ b/salt/filebeat/modules/nats.yml.disabled @@ -0,0 +1,11 @@ +# Module: nats +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-nats.html + +- module: nats + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/netflow.yml.disabled b/salt/filebeat/modules/netflow.yml.disabled new file mode 100644 index 000000000..781748b00 --- /dev/null +++ b/salt/filebeat/modules/netflow.yml.disabled @@ -0,0 +1,14 @@ +# Module: netflow +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-netflow.html + +- module: netflow + log: + enabled: true + var: + netflow_host: localhost + netflow_port: 2055 + # internal_networks specifies which networks are considered internal or private + # you can specify either a CIDR block or any of the special named ranges listed + # at: https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network + internal_networks: + - private diff --git a/salt/filebeat/modules/netscout.yml.disabled b/salt/filebeat/modules/netscout.yml.disabled new file mode 100644 index 000000000..215349046 --- /dev/null +++ b/salt/filebeat/modules/netscout.yml.disabled @@ -0,0 +1,22 @@ +# Module: netscout +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-netscout.html + +- module: netscout + sightline: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9502 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/nginx.yml.disabled b/salt/filebeat/modules/nginx.yml.disabled new file mode 100644 index 000000000..e2fa44a78 --- /dev/null +++ b/salt/filebeat/modules/nginx.yml.disabled @@ -0,0 +1,27 @@ +# Module: nginx +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-nginx.html + +- module: nginx + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs + ingress_controller: + enabled: false + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/o365.yml.disabled b/salt/filebeat/modules/o365.yml.disabled new file mode 100644 index 000000000..578ff365d --- /dev/null +++ b/salt/filebeat/modules/o365.yml.disabled @@ -0,0 +1,48 @@ +# Module: o365 +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-o365.html + +- module: o365 + audit: + enabled: true + + # Set the application_id (also known as client ID): + var.application_id: "" + + # Configure the tenants to monitor: + # Use the tenant ID (also known as directory ID) and the domain name. + # var.tenants: + # - id: "tenant_id_1" + # name: "mydomain.onmicrosoft.com" + # - id: "tenant_id_2" + # name: "mycompany.com" + var.tenants: + - id: "" + name: "mytenant.onmicrosoft.com" + + # List of content-types to fetch. By default all known content-types + # are retrieved: + # var.content_type: + # - "Audit.AzureActiveDirectory" + # - "Audit.Exchange" + # - "Audit.SharePoint" + # - "Audit.General" + # - "DLP.All" + + # Use the following settings to enable certificate-based authentication: + # var.certificate: "/path/to/certificate.pem" + # var.key: "/path/to/private_key.pem" + # var.key_passphrase: "myPrivateKeyPassword" + + # Client-secret based authentication: + # Comment the following line if using certificate authentication. + var.client_secret: "" + + # Advanced settings, use with care: + # var.api: + # # Settings for custom endpoints: + # authentication_endpoint: "https://login.microsoftonline.us/" + # resource: "https://manage.office365.us" + # + # max_retention: 168h + # max_requests_per_minute: 2000 + # poll_interval: 3m diff --git a/salt/filebeat/modules/okta.yml.disabled b/salt/filebeat/modules/okta.yml.disabled new file mode 100644 index 000000000..4fc943592 --- /dev/null +++ b/salt/filebeat/modules/okta.yml.disabled @@ -0,0 +1,10 @@ +# Module: okta +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-okta.html + +- module: okta + system: + enabled: true + # You must configure the URL with your Okta domain and provide an + # API token to access the logs API. + #var.url: https://yourOktaDomain/api/v1/logs + #var.api_key: 'yourApiTokenHere' diff --git a/salt/filebeat/modules/oracle.yml.disabled b/salt/filebeat/modules/oracle.yml.disabled new file mode 100644 index 000000000..3bd576ee1 --- /dev/null +++ b/salt/filebeat/modules/oracle.yml.disabled @@ -0,0 +1,13 @@ +# Module: oracle +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-oracle.html + +- module: oracle + database_audit: + enabled: true + + # Set which input to use between syslog or file (default). + #var.input: file + + # Set paths for the log files when file input is used. + # Should only be used together with file input + # var.paths: /home/user/oracleauditlogs/*.aud diff --git a/salt/filebeat/modules/osquery.yml.disabled b/salt/filebeat/modules/osquery.yml.disabled new file mode 100644 index 000000000..7a9a09dd8 --- /dev/null +++ b/salt/filebeat/modules/osquery.yml.disabled @@ -0,0 +1,15 @@ +# Module: osquery +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-osquery.html + +- module: osquery + result: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # If true, all fields created by this module are prefixed with + # `osquery.result`. Set to false to copy the fields in the root + # of the document. The default is true. + #var.use_namespace: true diff --git a/salt/filebeat/modules/panw.yml.disabled b/salt/filebeat/modules/panw.yml.disabled new file mode 100644 index 000000000..eb094a25a --- /dev/null +++ b/salt/filebeat/modules/panw.yml.disabled @@ -0,0 +1,22 @@ +# Module: panw +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-panw.html + +- module: panw + panos: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Set internal security zones. used to determine network.direction + # default "trust" + #var.internal_zones: + + # Set external security zones. used to determine network.direction + # default "untrust" + #var.external_zones: + diff --git a/salt/filebeat/modules/pensando.yml.disabled b/salt/filebeat/modules/pensando.yml.disabled new file mode 100644 index 000000000..66bd60d76 --- /dev/null +++ b/salt/filebeat/modules/pensando.yml.disabled @@ -0,0 +1,13 @@ +# Module: pensando +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-pensando.html + +- module: pensando +# Firewall logs + dfw: + enabled: true + var.syslog_host: 0.0.0.0 + var.syslog_port: 9001 + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + # var.paths: diff --git a/salt/filebeat/modules/postgresql.yml.disabled b/salt/filebeat/modules/postgresql.yml.disabled new file mode 100644 index 000000000..804b7f34f --- /dev/null +++ b/salt/filebeat/modules/postgresql.yml.disabled @@ -0,0 +1,11 @@ +# Module: postgresql +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-postgresql.html + +- module: postgresql + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/proofpoint.yml.disabled b/salt/filebeat/modules/proofpoint.yml.disabled new file mode 100644 index 000000000..9aeebd5fe --- /dev/null +++ b/salt/filebeat/modules/proofpoint.yml.disabled @@ -0,0 +1,22 @@ +# Module: proofpoint +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-proofpoint.html + +- module: proofpoint + emailsecurity: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9531 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/rabbitmq.yml.disabled b/salt/filebeat/modules/rabbitmq.yml.disabled new file mode 100644 index 000000000..e61a0a0c9 --- /dev/null +++ b/salt/filebeat/modules/rabbitmq.yml.disabled @@ -0,0 +1,11 @@ +# Module: rabbitmq +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-rabbitmq.html + +- module: rabbitmq + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: ["/var/log/rabbitmq/rabbit@localhost.log*"] diff --git a/salt/filebeat/modules/radware.yml.disabled b/salt/filebeat/modules/radware.yml.disabled new file mode 100644 index 000000000..f9ab3e519 --- /dev/null +++ b/salt/filebeat/modules/radware.yml.disabled @@ -0,0 +1,22 @@ +# Module: radware +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-radware.html + +- module: radware + defensepro: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9518 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/redis.yml.disabled b/salt/filebeat/modules/redis.yml.disabled new file mode 100644 index 000000000..9b621dc2d --- /dev/null +++ b/salt/filebeat/modules/redis.yml.disabled @@ -0,0 +1,21 @@ +# Module: redis +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-redis.html + +- module: redis + # Main logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: ["/var/log/redis/redis-server.log*"] + + # Slow logs, retrieved via the Redis API (SLOWLOG) + slowlog: + enabled: true + + # The Redis hosts to connect to. + #var.hosts: ["localhost:6379"] + + # Optional, the password to use when connecting to Redis. + #var.password: diff --git a/salt/filebeat/modules/santa.yml.disabled b/salt/filebeat/modules/santa.yml.disabled new file mode 100644 index 000000000..1a7363547 --- /dev/null +++ b/salt/filebeat/modules/santa.yml.disabled @@ -0,0 +1,9 @@ +# Module: santa +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-santa.html + +- module: santa + log: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the the default path. + #var.paths: diff --git a/salt/filebeat/modules/snort.yml.disabled b/salt/filebeat/modules/snort.yml.disabled new file mode 100644 index 000000000..8c9bcc471 --- /dev/null +++ b/salt/filebeat/modules/snort.yml.disabled @@ -0,0 +1,22 @@ +# Module: snort +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-snort.html + +- module: snort + log: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9532 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/snyk.yml.disabled b/salt/filebeat/modules/snyk.yml.disabled new file mode 100644 index 000000000..0b13f8155 --- /dev/null +++ b/salt/filebeat/modules/snyk.yml.disabled @@ -0,0 +1,112 @@ +# Module: snyk +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-snyk.html + +- module: snyk + audit: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + # + # What audit type to collect, can be either "group" or "organization". + #var.audit_type: organization + # + # The ID related to the audit_type. If audit type is group, then this value should be + # the group ID and if it is organization it should be the organization ID to collect from. + #var.audit_id: 1235432-asdfdf-2341234-asdgjhg + + # How often the API should be polled, defaults to 1 hour. + #var.interval: 1h + # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). + #var.first_interval: 24h + + # The API token that is created for a specific user, found in the Snyk management dashboard. + #var.api_token: + + # Event filtering. + # All configuration items below is OPTIONAL and the default options will be overwritten + # for each entry that is not commented out. + + # Will return only logs for this specific project. + #var.project_id: "" + # User public ID. Will fetch only audit logs originated from this user's actions. + #var.user_id: "" + # Will return only logs for this specific event. + #var.event: "" + # User email address. Will fetch only audit logs originated from this user's actions. + #var.email_address: "" + + vulnerabilities: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + + # How often the API should be polled. Data from the Snyk API is automatically updated + # once per day, so the default interval is 24 hours. + #var.interval: 24h + + # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). + #var.first_interval: 24h + + # The API token that is created for a specific user, found in the Snyk management dashboard. + #var.api_token: + + # The list of org IDs to filter the results by. + # One organization ID per line, starting with a - sign + #var.orgs: + # - 12354-asdfdf-123543-asdsdfg + # - 76554-jhggfd-654342-hgrfasd + + + # Event filtering. + # All configuration items below is OPTIONAL and the default options will be overwritten + # for each entry that is not commented out. + + # The severity levels of issues to filter the results by. + #var.included_severity: + # - high + # - medium + # - low + # + # The exploit maturity levels of issues to filter the results by. + #var.exploit_maturity: + # - mature + # - proof-of-concept + # - no-known-exploit + # - no-data + # + # The type of issues to filter the results by. + #var.types: + # - vuln + # - license + # + # The type of languages to filter the results by. + #var.languages: + # - javascript + # - ruby + # - java + # - scala + # - python + # - golang + # - php + # - dotnet + # - swift + # - docker + # + # Search term to filter issue name by, or an exact CVE or CWE. + #var.identifier: + # - "" + # + # If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + #var.ignored: false + #var.patched: false + #var.fixable: false + #var.is_fixed: false + #var.is_patchable: false + #var.is_pinnable: false + # + # The priority score ranging between 0-1000 + #var.min_priority_score: 0 + #var.max_priority_score: 1000 + diff --git a/salt/filebeat/modules/sonicwall.yml.disabled b/salt/filebeat/modules/sonicwall.yml.disabled new file mode 100644 index 000000000..de457109d --- /dev/null +++ b/salt/filebeat/modules/sonicwall.yml.disabled @@ -0,0 +1,22 @@ +# Module: sonicwall +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-sonicwall.html + +- module: sonicwall + firewall: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9519 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/sophos.yml.disabled b/salt/filebeat/modules/sophos.yml.disabled new file mode 100644 index 000000000..8fc346540 --- /dev/null +++ b/salt/filebeat/modules/sophos.yml.disabled @@ -0,0 +1,46 @@ +# Module: sophos +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-sophos.html + +- module: sophos + xg: + enabled: true + + # Set which input to use between tcp, udp (default) or file. + #var.input: udp + + # The interface to listen to syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The port to listen for syslog traffic. Defaults to 9004. + #var.syslog_port: 9005 + + # firewall default hostname + #var.default_host_name: firewall.localgroup.local + + # known firewalls + #var.known_devices: + #- serial_number: "1234567890123457" + # hostname: "a.host.local" + #- serial_number: "1234234590678557" + # hostname: "b.host.local" + + + utm: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9533 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/squid.yml.disabled b/salt/filebeat/modules/squid.yml.disabled new file mode 100644 index 000000000..a47807253 --- /dev/null +++ b/salt/filebeat/modules/squid.yml.disabled @@ -0,0 +1,22 @@ +# Module: squid +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-squid.html + +- module: squid + log: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9520 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/suricata.yml.disabled b/salt/filebeat/modules/suricata.yml.disabled new file mode 100644 index 000000000..1edd3f832 --- /dev/null +++ b/salt/filebeat/modules/suricata.yml.disabled @@ -0,0 +1,11 @@ +# Module: suricata +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html + +- module: suricata + # All logs + eve: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/system.yml.disabled b/salt/filebeat/modules/system.yml.disabled new file mode 100644 index 000000000..d633bac04 --- /dev/null +++ b/salt/filebeat/modules/system.yml.disabled @@ -0,0 +1,19 @@ +# Module: system +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-system.html + +- module: system + # Syslog + syslog: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Authorization logs + auth: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/threatintel.yml.disabled b/salt/filebeat/modules/threatintel.yml.disabled new file mode 100644 index 000000000..b461d91e2 --- /dev/null +++ b/salt/filebeat/modules/threatintel.yml.disabled @@ -0,0 +1,105 @@ +# Module: threatintel +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-threatintel.html + +- module: threatintel + abuseurl: + enabled: true + + # Input used for ingesting threat intel data. + var.input: httpjson + + # The URL used for Threat Intel API calls. + var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/ + + # The interval to poll the API for updates. + var.interval: 10m + + abusemalware: + enabled: true + + # Input used for ingesting threat intel data. + var.input: httpjson + + # The URL used for Threat Intel API calls. + var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/ + + # The interval to poll the API for updates. + var.interval: 10m + + misp: + enabled: true + + # Input used for ingesting threat intel data, defaults to JSON. + var.input: httpjson + + # The URL of the MISP instance, should end with "/events/restSearch". + var.url: https://SERVER/events/restSearch + + # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI. + var.api_token: API_KEY + + # Configures the type of SSL verification done, if MISP is running on self signed certificates + # then the certificate would either need to be trusted, or verification_mode set to none. + #var.ssl.verification_mode: none + + # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context. + # For examples please reference the filebeat module documentation. + #var.filters: + # - threat_level: [4, 5] + # - to_ids: true + + # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer + # than the last event that was already ingested. + var.first_interval: 300h + + # The interval to poll the API for updates. + var.interval: 5m + + otx: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The URL used for OTX Threat Intel API calls. + var.url: https://otx.alienvault.com/api/v1/indicators/export + + # The authentication token used to contact the OTX API, can be found on the OTX UI. + var.api_token: API_KEY + + # Optional filters that can be applied to retrieve only specific indicators. + #var.types: "domain,IPv4,hostname,url,FileHash-SHA256" + + # The timeout of the HTTP client connecting to the OTX API + #var.http_client_timeout: 120s + + # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module. + var.lookback_range: 1h + + # How far back to look once the beat starts up for the first time, the value has to be in hours. + var.first_interval: 400h + + # The interval to poll the API for updates + var.interval: 5m + + anomali: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending + # on the type of threat intel source that is needed. + var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects + + # The Username used by anomali Limo, defaults to guest. + #var.username: guest + + # The password used by anomali Limo, defaults to guest. + #var.password: guest + + # How far back to look once the beat starts up for the first time, the value has to be in hours. + var.first_interval: 400h + + # The interval to poll the API for updates + var.interval: 5m diff --git a/salt/filebeat/modules/tomcat.yml.disabled b/salt/filebeat/modules/tomcat.yml.disabled new file mode 100644 index 000000000..84f4619d5 --- /dev/null +++ b/salt/filebeat/modules/tomcat.yml.disabled @@ -0,0 +1,22 @@ +# Module: tomcat +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-tomcat.html + +- module: tomcat + log: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9501 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/traefik.yml.disabled b/salt/filebeat/modules/traefik.yml.disabled new file mode 100644 index 000000000..657d5ccd9 --- /dev/null +++ b/salt/filebeat/modules/traefik.yml.disabled @@ -0,0 +1,11 @@ +# Module: traefik +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-traefik.html + +- module: traefik + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/zeek.yml.disabled b/salt/filebeat/modules/zeek.yml.disabled new file mode 100644 index 000000000..0667c6e35 --- /dev/null +++ b/salt/filebeat/modules/zeek.yml.disabled @@ -0,0 +1,84 @@ +# Module: zeek +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html + +- module: zeek + capture_loss: + enabled: true + connection: + enabled: true + dce_rpc: + enabled: true + dhcp: + enabled: true + dnp3: + enabled: true + dns: + enabled: true + dpd: + enabled: true + files: + enabled: true + ftp: + enabled: true + http: + enabled: true + intel: + enabled: true + irc: + enabled: true + kerberos: + enabled: true + modbus: + enabled: true + mysql: + enabled: true + notice: + enabled: true + ntlm: + enabled: true + ocsp: + enabled: true + pe: + enabled: true + radius: + enabled: true + rdp: + enabled: true + rfb: + enabled: true + signature: + enabled: true + sip: + enabled: true + smb_cmd: + enabled: true + smb_files: + enabled: true + smb_mapping: + enabled: true + smtp: + enabled: true + snmp: + enabled: true + socks: + enabled: true + ssh: + enabled: true + ssl: + enabled: true + stats: + enabled: true + syslog: + enabled: true + traceroute: + enabled: true + tunnel: + enabled: true + weird: + enabled: true + x509: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/zoom.yml.disabled b/salt/filebeat/modules/zoom.yml.disabled new file mode 100644 index 000000000..15fa9d4b2 --- /dev/null +++ b/salt/filebeat/modules/zoom.yml.disabled @@ -0,0 +1,22 @@ +# Module: zoom +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zoom.html + +- module: zoom + webhook: + enabled: true + + # The type of input to use + #var.input: http_endpoint + + # The interface to listen for incoming HTTP requests. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.listen_address: localhost + + # The port to bind to + #var.listen_port: 80 + + # The header Zoom uses to send its secret token, defaults to "Authorization" + #secret.header: Authorization + + # The secret token value created by Zoom + #secret.value: ZOOMTOKEN diff --git a/salt/filebeat/modules/zscaler.yml.disabled b/salt/filebeat/modules/zscaler.yml.disabled new file mode 100644 index 000000000..accdec9ea --- /dev/null +++ b/salt/filebeat/modules/zscaler.yml.disabled @@ -0,0 +1,22 @@ +# Module: zscaler +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zscaler.html + +- module: zscaler + zia: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9521 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local From fcd1bea4a3b25b1b74c28363769d8256bca3a892 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 12:06:03 -0400 Subject: [PATCH 068/378] remove auto data migration, change duration from 0s to 0d --- salt/common/tools/sbin/soup | 8 -------- salt/influxdb/defaults.yaml | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 07a9804b2..1f6a6e5d7 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -307,14 +307,6 @@ post_2.3.30_to_2.3.40() { } post_2.3.5X_to_2.3.60() { - # These are post upgrade changes that need to take place after all the new states have been applied. - echo "" - echo "Migration of InfluxDB data will begin in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." - echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" - read -p "Press enter to continue." - echo "" - docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*" && echo "InfluxDB migration complete." || echo "InfluxDB migration failed."' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 - POSTVERSION=2.3.60 } diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 54e175024..0c9a09080 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -6,7 +6,7 @@ influxdb: shard_duration: 1d so_long_term: default: False - duration: 0s + duration: 0d shard_duration: 7d downsample: so_long_term: From c5f99b012e6e1995376d5c4deb603911c623d4bb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 13:58:53 -0400 Subject: [PATCH 069/378] comment out creation of cq to test data migration --- salt/influxdb/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index c6120d292..9a94b1c95 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -112,7 +112,7 @@ telegraf_database: - sls: salt.python3-influxdb {% endfor %} -{% for dest_rp in influxdb.downsample.keys() %} +{#% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq @@ -127,7 +127,7 @@ so_downsample_cq: - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - sls: salt.python3-influxdb -{% endfor %} +{% endfor %#} {% endif %} {% endif %} From a1a79719fc759577fe42ea0d368aa17f5b61bd39 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 May 2021 12:07:30 +0000 Subject: [PATCH 070/378] Add ignore above for message keyword field --- salt/elasticsearch/templates/so/so-common-template.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 8adbdcf13..214e5c691 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -320,7 +320,8 @@ "type":"text", "fields":{ "keyword":{ - "type":"keyword" + "type":"keyword", + "ignore_above": 32766 } } }, From 6a639edb052e328307a6bc90ec55041c486a8d4a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 08:33:31 -0400 Subject: [PATCH 071/378] have cq created once again --- salt/influxdb/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 9a94b1c95..c6120d292 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -112,7 +112,7 @@ telegraf_database: - sls: salt.python3-influxdb {% endfor %} -{#% for dest_rp in influxdb.downsample.keys() %} +{% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq @@ -127,7 +127,7 @@ so_downsample_cq: - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - sls: salt.python3-influxdb -{% endfor %#} +{% endfor %} {% endif %} {% endif %} From ce24781446c1ddbf736702dae33fb0d154fe7aeb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 13:29:24 -0400 Subject: [PATCH 072/378] first take at so-infludb-clean --- salt/common/tools/so-influxdb-clear | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 salt/common/tools/so-influxdb-clear diff --git a/salt/common/tools/so-influxdb-clear b/salt/common/tools/so-influxdb-clear new file mode 100644 index 000000000..38ef6491f --- /dev/null +++ b/salt/common/tools/so-influxdb-clear @@ -0,0 +1,42 @@ + +echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." +echo "The duration will need to be specified as an integer followed by the duration unit, without a space." +echo "For example, to purge all data, but retain the past 3 months, specify 3mo for the duration." +echo "The duration units are as follows:" +echo " mo - month(s)" +echo " w - week(s)" +echo " d - day(s)" + +while true; do +echo "" +read -p 'Enter the duration of past data that you would like to retain: ' duration +duration=$(echo $duration | tr '[:upper:]' '[:lower:]') + +originalIFS=$IFS + +case "${duration}" in + + [0-9]+mo$) + break + ;; + + [0-9]+w$) + break + ;; + + [0-9]+d$) + break + ;; + + *) + echo "" + echo "Invalid duration." + ;; + +esac +done + +IFS=$originalIFS + +echo "Cleaning InfluxDb and saving only the past ${duration}" +docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - ${duration}\"" \ No newline at end of file From e53280447415a81ab0063c1277c467798dece9ef Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 13:42:21 -0400 Subject: [PATCH 073/378] move to proper dir --- salt/common/tools/{ => sbin}/so-influxdb-clear | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) rename salt/common/tools/{ => sbin}/so-influxdb-clear (59%) diff --git a/salt/common/tools/so-influxdb-clear b/salt/common/tools/sbin/so-influxdb-clear similarity index 59% rename from salt/common/tools/so-influxdb-clear rename to salt/common/tools/sbin/so-influxdb-clear index 38ef6491f..c6d5446e9 100644 --- a/salt/common/tools/so-influxdb-clear +++ b/salt/common/tools/sbin/so-influxdb-clear @@ -1,3 +1,19 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." From a5ee8fb59d8a37ba71e1a596d20e78b103c97dfe Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 14:56:53 -0400 Subject: [PATCH 074/378] fix the issues with so-influxdb-clear --- salt/common/tools/sbin/so-influxdb-clear | 44 ++++++++++-------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clear b/salt/common/tools/sbin/so-influxdb-clear index c6d5446e9..809b17381 100644 --- a/salt/common/tools/sbin/so-influxdb-clear +++ b/salt/common/tools/sbin/so-influxdb-clear @@ -15,8 +15,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +modurregex="^[0-9]+mo$" +wdurregex="^[0-9]+w$" +ddurregex="^[0-9]+d$" + +echo "" echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." +echo "" echo "For example, to purge all data, but retain the past 3 months, specify 3mo for the duration." echo "The duration units are as follows:" echo " mo - month(s)" @@ -24,35 +30,21 @@ echo " w - week(s)" echo " d - day(s)" while true; do -echo "" -read -p 'Enter the duration of past data that you would like to retain: ' duration -duration=$(echo $duration | tr '[:upper:]' '[:lower:]') + echo "" + read -p 'Enter the duration of past data that you would like to retain: ' duration + duration=$(echo $duration | tr '[:upper:]' '[:lower:]') -originalIFS=$IFS - -case "${duration}" in - - [0-9]+mo$) + if [[ "$duration" =~ $modurregex ]] || [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then break - ;; + fi - [0-9]+w$) - break - ;; - - [0-9]+d$) - break - ;; - - *) - echo "" - echo "Invalid duration." - ;; - -esac + echo "" + echo "Invalid duration." done -IFS=$originalIFS - echo "Cleaning InfluxDb and saving only the past ${duration}" -docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - ${duration}\"" \ No newline at end of file +if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then + echo "InfluxDb clean complete." +else + echo "Something went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." +fi From 44de6110972dd9d857e11b7b066d04ba206c43a1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 14:57:39 -0400 Subject: [PATCH 075/378] rename to so-influxdb-clean --- salt/common/tools/sbin/{so-influxdb-clear => so-influxdb-clean} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/common/tools/sbin/{so-influxdb-clear => so-influxdb-clean} (100%) diff --git a/salt/common/tools/sbin/so-influxdb-clear b/salt/common/tools/sbin/so-influxdb-clean similarity index 100% rename from salt/common/tools/sbin/so-influxdb-clear rename to salt/common/tools/sbin/so-influxdb-clean From db98b7ed276cb0df0661798180771a7379562268 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 15:08:11 -0400 Subject: [PATCH 076/378] verify with user before proceedig to clean --- salt/common/tools/sbin/so-influxdb-clean | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 809b17381..453d5b2e4 100644 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -19,11 +19,9 @@ modurregex="^[0-9]+mo$" wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" -echo "" -echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." +echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." -echo "" -echo "For example, to purge all data, but retain the past 3 months, specify 3mo for the duration." +echo -e "\nFor example, to purge all data, but retain the past 3 months, specify 3mo for the duration." echo "The duration units are as follows:" echo " mo - month(s)" echo " w - week(s)" @@ -38,13 +36,18 @@ while true; do break fi - echo "" - echo "Invalid duration." + echo -e "\nInvalid duration." done -echo "Cleaning InfluxDb and saving only the past ${duration}" -if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then - echo "InfluxDb clean complete." +echo -e "\nInfluxDB will now be cleaned and leave only the past $duration worth of data." +read -r -p "Are you sure you want to continue? [y/N] " yorn +if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then + echo -e "\nCleaning InfluxDb and saving only the past $duration. This may could take several minutes depending on how much data needs to be cleaned." + if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then + echo -e "\nInfluxDb clean complete." + else + echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + fi else - echo "Something went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + echo -e "\nExiting as requested." fi From 0ea4c99102bbd3aecc11c164395ece5a8edd5dc8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 15:32:53 -0400 Subject: [PATCH 077/378] remove support for months as it isnt supported in InfluxQL --- salt/common/tools/sbin/so-influxdb-clean | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 453d5b2e4..954abfa8f 100644 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -15,15 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -modurregex="^[0-9]+mo$" wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." -echo -e "\nFor example, to purge all data, but retain the past 3 months, specify 3mo for the duration." +echo -e "\nFor example, to purge all data, but retain the past 12 weeks, specify 12w for the duration." echo "The duration units are as follows:" -echo " mo - month(s)" echo " w - week(s)" echo " d - day(s)" @@ -32,7 +30,7 @@ while true; do read -p 'Enter the duration of past data that you would like to retain: ' duration duration=$(echo $duration | tr '[:upper:]' '[:lower:]') - if [[ "$duration" =~ $modurregex ]] || [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then + if [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then break fi From 925be17d51be12a6a49caec3f3c5f3d194fa04ad Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 15:59:18 -0400 Subject: [PATCH 078/378] clean some commas in so-influxdb-clean --- salt/common/tools/sbin/so-influxdb-clean | 4 ++-- salt/common/tools/sbin/so-influxdb-migrate | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 salt/common/tools/sbin/so-influxdb-migrate diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 954abfa8f..635ac6a02 100644 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -19,8 +19,8 @@ wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." -echo "The duration will need to be specified as an integer followed by the duration unit, without a space." -echo -e "\nFor example, to purge all data, but retain the past 12 weeks, specify 12w for the duration." +echo "The duration will need to be specified as an integer followed by the duration unit without a space." +echo -e "\nFor example, to purge all data but retain the past 12 weeks, specify 12w for the duration." echo "The duration units are as follows:" echo " w - week(s)" echo " d - day(s)" diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate new file mode 100644 index 000000000..c27997bd5 --- /dev/null +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -0,0 +1,16 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . From 20188549f731ca133d4fc69402f3ea31bc4e0757 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 19:28:16 -0400 Subject: [PATCH 079/378] add the logic for so-influxdb-migrate --- salt/common/tools/sbin/so-influxdb-migrate | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index c27997bd5..3b21d099d 100644 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -14,3 +14,33 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + +echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy." + +echo -e "\nInfluxDB will now be migrated. This could take a few hours depending on how large the database is and hardware resources available." +read -r -p "Are you sure you want to continue? [y/N] " yorn +if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then + echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be moved." + day=0 + startdate=`date` + + while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do + # why 2020-07-21? + echo "Migration of 2020-07-21T00:00:00.0000000Z' + ${day}d started at $startdate and completed at `date`." + + echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" + if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then + echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" + else + echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + fi + + ((day=day+1)) + startdate=`date` + done + + echo -e "\nInfluxDb data migration complete." + +else + echo -e "\nExiting as requested." +fi From 9dbb9f519b40765972e86bc1993796d268a2c5d2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 09:14:49 -0400 Subject: [PATCH 080/378] create so_short_term rp as default so that autogen can just be dropped once data is downsampled --- salt/common/tools/sbin/so-influxdb-migrate | 14 +++++++------- salt/influxdb/defaults.yaml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index 3b21d099d..3411dc39f 100644 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -24,16 +24,16 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then day=0 startdate=`date` - while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do + while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? echo "Migration of 2020-07-21T00:00:00.0000000Z' + ${day}d started at $startdate and completed at `date`." - echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" - if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then - echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" - else - echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." - fi +# echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" +# if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then +# echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" +# else +# echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." +# fi ((day=day+1)) startdate=`date` diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 0c9a09080..7ba83dd6d 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -1,6 +1,6 @@ influxdb: retention_policies: - autogen: + so_short_term: default: True duration: 30d shard_duration: 1d From 37929dbd7d09d0da03919541c792eb0cefa25fde Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 May 2021 13:54:28 +0000 Subject: [PATCH 081/378] Add additional config for Filebeat modules --- pillar/zeek/init.sls | 3 +- salt/filebeat/etc/filebeat.yml | 78 ----------------- salt/filebeat/init.sls | 23 +++++ salt/filebeat/modules/suricata.yml.disabled | 11 --- salt/filebeat/modules/zeek.yml.disabled | 84 ------------------- .../config/so/9000_output_zeek.conf.jinja | 27 +++--- .../config/so/9400_output_suricata.conf.jinja | 6 +- 7 files changed, 41 insertions(+), 191 deletions(-) delete mode 100644 salt/filebeat/modules/suricata.yml.disabled delete mode 100644 salt/filebeat/modules/zeek.yml.disabled diff --git a/pillar/zeek/init.sls b/pillar/zeek/init.sls index 30a59284a..5eeb273b9 100644 --- a/pillar/zeek/init.sls +++ b/pillar/zeek/init.sls @@ -52,5 +52,4 @@ zeek: - frameworks/signatures/detect-windows-shells redef: - LogAscii::use_json = T; - - LogAscii::json_timestamps = JSON::TS_ISO8601; - - CaptureLoss::watch_interval = 5 mins; \ No newline at end of file + - CaptureLoss::watch_interval = 5 mins; diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 0f7c9c778..bd72bc583 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -105,84 +105,6 @@ filebeat.inputs: fields_under_root: true {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} - {%- if ZEEKVER != 'SURICATA' %} - {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} -- type: log - paths: - - /nsm/zeek/logs/current/{{ LOGNAME }}.log - fields: - module: zeek - dataset: {{ LOGNAME }} - category: network - processors: - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: true - close_removed: false - -- type: log - paths: - - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log - fields: - module: zeek - dataset: {{ LOGNAME }} - category: network - imported: true - processors: - - add_tags: - tags: ["import"] - - dissect: - tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}" - field: "log.file.path" - target_prefix: "" - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: false - close_removed: false - {%- endfor %} - {%- endif %} - -- type: log - paths: - - /nsm/suricata/eve*.json - fields: - module: suricata - dataset: common - category: network - - processors: - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: false - close_removed: false - -- type: log - paths: - - /nsm/import/*/suricata/eve*.json - fields: - module: suricata - dataset: common - category: network - imported: true - processors: - - add_tags: - tags: ["import"] - - dissect: - tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}" - field: "log.file.path" - target_prefix: "" - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: false - close_removed: false {%- if STRELKAENABLED == 1 %} - type: log diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 64cdc47fc..8ab200276 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -26,6 +26,12 @@ filebeatetcdir: - user: 939 - group: 939 - makedirs: True +filebeatmoduledir: + file.directory: + - name: /opt/so/conf/filebeat/modules + - user: root + - group: root + - makedirs: True filebeatlogdir: file.directory: - name: /opt/so/log/filebeat @@ -55,6 +61,21 @@ filebeatconfsync: - defaults: INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }} OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }} +# Filebeat module config file +filebeatmoduleconfsync: + file.managed: + - name: /opt/so/conf/filebeat/etc/module-setup.yml + - source: salt://filebeat/etc/module-setup.yml + - user: root + - group: root + - template: jinja +# Sync Filebeat modules +filebeatmodules: + file.recurse: + - name: /opt/so/conf/filebeat/modules + - source: salt://filebeat/modules + - user: root + - group: root so-filebeat: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }} @@ -65,8 +86,10 @@ so-filebeat: - /nsm:/nsm:ro - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - /opt/so/conf/filebeat/etc/module-setup.yml:/usr/share/filebeat/module-setup.yml:ro - /nsm/wazuh/logs/alerts:/wazuh/alerts:ro - /nsm/wazuh/logs/archives:/wazuh/archives:ro + - /opt/so/conf/filebeat/modules:/usr/share/filebeat/modules.d - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw diff --git a/salt/filebeat/modules/suricata.yml.disabled b/salt/filebeat/modules/suricata.yml.disabled deleted file mode 100644 index 1edd3f832..000000000 --- a/salt/filebeat/modules/suricata.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: suricata -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html - -- module: suricata - # All logs - eve: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/zeek.yml.disabled b/salt/filebeat/modules/zeek.yml.disabled deleted file mode 100644 index 0667c6e35..000000000 --- a/salt/filebeat/modules/zeek.yml.disabled +++ /dev/null @@ -1,84 +0,0 @@ -# Module: zeek -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html - -- module: zeek - capture_loss: - enabled: true - connection: - enabled: true - dce_rpc: - enabled: true - dhcp: - enabled: true - dnp3: - enabled: true - dns: - enabled: true - dpd: - enabled: true - files: - enabled: true - ftp: - enabled: true - http: - enabled: true - intel: - enabled: true - irc: - enabled: true - kerberos: - enabled: true - modbus: - enabled: true - mysql: - enabled: true - notice: - enabled: true - ntlm: - enabled: true - ocsp: - enabled: true - pe: - enabled: true - radius: - enabled: true - rdp: - enabled: true - rfb: - enabled: true - signature: - enabled: true - sip: - enabled: true - smb_cmd: - enabled: true - smb_files: - enabled: true - smb_mapping: - enabled: true - smtp: - enabled: true - snmp: - enabled: true - socks: - enabled: true - ssh: - enabled: true - ssl: - enabled: true - stats: - enabled: true - syslog: - enabled: true - traceroute: - enabled: true - tunnel: - enabled: true - weird: - enabled: true - x509: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index d17dc2b22..da798a79d 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -1,19 +1,20 @@ -{%- if grains['role'] == 'so-eval' -%} +%- if grains['role'] == 'so-eval' -%} {%- set ES = salt['pillar.get']('manager:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [module] =~ "zeek" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ ES }}" - index => "so-zeek" - template_name => "so-zeek" - template => "/templates/so-zeek-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } + if [event][module] == 'zeek' { + elasticsearch { + id => "zeek_logs" + pipeline => "filebeat-%{[agent][version]}-zeek-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-zeek-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 5013bafc1..258781f29 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -4,12 +4,12 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [module] =~ "suricata" and "import" not in [tags] { + if [event][module] =~ "suricata" and "import" not in [tags] { elasticsearch { - pipeline => "%{module}.%{dataset}" + pipeline => "filebeat-%{[agent][version]}-suricata-%{[fileset][name]}-pipeline" hosts => "{{ ES }}" index => "so-ids" - template_name => "so-ids" + template_name => "so-ids" template => "/templates/so-ids-template.json" ssl => true ssl_certificate_verification => false From 1b749cf004979b6cd8fc25959ea4d937b9b656fd Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 May 2021 13:55:07 +0000 Subject: [PATCH 082/378] Additional config --- .../tools/sbin/so-filebeat-module-setup | 56 +++++++++++++++++++ salt/filebeat/etc/module-setup.yml | 10 ++++ 2 files changed, 66 insertions(+) create mode 100755 salt/common/tools/sbin/so-filebeat-module-setup create mode 100644 salt/filebeat/etc/module-setup.yml diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup new file mode 100755 index 000000000..a42b0ac80 --- /dev/null +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -0,0 +1,56 @@ +{%- set mainint = salt['pillar.get']('host:mainint') %} +{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} + +#!/bin/bash +# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +default_conf_dir=/opt/so/conf +ELASTICSEARCH_HOST="{{ MYIP }}" +ELASTICSEARCH_PORT=9200 +#ELASTICSEARCH_AUTH="" + +# Define a default directory to load pipelines from +FB_MODULE_YML="/usr/share/filebeat/module-setup.yml" + +if [ "$1" == "" ]; then + echo "No module supplied. Exiting..." +else + # Wait for ElasticSearch to initialize + echo -n "Waiting for ElasticSearch..." + COUNT=0 + ELASTICSEARCH_CONNECTED="no" + while [[ "$COUNT" -le 240 ]]; do + curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi + done + if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo + fi + + echo "Setting up ingest pipeline(s) for $1..." + docker exec -it so-filebeat filebeat setup modules -pipelines -modules $1 -c $FB_MODULE_YML +fi + diff --git a/salt/filebeat/etc/module-setup.yml b/salt/filebeat/etc/module-setup.yml new file mode 100644 index 000000000..431e432b3 --- /dev/null +++ b/salt/filebeat/etc/module-setup.yml @@ -0,0 +1,10 @@ +{%- if grains['role'] in ['so-managersearch', 'so-heavynode', 'so-node'] %} +{%- set MANAGER = salt['grains.get']('host' '') %} +{%- else %} +{%- set MANAGER = salt['grains.get']('master') %} +{%- endif %} + +output.elasticsearch: + enabled: true + hosts: ["https://{{ MANAGER }}:9200"] + ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"] From ee92ba20b04a025eb4e75d49000930724bd772c0 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 May 2021 13:56:39 +0000 Subject: [PATCH 083/378] Add modules path reference --- salt/filebeat/etc/filebeat.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index bd72bc583..46a59f772 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -71,7 +71,13 @@ logging.files: # Set to true to log messages in json format. #logging.json: false + + #========================== Modules configuration ============================ +filebeat.config.modules: + enabled: true + path: ${path.config}/modules.d/*.yml + filebeat.modules: #=========================== Filebeat prospectors ============================= From 728d1f75406be60b5acda5a8eeb6a0b038a8950a Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 May 2021 14:06:17 +0000 Subject: [PATCH 084/378] Make Zeek and Suricata great again --- salt/filebeat/modules/suricata.yml | 12 +++ salt/filebeat/modules/zeek.yml | 122 +++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 salt/filebeat/modules/suricata.yml create mode 100644 salt/filebeat/modules/zeek.yml diff --git a/salt/filebeat/modules/suricata.yml b/salt/filebeat/modules/suricata.yml new file mode 100644 index 000000000..b7cc11e85 --- /dev/null +++ b/salt/filebeat/modules/suricata.yml @@ -0,0 +1,12 @@ +# Module: suricata +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html + +- module: suricata + # All logs + eve: + enabled: true + var.paths: ["/nsm/suricata/eve*.json"] + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/zeek.yml b/salt/filebeat/modules/zeek.yml new file mode 100644 index 000000000..9fd61c448 --- /dev/null +++ b/salt/filebeat/modules/zeek.yml @@ -0,0 +1,122 @@ +# Module: zeek +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html + +- module: zeek + capture_loss: + enabled: false + var.paths: ["/nsm/zeek/logs/current/capture_loss.log"] + connection: + enabled: true + var.paths: ["/nsm/zeek/logs/current/conn.log"] + dce_rpc: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dce_rpc.log"] + dhcp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dhcp.log"] + dnp3: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dnp3.log"] + dns: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dns.log"] + dpd: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dpd.log"] + files: + enabled: true + var.paths: ["/nsm/zeek/logs/current/files.log"] + ftp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/ftp.log"] + http: + enabled: true + var.paths: ["/nsm/zeek/logs/current/http.log"] + intel: + enabled: true + var.paths: ["/nsm/zeek/logs/current/intel.log"] + irc: + enabled: true + var.paths: ["/nsm/zeek/logs/current/irc.log"] + kerberos: + enabled: true + var.paths: ["/nsm/zeek/logs/current/kerberos.log"] + modbus: + enabled: true + var.paths: ["/nsm/zeek/logs/current/modbus.log"] + mysql: + enabled: true + var.paths: ["/nsm/zeek/logs/current/mysql.log"] + notice: + enabled: true + var.paths: ["/nsm/zeek/logs/current/notice.log"] + ntlm: + enabled: true + var.paths: ["/nsm/zeek/logs/current/ntlm.log"] + ocsp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/oscp.log"] + pe: + enabled: true + var.paths: ["/nsm/zeek/logs/current/pe.log"] + radius: + enabled: true + var.paths: ["/nsm/zeek/logs/current/radius.log"] + rdp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/rdp.log"] + rfb: + enabled: true + var.paths: ["/nsm/zeek/logs/current/rfb.log"] + signature: + enabled: true + var.paths: ["/nsm/zeek/logs/current/signature.log"] + sip: + enabled: true + var.paths: ["/nsm/zeek/logs/current/sip.log"] + smb_cmd: + enabled: true + var.paths: ["/nsm/zeek/logs/current/smb_cmd.log"] + smb_files: + enabled: true + var.paths: ["/nsm/zeek/logs/current/smb_files.log"] + smb_mapping: + enabled: true + var.paths: ["/nsm/zeek/logs/current/smb_mapping.log"] + smtp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/smtp.log"] + snmp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/snmp.log"] + socks: + enabled: true + var.paths: ["/nsm/zeek/logs/current/socks.log"] + ssh: + enabled: true + var.paths: ["/nsm/zeek/logs/current/ssh.log"] + ssl: + enabled: true + var.paths: ["/nsm/zeek/logs/current/ssl.log"] + stats: + enabled: false + var.paths: ["/nsm/zeek/logs/current/stats.log"] + syslog: + enabled: false + var.paths: ["/nsm/zeek/logs/current/syslog.log"] + traceroute: + enabled: false + var.paths: ["/nsm/zeek/logs/current/traceroute.log.log"] + tunnel: + enabled: true + var.paths: ["/nsm/zeek/logs/current/tunnel.log"] + weird: + enabled: true + var.paths: ["/nsm/zeek/logs/current/weird.log"] + x509: + enabled: true + var.paths: ["/nsm/zeek/logs/current/x509.log"] + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: From da528e802f2797cdcc4a6b920b87f3476bd74399 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 12:52:47 -0400 Subject: [PATCH 085/378] ensure migration script doesnt migrate the current days data and fix downsample cq to move from so_short_term rp --- salt/common/tools/sbin/so-influxdb-migrate | 21 +++++++++++---------- salt/influxdb/init.sls | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index 3411dc39f..4d65cba95 100644 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -26,17 +26,18 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? - echo "Migration of 2020-07-21T00:00:00.0000000Z' + ${day}d started at $startdate and completed at `date`." + migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` + echo "Migration of $migrationdate started at $startdate and completed at `date`." -# echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" -# if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then -# echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" -# else -# echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." -# fi - - ((day=day+1)) - startdate=`date` + newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s") + today=$(date +"%s") + if [ $newdaytomigrate -ge $today ]; then + break + else + ((day=day+1)) + startdate=`date` + echo -e "\nMigrating the next day's worth of data." + fi done echo -e "\nInfluxDb data migration complete." diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index c6120d292..2617578a4 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,7 +117,7 @@ so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM \"so_short_term\"./.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] From 78240b4b5232513e32521bed339157df32940a9c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 17:49:02 -0400 Subject: [PATCH 086/378] change retry command --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 965b9369c..6cb292aba 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that there is at least 1 minion connected - retry 50 10 "(( $(salt-run manage.up -l error | wc -l) >= 1 ))" + retry 50 10 "salt-run manage.up -l error" $MINIONID echo "" echo "Ensuring python modules for Salt are installed and patched." From b8ad80ae35c0cc352ab3b942efcd4ca82c1329f9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 17:49:40 -0400 Subject: [PATCH 087/378] update comment --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6cb292aba..bf09e2b2e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -801,7 +801,7 @@ else echo "Starting Salt Master service." systemctl start salt-master - # Testing that that salt-master is up by checking that there is at least 1 minion connected + # Testing that that salt-master is up by checking that is it connected to itself retry 50 10 "salt-run manage.up -l error" $MINIONID echo "" From 6474c296e18240ee028c234f83f69eeb9e23bfaf Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 20:26:13 -0400 Subject: [PATCH 088/378] dont need to specify dest rp --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 2617578a4..c6120d292 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,7 +117,7 @@ so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM \"so_short_term\"./.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] From 5d3b147b42aa1040e11af75dae0fec34494c2e8d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 20:32:26 -0400 Subject: [PATCH 089/378] change retry command --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index bf09e2b2e..751a2413c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-run manage.up -l error" $MINIONID + retry 50 10 "salt-run manage.up -l error | grep $MINIONID" echo "" echo "Ensuring python modules for Salt are installed and patched." From 46223e0b3055e8c0af6d424ab23df399421d0627 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 08:59:47 -0400 Subject: [PATCH 090/378] add quotes around minionid --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 751a2413c..6fb139909 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-run manage.up -l error | grep $MINIONID" + retry 50 10 "salt-run manage.up -l error | grep \"$MINIONID\"" echo "" echo "Ensuring python modules for Salt are installed and patched." From d0e084b8eab150719b63925714a8dbc5fce19927 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 10:20:04 -0400 Subject: [PATCH 091/378] change command to test if salt-master is accepting connections --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6fb139909..77a2b6018 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-run manage.up -l error | grep \"$MINIONID\"" + retry 50 10 "salt-call state.show_top -l error" echo "" echo "Ensuring python modules for Salt are installed and patched." From 7b29c6427b5f5efc1ef53052c65e588d992b4a90 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 7 May 2021 10:55:17 -0400 Subject: [PATCH 092/378] Add preliminary error handling in soup --- salt/common/tools/sbin/so-common | 15 ++++ salt/common/tools/sbin/so-image-common | 13 ++- salt/common/tools/sbin/soup | 107 +++++++++++++++++++++---- 3 files changed, 117 insertions(+), 18 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 22c7543ea..f78919226 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -324,6 +324,21 @@ retry() { return 1 } +run_check_net_err() { + local cmd=$1 + local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable + + local exit_code + retry 50 10 "$cmd" + exit_code=$? + + if [[ $exit_code -ne 0 ]]; then + echo "Command failed with error $exit_code" + echo "$err_msg" + exit $exit_code + fi +} + set_os() { if [ -f /etc/redhat-release ]; then OS=centos diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index be5a327f0..c855abd15 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -18,6 +18,7 @@ # NOTE: This script depends on so-common IMAGEREPO=security-onion-solutions +# shellcheck disable=SC2120 container_list() { MANAGERCHECK=$1 @@ -128,7 +129,9 @@ update_docker_containers() { mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 # Let's make sure we have the public key - retry 50 10 "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1 + run_check_net_err \ + "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ + "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" >> "$LOG_FILE" 2>&1 result=$? if [[ $result -eq 0 ]]; then cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 @@ -148,10 +151,14 @@ update_docker_containers() { # Pull down the trusted docker image local image=$i:$VERSION$IMAGE_TAG_SUFFIX - retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1 + run_check_net_err \ + "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \ + "Could not pull pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 # Get signature - retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1 + run_check_net_err \ + "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ + "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 exit 1 diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 8e7f70517..f43a31740 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -20,11 +20,83 @@ UPDATE_DIR=/tmp/sogh/securityonion INSTALLEDVERSION=$(cat /etc/soversion) POSTVERSION=$INSTALLEDVERSION -INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) +INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}') BATCHSIZE=5 SOUP_LOG=/root/soup.log WHATWOULDYOUSAYYAHDOHERE=soup +set -e + +trap 'check_err $? $BASH_LINENO $BASH_COMMAND' EXIT + +check_err() { + local exit_code=$1 + local lineno=$2 + local cmd=$3 + local err_msg="Unknown error occured, please check $SOUP_LOG for details." + + if [[ $exit_code -ne 0 ]]; then + printf '%s' "Soup failed on line $lineno with error $exit_code: " + case $exit_code in + 2) + echo 'No such file or directory' + ;; + 5) + echo 'Interrupted system call' + ;; + 12) + echo 'Out of memory' + ;; + 28) + echo 'No space left on device' + echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html' + ;; + 30) + echo 'Read-only file system' + ;; + 35) + echo 'Resource temporarily unavailable' + ;; + 64) + echo 'Machine is not on the network' + ;; + 67) + echo 'Link has been severed' + ;; + 100) + echo 'Netowrk is down' + ;; + 101) + echo 'Network is unreachable' + ;; + 102) + echo 'Network reset' + ;; + 110) + echo 'Connection timed out' + ;; + 111) + echo 'Connection refused' + ;; + 112) + echo 'Host is down' + ;; + 113) + echo 'No route to host' + ;; + *) + echo '' + echo "$err_msg" + ;; + esac + if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then + echo "$err_msg" + fi + exit $exit_code + fi + +} + add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ @@ -87,9 +159,9 @@ airgap_update_dockers() { docker stop so-dockerregistry docker rm so-dockerregistry echo "Copying the new dockers over" - tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker + tar xvf "$AGDOCKER/registry.tar" -C /nsm/docker-registry/docker echo "Add Registry back" - docker load -i $AGDOCKER/registry_image.tar + docker load -i "$AGDOCKER/registry_image.tar" fi fi } @@ -101,9 +173,9 @@ update_registry() { } check_sudoers() { - if grep -q "so-setup" /etc/sudoers; then - echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." - fi + if grep -q "so-setup" /etc/sudoers; then + echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." + fi } check_log_size_limit() { @@ -177,7 +249,7 @@ check_os_updates() { echo "Continuing without updating packages" elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" - salt \* -b 5 state.apply patch.os queue=True + run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' else echo "Exiting soup" exit 0 @@ -205,7 +277,7 @@ clone_to_tmp() { if [ -n "$BRANCH" ]; then SOUP_BRANCH="-b $BRANCH" fi - git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git + run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com" cd /tmp if [ ! -f $UPDATE_DIR/VERSION ]; then echo "Update was unable to pull from github. Please check your internet." @@ -586,13 +658,14 @@ upgrade_check() { } upgrade_check_salt() { - NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'}) + NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk '{print $2}') if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then echo "You are already running the correct version of Salt for Security Onion." else UPGRADESALT=1 fi } + upgrade_salt() { SALTUPGRADED=True echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." @@ -604,7 +677,9 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" + run_check_net_err \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "Could not update soup, please check $SOUP_LOG for details." echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" @@ -617,7 +692,9 @@ upgrade_salt() { apt-mark unhold "salt-minion" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" + run_check_net_err \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "Could not update soup, please check $SOUP_LOG for details." echo "Applying apt hold for Salt." echo "" apt-mark hold "salt-common" @@ -650,7 +727,7 @@ verify_latest_update_script() { } main () { -echo "### Preparing soup at `date` ###" +echo "### Preparing soup at $(date) ###" while getopts ":b" opt; do case "$opt" in b ) # process option b @@ -756,7 +833,7 @@ else echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded - SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'}) + SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}') if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." @@ -874,7 +951,7 @@ EOF fi fi -echo "### soup has been served at `date` ###" +echo "### soup has been served at $(date) ###" } cat << EOF @@ -889,6 +966,6 @@ Press Enter to continue or Ctrl-C to cancel. EOF -read input +read -r input main "$@" | tee -a $SOUP_LOG From b39c8c1f1f19cf9a62a5a3a54438817d84b84853 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 11:02:23 -0400 Subject: [PATCH 093/378] exit after 50 tries if manager cant connect to iteself via salt --- salt/common/tools/sbin/soup | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 77a2b6018..620737c16 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" + retry 50 10 "salt-call state.show_top -l error" || exit 1 echo "" echo "Ensuring python modules for Salt are installed and patched." @@ -834,6 +834,10 @@ else echo "" echo "Starting Salt Master service." systemctl start salt-master + + # Testing that that salt-master is up by checking that is it connected to itself + retry 50 10 "salt-call state.show_top -l error" || exit 1 + echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes From 009f7617c1c77426c8a20c7ce62b3ec97dec6472 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 12:47:22 -0400 Subject: [PATCH 094/378] check salt-master is responding --- salt/common/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 77a2b6018..51b7bc29c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -834,6 +834,10 @@ else echo "" echo "Starting Salt Master service." systemctl start salt-master + + # Testing that that salt-master is up by checking that is it connected to itself + retry 50 10 "salt-call state.show_top -l error" + echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes From 2e228c8355473c7f366ea2f1e35defab75d8591a Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 7 May 2021 13:22:03 -0400 Subject: [PATCH 095/378] FEATURE: Pivot from Alerts/Hunt to CyberChef #4081 --- salt/soc/files/soc/alerts.actions.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/alerts.actions.json b/salt/soc/files/soc/alerts.actions.json index c0543d8fc..558d10a36 100644 --- a/salt/soc/files/soc/alerts.actions.json +++ b/salt/soc/files/soc/alerts.actions.json @@ -18,6 +18,10 @@ "/joblookup?esid={:soc_id}", "/joblookup?ncid={:network.community_id}" ]}, + { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", + "links": [ + "/cyberchef/#input={value|base64}" + ]}, { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", "links": [ "https://www.google.com/search?q={value}" @@ -26,4 +30,4 @@ "links": [ "https://www.virustotal.com/gui/search/{value}" ]} -] \ No newline at end of file +] From 293fb0a76d87feb207c8eb6db6b0a85aca49b928 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 7 May 2021 13:23:46 -0400 Subject: [PATCH 096/378] FEATURE: Pivot from Alerts/Hunt to CyberChef #4081 --- salt/soc/files/soc/hunt.actions.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.actions.json b/salt/soc/files/soc/hunt.actions.json index c0543d8fc..558d10a36 100644 --- a/salt/soc/files/soc/hunt.actions.json +++ b/salt/soc/files/soc/hunt.actions.json @@ -18,6 +18,10 @@ "/joblookup?esid={:soc_id}", "/joblookup?ncid={:network.community_id}" ]}, + { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", + "links": [ + "/cyberchef/#input={value|base64}" + ]}, { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", "links": [ "https://www.google.com/search?q={value}" @@ -26,4 +30,4 @@ "links": [ "https://www.virustotal.com/gui/search/{value}" ]} -] \ No newline at end of file +] From 9eb63b17f9f571282836b48a1343a1f6c265ff6d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 14:48:02 -0400 Subject: [PATCH 097/378] exit if retry fails --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 51b7bc29c..620737c16 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" + retry 50 10 "salt-call state.show_top -l error" || exit 1 echo "" echo "Ensuring python modules for Salt are installed and patched." @@ -836,7 +836,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" + retry 50 10 "salt-call state.show_top -l error" || exit 1 echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True From f8dd6890b27f6109d8f31a7dbbf3db709d312ea9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 7 May 2021 16:50:59 -0400 Subject: [PATCH 098/378] Unset/set exit on command fail for retries --- salt/common/tools/sbin/soup | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f43a31740..b02a63fc6 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -249,7 +249,9 @@ check_os_updates() { echo "Continuing without updating packages" elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" + set +e run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' + set -e else echo "Exiting soup" exit 0 @@ -277,7 +279,9 @@ clone_to_tmp() { if [ -n "$BRANCH" ]; then SOUP_BRANCH="-b $BRANCH" fi + set +e run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com" + set -e cd /tmp if [ ! -f $UPDATE_DIR/VERSION ]; then echo "Update was unable to pull from github. Please check your internet." @@ -677,9 +681,11 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" + set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." + set -e echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" @@ -692,9 +698,11 @@ upgrade_salt() { apt-mark unhold "salt-minion" echo "Updating Salt packages and restarting services." echo "" + set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." + set -e echo "Applying apt hold for Salt." echo "" apt-mark hold "salt-common" @@ -808,7 +816,9 @@ else check_os_updates else update_registry + set +e update_docker_containers "soup" + set -e fi echo "" From 8b3730748b7a26d58489a9b803337b54e3f5827c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 10:58:29 -0400 Subject: [PATCH 099/378] Add debug line and remove exit command on retry failure --- salt/common/tools/sbin/soup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 398aed29c..fdc983d30 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -662,6 +662,7 @@ upgrade_check() { is_hotfix=true fi else + echo "DEBUG: here" is_hotfix=false fi @@ -889,7 +890,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" || exit 1 + retry 50 10 "salt-call state.show_top -l error" echo "" echo "Ensuring python modules for Salt are installed and patched." @@ -923,7 +924,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" || exit 1 + retry 50 10 "salt-call state.show_top -l error" echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True From a75d4841d0a29eb06ec492fbf0518f3c6bb86b45 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 11:05:24 -0400 Subject: [PATCH 100/378] Add debug lines --- salt/common/tools/sbin/soup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index fdc983d30..d1d3f5e6e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -734,7 +734,7 @@ verify_latest_update_script() { cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - salt-call state.apply common queue=True + salt-call state.apply -l info common queue=True echo "" echo "soup has been updated. Please run soup again." exit 0 @@ -796,7 +796,9 @@ add_common fi echo "Let's see if we need to update Security Onion." +echo "DEBUG: before upgrade_check" upgrade_check +echo "DEBUG: after upgrade_check" upgrade_space echo "Checking for Salt Master and Minion updates." From d3b08beb534a2b3cd336d7d2dcdb9dc8ab8dc7b0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 11:11:54 -0400 Subject: [PATCH 101/378] Only cat file if it exists --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d1d3f5e6e..6c4af361b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -651,7 +651,7 @@ upgrade_check() { # Let's make sure we actually need to update. NEWVERSION=$(cat $UPDATE_DIR/VERSION) HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX) - CURRENTHOTFIX=$(cat /etc/sohotfix 2>/dev/null) + [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix) if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then echo "Checking to see if there are hotfixes needed" if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then From fec868432f3d240a216b42647cc633a6cd9c168c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 11:59:22 -0400 Subject: [PATCH 102/378] Try to fix bash trap --- salt/common/tools/sbin/soup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6c4af361b..364a769c0 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -28,13 +28,12 @@ WHATWOULDYOUSAYYAHDOHERE=soup set -e -trap 'check_err $? $BASH_LINENO $BASH_COMMAND' EXIT +trap 'check_err $? $BASH_LINENO' EXIT check_err() { local exit_code=$1 local lineno=$2 - local cmd=$3 - local err_msg="Unknown error occured, please check $SOUP_LOG for details." + local err_msg="Unhandled error occured, please check $SOUP_LOG for details." if [[ $exit_code -ne 0 ]]; then printf '%s' "Soup failed on line $lineno with error $exit_code: " @@ -86,7 +85,7 @@ check_err() { echo 'No route to host' ;; *) - echo '' + echo 'Unhandled error' echo "$err_msg" ;; esac From f793450d972cb69743fe7cac96644e934ac275b6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 13:22:13 -0400 Subject: [PATCH 103/378] Return actual exit code from retry --- salt/common/tools/sbin/so-common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index f78919226..2b2bbf944 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -302,6 +302,7 @@ retry() { cmd=$3 expectedOutput=$4 attempt=0 + local exitcode=0 while [[ $attempt -lt $maxAttempts ]]; do attempt=$((attempt+1)) echo "Executing command with retry support: $cmd" @@ -321,7 +322,7 @@ retry() { sleep $sleepDelay done echo "Command continues to fail; giving up." - return 1 + return $exitcode } run_check_net_err() { From 01a121e0298243dc664be36caa2661b13b9bc020 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 May 2021 15:29:50 -0400 Subject: [PATCH 104/378] Add defaults.yml --- salt/filebeat/thirdpartydefaults.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 salt/filebeat/thirdpartydefaults.yml diff --git a/salt/filebeat/thirdpartydefaults.yml b/salt/filebeat/thirdpartydefaults.yml new file mode 100644 index 000000000..9e5fef988 --- /dev/null +++ b/salt/filebeat/thirdpartydefaults.yml @@ -0,0 +1,17 @@ +thirtd_party_filebeat: + modules: + fortinet: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9004 + clientendpoint: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9510 + fortimail: + enabled: false + var.input: udp + var.syslog_port: 9350 \ No newline at end of file From efc028d0a55b39a39ea484cb9c1a5cd425e63dd6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 10 May 2021 18:08:47 -0400 Subject: [PATCH 105/378] handle the docker port bindings for filebeat modules --- salt/filebeat/init.sls | 9 +++++++++ salt/filebeat/map.jinja | 2 ++ salt/filebeat/thirdpartydefaults.yml | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 salt/filebeat/map.jinja diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 8ab200276..eb4dc116a 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -20,6 +20,8 @@ {% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set MANAGER = salt['grains.get']('master') %} {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} + filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc @@ -98,6 +100,13 @@ so-filebeat: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp - 0.0.0.0:5066:5066/tcp +{% for module in THIRDPARTY.modules.keys() %} + {% for submodule in THIRDPARTY.modules[module] %} + {% if THIRDPARTY.modules[module][submodule].enabled %} + - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/{{ THIRDPARTY.modules[module][submodule]["var.input"] }} + {% endif %} + {% endfor %} +{% endfor %} - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja new file mode 100644 index 000000000..668889227 --- /dev/null +++ b/salt/filebeat/map.jinja @@ -0,0 +1,2 @@ +{% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %} +{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} \ No newline at end of file diff --git a/salt/filebeat/thirdpartydefaults.yml b/salt/filebeat/thirdpartydefaults.yml index 9e5fef988..1e2eb8c23 100644 --- a/salt/filebeat/thirdpartydefaults.yml +++ b/salt/filebeat/thirdpartydefaults.yml @@ -1,4 +1,4 @@ -thirtd_party_filebeat: +third_party_filebeat: modules: fortinet: firewall: From 4012a8276c93e3b6dcf9ff3c22e3085910c5d249 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 May 2021 12:22:25 -0400 Subject: [PATCH 106/378] add template for module .yml file --- salt/filebeat/etc/thirdparty.yml.jinja | 16 ++++++++++++++++ salt/filebeat/init.sls | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 salt/filebeat/etc/thirdparty.yml.jinja diff --git a/salt/filebeat/etc/thirdparty.yml.jinja b/salt/filebeat/etc/thirdparty.yml.jinja new file mode 100644 index 000000000..8c081828e --- /dev/null +++ b/salt/filebeat/etc/thirdparty.yml.jinja @@ -0,0 +1,16 @@ +# DO NOT EDIT THIS FILE +{% for module in THIRDPARTY.modules.keys() -%} +- module: {{ module }} + {%- for fileset in THIRDPARTY.modules[module] %} + {{ fileset }}: + enabled: {{ THIRDPARTY.modules[module][fileset].enabled }} + {#- only manage the settings if the fileset is enabled #} + {%- if THIRDPARTY.modules[module][fileset].enabled %} + {%- for var, value in THIRDPARTY.modules[module][fileset].items() %} + {%- if var|lower != 'enabled' %} + {{ var }}: {{ value }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endfor %} +{% endfor %} \ No newline at end of file diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index eb4dc116a..b3bce806c 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -22,24 +22,28 @@ {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} + filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc - user: 939 - group: 939 - makedirs: True + filebeatmoduledir: file.directory: - name: /opt/so/conf/filebeat/modules - user: root - group: root - makedirs: True + filebeatlogdir: file.directory: - name: /opt/so/log/filebeat - user: 939 - group: 939 - makedirs: True + filebeatpkidir: file.directory: - name: /opt/so/conf/filebeat/etc/pki @@ -52,6 +56,7 @@ fileregistrydir: - user: 939 - group: 939 - makedirs: True + # This needs to be owned by root filebeatconfsync: file.managed: @@ -63,6 +68,7 @@ filebeatconfsync: - defaults: INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }} OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }} + # Filebeat module config file filebeatmoduleconfsync: file.managed: @@ -71,6 +77,7 @@ filebeatmoduleconfsync: - user: root - group: root - template: jinja + # Sync Filebeat modules filebeatmodules: file.recurse: @@ -78,6 +85,15 @@ filebeatmodules: - source: salt://filebeat/modules - user: root - group: root + +thirdparty_module_conf: + file.managed: + - name: /opt/so/conf/filebeat/etc/thirdparty.yml + - source: salt://filebeat/etc/thirdparty.yml.jinja + - template: jinja + - defaults: + THIRDPARTY: {{ THIRDPARTY }} + so-filebeat: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }} From ad67167e9778908534b921c38f27b3bdbc88d162 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 May 2021 12:58:21 -0400 Subject: [PATCH 107/378] remove whitespace control --- salt/filebeat/etc/thirdparty.yml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/thirdparty.yml.jinja b/salt/filebeat/etc/thirdparty.yml.jinja index 8c081828e..186115af4 100644 --- a/salt/filebeat/etc/thirdparty.yml.jinja +++ b/salt/filebeat/etc/thirdparty.yml.jinja @@ -1,5 +1,5 @@ # DO NOT EDIT THIS FILE -{% for module in THIRDPARTY.modules.keys() -%} +{% for module in THIRDPARTY.modules.keys() %} - module: {{ module }} {%- for fileset in THIRDPARTY.modules[module] %} {{ fileset }}: From c379822bf0df0fb52083392498feebc8e22ddc1e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 11 May 2021 12:59:49 -0400 Subject: [PATCH 108/378] Set variable to skip trap if error already handled --- salt/common/tools/sbin/so-common | 1 + salt/common/tools/sbin/soup | 1 + 2 files changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 2b2bbf944..2b22ebbeb 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -334,6 +334,7 @@ run_check_net_err() { exit_code=$? if [[ $exit_code -ne 0 ]]; then + ERR_HANDLED=true echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 364a769c0..c8601fe1a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -35,6 +35,7 @@ check_err() { local lineno=$2 local err_msg="Unhandled error occured, please check $SOUP_LOG for details." + [[ $ERR_HANDLED == true ]] && exit $exit_code if [[ $exit_code -ne 0 ]]; then printf '%s' "Soup failed on line $lineno with error $exit_code: " case $exit_code in From 7bd04deae78f0199afb706afefb8ca4b20588b5e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 12 May 2021 10:45:03 -0400 Subject: [PATCH 109/378] Unset exit on failure for pkill command --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c8601fe1a..7c97a540a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -834,7 +834,9 @@ else echo "Stopping Salt Minion service." systemctl stop salt-minion echo "Killing any remaining Salt Minion processes." + set +e pkill -9 -ef /usr/bin/salt-minion + set -e echo "" echo "Stopping Salt Master service." systemctl stop salt-master From 807b525c797bd280036bf06d0a63a17693894f44 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 12 May 2021 11:19:33 -0400 Subject: [PATCH 110/378] Temp remove exit on failure + bash trap --- salt/common/tools/sbin/soup | 83 ------------------------------------- 1 file changed, 83 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 7c97a540a..53dd7f176 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -26,77 +26,6 @@ SOUP_LOG=/root/soup.log INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup -set -e - -trap 'check_err $? $BASH_LINENO' EXIT - -check_err() { - local exit_code=$1 - local lineno=$2 - local err_msg="Unhandled error occured, please check $SOUP_LOG for details." - - [[ $ERR_HANDLED == true ]] && exit $exit_code - if [[ $exit_code -ne 0 ]]; then - printf '%s' "Soup failed on line $lineno with error $exit_code: " - case $exit_code in - 2) - echo 'No such file or directory' - ;; - 5) - echo 'Interrupted system call' - ;; - 12) - echo 'Out of memory' - ;; - 28) - echo 'No space left on device' - echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html' - ;; - 30) - echo 'Read-only file system' - ;; - 35) - echo 'Resource temporarily unavailable' - ;; - 64) - echo 'Machine is not on the network' - ;; - 67) - echo 'Link has been severed' - ;; - 100) - echo 'Netowrk is down' - ;; - 101) - echo 'Network is unreachable' - ;; - 102) - echo 'Network reset' - ;; - 110) - echo 'Connection timed out' - ;; - 111) - echo 'Connection refused' - ;; - 112) - echo 'Host is down' - ;; - 113) - echo 'No route to host' - ;; - *) - echo 'Unhandled error' - echo "$err_msg" - ;; - esac - if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then - echo "$err_msg" - fi - exit $exit_code - fi - -} add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ @@ -250,9 +179,7 @@ check_os_updates() { echo "Continuing without updating packages" elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" - set +e run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' - set -e else echo "Exiting soup" exit 0 @@ -280,9 +207,7 @@ clone_to_tmp() { if [ -n "$BRANCH" ]; then SOUP_BRANCH="-b $BRANCH" fi - set +e run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com" - set -e cd /tmp if [ ! -f $UPDATE_DIR/VERSION ]; then echo "Update was unable to pull from github. Please check your internet." @@ -688,11 +613,9 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." - set -e echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" @@ -705,11 +628,9 @@ upgrade_salt() { apt-mark unhold "salt-minion" echo "Updating Salt packages and restarting services." echo "" - set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." - set -e echo "Applying apt hold for Salt." echo "" apt-mark hold "salt-common" @@ -825,18 +746,14 @@ else check_os_updates else update_registry - set +e update_docker_containers "soup" - set -e fi echo "" echo "Stopping Salt Minion service." systemctl stop salt-minion echo "Killing any remaining Salt Minion processes." - set +e pkill -9 -ef /usr/bin/salt-minion - set -e echo "" echo "Stopping Salt Master service." systemctl stop salt-master From 9ced391c11f255a7fe22b4e740b436398de0983d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 12 May 2021 13:20:59 -0400 Subject: [PATCH 111/378] Fix indent in main(), re-add trap, remove ERR_HANDLED variable --- salt/common/tools/sbin/so-common | 1 - salt/common/tools/sbin/soup | 534 ++++++++++++++++++------------- 2 files changed, 309 insertions(+), 226 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 2b22ebbeb..2b2bbf944 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -334,7 +334,6 @@ run_check_net_err() { exit_code=$? if [[ $exit_code -ne 0 ]]; then - ERR_HANDLED=true echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 53dd7f176..814f9fd1b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -26,6 +26,72 @@ SOUP_LOG=/root/soup.log INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup +check_err() { + local exit_code=$1 + local lineno=$2 + local err_msg="Unhandled error occured, please check $SOUP_LOG for details." + + if [[ $exit_code -ne 0 ]]; then + printf '%s' "Soup failed on line $lineno with error $exit_code: " + case $exit_code in + 2) + echo 'No such file or directory' + ;; + 5) + echo 'Interrupted system call' + ;; + 12) + echo 'Out of memory' + ;; + 28) + echo 'No space left on device' + echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html' + ;; + 30) + echo 'Read-only file system' + ;; + 35) + echo 'Resource temporarily unavailable' + ;; + 64) + echo 'Machine is not on the network' + ;; + 67) + echo 'Link has been severed' + ;; + 100) + echo 'Netowrk is down' + ;; + 101) + echo 'Network is unreachable' + ;; + 102) + echo 'Network reset' + ;; + 110) + echo 'Connection timed out' + ;; + 111) + echo 'Connection refused' + ;; + 112) + echo 'Host is down' + ;; + 113) + echo 'No route to host' + ;; + *) + echo 'Unhandled error' + echo "$err_msg" + ;; + esac + if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then + echo "$err_msg" + fi + exit $exit_code + fi + +} add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ @@ -48,8 +114,8 @@ airgap_mounted() { echo "Example: /home/user/securityonion-2.X.0.iso" echo "Example: /dev/sdx1" echo "" - read -p 'Enter the location of the iso: ' ISOLOC - if [ -f $ISOLOC ]; then + read -rp 'Enter the location of the iso: ' ISOLOC + if [[ -f $ISOLOC ]]; then # Mounting the ISO image mkdir -p /tmp/soagupdate mount -t iso9660 -o loop $ISOLOC /tmp/soagupdate @@ -61,7 +127,7 @@ airgap_mounted() { else echo "ISO has been mounted!" fi - elif [ -f $ISOLOC/SecurityOnion/VERSION ]; then + elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then ln -s $ISOLOC /tmp/soagupdate echo "Found the update content" else @@ -79,9 +145,9 @@ airgap_mounted() { } airgap_update_dockers() { - if [ $is_airgap -eq 0 ]; then + if [[ $is_airgap -eq 0 ]]; then # Let's copy the tarball - if [ ! -f $AGDOCKER/registry.tar ]; then + if [[ ! -f $AGDOCKER/registry.tar ]]; then echo "Unable to locate registry. Exiting" exit 1 else @@ -179,7 +245,9 @@ check_os_updates() { echo "Continuing without updating packages" elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" + set +e run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' + set -e else echo "Exiting soup" exit 0 @@ -207,7 +275,9 @@ clone_to_tmp() { if [ -n "$BRANCH" ]; then SOUP_BRANCH="-b $BRANCH" fi + set +e run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com" + set -e cd /tmp if [ ! -f $UPDATE_DIR/VERSION ]; then echo "Update was unable to pull from github. Please check your internet." @@ -253,7 +323,11 @@ preupgrade_changes_2.3.50_repo() { # We made repo changes in 2.3.50 and this prepares for that on upgrade echo "Checking to see if 2.3.50 repo changes are needed." - [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo + if [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]]; then + up_2.3.3X_to_2.3.50_repo + else + echo "No changes needed." + fi } preupgrade_changes() { @@ -433,7 +507,7 @@ up_2.3.2X_to_2.3.30() { sed -i "/ imagerepo: securityonion/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls # Strelka rule repo pillar addition - if [ $is_airgap -eq 0 ]; then + if [[ $is_airgap -eq 0 ]]; then # Add manager as default Strelka YARA rule repo sed -i "/^strelka:/a \\ repos: \n - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls; else @@ -460,7 +534,7 @@ up_2.3.3X_to_2.3.50_repo() { rm -f "/etc/yum.repos.d/$DELREPO.repo" fi done - if [ $is_airgap -eq 1 ]; then + if [[ $is_airgap -eq 1 ]]; then # Copy the new repo file if not airgap cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ yum clean all @@ -613,9 +687,11 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" + set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." + set -e echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" @@ -628,9 +704,11 @@ upgrade_salt() { apt-mark unhold "salt-minion" echo "Updating Salt packages and restarting services." echo "" + set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." + set -e echo "Applying apt hold for Salt." echo "" apt-mark hold "salt-common" @@ -662,229 +740,235 @@ verify_latest_update_script() { fi } -main () { -echo "### Preparing soup at $(date) ###" -while getopts ":b" opt; do - case "$opt" in - b ) # process option b - shift - BATCHSIZE=$1 - if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then - echo "Batch size must be a number greater than 0." - exit 1 - fi - ;; - \? ) - echo "Usage: cmd [-b]" - ;; - esac -done - -echo "Checking to see if this is a manager." -echo "" -require_manager -set_minionid -echo "Checking to see if this is an airgap install" -echo "" -check_airgap -echo "Found that Security Onion $INSTALLEDVERSION is currently installed." -echo "" -set_os -set_palette -check_elastic_license -echo "" -if [ $is_airgap -eq 0 ]; then - # Let's mount the ISO since this is airgap - airgap_mounted -else - echo "Cloning Security Onion github repo into $UPDATE_DIR." - echo "Removing previous upgrade sources." - rm -rf $UPDATE_DIR - clone_to_tmp -fi -check_os_updates -echo "" -echo "Verifying we have the latest soup script." -verify_latest_update_script -echo "" - -echo "Generating new repo archive" -generate_and_clean_tarballs -if [ -f /usr/sbin/so-image-common ]; then - . /usr/sbin/so-image-common -else -add_common -fi - -echo "Let's see if we need to update Security Onion." -echo "DEBUG: before upgrade_check" -upgrade_check -echo "DEBUG: after upgrade_check" -upgrade_space - -echo "Checking for Salt Master and Minion updates." -upgrade_check_salt - - -if [ "$is_hotfix" == "true" ]; then - echo "Applying $HOTFIXVERSION" - copy_new_files - echo "" - update_version - salt-call state.highstate -l info queue=True +main() { + set -e + trap 'check_err $? $BASH_LINENO' EXIT -else - echo "" - echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." - echo "" - - echo "Updating dockers to $NEWVERSION." - if [ $is_airgap -eq 0 ]; then - airgap_update_dockers - update_centos_repo - yum clean all - check_os_updates - else - update_registry - update_docker_containers "soup" - fi - - echo "" - echo "Stopping Salt Minion service." - systemctl stop salt-minion - echo "Killing any remaining Salt Minion processes." - pkill -9 -ef /usr/bin/salt-minion - echo "" - echo "Stopping Salt Master service." - systemctl stop salt-master - echo "" - - preupgrade_changes_2.3.50_repo - - # Does salt need upgraded. If so update it. - if [ "$UPGRADESALT" == "1" ]; then - echo "Upgrading Salt" - # Update the repo files so it can actually upgrade - upgrade_salt - fi - - echo "Checking if Salt was upgraded." - echo "" - # Check that Salt was upgraded - SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}') - if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then - echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." - echo "Once the issue is resolved, run soup again." - echo "Exiting." - echo "" - exit 1 - else - echo "Salt upgrade success." - echo "" - fi - - preupgrade_changes - echo "" - - if [ $is_airgap -eq 0 ]; then - echo "Updating Rule Files to the Latest." - update_airgap_rules - fi - - # Only update the repo if its airgap - if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then - update_centos_repo - fi - - echo "" - echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR." - copy_new_files - echo "" - update_version - - echo "" - echo "Locking down Salt Master for upgrade" - masterlock - - echo "" - echo "Starting Salt Master service." - systemctl start salt-master - - # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" - - echo "" - echo "Ensuring python modules for Salt are installed and patched." - salt-call state.apply salt.python3-influxdb -l info queue=True - echo "" - - # Only regenerate osquery packages if Fleet is enabled - FLEET_MANAGER=$(lookup_pillar fleet_manager) - FLEET_NODE=$(lookup_pillar fleet_node) - if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then - echo "" - echo "Regenerating Osquery Packages.... This will take several minutes." - salt-call state.apply fleet.event_gen-packages -l info queue=True - echo "" - fi - - echo "" - echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." - salt-call state.highstate -l info queue=True - echo "" - echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete." - - echo "" - echo "Stopping Salt Master to remove ACL" - systemctl stop salt-master - - masterunlock - - echo "" - echo "Starting Salt Master service." - systemctl start salt-master - - # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" - - echo "Running a highstate. This could take several minutes." - salt-call state.highstate -l info queue=True - postupgrade_changes - unmount_update - thehive_maint - - if [ "$UPGRADESALT" == "1" ]; then - if [ $is_airgap -eq 0 ]; then - echo "" - echo "Cleaning repos on remote Security Onion nodes." - salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all" - echo "" - fi - fi - - check_sudoers - - if [[ -n $lsl_msg ]]; then - case $lsl_msg in - 'distributed') - echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect." - echo " -> We recommend checking and adjusting the values as necessary." - echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/" + echo "### Preparing soup at $(date) ###" + while getopts ":b" opt; do + case "$opt" in + b ) # process option b + shift + BATCHSIZE=$1 + if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then + echo "Batch size must be a number greater than 0." + exit 1 + fi ;; - 'single-node') - # We can assume the lsl_details array has been set if lsl_msg has this value - echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}." - echo " -> We recommend checking and adjusting the value as necessary." - echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls" + \? ) + echo "Usage: cmd [-b]" ;; esac + done + + echo "Checking to see if this is a manager." + echo "" + require_manager + set_minionid + echo "Checking to see if this is an airgap install" + echo "" + check_airgap + echo "Found that Security Onion $INSTALLEDVERSION is currently installed." + echo "" + set_os + set_palette + check_elastic_license + echo "" + if [[ $is_airgap -eq 0 ]]; then + # Let's mount the ISO since this is airgap + airgap_mounted + else + echo "Cloning Security Onion github repo into $UPDATE_DIR." + echo "Removing previous upgrade sources." + rm -rf $UPDATE_DIR + clone_to_tmp + fi + check_os_updates + echo "" + echo "Verifying we have the latest soup script." + verify_latest_update_script + echo "" + + echo "Generating new repo archive" + generate_and_clean_tarballs + if [ -f /usr/sbin/so-image-common ]; then + . /usr/sbin/so-image-common + else + add_common fi - NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) + echo "Let's see if we need to update Security Onion." + echo "DEBUG: before upgrade_check" + upgrade_check + echo "DEBUG: after upgrade_check" + upgrade_space - if [ $NUM_MINIONS -gt 1 ]; then + echo "Checking for Salt Master and Minion updates." + upgrade_check_salt - cat << EOF + + if [ "$is_hotfix" == "true" ]; then + echo "Applying $HOTFIXVERSION" + copy_new_files + echo "" + update_version + salt-call state.highstate -l info queue=True + else + echo "" + echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." + echo "" + + echo "Updating dockers to $NEWVERSION." + if [[ $is_airgap -eq 0 ]]; then + airgap_update_dockers + update_centos_repo + yum clean all + check_os_updates + else + update_registry + set +e + update_docker_containers "soup" + set -e + fi + + echo "" + echo "Stopping Salt Minion service." + systemctl stop salt-minion + echo "Killing any remaining Salt Minion processes." + set +e + pkill -9 -ef /usr/bin/salt-minion + set -e + echo "" + echo "Stopping Salt Master service." + systemctl stop salt-master + echo "" + + preupgrade_changes_2.3.50_repo + + # Does salt need upgraded. If so update it. + if [[ $UPGRADESALT -eq 1 ]]; then + echo "Upgrading Salt" + # Update the repo files so it can actually upgrade + upgrade_salt + fi + + echo "Checking if Salt was upgraded." + echo "" + # Check that Salt was upgraded + SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}') + if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then + echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." + echo "Once the issue is resolved, run soup again." + echo "Exiting." + echo "" + exit 1 + else + echo "Salt upgrade success." + echo "" + fi + + preupgrade_changes + echo "" + + if [[ $is_airgap -eq 0 ]]; then + echo "Updating Rule Files to the Latest." + update_airgap_rules + fi + + # Only update the repo if its airgap + if [[ $is_airgap -eq 0 ]] && [[ $UPGRADESALT -ne 1 ]]; then + update_centos_repo + fi + + echo "" + echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR." + copy_new_files + echo "" + update_version + + echo "" + echo "Locking down Salt Master for upgrade" + masterlock + + echo "" + echo "Starting Salt Master service." + systemctl start salt-master + + # Testing that that salt-master is up by checking that is it connected to itself + retry 50 10 "salt-call state.show_top -l error" + + echo "" + echo "Ensuring python modules for Salt are installed and patched." + salt-call state.apply salt.python3-influxdb -l info queue=True + echo "" + + # Only regenerate osquery packages if Fleet is enabled + FLEET_MANAGER=$(lookup_pillar fleet_manager) + FLEET_NODE=$(lookup_pillar fleet_node) + if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then + echo "" + echo "Regenerating Osquery Packages.... This will take several minutes." + salt-call state.apply fleet.event_gen-packages -l info queue=True + echo "" + fi + + echo "" + echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." + salt-call state.highstate -l info queue=True + echo "" + echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete." + + echo "" + echo "Stopping Salt Master to remove ACL" + systemctl stop salt-master + + masterunlock + + echo "" + echo "Starting Salt Master service." + systemctl start salt-master + + # Testing that that salt-master is up by checking that is it connected to itself + retry 50 10 "salt-call state.show_top -l error" + + echo "Running a highstate. This could take several minutes." + salt-call state.highstate -l info queue=True + postupgrade_changes + unmount_update + thehive_maint + + if [[ $UPGRADESALT -eq 1 ]]; then + if [[ $is_airgap -eq 0 ]]; then + echo "" + echo "Cleaning repos on remote Security Onion nodes." + salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all" + echo "" + fi + fi + + check_sudoers + + if [[ -n $lsl_msg ]]; then + case $lsl_msg in + 'distributed') + echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect." + echo " -> We recommend checking and adjusting the values as necessary." + echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/" + ;; + 'single-node') + # We can assume the lsl_details array has been set if lsl_msg has this value + echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}." + echo " -> We recommend checking and adjusting the value as necessary." + echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls" + ;; + esac + fi + + NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) + + if [[ $NUM_MINIONS -gt 1 ]]; then + + cat << EOF @@ -898,10 +982,10 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html EOF + fi fi -fi -echo "### soup has been served at $(date) ###" + echo "### soup has been served at $(date) ###" } cat << EOF From 192b5db25a5c3d2746da34f8cf8d79c8a388ba28 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 12 May 2021 15:26:39 -0400 Subject: [PATCH 112/378] Add true to end of functions ending with shorthand comparison Functions ending with test using [[ ]] && will trip set -e, so adding true to the last line of the function will prevent the function from returning a nonzero code --- salt/common/tools/sbin/soup | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 814f9fd1b..e5c09a487 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -28,11 +28,10 @@ WHATWOULDYOUSAYYAHDOHERE=soup check_err() { local exit_code=$1 - local lineno=$2 local err_msg="Unhandled error occured, please check $SOUP_LOG for details." if [[ $exit_code -ne 0 ]]; then - printf '%s' "Soup failed on line $lineno with error $exit_code: " + printf '%s' "Soup failed with error $exit_code: " case $exit_code in 2) echo 'No such file or directory' @@ -323,11 +322,8 @@ preupgrade_changes_2.3.50_repo() { # We made repo changes in 2.3.50 and this prepares for that on upgrade echo "Checking to see if 2.3.50 repo changes are needed." - if [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]]; then - up_2.3.3X_to_2.3.50_repo - else - echo "No changes needed." - fi + [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo + true } preupgrade_changes() { @@ -340,6 +336,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50 + true } postupgrade_changes() { @@ -350,6 +347,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30 [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40 [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60 + true } post_rc1_to_2.3.21() { @@ -661,7 +659,6 @@ upgrade_check() { is_hotfix=true fi else - echo "DEBUG: here" is_hotfix=false fi @@ -742,7 +739,7 @@ verify_latest_update_script() { main() { set -e - trap 'check_err $? $BASH_LINENO' EXIT + trap 'check_err $?' EXIT echo "### Preparing soup at $(date) ###" while getopts ":b" opt; do @@ -876,7 +873,7 @@ main() { fi # Only update the repo if its airgap - if [[ $is_airgap -eq 0 ]] && [[ $UPGRADESALT -ne 1 ]]; then + if [[ $is_airgap -eq 0 && $UPGRADESALT -ne 1 ]]; then update_centos_repo fi From 68a667ee7cb6de7c8e829939f9e4042c5cc63890 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 May 2021 15:31:19 -0400 Subject: [PATCH 113/378] Add thirfpartydefaults.yml --- ...efaults.yml => securityoniondefaults.yaml} | 0 salt/filebeat/thirdpartydefaults.yaml | 275 ++++++++++++++++++ 2 files changed, 275 insertions(+) rename salt/filebeat/{thirdpartydefaults.yml => securityoniondefaults.yaml} (100%) create mode 100644 salt/filebeat/thirdpartydefaults.yaml diff --git a/salt/filebeat/thirdpartydefaults.yml b/salt/filebeat/securityoniondefaults.yaml similarity index 100% rename from salt/filebeat/thirdpartydefaults.yml rename to salt/filebeat/securityoniondefaults.yaml diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml new file mode 100644 index 000000000..027ec4595 --- /dev/null +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -0,0 +1,275 @@ +third_party_filebeat: + modules: + aws: + cloudtrail: + enabled: false + cloudwatch: + enabled: false + ec2: + enabled: false + elb: + enabled: false + s3access: + enabled: false + vpcflow: + enabled: false + azure: + activitylogs: + enabled: false + platformlogs: + enabled: false + auditlogs: + enabled: false + signinlogs: + enabled: false + barracuda: + waf: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9503 + spamfirewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9524 + bluecoat: + director: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9505 + cef: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9003 + checkpoint: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9505 + cisco: + asa: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9001 + ftd: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9003 + ios: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9002 + nexus: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9506 + meraki: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9525 + umbrella: + enabled: false + amp: + enabled: false + cyberark: + corepas: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9527 + cylance: + protect: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9508 + f5: + bigipapm: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9504 + bigipafm: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9528 + fortinet: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9004 + clientendpoint: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9510 + fortimail: + enabled: false + var.input: udp + var.syslog_port: 9350 + gcp: + vpcflow: + enabled: false + firewall: + enabled: false + audit: + enabled: false + google_workspace: + saml: + enabled: false + user_accounts: + enabled: false + login: + enabled: false + admin: + enabled: false + drive: + enabled: false + groups: + enabled: false + imperva: + securesphere: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9511 + infoblox: + nios: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9512 + juniper: + junos: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9513 + netscreen: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9523 + srx: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9006 + microsoft: + defender_atp: + enabled: false + m365_defender: + enabled: false + dhcp: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9515 + misp: + threat: + enabled: false + netflow: + log: + enabled: false + var.netflow_host: 0.0.0.0 + var.netflow_port: 2055 + var.internal_networks: + - private + netscout: + sightline: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9502 + o365: + audit: + enabled: false + okta: + enabled: false + pesando: + dfw: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9001 + proofpoint: + emailsecurity: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9531 + radware: + defensepro: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9518 + snort: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9532 + snyk: + audit: + enabled: false + vulnerabilities: + enabled: false + sonicwall: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9519 + sophos: + xg: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9005 + utm: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9533 + squid: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9520 + threatintel: + abuseurl: + enabled: false + abusemalware: + enabled: false + misp: + enabled: false + otx: + enabled: false + anomali: + enabled: false + tomcat: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9501 + zscaler: + zia: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9521 From 74f2a61b25c3697908283660c73ab7b463bfb0e5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 May 2021 09:06:47 -0400 Subject: [PATCH 114/378] install influxdb and grafana during setup prior to final highstate --- salt/influxdb/init.sls | 4 ++-- setup/so-setup | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index c6120d292..9ef9aaae2 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -1,6 +1,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} + {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} {% set MANAGER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} @@ -11,8 +13,6 @@ {% from 'salt/map.jinja' import PYTHONINFLUXVERSION with context %} {% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) %} -{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} - include: - salt.minion - salt.python3-influxdb diff --git a/setup/so-setup b/setup/so-setup index 1d8eba9ed..4573da330 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -837,6 +837,13 @@ echo "1" > /root/accept_changes set_progress_str 73 "Update playbook rules" so-playbook-ruleupdate >> /root/setup_playbook_rule_update.log 2>&1 & fi + + if [[ "$GRAFANA" = 1 ]]; then + set_progress_str 74 "Installing InfluxDB and Grafana" + salt-call state.apply -l info influxdb >> $setup_log 2>&1 + salt-call state.apply -l info grafana >> $setup_log 2>&1 + fi + fi if [[ "$OSQUERY" = 1 ]]; then From d3af06e7a4a47d3ff902690834a1f2ec6da7cccc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 May 2021 11:00:42 -0400 Subject: [PATCH 115/378] handle exception if influxdb module doesnt exist --- salt/influxdb/init.sls | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 9ef9aaae2..3a309db47 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -1,9 +1,10 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} +{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} + {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} -{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} {% set MANAGER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} @@ -11,7 +12,7 @@ {% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} {% from 'salt/map.jinja' import PYTHON3INFLUX with context %} {% from 'salt/map.jinja' import PYTHONINFLUXVERSION with context %} -{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) %} +{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c \"exec('try:import influxdb; print (influxdb.__version__)\\nexcept:print(\\'Module Not Found\\')')\"", python_shell=True) %} include: - salt.minion From 71a74a665653f209c31354eed53ee345007b213d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 13 May 2021 13:07:16 -0400 Subject: [PATCH 116/378] Added updated script and core modules --- .../tools/sbin/so-filebeat-module-setup | 50 +++++++++---------- salt/filebeat/securityoniondefaults.yaml | 45 +++++++++++------ 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index a42b0ac80..6616854eb 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -25,32 +25,30 @@ ELASTICSEARCH_PORT=9200 # Define a default directory to load pipelines from FB_MODULE_YML="/usr/share/filebeat/module-setup.yml" -if [ "$1" == "" ]; then - echo "No module supplied. Exiting..." -else - # Wait for ElasticSearch to initialize - echo -n "Waiting for ElasticSearch..." - COUNT=0 - ELASTICSEARCH_CONNECTED="no" - while [[ "$COUNT" -le 240 ]]; do - curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi - done - if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo - fi - echo "Setting up ingest pipeline(s) for $1..." - docker exec -it so-filebeat filebeat setup modules -pipelines -modules $1 -c $FB_MODULE_YML +# Wait for ElasticSearch to initialize +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 240 ]]; do + curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo fi +echo "Setting up ingest pipeline(s)" +docker exec -it so-filebeat filebeat setup modules -pipelines -modules activemq,apache,auditd,aws,azure,barracuda,bluecoat,cef,checkpoint,cisco,coredns,crowdstrike,cyberark,cylance,elasticsearch,envoyproxy,f5,fortinet,gcp,google_workspace,googlecloud,gsuite,haproxy,ibmmq,icinga,iis,imperva,infoblox,iptables,juniper,kafka,kibana,logstash,microsoft,misp,mondogb,mssql,mysql,mysqlenterprise,nats,netflow,netscout,nginx,o365,okta,osquery,panw,pensando,postgresql,rabbitmq,radware,redis,santa,snort,snyk,sonicwall,sophos,squid,suricata,system,threatintel,tomcat,traefik,zeek,zoom,zscaler -c $FB_MODULE_YML + + diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 1e2eb8c23..e6406af8c 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -1,17 +1,32 @@ -third_party_filebeat: +securityonion_filebeat: modules: - fortinet: - firewall: - enabled: false - var.input: udp - var.syslog_host: 0.0.0.0 - var.syslog_port: 9004 - clientendpoint: + elasticsearch: + server: + enabled: true + var.paths: ["/logs/elasticsearch/*.log"] + kibana: + log: + enabled: true + var.paths: ["/logs/kibana/kibana.log"] + logstash: + log: + enabled: true + var.paths: ["/logs/logstash.log"] + redis: + log: + enabled: true + var.paths: ["/logs/redis.log"] + suricata: + eve: + enabled: true + var.paths: ["/nsm/suricata/eve*.json"] + {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} + {%- if ZEEKVER != 'SURICATA' %} + zeek: + {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} + {{ LOGNAME }}: enabled: false - var.input: udp - var.syslog_host: 0.0.0.0 - var.syslog_port: 9510 - fortimail: - enabled: false - var.input: udp - var.syslog_port: 9350 \ No newline at end of file + var.paths: ["/nsm/zeek/logs/current/{{ LOGNAME }}.log"] + {%- endfor %} + {%- endif %} + {%- endif %} \ No newline at end of file From 3859f6464ab0ab6b418c2d39561196a848e28df5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 May 2021 08:56:42 -0400 Subject: [PATCH 117/378] dont be quiet on first grep --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 4573da330..4c364ded6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -926,7 +926,7 @@ if [[ $success != 0 ]]; then SO_ERROR=1; fi # Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox # Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry -if grep -q -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then +if grep -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then SO_ERROR=1 grep --color=never "ERROR" "$setup_log" | grep -qvE "Status .* was not found" > "$error_log" From 231e07dbbda9a33091be927c70965ddcab26cec7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 May 2021 15:19:45 -0400 Subject: [PATCH 118/378] circumvent file.patch putting ERROR in log if patch doesnt need applied --- salt/salt/python3-influxdb.sls | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index acadf2161..055390a27 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -15,6 +15,9 @@ python3_influxdb: {{PYTHONINSTALLER}}.installed: - name: {{ PYTHON3INFLUX }} +# We circumvent the file.patch state putting ERROR in the log by using the unless and file.touch below +# https://github.com/saltstack/salt/pull/47010 and https://github.com/saltstack/salt/issues/52329 + #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: @@ -23,8 +26,15 @@ influxdb_continuous_query.present_patch: - require: - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package + - unless: ls /opt/so/state/influxdb_continuous_query.py.patched -#https://github.com/saltstack/salt/issues/59761 +influxdb_continuous_query.py.patched: + file.touch: + - name: /opt/so/state/influxdb_continuous_query.py.patched + - onchanges: + - file: influxdb_continuous_query.present_patch + + #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py @@ -32,6 +42,13 @@ influxdb_retention_policy.present_patch: - require: - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package + - unless: ls /opt/so/state/influxdb_retention_policy.py.patched + +influxdb_retention_policy.py.patched: + file.touch: + - name: /opt/so/state/influxdb_retention_policy.py.patched + - onchanges: + - file: influxdb_retention_policy.present_patch # We should be able to set reload_modules: True in this state in order to tell salt to reload its python modules due to us possibly installing # and possibly modifying modules in this state. This is bugged according to https://github.com/saltstack/salt/issues/24925 @@ -41,4 +58,13 @@ influxdbmod.py_shard_duration_patch: - source: salt://salt/files/influxdbmod.py.patch - require: - {{PYTHONINSTALLER}}: python3_influxdb - - pkg: patch_package \ No newline at end of file + - pkg: patch_package + - unless: ls /opt/so/state/influxdbmod.py.patched + +influxdbmod.py.patched: + file.touch: + - name: /opt/so/state/influxdbmod.py.patched + - onchanges: + - file: influxdbmod.py_shard_duration_patch + + From 54f9e3ff9df127d1ddd2706db5109f414dc4bd4c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 May 2021 16:24:16 -0400 Subject: [PATCH 119/378] remove leading space on comment line --- salt/salt/python3-influxdb.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 055390a27..7a59c4303 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -34,7 +34,7 @@ influxdb_continuous_query.py.patched: - onchanges: - file: influxdb_continuous_query.present_patch - #https://github.com/saltstack/salt/issues/59761 +#https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py From 3625453668953d6d6a7c18cea26e4c16909a6344 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 11:00:28 -0400 Subject: [PATCH 120/378] Don't unmount airgap directory if not airgap --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e5c09a487..676c5b0bc 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -931,7 +931,7 @@ main() { echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes - unmount_update + [[ $is_airgap -eq 0 ]] && unmount_update thehive_maint if [[ $UPGRADESALT -eq 1 ]]; then From 2bcd51b21c3771e3ee488a7a63b6342e8481bebe Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 15:10:57 -0400 Subject: [PATCH 121/378] Fix error message --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 676c5b0bc..323e755b2 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -687,7 +687,7 @@ upgrade_salt() { set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ - "Could not update soup, please check $SOUP_LOG for details." + "Could not update salt, please check $SOUP_LOG or scrollback for details." set -e echo "Applying yum versionlock for Salt." echo "" @@ -704,7 +704,7 @@ upgrade_salt() { set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ - "Could not update soup, please check $SOUP_LOG for details." + "Could not update salt, please check $SOUP_LOG or scrollback for details." set -e echo "Applying apt hold for Salt." echo "" From 992b76a0f09c40b6e7631767397e94039c5a79f0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 15:38:10 -0400 Subject: [PATCH 122/378] Remove debug lines --- salt/common/tools/sbin/soup | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 323e755b2..2f27939e6 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -795,9 +795,7 @@ main() { fi echo "Let's see if we need to update Security Onion." - echo "DEBUG: before upgrade_check" upgrade_check - echo "DEBUG: after upgrade_check" upgrade_space echo "Checking for Salt Master and Minion updates." From b89091cc7d441361e571f31d1dd723606088ebb7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 15:58:25 -0400 Subject: [PATCH 123/378] Try retrying in curl instead of shell function --- salt/common/tools/sbin/so-common | 10 ++++++++-- salt/common/tools/sbin/so-image-common | 10 ++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 2b2bbf944..d5450f096 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -328,10 +328,16 @@ retry() { run_check_net_err() { local cmd=$1 local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable + local no_retry=$3 local exit_code - retry 50 10 "$cmd" - exit_code=$? + if [[ -z $no_retry ]]; then + retry 50 10 "$cmd" + exit_code=$? + else + eval "$cmd" + exit_code=$? + fi if [[ $exit_code -ne 0 ]]; then echo "Command failed with error $exit_code" diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index c855abd15..fa1d6c6dd 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -130,8 +130,9 @@ update_docker_containers() { # Let's make sure we have the public key run_check_net_err \ - "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ - "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" >> "$LOG_FILE" 2>&1 + "curl --retry 20 --retry-delay 30 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ + "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \ + noretry >> "$LOG_FILE" 2>&1 result=$? if [[ $result -eq 0 ]]; then cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 @@ -157,8 +158,9 @@ update_docker_containers() { # Get signature run_check_net_err \ - "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ - "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " >> "$LOG_FILE" 2>&1 + "curl --retry 20 --retry-delay 30 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ + "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " \ + noretry >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 exit 1 From 3f238f7a4ad911e605c5c859d9edaf564a012aa4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 16:02:52 -0400 Subject: [PATCH 124/378] Set flag so trap doesn't repeat info --- salt/common/tools/sbin/so-common | 1 + salt/common/tools/sbin/soup | 1 + 2 files changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index d5450f096..0390a2bf2 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -340,6 +340,7 @@ run_check_net_err() { fi if [[ $exit_code -ne 0 ]]; then + ERR_HANDLED=true echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2f27939e6..9a1afb2bb 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -30,6 +30,7 @@ check_err() { local exit_code=$1 local err_msg="Unhandled error occured, please check $SOUP_LOG for details." + [[ $ERR_HANDLED == true ]] && exit $exit_code if [[ $exit_code -ne 0 ]]; then printf '%s' "Soup failed with error $exit_code: " case $exit_code in From 4e1fba5b38183cad4c479fab7149946c06c8be83 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 16:04:13 -0400 Subject: [PATCH 125/378] Only echo error code if not using retry --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 0390a2bf2..0b8cd97aa 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -341,7 +341,7 @@ run_check_net_err() { if [[ $exit_code -ne 0 ]]; then ERR_HANDLED=true - echo "Command failed with error $exit_code" + [[ -z $no_retry ]] && echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code fi From ce9554281ee2d83567a9cd1f2b5a9a7a3f1eab13 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 16:08:34 -0400 Subject: [PATCH 126/378] Fix backwards logic --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 0b8cd97aa..c6b0c0bd8 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -341,7 +341,7 @@ run_check_net_err() { if [[ $exit_code -ne 0 ]]; then ERR_HANDLED=true - [[ -z $no_retry ]] && echo "Command failed with error $exit_code" + [[ -z $no_retry ]] || echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code fi From dbd5ef70c995eb6a8065487d9e3908307e41bf26 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 16:19:31 -0400 Subject: [PATCH 127/378] Change retry delay + count --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index c6b0c0bd8..3ab260eb8 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -332,7 +332,7 @@ run_check_net_err() { local exit_code if [[ -z $no_retry ]]; then - retry 50 10 "$cmd" + retry 20 30 "$cmd" exit_code=$? else eval "$cmd" From bdd53ed5e3a806ed26786a43df20b85ce9002294 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 18 May 2021 12:23:40 -0400 Subject: [PATCH 128/378] Change retry delay + count --- salt/common/tools/sbin/so-common | 2 +- salt/common/tools/sbin/so-image-common | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 3ab260eb8..55350d702 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -332,7 +332,7 @@ run_check_net_err() { local exit_code if [[ -z $no_retry ]]; then - retry 20 30 "$cmd" + retry 5 60 "$cmd" exit_code=$? else eval "$cmd" diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index fa1d6c6dd..2c61838b5 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -130,7 +130,7 @@ update_docker_containers() { # Let's make sure we have the public key run_check_net_err \ - "curl --retry 20 --retry-delay 30 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ + "curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \ noretry >> "$LOG_FILE" 2>&1 result=$? @@ -158,7 +158,7 @@ update_docker_containers() { # Get signature run_check_net_err \ - "curl --retry 20 --retry-delay 30 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ + "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " \ noretry >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then From f445186f1ebb2b28fd3f6592e8b07c98fce06fd5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 18 May 2021 13:38:55 -0400 Subject: [PATCH 129/378] Remove redundant error messages --- salt/common/tools/sbin/so-image-common | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 2c61838b5..f840e07ab 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -136,9 +136,6 @@ update_docker_containers() { result=$? if [[ $result -eq 0 ]]; then cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 - else - echo "Failed to pull signature key file: $result" - exit 1 fi # Download the containers from the interwebs @@ -159,12 +156,8 @@ update_docker_containers() { # Get signature run_check_net_err \ "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ - "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " \ + "Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \ noretry >> "$LOG_FILE" 2>&1 - if [[ $? -ne 0 ]]; then - echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 - exit 1 - fi # Dump our hash values DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image) From ac42cba50b6e4679d5dd976bfb1f11fa8428eec6 Mon Sep 17 00:00:00 2001 From: Masaya-A <68965261+Masaya-A@users.noreply.github.com> Date: Wed, 19 May 2021 09:06:02 +0900 Subject: [PATCH 130/378] Adding MAC Address for NIC List --- setup/so-functions | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index dfe79f674..bc5d9d1ca 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1348,15 +1348,16 @@ filter_unused_nics() { nic_list=() for nic in "${filtered_nics[@]}"; do + local nic_mac=$(cat "/sys/class/net/${nic}/address") case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in 1) - nic_list+=("$nic" "Link UP " "OFF") + nic_list+=("$nic" "$nic_mac" "Link UP " "OFF") ;; 0) - nic_list+=("$nic" "Link DOWN " "OFF") + nic_list+=("$nic" "$nic_mac" "Link DOWN " "OFF") ;; *) - nic_list+=("$nic" "Link UNKNOWN " "OFF") + nic_list+=("$nic" "$nic_mac" "Link UNKNOWN " "OFF") ;; esac done From 6af126b87200e616864e5788798790919928b84a Mon Sep 17 00:00:00 2001 From: Masaya-A <68965261+Masaya-A@users.noreply.github.com> Date: Wed, 19 May 2021 10:02:19 +0900 Subject: [PATCH 131/378] Fix array --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index bc5d9d1ca..8cdbc2596 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1351,13 +1351,13 @@ filter_unused_nics() { local nic_mac=$(cat "/sys/class/net/${nic}/address") case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in 1) - nic_list+=("$nic" "$nic_mac" "Link UP " "OFF") + nic_list+=("$nic" "$nic_mac Link UP " "OFF") ;; 0) - nic_list+=("$nic" "$nic_mac" "Link DOWN " "OFF") + nic_list+=("$nic" "$nic_mac Link DOWN " "OFF") ;; *) - nic_list+=("$nic" "$nic_mac" "Link UNKNOWN " "OFF") + nic_list+=("$nic" "$nic_mac Link UNKNOWN " "OFF") ;; esac done From 3450219bc7073fb692259ed185fc88eca2dd516e Mon Sep 17 00:00:00 2001 From: Masaya-A <68965261+Masaya-A@users.noreply.github.com> Date: Thu, 20 May 2021 09:33:14 +0900 Subject: [PATCH 132/378] Drop error to /dev/null --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 8cdbc2596..5ce3d6dee 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1348,7 +1348,7 @@ filter_unused_nics() { nic_list=() for nic in "${filtered_nics[@]}"; do - local nic_mac=$(cat "/sys/class/net/${nic}/address") + local nic_mac=$(cat "/sys/class/net/${nic}/address" 2>/dev/null) case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in 1) nic_list+=("$nic" "$nic_mac Link UP " "OFF") From c1dd4dafe47001101e949d5a8fbefaca0d97f987 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 21 May 2021 12:41:10 -0400 Subject: [PATCH 133/378] Fix influx state --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index bafcbd53b..346d971fd 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -3,7 +3,7 @@ {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} -{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %} {% set MANAGER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} From 71032150c5a08c6889e828b6500c79a3cd86929f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 21 May 2021 17:27:00 -0400 Subject: [PATCH 134/378] Add secure HTTP headers to all SO application responses to reduce exposure to browser and other HTTP-related vulnerabilities --- salt/nginx/etc/nginx.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index ea820442b..e9f67e065 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -149,6 +149,12 @@ http { root /opt/socore/html; index index.html; + add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' 'unsafe-eval' https: data:; frame-ancestors 'self'"; + add_header X-Frame-Options SAMEORIGIN; + add_header X-XSS-Protection "1; mode=block"; + add_header X-Content-Type-Options nosniff; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"; + ssl_certificate "/etc/pki/nginx/server.crt"; ssl_certificate_key "/etc/pki/nginx/server.key"; ssl_session_cache shared:SSL:1m; From 0572ea4095b7388d7d88c8e4740be18a56fb73ab Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 21 May 2021 17:27:11 -0400 Subject: [PATCH 135/378] Fail curl command if a failing status code is returned by the remote server --- salt/common/tools/sbin/so-image-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index be5a327f0..47b04f241 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -128,7 +128,7 @@ update_docker_containers() { mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 # Let's make sure we have the public key - retry 50 10 "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1 + retry 50 10 "curl -f -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1 result=$? if [[ $result -eq 0 ]]; then cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 @@ -151,7 +151,7 @@ update_docker_containers() { retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1 # Get signature - retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1 + retry 50 10 "curl -f -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 exit 1 From e2d5102a0e70f8f209b46e385031978fc9328390 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 May 2021 10:13:29 -0400 Subject: [PATCH 136/378] changes for script to auth to elastic --- salt/common/init.sls | 9 ++++++++ salt/common/tools/sbin/so-elastic-clear | 6 ++--- .../tools/sbin/so-elasticsearch-indices-list | 2 +- .../tools/sbin/so-elasticsearch-indices-rw | 4 ++-- .../sbin/so-elasticsearch-pipeline-stats | 4 ++-- .../tools/sbin/so-elasticsearch-pipeline-view | 4 ++-- .../sbin/so-elasticsearch-pipelines-list | 4 ++-- .../tools/sbin/so-elasticsearch-shards-list | 2 +- .../sbin/so-elasticsearch-template-remove | 2 +- .../tools/sbin/so-elasticsearch-template-view | 4 ++-- .../sbin/so-elasticsearch-templates-list | 4 ++-- salt/common/tools/sbin/so-index-list | 2 +- salt/common/tools/sbin/soup | 4 ++-- salt/firewall/portgroups.yaml | 3 +++ salt/utility/bin/crossthestreams | 6 ++--- salt/utility/bin/eval | 4 ++-- salt/utility/init.sls | 23 +++++++++++++++---- 17 files changed, 57 insertions(+), 30 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 33a8b9984..948adee99 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -2,6 +2,8 @@ {% if sls in allowed_states %} {% set role = grains.id.split('_') | last %} +{% set ELASTICUSER = salt['pillar.get']('elasticsearch:auth:user', '' ) %} +{% set ELASTICPASS = salt['pillar.get']('elasticsearch:auth:pass', '' ) %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -178,6 +180,13 @@ utilsyncscripts: - file_mode: 755 - template: jinja - source: salt://common/tools/sbin + - defaults: + ELASTICCURL: "curl" +{% if salt['pillar.get']('elasticsearch:auth_enabled', False) %} + - context: + ELASTICCURL: "curl --user {{ELASTICUSER}}:{{ELASTICPASS}}" +{% endif %} + {% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %} # Add sensor cleanup diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear index 4c7271272..56b5c3d2c 100755 --- a/salt/common/tools/sbin/so-elastic-clear +++ b/salt/common/tools/sbin/so-elastic-clear @@ -50,7 +50,7 @@ done if [ $SKIP -ne 1 ]; then # List indices echo - curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v + {{ ELASTICCURL }} -k -L https://{{ NODEIP }}:9200/_cat/indices?v echo # Inform user we are about to delete all data echo @@ -89,10 +89,10 @@ fi # Delete data echo "Deleting data..." -INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') +INDXS=$({{ ELASTICCURL }} -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }') for INDX in ${INDXS} do - curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 + {{ ELASTICCURL }} -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 done #Start Logstash/Filebeat diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-list b/salt/common/tools/sbin/so-elasticsearch-indices-list index c9df67a25..b5cd1b359 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-list +++ b/salt/common/tools/sbin/so-elasticsearch-indices-list @@ -18,4 +18,4 @@ . /usr/sbin/so-common -curl -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty +{{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_cat/indices?pretty diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw index 6b123bd0d..f5296f2b8 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-rw +++ b/salt/common/tools/sbin/so-elasticsearch-indices-rw @@ -21,5 +21,5 @@ THEHIVEESPORT=9400 echo "Removing read only attributes for indices..." echo -curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; -curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; +{{ ELASTICCURL }} -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; +{{ ELASTICCURL }} -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats index 146196917..2f9edb6c1 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-stats @@ -19,7 +19,7 @@ . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" else - curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" fi diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/common/tools/sbin/so-elasticsearch-pipeline-view index 04901e122..9f799c07f 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-view +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-view @@ -19,7 +19,7 @@ . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . else - curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . fi diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/common/tools/sbin/so-elasticsearch-pipelines-list index 565f90071..f6ef516ef 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list +++ b/salt/common/tools/sbin/so-elasticsearch-pipelines-list @@ -17,7 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' else - curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq fi diff --git a/salt/common/tools/sbin/so-elasticsearch-shards-list b/salt/common/tools/sbin/so-elasticsearch-shards-list index 9d28ed95b..a240f993f 100755 --- a/salt/common/tools/sbin/so-elasticsearch-shards-list +++ b/salt/common/tools/sbin/so-elasticsearch-shards-list @@ -18,4 +18,4 @@ . /usr/sbin/so-common -curl -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty +{{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty diff --git a/salt/common/tools/sbin/so-elasticsearch-template-remove b/salt/common/tools/sbin/so-elasticsearch-template-remove index f7c3e6812..fe19a9d03 100755 --- a/salt/common/tools/sbin/so-elasticsearch-template-remove +++ b/salt/common/tools/sbin/so-elasticsearch-template-remove @@ -18,4 +18,4 @@ . /usr/sbin/so-common -curl -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1 +{{ ELASTICCURL }} -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1 diff --git a/salt/common/tools/sbin/so-elasticsearch-template-view b/salt/common/tools/sbin/so-elasticsearch-template-view index c9f3ec199..1083cb762 100755 --- a/salt/common/tools/sbin/so-elasticsearch-template-view +++ b/salt/common/tools/sbin/so-elasticsearch-template-view @@ -19,7 +19,7 @@ . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_template/* | jq . else - curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq . fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list index 494ca5770..6a7c4d039 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-list +++ b/salt/common/tools/sbin/so-elasticsearch-templates-list @@ -17,7 +17,7 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' else - curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq fi diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list index cf9232150..e24599f0e 100755 --- a/salt/common/tools/sbin/so-index-list +++ b/salt/common/tools/sbin/so-index-list @@ -15,4 +15,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -curl -X GET -k -L "https://localhost:9200/_cat/indices?v&s=index" +{{ ELASTICCURL }} -X GET -k -L "https://localhost:9200/_cat/indices?v&s=index" diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 620737c16..14e2148d0 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -326,7 +326,7 @@ rc1_to_rc2() { local NAME=$(echo $p | awk '{print $1}') local IP=$(echo $p | awk '{print $2}') echo "Removing the old cross cluster config for $NAME" - curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}' + {{ ELASTICCURL }} -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}' done Date: Mon, 24 May 2021 10:52:54 -0400 Subject: [PATCH 137/378] Switch Kratos config from argon2 to bcrypt12 --- salt/soc/files/kratos/kratos.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/salt/soc/files/kratos/kratos.yaml b/salt/soc/files/kratos/kratos.yaml index c26aeec3f..a0a72b3ab 100644 --- a/salt/soc/files/kratos/kratos.yaml +++ b/salt/soc/files/kratos/kratos.yaml @@ -41,12 +41,8 @@ serve: base_url: https://{{ WEBACCESS }}/kratos/ hashers: - argon2: - parallelism: 2 - memory: 16384 - iterations: 3 - salt_length: 16 - key_length: 32 + bcrypt: + cost: 12 identity: default_schema_url: file:///kratos-conf/schema.json From 99d41d1606245cc0d75520545e55bf9b27dd1e9b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 11:29:40 -0400 Subject: [PATCH 138/378] Add ending newline to soup --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 9a1afb2bb..eb15305aa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -999,3 +999,4 @@ EOF read -r input main "$@" | tee -a $SOUP_LOG + \ No newline at end of file From 409eea677ddc6f57bc77ecbbbe6610f7b9cb5f7c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 24 May 2021 11:50:53 -0400 Subject: [PATCH 139/378] Continue removal of argon hashing --- salt/common/init.sls | 2 -- salt/common/tools/sbin/so-user | 11 ++++------- setup/so-functions | 4 ++-- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 33a8b9984..79a7c5300 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -95,7 +95,6 @@ commonpkgs: - netcat - python3-mysqldb - sqlite3 - - argon2 - libssl-dev - python3-dateutil - python3-m2crypto @@ -128,7 +127,6 @@ commonpkgs: - net-tools - curl - sqlite - - argon2 - mariadb-devel - nmap-ncat - python3 diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index b97cc8a8b..edfa4b303 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -39,10 +39,7 @@ email=$2 kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434} databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite} -argon2Iterations=${ARGON2_ITERATIONS:-3} -argon2Memory=${ARGON2_MEMORY:-14} -argon2Parallelism=${ARGON2_PARALLELISM:-2} -argon2HashSize=${ARGON2_HASH_SIZE:-32} +bcryptRounds=${BCRYPT_ROUNDS:-12} function fail() { msg=$1 @@ -58,7 +55,7 @@ function require() { # Verify this environment is capable of running this script function verifyEnvironment() { - require "argon2" + require "htpasswd" require "jq" require "curl" require "openssl" @@ -111,8 +108,8 @@ function updatePassword() { if [[ -n $identityId ]]; then # Generate password hash - salt=$(openssl rand -hex 8) - passwordHash=$(echo "${password}" | argon2 ${salt} -id -t $argon2Iterations -m $argon2Memory -p $argon2Parallelism -l $argon2HashSize -e) + passwordHash=$(echo "${password}" | htpasswd -niBC $bcryptRounds SOUSER) | cut -c 11- + passwordHash="\$2a${passwordHash} # Update DB with new hash echo "update identity_credentials set config=CAST('{\"hashed_password\":\"${passwordHash}\"}' as BLOB) where identity_id=${identityId};" | sqlite3 "$databasePath" diff --git a/setup/so-functions b/setup/so-functions index 5ce3d6dee..33e0514fc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2079,7 +2079,7 @@ saltify() { 'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE'| 'IMPORT') reserve_group_ids >> "$setup_log" 2>&1 if [[ ! $is_iso ]]; then - logCmd "yum -y install sqlite argon2 curl mariadb-devel" + logCmd "yum -y install sqlite curl mariadb-devel" fi # Download Ubuntu Keys in case manager updates = 1 mkdir -p /opt/so/gpg >> "$setup_log" 2>&1 @@ -2175,7 +2175,7 @@ saltify() { retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1 set_progress_str 6 'Installing various dependencies' - retry 50 10 "apt-get -y install sqlite3 argon2 libssl-dev" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install sqlite3 libssl-dev" >> "$setup_log" 2>&1 || exit 1 set_progress_str 7 'Installing salt-master' retry 50 10 "apt-get -y install salt-master=3003+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 From 686c7c5a6c5cf5010851381900f1cd8f9ea3a2f5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 13:26:43 -0400 Subject: [PATCH 140/378] Add exception handling for docker API error to so-docker-prune --- salt/common/tools/sbin/so-docker-prune | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index f6c043ef3..754a0f171 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -38,7 +38,7 @@ def get_image_version(string) -> str: Version(ver) except InvalidVersion: # Strip the last substring following a hyphen for automated branches - ver = '-'.join(ver.split('-')[:-1]) + ver = '-'.join(ver.split('-')[:-1]) return ver @@ -73,8 +73,12 @@ def main(quiet): for tag in group: if not quiet: print(f'Removing image {tag}') client.images.remove(tag) - except InvalidVersion as e: - print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr) + except (docker.errors.APIError, InvalidVersion) as e: + print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr) + exit(1) + except Exception as e: + print('Unhandled exception occurred:') + print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr) exit(1) if no_prunable and not quiet: @@ -86,4 +90,4 @@ if __name__ == "__main__": main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False) args = main_parser.parse_args(sys.argv[1:]) - main(args.quiet) \ No newline at end of file + main(args.quiet) From dcb89b704a373cf804b318b2c261e97e39dcb4e2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 13:45:12 -0400 Subject: [PATCH 141/378] Move sensoroni state out of the * block of top.sls Resolves #3559 --- salt/top.sls | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/top.sls b/salt/top.sls index 8a12aaa26..f3ccc9dda 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -38,7 +38,6 @@ base: - patch.os.schedule - motd - salt.minion-check - - sensoroni - salt.lasthighstate '*_helixsensor and G@saltversion:{{saltversion}}': @@ -47,6 +46,7 @@ base: - ca - ssl - registry + - sensoroni - telegraf - firewall - idstools @@ -66,6 +66,7 @@ base: - match: compound - ca - ssl + - sensoroni - telegraf - firewall - nginx @@ -92,6 +93,7 @@ base: - ca - ssl - registry + - sensoroni - manager - nginx - telegraf @@ -160,6 +162,7 @@ base: - ca - ssl - registry + - sensoroni - nginx - telegraf - influxdb @@ -220,6 +223,7 @@ base: - ca - ssl - registry + - sensoroni - manager - nginx - telegraf @@ -290,6 +294,7 @@ base: - match: compound - ca - ssl + - sensoroni - nginx - telegraf - firewall @@ -320,6 +325,7 @@ base: - ca - ssl - registry + - sensoroni - nginx - telegraf - influxdb @@ -382,6 +388,7 @@ base: - match: compound - ca - ssl + - sensoroni - nginx - telegraf - firewall @@ -424,6 +431,7 @@ base: - match: compound - ca - ssl + - sensoroni - nginx - telegraf - firewall @@ -441,6 +449,7 @@ base: - ca - ssl - registry + - sensoroni - manager - nginx - soc From a4226cc39a72d26f58d84d9199c5c331b049f526 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 May 2021 15:14:05 -0400 Subject: [PATCH 142/378] use elastic map file --- salt/common/init.sls | 9 ++------- .../tools/sbin/so-elasticsearch-templates-load | 4 ++-- .../files/bin/so-curator-closed-delete-delete | 6 +++--- salt/curator/init.sls | 5 ++++- salt/elasticsearch/auth.map.jinja | 7 +++++++ .../files/so-elasticsearch-pipelines | 6 +++--- salt/utility/init.sls | 15 +++------------ 7 files changed, 24 insertions(+), 28 deletions(-) create mode 100644 salt/elasticsearch/auth.map.jinja diff --git a/salt/common/init.sls b/salt/common/init.sls index 948adee99..389b0b8a3 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -2,8 +2,7 @@ {% if sls in allowed_states %} {% set role = grains.id.split('_') | last %} -{% set ELASTICUSER = salt['pillar.get']('elasticsearch:auth:user', '' ) %} -{% set ELASTICPASS = salt['pillar.get']('elasticsearch:auth:pass', '' ) %} +{% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -181,11 +180,7 @@ utilsyncscripts: - template: jinja - source: salt://common/tools/sbin - defaults: - ELASTICCURL: "curl" -{% if salt['pillar.get']('elasticsearch:auth_enabled', False) %} - - context: - ELASTICCURL: "curl --user {{ELASTICUSER}}:{{ELASTICPASS}}" -{% endif %} + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} {% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %} diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load index 42a836854..30ab66b48 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-load +++ b/salt/common/tools/sbin/so-elasticsearch-templates-load @@ -30,7 +30,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -51,7 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES} echo "Loading templates..." -for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done +for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; {{ ELASTICCURL }} -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done echo cd - >/dev/null diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index 9cc94833c..7dd7b82e7 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -34,7 +34,7 @@ overlimit() { closedindices() { - INDICES=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null) + INDICES=$({{ ELASTICCURL }} -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed 2> /dev/null) [ $? -eq 1 ] && return false echo ${INDICES} | grep -q -E "(logstash-|so-)" } @@ -49,10 +49,10 @@ while overlimit && closedindices; do # First, get the list of closed indices using _cat/indices?h=index\&expand_wildcards=closed. # Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field. # Finally, select the first entry in that sorted list. - OLDEST_INDEX=$(curl -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1) + OLDEST_INDEX=$({{ ELASTICCURL }} -s -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices?h=index\&expand_wildcards=closed | grep -E "(logstash-|so-)" | sort -t- -k3 | head -1) # Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it. - curl -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} + {{ ELASTICCURL }} -XDELETE -k https://{{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} # Finally, write a log entry that says we deleted it. echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT ({{LOG_SIZE_LIMIT}} GB) - Index ${OLDEST_INDEX} deleted ..." >> ${LOG} diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 245b700d0..966b0b857 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -5,6 +5,7 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone'] %} + {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} # Curator # Create the group curatorgroup: @@ -66,6 +67,8 @@ curcloseddeldel: - group: 939 - mode: 755 - template: jinja + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} curclose: file.managed: @@ -147,4 +150,4 @@ append_so-curator_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/elasticsearch/auth.map.jinja b/salt/elasticsearch/auth.map.jinja new file mode 100644 index 000000000..af13729ef --- /dev/null +++ b/salt/elasticsearch/auth.map.jinja @@ -0,0 +1,7 @@ +{% set ELASTICAUTH = salt['pillar.filter_by']({ + True: { + 'user': salt['pillar.get']('elasticsearch:auth:user'), + 'pass': salt['pillar.get']('elasticsearch:auth:pass'), + 'elasticcurl':'curl --user {{ELASTICAUTH.user}}:{{ELASTICAUTH.pass}}'}, + False: {'elasticcurl': 'curl'}, +}, pillar='elasticsearch:auth:enabled') %} diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index fca50b7d4..5d103963e 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -27,7 +27,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -47,9 +47,9 @@ fi cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." -for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done +for i in *; do echo $i; RESPONSE=$({{ ELASTICCURL }} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done echo cd - >/dev/null -exit $RETURN_CODE \ No newline at end of file +exit $RETURN_CODE diff --git a/salt/utility/init.sls b/salt/utility/init.sls index 5558139de..0b4f0f2c3 100644 --- a/salt/utility/init.sls +++ b/salt/utility/init.sls @@ -1,8 +1,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} - {% set ELASTICUSER = salt['pillar.get']('elasticsearch:auth:user', '' ) %} - {% set ELASTICPASS = salt['pillar.get']('elasticsearch:auth:pass', '' ) %} + {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} # This state is for checking things {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %} @@ -15,11 +14,7 @@ crossclusterson: - source: salt://utility/bin/crossthestreams - template: jinja - defaults: - ELASTICCURL: "curl" - {% if salt['pillar.get']('elasticsearch:auth_enabled', False) %} - - context: - ELASTICCURL: "curl --user {{ELASTICUSER}}:{{ELASTICPASS}}" - {% endif %} + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} {% endif %} {% if grains['role'] in ['so-eval', 'so-import'] %} @@ -31,11 +26,7 @@ fixsearch: - source: salt://utility/bin/eval - template: jinja - defaults: - ELASTICCURL: "curl" - {% if salt['pillar.get']('elasticsearch:auth_enabled', False) %} - - context: - ELASTICCURL: "curl --user {{ELASTICUSER}}:{{ELASTICPASS}}" - {% endif %} + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} {% endif %} {% else %} From ffd5bfc48063aeb501def0030748b93091dd6678 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 15:25:03 -0400 Subject: [PATCH 143/378] Force images from automated branches to a very high semver --- salt/common/tools/sbin/so-docker-prune | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 754a0f171..2dd183c10 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -37,8 +37,10 @@ def get_image_version(string) -> str: try: Version(ver) except InvalidVersion: - # Strip the last substring following a hyphen for automated branches - ver = '-'.join(ver.split('-')[:-1]) + # Also return a very high (but less than 'latest') semver for automated branches + # since the image will most likely be the latest version + if any(substr in ver for substr in ['bravo', 'delta', 'foxtrot', 'kilo']): + return '99999.9.9' return ver From ba3a51387caeb293c2fbbc134931a18e70b885d8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 May 2021 15:31:46 -0400 Subject: [PATCH 144/378] set default to False --- salt/elasticsearch/auth.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/auth.map.jinja b/salt/elasticsearch/auth.map.jinja index af13729ef..f216ef3c2 100644 --- a/salt/elasticsearch/auth.map.jinja +++ b/salt/elasticsearch/auth.map.jinja @@ -4,4 +4,4 @@ 'pass': salt['pillar.get']('elasticsearch:auth:pass'), 'elasticcurl':'curl --user {{ELASTICAUTH.user}}:{{ELASTICAUTH.pass}}'}, False: {'elasticcurl': 'curl'}, -}, pillar='elasticsearch:auth:enabled') %} +}, pillar='elasticsearch:auth:enabled', default=False) %} From 87609ba5d1afab1179e6e15105f4ade629d8d34a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 May 2021 15:44:01 -0400 Subject: [PATCH 145/378] fix elasticcurl if auth is enabled --- salt/elasticsearch/auth.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/auth.map.jinja b/salt/elasticsearch/auth.map.jinja index f216ef3c2..147b0157b 100644 --- a/salt/elasticsearch/auth.map.jinja +++ b/salt/elasticsearch/auth.map.jinja @@ -2,6 +2,6 @@ True: { 'user': salt['pillar.get']('elasticsearch:auth:user'), 'pass': salt['pillar.get']('elasticsearch:auth:pass'), - 'elasticcurl':'curl --user {{ELASTICAUTH.user}}:{{ELASTICAUTH.pass}}'}, + 'elasticcurl':'curl --user ' ~ salt['pillar.get']('elasticsearch:auth:user') ~ ':' ~ salt['pillar.get']('elasticsearch:auth:pass') }, False: {'elasticcurl': 'curl'}, }, pillar='elasticsearch:auth:enabled', default=False) %} From 8c6489a49a945ee08c2906bd9955bc858bbb489b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 24 May 2021 15:48:05 -0400 Subject: [PATCH 146/378] Initial pass at synchronizing users file --- salt/common/tools/sbin/so-user | 21 ++++++++++++++++++--- salt/elasticsearch/init.sls | 2 ++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index edfa4b303..dbd755bc4 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -40,6 +40,7 @@ email=$2 kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434} databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite} bcryptRounds=${BCRYPT_ROUNDS:-12} +extractedHashFile=${EXTRACTED_HASH_FILE:-/opt/so/conf/elasticsearch/users} function fail() { msg=$1 @@ -108,15 +109,19 @@ function updatePassword() { if [[ -n $identityId ]]; then # Generate password hash - passwordHash=$(echo "${password}" | htpasswd -niBC $bcryptRounds SOUSER) | cut -c 11- - passwordHash="\$2a${passwordHash} - + passwordHash=$(echo "${password}" | htpasswd -niBC $bcryptRounds SOUSER) + passwordHash=$(echo "$passwordHash" | cut -c 11-) + passwordHash="\$2a${passwordHash}" # Update DB with new hash echo "update identity_credentials set config=CAST('{\"hashed_password\":\"${passwordHash}\"}' as BLOB) where identity_id=${identityId};" | sqlite3 "$databasePath" [[ $? != 0 ]] && fail "Unable to update password" fi } +function extractHashes() { + echo "select ici.identifier || ':' || json_extract(ic.config, '$.hashed_password') from identity_credential_identifiers ici, identity_credentials ic where ici.identity_credential_id=ic.id and json_extract(ic.config, '$.hashed_password') is not null order by ici.identifier" | sqlite3 "$databasePath" > "$extractedHashFile" +} + function listUsers() { response=$(curl -Ss -L ${kratosUrl}/identities) [[ $? != 0 ]] && fail "Unable to communicate with Kratos" @@ -208,6 +213,7 @@ case "${operation}" in validateEmail "$email" updatePassword createUser "$email" + extractHashes echo "Successfully added new user to SOC" check_container thehive && echo $password | so-thehive-user-add "$email" check_container fleet && echo $password | so-fleet-user-add "$email" @@ -223,6 +229,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateUser "$email" + extractHashes echo "Successfully updated user" ;; @@ -231,6 +238,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateStatus "$email" 'active' + extractHashes echo "Successfully enabled user" check_container thehive && so-thehive-user-enable "$email" true check_container fleet && so-fleet-user-enable "$email" true @@ -241,6 +249,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateStatus "$email" 'locked' + extractHashes echo "Successfully disabled user" check_container thehive && so-thehive-user-enable "$email" false check_container fleet && so-fleet-user-enable "$email" false @@ -251,11 +260,17 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" deleteUser "$email" + extractHashes echo "Successfully deleted user" check_container thehive && so-thehive-user-enable "$email" false check_container fleet && so-fleet-user-enable "$email" false ;; + "sync") + extractHashes + echo "Synchronization complete" + ;; + "validate") validateEmail "$email" updatePassword diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index df297986a..c6ac600ef 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -202,6 +202,8 @@ so-elasticsearch: - binds: - /opt/so/conf/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro - /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro + - /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro + - /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro From 2deb703272761598aba2557f05c132244c4989c3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 May 2021 16:30:55 -0400 Subject: [PATCH 147/378] map users_roles and users conf into docker container --- salt/elasticsearch/init.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index df297986a..7e0e700e9 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -35,6 +35,7 @@ {% endif %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} +{% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} vm.max_map_count: sysctl.present: @@ -213,6 +214,10 @@ so-elasticsearch: - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro + {% if salt['pillar.get']('elasticsearch:auth:enabled', False) %} + - /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro + - /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro + {% endif %} - watch: - file: cacertz - file: esyml From bd301880ade0360deef8cf9362a9a598533a63aa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 24 May 2021 16:32:30 -0400 Subject: [PATCH 148/378] define the default --- salt/elasticsearch/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 7e0e700e9..0a3ea0719 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -237,6 +237,8 @@ so-elasticsearch-pipelines-file: - group: 939 - mode: 754 - template: jinja + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} so-elasticsearch-pipelines: cmd.run: From 3bc0def02af00ab5741f903722f8316e68d3bfe1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 16:45:05 -0400 Subject: [PATCH 149/378] Add failure message to salt-master check --- salt/common/tools/sbin/soup | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index eb15305aa..d2b96b194 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -890,8 +890,10 @@ main() { echo "Starting Salt Master service." systemctl start salt-master - # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" + # Testing that salt-master is up by checking that is it connected to itself + set +e + retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details." + set -e echo "" echo "Ensuring python modules for Salt are installed and patched." @@ -924,9 +926,11 @@ main() { echo "Starting Salt Master service." systemctl start salt-master - # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" - + # Testing that salt-master is up by checking that is it connected to itself + set +e + retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details." + set -e + echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes From 5e5d30a3774a49b2e2128e2c554b5eb6a7a4f53b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 08:26:25 -0400 Subject: [PATCH 150/378] Fix 3rd party modules --- salt/filebeat/thirdpartydefaults.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml index 027ec4595..cfb8d10d9 100644 --- a/salt/filebeat/thirdpartydefaults.yaml +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -195,9 +195,10 @@ third_party_filebeat: var.syslog_port: 9502 o365: audit: - enabled: false + enabled: false okta: - enabled: false + system: + enabled: false pesando: dfw: enabled: false From 5a1e8d9fe96d2d553490d41a882f0e6b4949873e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 08:50:55 -0400 Subject: [PATCH 151/378] update kibana scripts for elastic auth --- salt/common/tools/sbin/so-common | 3 ++- salt/common/tools/sbin/so-kibana-space-defaults | 8 ++++---- salt/kibana/bin/so-kibana-config-load | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 22c7543ea..21948e67d 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -486,12 +486,13 @@ wait_for_web_response() { url=$1 expected=$2 maxAttempts=${3:-300} + curlcmd=${4:-curl} logfile=/root/wait_for_web_response.log attempt=0 while [[ $attempt -lt $maxAttempts ]]; do attempt=$((attempt+1)) echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)" - result=$(curl -ks -L $url) + result=$($curlcmd -ks -L $url) exitcode=$? echo "--------------------------------------------------" >> $logfile diff --git a/salt/common/tools/sbin/so-kibana-space-defaults b/salt/common/tools/sbin/so-kibana-space-defaults index edf356d45..48225e2f4 100755 --- a/salt/common/tools/sbin/so-kibana-space-defaults +++ b/salt/common/tools/sbin/so-kibana-space-defaults @@ -1,13 +1,13 @@ . /usr/sbin/so-common -wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" +wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" ## This hackery will be removed if using Elastic Auth ## # Let's snag a cookie from Kibana -THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') +THECOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') # Disable certain Features from showing up in the Kibana UI echo echo "Setting up default Space:" -curl -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log -echo \ No newline at end of file +{{ ELASTICCURL }} -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log +echo diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index f07377018..d0e6f1edb 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -16,11 +16,11 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o # SOCtopus and Manager sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson -wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" +wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"" ## This hackery will be removed if using Elastic Auth ## # Let's snag a cookie from Kibana -THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') +THECOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') # Load saved objects -curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log \ No newline at end of file +{{ ELASTICCURL }} -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log From dbc4ffd69ad757d59d9e3d8351a7f1c1510fa0e6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 25 May 2021 09:20:45 -0400 Subject: [PATCH 152/378] Fix typo --- salt/common/tools/sbin/so-image-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index f840e07ab..9b6e2174a 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -151,7 +151,7 @@ update_docker_containers() { local image=$i:$VERSION$IMAGE_TAG_SUFFIX run_check_net_err \ "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \ - "Could not pull pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 + "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 # Get signature run_check_net_err \ From 7e008378bae365a16ae4d9741e3346eebcae04c0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 25 May 2021 09:23:44 -0400 Subject: [PATCH 153/378] Replace string with variable, remove unnecessary text --- salt/common/tools/sbin/soup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d2b96b194..7fca8ceb9 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -688,7 +688,7 @@ upgrade_salt() { set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ - "Could not update salt, please check $SOUP_LOG or scrollback for details." + "Could not update salt, please check $SOUP_LOG for details." set -e echo "Applying yum versionlock for Salt." echo "" @@ -705,7 +705,7 @@ upgrade_salt() { set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ - "Could not update salt, please check $SOUP_LOG or scrollback for details." + "Could not update salt, please check $SOUP_LOG for details." set -e echo "Applying apt hold for Salt." echo "" @@ -892,7 +892,7 @@ main() { # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details." + retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "" @@ -928,7 +928,7 @@ main() { # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details." + retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "Running a highstate. This could take several minutes." From 1e564c2140d0f53e3b5c90ba89bd3ec8b7e3471b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 10:22:36 -0400 Subject: [PATCH 154/378] Fix zeek jinja --- salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index da798a79d..486d22bfe 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -1,4 +1,4 @@ -%- if grains['role'] == 'so-eval' -%} +{%- if grains['role'] == 'so-eval' -%} {%- set ES = salt['pillar.get']('manager:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} From 192cec182550135ad711b4c58b61cbe86c5ae7bb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 25 May 2021 11:25:12 -0400 Subject: [PATCH 155/378] Change how version with dashes are handled by so-docker-prune --- salt/common/tools/sbin/so-docker-prune | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 2dd183c10..b1c359b58 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -32,15 +32,15 @@ def get_image_version(string) -> str: ver = string.split(':')[-1] if ver == 'latest': # Version doesn't like "latest", so use a high semver - return '999999.9.9' + return '99999.9.9' else: try: Version(ver) except InvalidVersion: - # Also return a very high (but less than 'latest') semver for automated branches - # since the image will most likely be the latest version - if any(substr in ver for substr in ['bravo', 'delta', 'foxtrot', 'kilo']): - return '99999.9.9' + # Also return a very high semver for any version + # with a dash in it since it will likely be a dev version of some kind + if '-' in ver: + return '999999.9.9' return ver From 94af55a95104cd4229a315343973c8167b6cadd6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 25 May 2021 11:25:37 -0400 Subject: [PATCH 156/378] Fix typo --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 7fca8ceb9..d1261fe30 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -60,7 +60,7 @@ check_err() { echo 'Link has been severed' ;; 100) - echo 'Netowrk is down' + echo 'Network is down' ;; 101) echo 'Network is unreachable' From 8d9d5a267ac7c3f34bd0e482f5a107c06161cc64 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 11:52:58 -0400 Subject: [PATCH 157/378] generate elasticsearch.auth pillar if it doesnt exist --- pillar/top.sls | 5 ++++- salt/elasticsearch/auth.sls | 9 +++++++++ salt/elasticsearch/files/curl.config | 1 + salt/elasticsearch/init.sls | 8 ++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/auth.sls create mode 100644 salt/elasticsearch/files/curl.config diff --git a/pillar/top.sls b/pillar/top.sls index a795e03c1..80e02902e 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -46,6 +46,9 @@ base: - logstash.manager - logstash.search - elasticsearch.search +{% if salt['file.exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} + - elasticsearch.auth +{% endif %} - data.* - zeeklogs - secrets @@ -89,4 +92,4 @@ base: - secrets - elasticsearch.eval - global - - minions.{{ grains.id }} \ No newline at end of file + - minions.{{ grains.id }} diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls new file mode 100644 index 000000000..a51abca1c --- /dev/null +++ b/salt/elasticsearch/auth.sls @@ -0,0 +1,9 @@ +elastic_auth_pillar: + file.managed: + - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls + - contents: | + elasticsearch: + auth: + enabled: False + user: so_elastic + pass: {{ salt['random.get_str'](20) }} diff --git a/salt/elasticsearch/files/curl.config b/salt/elasticsearch/files/curl.config new file mode 100644 index 000000000..e4633df6c --- /dev/null +++ b/salt/elasticsearch/files/curl.config @@ -0,0 +1 @@ +user = "salt['pillar.get']('elasticsearch:auth:user'):salt['pillar.get']('elasticsearch:auth:pass')" diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 0a3ea0719..f0eee6d44 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -256,6 +256,14 @@ so-elasticsearch-templates: - template: jinja {% endif %} +elastic_curl_config: + file.managed: + - name: /opt/so/conf/elasticsearch/curl.config + - mode: 600 + # since we are generating a random password, and we don't want that to happen everytime + # a highstate runs, we only manage the file if it doesn't exist + - unless: ls /opt/so/conf/elasticsearch/curl.config + {% endif %} {# if grains['role'] != 'so-helix' #} {% else %} From 6da0b57ce1960858da182cc49c4a4d4eaa6ca706 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 11:55:22 -0400 Subject: [PATCH 158/378] fix file.file_exists --- pillar/top.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/top.sls b/pillar/top.sls index 80e02902e..8bd67c174 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -46,7 +46,7 @@ base: - logstash.manager - logstash.search - elasticsearch.search -{% if salt['file.exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} +{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} - data.* From 58ec31d6c7c3cb9cacd76e907aaf59736814cabc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 12:02:41 -0400 Subject: [PATCH 159/378] pass ELASTICAUTH to script --- salt/kibana/init.sls | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 75b96b72a..61498cfeb 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -4,6 +4,7 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} +{% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} # Add ES Group kibanasearchgroup: @@ -63,6 +64,8 @@ kibanabin: - source: salt://kibana/bin/so-kibana-config-load - mode: 755 - template: jinja + - defaults: + ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} # Start the kibana docker so-kibana: @@ -113,4 +116,4 @@ so-kibana-config-load: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From 35cc7b27e9f66716621ae9c33a0a9045acba934c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 13:12:30 -0400 Subject: [PATCH 160/378] remove extra quote --- salt/kibana/bin/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index d0e6f1edb..58535a886 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -16,7 +16,7 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o # SOCtopus and Manager sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson -wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"" +wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" ## This hackery will be removed if using Elastic Auth ## # Let's snag a cookie from Kibana From 747dc77c92032d0a569e80706d65d3731ef49b77 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 13:23:26 -0400 Subject: [PATCH 161/378] comment out the hackery --- salt/kibana/bin/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index 58535a886..8f80b5c3d 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -16,7 +16,7 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o # SOCtopus and Manager sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson -wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" +#wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" ## This hackery will be removed if using Elastic Auth ## # Let's snag a cookie from Kibana From e261c197f3aacc639ca4583e343a1a6daafc6dab Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 13:46:18 -0400 Subject: [PATCH 162/378] add elasticsearch.auth state to statnalone node --- salt/elasticsearch/auth.sls | 3 +++ salt/elasticsearch/init.sls | 3 --- salt/top.sls | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index a51abca1c..e8ab1e378 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -7,3 +7,6 @@ elastic_auth_pillar: enabled: False user: so_elastic pass: {{ salt['random.get_str'](20) }} + # since we are generating a random password, and we don't want that to happen everytime + # a highstate runs, we only manage the file if it doesn't exist + - unless: ls /opt/so/saltstack/local/pillar/elasticsearch/auth.sls diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index f0eee6d44..7005a66c5 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -260,9 +260,6 @@ elastic_curl_config: file.managed: - name: /opt/so/conf/elasticsearch/curl.config - mode: 600 - # since we are generating a random password, and we don't want that to happen everytime - # a highstate runs, we only manage the file if it doesn't exist - - unless: ls /opt/so/conf/elasticsearch/curl.config {% endif %} {# if grains['role'] != 'so-helix' #} diff --git a/salt/top.sls b/salt/top.sls index 8a12aaa26..f3f2958ae 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -238,6 +238,7 @@ base: {%- endif %} {%- if ELASTICSEARCH %} - elasticsearch + - elasticsearch.auth {%- endif %} {%- if LOGSTASH %} - logstash From c3b2e1e8b22d65d05f5a0e87a615ef28af1530c8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 16:16:57 -0400 Subject: [PATCH 163/378] dont show changes --- salt/elasticsearch/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 7005a66c5..65a3d2bca 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -260,6 +260,7 @@ elastic_curl_config: file.managed: - name: /opt/so/conf/elasticsearch/curl.config - mode: 600 + - show_changes: False {% endif %} {# if grains['role'] != 'so-helix' #} From 2aacd5b9b6cf339052d674595531f515c3842fff Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 16:40:50 -0400 Subject: [PATCH 164/378] so defaults filebeat modules --- salt/filebeat/etc/module_config.yml.jinja | 16 ++++++++++++++++ salt/filebeat/etc/thirdparty.yml.jinja | 16 ---------------- salt/filebeat/init.sls | 19 ++++++++++--------- salt/filebeat/map.jinja | 5 ++++- 4 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 salt/filebeat/etc/module_config.yml.jinja delete mode 100644 salt/filebeat/etc/thirdparty.yml.jinja diff --git a/salt/filebeat/etc/module_config.yml.jinja b/salt/filebeat/etc/module_config.yml.jinja new file mode 100644 index 000000000..7cd624895 --- /dev/null +++ b/salt/filebeat/etc/module_config.yml.jinja @@ -0,0 +1,16 @@ +# DO NOT EDIT THIS FILE +{% for module in MODULES.modules.keys() %} +- module: {{ module }} + {%- for fileset in MODULES.modules[module] %} + {{ fileset }}: + enabled: {{ MODULES.modules[module][fileset].enabled }} + {#- only manage the settings if the fileset is enabled #} + {%- if MODULES.modules[module][fileset].enabled %} + {%- for var, value in MODULES.modules[module][fileset].items() %} + {%- if var|lower != 'enabled' %} + {{ var }}: {{ value }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endfor %} +{% endfor %} diff --git a/salt/filebeat/etc/thirdparty.yml.jinja b/salt/filebeat/etc/thirdparty.yml.jinja deleted file mode 100644 index 186115af4..000000000 --- a/salt/filebeat/etc/thirdparty.yml.jinja +++ /dev/null @@ -1,16 +0,0 @@ -# DO NOT EDIT THIS FILE -{% for module in THIRDPARTY.modules.keys() %} -- module: {{ module }} - {%- for fileset in THIRDPARTY.modules[module] %} - {{ fileset }}: - enabled: {{ THIRDPARTY.modules[module][fileset].enabled }} - {#- only manage the settings if the fileset is enabled #} - {%- if THIRDPARTY.modules[module][fileset].enabled %} - {%- for var, value in THIRDPARTY.modules[module][fileset].items() %} - {%- if var|lower != 'enabled' %} - {{ var }}: {{ value }} - {%- endif %} - {%- endfor %} - {%- endif %} - {%- endfor %} -{% endfor %} \ No newline at end of file diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index b3bce806c..b1a91b133 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -21,6 +21,7 @@ {% set MANAGER = salt['grains.get']('master') %} {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} filebeatetcdir: @@ -78,21 +79,21 @@ filebeatmoduleconfsync: - group: root - template: jinja -# Sync Filebeat modules -filebeatmodules: - file.recurse: - - name: /opt/so/conf/filebeat/modules - - source: salt://filebeat/modules - - user: root - - group: root +sodefaults_module_conf: + file.managed: + - name: /opt/so/conf/filebeat/etc/securityonion.yml + - source: salt://filebeat/etc/module_config.yml.jinja + - template: jinja + - defaults: + MODULES: {{ SO }} thirdparty_module_conf: file.managed: - name: /opt/so/conf/filebeat/etc/thirdparty.yml - - source: salt://filebeat/etc/thirdparty.yml.jinja + - source: salt://filebeat/etc/module_config.yml.jinja - template: jinja - defaults: - THIRDPARTY: {{ THIRDPARTY }} + MODULES: {{ THIRDPARTY }} so-filebeat: docker_container.running: diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index 668889227..aaae60f31 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -1,2 +1,5 @@ {% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %} -{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} \ No newline at end of file +{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} + +{% import_yaml 'filebeat/securityoniondefaults.yaml' as SO %} +{#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} From 9363fc153c3c7b45bdf1c82530530e7c8b92398f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 16:44:13 -0400 Subject: [PATCH 165/378] Fix pillar for module --- salt/filebeat/securityoniondefaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index e6406af8c..8bcecd618 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -1,3 +1,4 @@ +{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %} securityonion_filebeat: modules: elasticsearch: From 0de5c6f204186342d90b928fca5b1019b22b26cb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 16:52:02 -0400 Subject: [PATCH 166/378] fix sodefault modules --- salt/filebeat/map.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index aaae60f31..6ae6e7cff 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -1,5 +1,6 @@ {% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %} {% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} -{% import_yaml 'filebeat/securityoniondefaults.yaml' as SO %} +{% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} +{% set SO = SODEFAULTS.securityonion_filebeat %} {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} From cd3e355f848536e2c7d7241c498f9c51315d8ebe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 16:54:20 -0400 Subject: [PATCH 167/378] Fix zeek depth --- salt/zeek/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 02c1cc1ba..4e597f597 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -76,9 +76,9 @@ zeekpolicysync: # Ensure the zeek spool tree (and state.db) ownership is correct zeekspoolownership: file.directory: - - name: /nsm/zeek/spool + - name: /nsm/zeek - user: 937 - - max_depth: 0 + - max_depth: 1 - recurse: - user From 543154f037453377c0780d4375dc4b52613b14db Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 16:58:18 -0400 Subject: [PATCH 168/378] Remove old modules --- salt/filebeat/init.sls | 4 +- salt/filebeat/modules/activemq.yml.disabled | 19 -- salt/filebeat/modules/apache.yml.disabled | 19 -- salt/filebeat/modules/auditd.yml.disabled | 10 - salt/filebeat/modules/aws.yml.disabled | 255 ------------------ salt/filebeat/modules/azure.yml.disabled | 45 ---- salt/filebeat/modules/barracuda.yml.disabled | 41 --- salt/filebeat/modules/bluecoat.yml.disabled | 22 -- salt/filebeat/modules/cef.yml.disabled | 17 -- salt/filebeat/modules/checkpoint.yml.disabled | 24 -- salt/filebeat/modules/cisco.yml.disabled | 142 ---------- salt/filebeat/modules/coredns.yml.disabled | 11 - .../filebeat/modules/crowdstrike.yml.disabled | 11 - salt/filebeat/modules/cyberark.yml.disabled | 22 -- salt/filebeat/modules/cylance.yml.disabled | 22 -- .../modules/elasticsearch.yml.disabled | 35 --- salt/filebeat/modules/envoyproxy.yml.disabled | 11 - salt/filebeat/modules/f5.yml.disabled | 41 --- salt/filebeat/modules/fortinet.yml.disabled | 83 ------ salt/filebeat/modules/gcp.yml.disabled | 76 ------ .../modules/google_workspace.yml.disabled | 53 ---- .../filebeat/modules/googlecloud.yml.disabled | 58 ---- salt/filebeat/modules/gsuite.yml.disabled | 53 ---- salt/filebeat/modules/haproxy.yml.disabled | 14 - salt/filebeat/modules/ibmmq.yml.disabled | 11 - salt/filebeat/modules/icinga.yml.disabled | 27 -- salt/filebeat/modules/iis.yml.disabled | 20 -- salt/filebeat/modules/imperva.yml.disabled | 22 -- salt/filebeat/modules/infoblox.yml.disabled | 22 -- salt/filebeat/modules/iptables.yml.disabled | 13 - salt/filebeat/modules/juniper.yml.disabled | 54 ---- salt/filebeat/modules/kafka.yml.disabled | 15 -- salt/filebeat/modules/kibana.yml.disabled | 19 -- salt/filebeat/modules/logstash.yml.disabled | 18 -- salt/filebeat/modules/microsoft.yml.disabled | 49 ---- salt/filebeat/modules/misp.yml.disabled | 17 -- salt/filebeat/modules/mongodb.yml.disabled | 11 - salt/filebeat/modules/mssql.yml.disabled | 11 - salt/filebeat/modules/mysql.yml.disabled | 19 -- .../modules/mysqlenterprise.yml.disabled | 14 - salt/filebeat/modules/nats.yml.disabled | 11 - salt/filebeat/modules/netflow.yml.disabled | 14 - salt/filebeat/modules/netscout.yml.disabled | 22 -- salt/filebeat/modules/nginx.yml.disabled | 27 -- salt/filebeat/modules/o365.yml.disabled | 48 ---- salt/filebeat/modules/okta.yml.disabled | 10 - salt/filebeat/modules/oracle.yml.disabled | 13 - salt/filebeat/modules/osquery.yml.disabled | 15 -- salt/filebeat/modules/panw.yml.disabled | 22 -- salt/filebeat/modules/pensando.yml.disabled | 13 - salt/filebeat/modules/postgresql.yml.disabled | 11 - salt/filebeat/modules/proofpoint.yml.disabled | 22 -- salt/filebeat/modules/rabbitmq.yml.disabled | 11 - salt/filebeat/modules/radware.yml.disabled | 22 -- salt/filebeat/modules/redis.yml.disabled | 21 -- salt/filebeat/modules/santa.yml.disabled | 9 - salt/filebeat/modules/snort.yml.disabled | 22 -- salt/filebeat/modules/snyk.yml.disabled | 112 -------- salt/filebeat/modules/sonicwall.yml.disabled | 22 -- salt/filebeat/modules/sophos.yml.disabled | 46 ---- salt/filebeat/modules/squid.yml.disabled | 22 -- salt/filebeat/modules/suricata.yml | 12 - salt/filebeat/modules/system.yml.disabled | 19 -- .../filebeat/modules/threatintel.yml.disabled | 105 -------- salt/filebeat/modules/tomcat.yml.disabled | 22 -- salt/filebeat/modules/traefik.yml.disabled | 11 - salt/filebeat/modules/zeek.yml | 122 --------- salt/filebeat/modules/zoom.yml.disabled | 22 -- salt/filebeat/modules/zscaler.yml.disabled | 22 -- salt/filebeat/securityoniondefaults.yaml | 2 +- 70 files changed, 3 insertions(+), 2279 deletions(-) delete mode 100644 salt/filebeat/modules/activemq.yml.disabled delete mode 100644 salt/filebeat/modules/apache.yml.disabled delete mode 100644 salt/filebeat/modules/auditd.yml.disabled delete mode 100644 salt/filebeat/modules/aws.yml.disabled delete mode 100644 salt/filebeat/modules/azure.yml.disabled delete mode 100644 salt/filebeat/modules/barracuda.yml.disabled delete mode 100644 salt/filebeat/modules/bluecoat.yml.disabled delete mode 100644 salt/filebeat/modules/cef.yml.disabled delete mode 100644 salt/filebeat/modules/checkpoint.yml.disabled delete mode 100644 salt/filebeat/modules/cisco.yml.disabled delete mode 100644 salt/filebeat/modules/coredns.yml.disabled delete mode 100644 salt/filebeat/modules/crowdstrike.yml.disabled delete mode 100644 salt/filebeat/modules/cyberark.yml.disabled delete mode 100644 salt/filebeat/modules/cylance.yml.disabled delete mode 100644 salt/filebeat/modules/elasticsearch.yml.disabled delete mode 100644 salt/filebeat/modules/envoyproxy.yml.disabled delete mode 100644 salt/filebeat/modules/f5.yml.disabled delete mode 100644 salt/filebeat/modules/fortinet.yml.disabled delete mode 100644 salt/filebeat/modules/gcp.yml.disabled delete mode 100644 salt/filebeat/modules/google_workspace.yml.disabled delete mode 100644 salt/filebeat/modules/googlecloud.yml.disabled delete mode 100644 salt/filebeat/modules/gsuite.yml.disabled delete mode 100644 salt/filebeat/modules/haproxy.yml.disabled delete mode 100644 salt/filebeat/modules/ibmmq.yml.disabled delete mode 100644 salt/filebeat/modules/icinga.yml.disabled delete mode 100644 salt/filebeat/modules/iis.yml.disabled delete mode 100644 salt/filebeat/modules/imperva.yml.disabled delete mode 100644 salt/filebeat/modules/infoblox.yml.disabled delete mode 100644 salt/filebeat/modules/iptables.yml.disabled delete mode 100644 salt/filebeat/modules/juniper.yml.disabled delete mode 100644 salt/filebeat/modules/kafka.yml.disabled delete mode 100644 salt/filebeat/modules/kibana.yml.disabled delete mode 100644 salt/filebeat/modules/logstash.yml.disabled delete mode 100644 salt/filebeat/modules/microsoft.yml.disabled delete mode 100644 salt/filebeat/modules/misp.yml.disabled delete mode 100644 salt/filebeat/modules/mongodb.yml.disabled delete mode 100644 salt/filebeat/modules/mssql.yml.disabled delete mode 100644 salt/filebeat/modules/mysql.yml.disabled delete mode 100644 salt/filebeat/modules/mysqlenterprise.yml.disabled delete mode 100644 salt/filebeat/modules/nats.yml.disabled delete mode 100644 salt/filebeat/modules/netflow.yml.disabled delete mode 100644 salt/filebeat/modules/netscout.yml.disabled delete mode 100644 salt/filebeat/modules/nginx.yml.disabled delete mode 100644 salt/filebeat/modules/o365.yml.disabled delete mode 100644 salt/filebeat/modules/okta.yml.disabled delete mode 100644 salt/filebeat/modules/oracle.yml.disabled delete mode 100644 salt/filebeat/modules/osquery.yml.disabled delete mode 100644 salt/filebeat/modules/panw.yml.disabled delete mode 100644 salt/filebeat/modules/pensando.yml.disabled delete mode 100644 salt/filebeat/modules/postgresql.yml.disabled delete mode 100644 salt/filebeat/modules/proofpoint.yml.disabled delete mode 100644 salt/filebeat/modules/rabbitmq.yml.disabled delete mode 100644 salt/filebeat/modules/radware.yml.disabled delete mode 100644 salt/filebeat/modules/redis.yml.disabled delete mode 100644 salt/filebeat/modules/santa.yml.disabled delete mode 100644 salt/filebeat/modules/snort.yml.disabled delete mode 100644 salt/filebeat/modules/snyk.yml.disabled delete mode 100644 salt/filebeat/modules/sonicwall.yml.disabled delete mode 100644 salt/filebeat/modules/sophos.yml.disabled delete mode 100644 salt/filebeat/modules/squid.yml.disabled delete mode 100644 salt/filebeat/modules/suricata.yml delete mode 100644 salt/filebeat/modules/system.yml.disabled delete mode 100644 salt/filebeat/modules/threatintel.yml.disabled delete mode 100644 salt/filebeat/modules/tomcat.yml.disabled delete mode 100644 salt/filebeat/modules/traefik.yml.disabled delete mode 100644 salt/filebeat/modules/zeek.yml delete mode 100644 salt/filebeat/modules/zoom.yml.disabled delete mode 100644 salt/filebeat/modules/zscaler.yml.disabled diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index b1a91b133..18ca9b8c1 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -81,7 +81,7 @@ filebeatmoduleconfsync: sodefaults_module_conf: file.managed: - - name: /opt/so/conf/filebeat/etc/securityonion.yml + - name: /opt/so/conf/filebeat/modules/securityonion.yml - source: salt://filebeat/etc/module_config.yml.jinja - template: jinja - defaults: @@ -89,7 +89,7 @@ sodefaults_module_conf: thirdparty_module_conf: file.managed: - - name: /opt/so/conf/filebeat/etc/thirdparty.yml + - name: /opt/so/conf/filebeat/modules/thirdparty.yml - source: salt://filebeat/etc/module_config.yml.jinja - template: jinja - defaults: diff --git a/salt/filebeat/modules/activemq.yml.disabled b/salt/filebeat/modules/activemq.yml.disabled deleted file mode 100644 index 43536ecbc..000000000 --- a/salt/filebeat/modules/activemq.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: activemq -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-activemq.html - -- module: activemq - # Audit logs - audit: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Application logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/apache.yml.disabled b/salt/filebeat/modules/apache.yml.disabled deleted file mode 100644 index b923dd581..000000000 --- a/salt/filebeat/modules/apache.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: apache -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-apache.html - -- module: apache - # Access logs - access: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Error logs - error: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/auditd.yml.disabled b/salt/filebeat/modules/auditd.yml.disabled deleted file mode 100644 index 76296ec85..000000000 --- a/salt/filebeat/modules/auditd.yml.disabled +++ /dev/null @@ -1,10 +0,0 @@ -# Module: auditd -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-auditd.html - -- module: auditd - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/aws.yml.disabled b/salt/filebeat/modules/aws.yml.disabled deleted file mode 100644 index 904bd976c..000000000 --- a/salt/filebeat/modules/aws.yml.disabled +++ /dev/null @@ -1,255 +0,0 @@ -# Module: aws -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-aws.html - -- module: aws - cloudtrail: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Process CloudTrail logs - # default is true, set to false to skip Cloudtrail logs - # var.process_cloudtrail_logs: false - - # Process CloudTrail Digest logs - # default true, set to false to skip CloudTrail Digest logs - # var.process_digest_logs: false - - # Process CloudTrail Insight logs - # default true, set to false to skip CloudTrail Insight logs - # var.process_insight_logs: false - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - cloudwatch: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - ec2: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - elb: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - s3access: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - vpcflow: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 diff --git a/salt/filebeat/modules/azure.yml.disabled b/salt/filebeat/modules/azure.yml.disabled deleted file mode 100644 index 3b2bc1ecf..000000000 --- a/salt/filebeat/modules/azure.yml.disabled +++ /dev/null @@ -1,45 +0,0 @@ -# Module: azure -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-azure.html - -- module: azure - # All logs - activitylogs: - enabled: true - var: - # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub - eventhub: "insights-operational-logs" - # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module - consumer_group: "$Default" - # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string - connection_string: "" - # the name of the storage account the state/offsets will be stored and updated - storage_account: "" - # the storage account key, this key will be used to authorize access to data in your storage account - storage_account_key: "" - - platformlogs: - enabled: false - # var: - # eventhub: "" - # consumer_group: "$Default" - # connection_string: "" - # storage_account: "" - # storage_account_key: "" - - - auditlogs: - enabled: false - # var: - # eventhub: "insights-logs-auditlogs" - # consumer_group: "$Default" - # connection_string: "" - # storage_account: "" - # storage_account_key: "" - signinlogs: - enabled: false - # var: - # eventhub: "insights-logs-signinlogs" - # consumer_group: "$Default" - # connection_string: "" - # storage_account: "" - # storage_account_key: "" diff --git a/salt/filebeat/modules/barracuda.yml.disabled b/salt/filebeat/modules/barracuda.yml.disabled deleted file mode 100644 index 99ff85036..000000000 --- a/salt/filebeat/modules/barracuda.yml.disabled +++ /dev/null @@ -1,41 +0,0 @@ -# Module: barracuda -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-barracuda.html - -- module: barracuda - waf: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9503 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - spamfirewall: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9524 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/bluecoat.yml.disabled b/salt/filebeat/modules/bluecoat.yml.disabled deleted file mode 100644 index 6550c8eed..000000000 --- a/salt/filebeat/modules/bluecoat.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: bluecoat -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-bluecoat.html - -- module: bluecoat - director: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9505 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/cef.yml.disabled b/salt/filebeat/modules/cef.yml.disabled deleted file mode 100644 index 2de22edcc..000000000 --- a/salt/filebeat/modules/cef.yml.disabled +++ /dev/null @@ -1,17 +0,0 @@ -# Module: cef -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cef.html - -- module: cef - log: - enabled: true - var: - syslog_host: localhost - syslog_port: 9003 - - # Set internal security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.internal_zones: [ "Internal" ] - - # Set external security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.external_zones: [ "External" ] diff --git a/salt/filebeat/modules/checkpoint.yml.disabled b/salt/filebeat/modules/checkpoint.yml.disabled deleted file mode 100644 index 9d34b8d72..000000000 --- a/salt/filebeat/modules/checkpoint.yml.disabled +++ /dev/null @@ -1,24 +0,0 @@ -# Module: checkpoint -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-checkpoint.html - -- module: checkpoint - firewall: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: syslog - - # The interface to listen to UDP based syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The UDP port to listen for syslog traffic. Defaults to 9001. - #var.syslog_port: 9001 - - # Set internal security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.internal_zones: [ "Internal" ] - - # Set external security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.external_zones: [ "External" ] diff --git a/salt/filebeat/modules/cisco.yml.disabled b/salt/filebeat/modules/cisco.yml.disabled deleted file mode 100644 index 9e4658045..000000000 --- a/salt/filebeat/modules/cisco.yml.disabled +++ /dev/null @@ -1,142 +0,0 @@ -# Module: cisco -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cisco.html - -- module: cisco - asa: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: syslog - - # The interface to listen to UDP based syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The UDP port to listen for syslog traffic. Defaults to 9001. - #var.syslog_port: 9001 - - # Set the log level from 1 (alerts only) to 7 (include all messages). - # Messages with a log level higher than the specified will be dropped. - # See https://www.cisco.com/c/en/us/td/docs/security/asa/syslog/b_syslog/syslogs-sev-level.html - #var.log_level: 7 - - # Set internal security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.internal_zones: [ "Internal" ] - - # Set external security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.external_zones: [ "External" ] - - ftd: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: syslog - - # The interface to listen to UDP based syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The UDP port to listen for syslog traffic. Defaults to 9003. - #var.syslog_port: 9003 - - # Set the log level from 1 (alerts only) to 7 (include all messages). - # Messages with a log level higher than the specified will be dropped. - # See https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide/syslogs-sev-level.html - #var.log_level: 7 - - # Set internal security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.internal_zones: [ "Internal" ] - - # Set external security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.external_zones: [ "External" ] - - ios: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: syslog - - # The interface to listen to UDP based syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The UDP port to listen for syslog traffic. Defaults to 9002. - #var.syslog_port: 9002 - - # Set custom paths for the log files when using file input. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - nexus: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9506 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - meraki: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9525 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - umbrella: - enabled: true - - #var.input: aws-s3 - # AWS SQS queue url - #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue - # Access ID to authenticate with the S3 input - #var.access_key_id: 123456 - # Access key to authenticate with the S3 input - #var.secret_access_key: PASSWORD - # The duration that the received messages are hidden from ReceiveMessage request - #var.visibility_timeout: 300s - # Maximum duration before AWS API request will be interrupted - #var.api_timeout: 120s - - amp: - enabled: true - - # Set which input to use between httpjson (default) or file. - #var.input: httpjson - - # The API URL - #var.url: https://api.amp.cisco.com/v1/events - # The client ID used as a username for the API requests. - #var.client_id: - # The API key related to the client ID. - #var.api_key: - # How far to look back the first time the module is started. Expects an amount of hours. - #var.first_interval: 24h - # Overriding the default request timeout, optional. - #var.request_timeout: 60s diff --git a/salt/filebeat/modules/coredns.yml.disabled b/salt/filebeat/modules/coredns.yml.disabled deleted file mode 100644 index 46e9e55c1..000000000 --- a/salt/filebeat/modules/coredns.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: coredns -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-coredns.html - -- module: coredns - # Fileset for native deployment - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/crowdstrike.yml.disabled b/salt/filebeat/modules/crowdstrike.yml.disabled deleted file mode 100644 index 8d2c8531d..000000000 --- a/salt/filebeat/modules/crowdstrike.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: crowdstrike -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-crowdstrike.html - -- module: crowdstrike - - falcon: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/cyberark.yml.disabled b/salt/filebeat/modules/cyberark.yml.disabled deleted file mode 100644 index e97955adf..000000000 --- a/salt/filebeat/modules/cyberark.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: cyberark -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cyberark.html - -- module: cyberark - corepas: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9527 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/cylance.yml.disabled b/salt/filebeat/modules/cylance.yml.disabled deleted file mode 100644 index 342d654d2..000000000 --- a/salt/filebeat/modules/cylance.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: cylance -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cylance.html - -- module: cylance - protect: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9508 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/elasticsearch.yml.disabled b/salt/filebeat/modules/elasticsearch.yml.disabled deleted file mode 100644 index e6074c05e..000000000 --- a/salt/filebeat/modules/elasticsearch.yml.disabled +++ /dev/null @@ -1,35 +0,0 @@ -# Module: elasticsearch -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-elasticsearch.html - -- module: elasticsearch - # Server log - server: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - gc: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - audit: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - slowlog: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - deprecation: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/envoyproxy.yml.disabled b/salt/filebeat/modules/envoyproxy.yml.disabled deleted file mode 100644 index 543b17be5..000000000 --- a/salt/filebeat/modules/envoyproxy.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: envoyproxy -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-envoyproxy.html - -- module: envoyproxy - # Fileset for native deployment - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/f5.yml.disabled b/salt/filebeat/modules/f5.yml.disabled deleted file mode 100644 index 959842174..000000000 --- a/salt/filebeat/modules/f5.yml.disabled +++ /dev/null @@ -1,41 +0,0 @@ -# Module: f5 -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-f5.html - -- module: f5 - bigipapm: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9504 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - bigipafm: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9528 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/fortinet.yml.disabled b/salt/filebeat/modules/fortinet.yml.disabled deleted file mode 100644 index 281b7d788..000000000 --- a/salt/filebeat/modules/fortinet.yml.disabled +++ /dev/null @@ -1,83 +0,0 @@ -# Module: fortinet -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-fortinet.html - -- module: fortinet - firewall: - enabled: true - - # Set which input to use between tcp, udp (default) or file. - #var.input: udp - - # The interface to listen to syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The port to listen for syslog traffic. Defaults to 9004. - #var.syslog_port: 9004 - - # Set internal interfaces. used to override parsed network.direction - # based on a tagged interface. Both internal and external interfaces must be - # set to leverage this functionality. - #var.internal_interfaces: [ "LAN" ] - - # Set external interfaces. used to override parsed network.direction - # based on a tagged interface. Both internal and external interfaces must be - # set to leverage this functionality. - #var.external_interfaces: [ "WAN" ] - - clientendpoint: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9510 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - fortimail: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9529 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - fortimanager: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9530 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/gcp.yml.disabled b/salt/filebeat/modules/gcp.yml.disabled deleted file mode 100644 index a09d0fe36..000000000 --- a/salt/filebeat/modules/gcp.yml.disabled +++ /dev/null @@ -1,76 +0,0 @@ -# Module: gcp -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-gcp.html - -- module: gcp - vpcflow: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be - # configured to use this topic as a sink for VPC flow logs. - var.topic: gcp-vpc-flowlogs - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-vpc-flowlogs-sub - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json - - # Set internal networks. This is used to classify network.direction based - # off of what networks are considered "internal" either base off of a CIDR - # block or named network conditions. If this is not specified, then traffic - # direction is determined by whether it is between source and destination - # instance information rather than IP. - # - # For a full list of network conditions see: - # https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network - #var.internal_networks: [ "private" ] - - firewall: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing firewall logs. Stackdriver must be - # configured to use this topic as a sink for firewall logs. - var.topic: gcp-vpc-firewall - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-firewall-sub - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json - - # Set internal networks. This is used to classify network.direction based - # off of what networks are considered "internal" either base off of a CIDR - # block or named network conditions. If this is not specified, then traffic - # is taken from the direction data in the rule_details event payload. - # - # For a full list of network conditions see: - # https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network - #var.internal_networks: [ "private" ] - - audit: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing firewall logs. Stackdriver must be - # configured to use this topic as a sink for firewall logs. - var.topic: gcp-vpc-audit - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-audit - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/salt/filebeat/modules/google_workspace.yml.disabled b/salt/filebeat/modules/google_workspace.yml.disabled deleted file mode 100644 index 6d364af98..000000000 --- a/salt/filebeat/modules/google_workspace.yml.disabled +++ /dev/null @@ -1,53 +0,0 @@ -# Module: google_workspace -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-google_workspace.html - -- module: google_workspace - saml: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - user_accounts: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - login: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - admin: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - drive: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - groups: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - diff --git a/salt/filebeat/modules/googlecloud.yml.disabled b/salt/filebeat/modules/googlecloud.yml.disabled deleted file mode 100644 index 9a28dc036..000000000 --- a/salt/filebeat/modules/googlecloud.yml.disabled +++ /dev/null @@ -1,58 +0,0 @@ -# Module: googlecloud -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-googlecloud.html - -# googlecloud module is deprecated, please use gcp instead -- module: gcp - vpcflow: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be - # configured to use this topic as a sink for VPC flow logs. - var.topic: gcp-vpc-flowlogs - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-vpc-flowlogs-sub - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json - - firewall: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing firewall logs. Stackdriver must be - # configured to use this topic as a sink for firewall logs. - var.topic: gcp-vpc-firewall - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-firewall-sub - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json - - audit: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing firewall logs. Stackdriver must be - # configured to use this topic as a sink for firewall logs. - var.topic: gcp-vpc-audit - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-audit - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/salt/filebeat/modules/gsuite.yml.disabled b/salt/filebeat/modules/gsuite.yml.disabled deleted file mode 100644 index 6aec3b65d..000000000 --- a/salt/filebeat/modules/gsuite.yml.disabled +++ /dev/null @@ -1,53 +0,0 @@ -# Module: gsuite -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-gsuite.html - -# Gsuite module is deprecated and will be removed in future releases. Please use Google Workspace module instead. -- module: gsuite - saml: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - user_accounts: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - login: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - admin: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - drive: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - groups: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h diff --git a/salt/filebeat/modules/haproxy.yml.disabled b/salt/filebeat/modules/haproxy.yml.disabled deleted file mode 100644 index b2615dbb8..000000000 --- a/salt/filebeat/modules/haproxy.yml.disabled +++ /dev/null @@ -1,14 +0,0 @@ -# Module: haproxy -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-haproxy.html - -- module: haproxy - # All logs - log: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/ibmmq.yml.disabled b/salt/filebeat/modules/ibmmq.yml.disabled deleted file mode 100644 index bfaf3792d..000000000 --- a/salt/filebeat/modules/ibmmq.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: ibmmq -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-ibmmq.html - -- module: ibmmq - # All logs - errorlog: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/icinga.yml.disabled b/salt/filebeat/modules/icinga.yml.disabled deleted file mode 100644 index a7c3ac6e1..000000000 --- a/salt/filebeat/modules/icinga.yml.disabled +++ /dev/null @@ -1,27 +0,0 @@ -# Module: icinga -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-icinga.html - -- module: icinga - # Main logs - main: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Debug logs - debug: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Startup logs - startup: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/iis.yml.disabled b/salt/filebeat/modules/iis.yml.disabled deleted file mode 100644 index 44c200ba1..000000000 --- a/salt/filebeat/modules/iis.yml.disabled +++ /dev/null @@ -1,20 +0,0 @@ -# Module: iis -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-iis.html - -- module: iis - # Access logs - access: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Error logs - error: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - \ No newline at end of file diff --git a/salt/filebeat/modules/imperva.yml.disabled b/salt/filebeat/modules/imperva.yml.disabled deleted file mode 100644 index 8e53deaa6..000000000 --- a/salt/filebeat/modules/imperva.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: imperva -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-imperva.html - -- module: imperva - securesphere: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9511 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/infoblox.yml.disabled b/salt/filebeat/modules/infoblox.yml.disabled deleted file mode 100644 index 9e82f8340..000000000 --- a/salt/filebeat/modules/infoblox.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: infoblox -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-infoblox.html - -- module: infoblox - nios: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9512 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/iptables.yml.disabled b/salt/filebeat/modules/iptables.yml.disabled deleted file mode 100644 index 1147e14dd..000000000 --- a/salt/filebeat/modules/iptables.yml.disabled +++ /dev/null @@ -1,13 +0,0 @@ -# Module: iptables -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-iptables.html - -- module: iptables - log: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/juniper.yml.disabled b/salt/filebeat/modules/juniper.yml.disabled deleted file mode 100644 index 71112679d..000000000 --- a/salt/filebeat/modules/juniper.yml.disabled +++ /dev/null @@ -1,54 +0,0 @@ -# Module: juniper -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-juniper.html - -- module: juniper - junos: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9513 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - netscreen: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9523 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - srx: - enabled: true - - # Set which input to use between tcp, udp (default) or file. - #var.input: udp - - # The interface to listen to syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The port to listen for syslog traffic. Defaults to 9006. - #var.syslog_port: 9006 diff --git a/salt/filebeat/modules/kafka.yml.disabled b/salt/filebeat/modules/kafka.yml.disabled deleted file mode 100644 index 23362c8a1..000000000 --- a/salt/filebeat/modules/kafka.yml.disabled +++ /dev/null @@ -1,15 +0,0 @@ -# Module: kafka -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-kafka.html - -- module: kafka - # All logs - log: - enabled: true - - # Set custom paths for Kafka. If left empty, - # Filebeat will look under /opt. - #var.kafka_home: - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/kibana.yml.disabled b/salt/filebeat/modules/kibana.yml.disabled deleted file mode 100644 index a4956c4b6..000000000 --- a/salt/filebeat/modules/kibana.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: kibana -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-kibana.html - -- module: kibana - # Server logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Audit logs - audit: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/logstash.yml.disabled b/salt/filebeat/modules/logstash.yml.disabled deleted file mode 100644 index f14229409..000000000 --- a/salt/filebeat/modules/logstash.yml.disabled +++ /dev/null @@ -1,18 +0,0 @@ -# Module: logstash -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-logstash.html - -- module: logstash - # logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Slow logs - slowlog: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/microsoft.yml.disabled b/salt/filebeat/modules/microsoft.yml.disabled deleted file mode 100644 index b0a1b10c6..000000000 --- a/salt/filebeat/modules/microsoft.yml.disabled +++ /dev/null @@ -1,49 +0,0 @@ -# Module: microsoft -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-microsoft.html - -- module: microsoft - # ATP configuration - defender_atp: - enabled: true - # How often the API should be polled - #var.interval: 5m - - # Oauth Client ID - #var.oauth2.client.id: "" - - # Oauth Client Secret - #var.oauth2.client.secret: "" - - # Oauth Token URL, should include the tenant ID - #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" - m365_defender: - enabled: true - # How often the API should be polled - #var.interval: 5m - - # Oauth Client ID - #var.oauth2.client.id: "" - - # Oauth Client Secret - #var.oauth2.client.secret: "" - - # Oauth Token URL, should include the tenant ID - #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" - dhcp: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9515 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/misp.yml.disabled b/salt/filebeat/modules/misp.yml.disabled deleted file mode 100644 index 9a489fa0f..000000000 --- a/salt/filebeat/modules/misp.yml.disabled +++ /dev/null @@ -1,17 +0,0 @@ -# Module: misp -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-misp.html - -- module: misp - threat: - enabled: true - # API key to access MISP - #var.api_key - - # Array object in MISP response - #var.http_request_body.limit: 1000 - - # URL of the MISP REST API - #var.url - - # You can also pass SSL options. For example: - #var.ssl.verification_mode: none diff --git a/salt/filebeat/modules/mongodb.yml.disabled b/salt/filebeat/modules/mongodb.yml.disabled deleted file mode 100644 index 266d2e4e8..000000000 --- a/salt/filebeat/modules/mongodb.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: mongodb -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mongodb.html - -- module: mongodb - # All logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/mssql.yml.disabled b/salt/filebeat/modules/mssql.yml.disabled deleted file mode 100644 index bfe4c6e64..000000000 --- a/salt/filebeat/modules/mssql.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: mssql -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mssql.html - -- module: mssql - # Fileset for native deployment - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] diff --git a/salt/filebeat/modules/mysql.yml.disabled b/salt/filebeat/modules/mysql.yml.disabled deleted file mode 100644 index e6be4045b..000000000 --- a/salt/filebeat/modules/mysql.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: mysql -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mysql.html - -- module: mysql - # Error logs - error: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Slow logs - slowlog: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/mysqlenterprise.yml.disabled b/salt/filebeat/modules/mysqlenterprise.yml.disabled deleted file mode 100644 index 37e10d0eb..000000000 --- a/salt/filebeat/modules/mysqlenterprise.yml.disabled +++ /dev/null @@ -1,14 +0,0 @@ -# Module: mysqlenterprise -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mysqlenterprise.html - -- module: mysqlenterprise - audit: - enabled: true - - # Sets the input type. Currently only supports file - #var.input: file - - # Set paths for the log files when file input is used. - # Should only be used together with file input - # var.paths: - # - /home/user/mysqlauditlogs/audit.*.log diff --git a/salt/filebeat/modules/nats.yml.disabled b/salt/filebeat/modules/nats.yml.disabled deleted file mode 100644 index 65e44962d..000000000 --- a/salt/filebeat/modules/nats.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: nats -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-nats.html - -- module: nats - # All logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/netflow.yml.disabled b/salt/filebeat/modules/netflow.yml.disabled deleted file mode 100644 index 781748b00..000000000 --- a/salt/filebeat/modules/netflow.yml.disabled +++ /dev/null @@ -1,14 +0,0 @@ -# Module: netflow -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-netflow.html - -- module: netflow - log: - enabled: true - var: - netflow_host: localhost - netflow_port: 2055 - # internal_networks specifies which networks are considered internal or private - # you can specify either a CIDR block or any of the special named ranges listed - # at: https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network - internal_networks: - - private diff --git a/salt/filebeat/modules/netscout.yml.disabled b/salt/filebeat/modules/netscout.yml.disabled deleted file mode 100644 index 215349046..000000000 --- a/salt/filebeat/modules/netscout.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: netscout -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-netscout.html - -- module: netscout - sightline: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9502 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/nginx.yml.disabled b/salt/filebeat/modules/nginx.yml.disabled deleted file mode 100644 index e2fa44a78..000000000 --- a/salt/filebeat/modules/nginx.yml.disabled +++ /dev/null @@ -1,27 +0,0 @@ -# Module: nginx -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-nginx.html - -- module: nginx - # Access logs - access: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Error logs - error: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs - ingress_controller: - enabled: false - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/o365.yml.disabled b/salt/filebeat/modules/o365.yml.disabled deleted file mode 100644 index 578ff365d..000000000 --- a/salt/filebeat/modules/o365.yml.disabled +++ /dev/null @@ -1,48 +0,0 @@ -# Module: o365 -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-o365.html - -- module: o365 - audit: - enabled: true - - # Set the application_id (also known as client ID): - var.application_id: "" - - # Configure the tenants to monitor: - # Use the tenant ID (also known as directory ID) and the domain name. - # var.tenants: - # - id: "tenant_id_1" - # name: "mydomain.onmicrosoft.com" - # - id: "tenant_id_2" - # name: "mycompany.com" - var.tenants: - - id: "" - name: "mytenant.onmicrosoft.com" - - # List of content-types to fetch. By default all known content-types - # are retrieved: - # var.content_type: - # - "Audit.AzureActiveDirectory" - # - "Audit.Exchange" - # - "Audit.SharePoint" - # - "Audit.General" - # - "DLP.All" - - # Use the following settings to enable certificate-based authentication: - # var.certificate: "/path/to/certificate.pem" - # var.key: "/path/to/private_key.pem" - # var.key_passphrase: "myPrivateKeyPassword" - - # Client-secret based authentication: - # Comment the following line if using certificate authentication. - var.client_secret: "" - - # Advanced settings, use with care: - # var.api: - # # Settings for custom endpoints: - # authentication_endpoint: "https://login.microsoftonline.us/" - # resource: "https://manage.office365.us" - # - # max_retention: 168h - # max_requests_per_minute: 2000 - # poll_interval: 3m diff --git a/salt/filebeat/modules/okta.yml.disabled b/salt/filebeat/modules/okta.yml.disabled deleted file mode 100644 index 4fc943592..000000000 --- a/salt/filebeat/modules/okta.yml.disabled +++ /dev/null @@ -1,10 +0,0 @@ -# Module: okta -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-okta.html - -- module: okta - system: - enabled: true - # You must configure the URL with your Okta domain and provide an - # API token to access the logs API. - #var.url: https://yourOktaDomain/api/v1/logs - #var.api_key: 'yourApiTokenHere' diff --git a/salt/filebeat/modules/oracle.yml.disabled b/salt/filebeat/modules/oracle.yml.disabled deleted file mode 100644 index 3bd576ee1..000000000 --- a/salt/filebeat/modules/oracle.yml.disabled +++ /dev/null @@ -1,13 +0,0 @@ -# Module: oracle -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-oracle.html - -- module: oracle - database_audit: - enabled: true - - # Set which input to use between syslog or file (default). - #var.input: file - - # Set paths for the log files when file input is used. - # Should only be used together with file input - # var.paths: /home/user/oracleauditlogs/*.aud diff --git a/salt/filebeat/modules/osquery.yml.disabled b/salt/filebeat/modules/osquery.yml.disabled deleted file mode 100644 index 7a9a09dd8..000000000 --- a/salt/filebeat/modules/osquery.yml.disabled +++ /dev/null @@ -1,15 +0,0 @@ -# Module: osquery -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-osquery.html - -- module: osquery - result: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # If true, all fields created by this module are prefixed with - # `osquery.result`. Set to false to copy the fields in the root - # of the document. The default is true. - #var.use_namespace: true diff --git a/salt/filebeat/modules/panw.yml.disabled b/salt/filebeat/modules/panw.yml.disabled deleted file mode 100644 index eb094a25a..000000000 --- a/salt/filebeat/modules/panw.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: panw -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-panw.html - -- module: panw - panos: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Set internal security zones. used to determine network.direction - # default "trust" - #var.internal_zones: - - # Set external security zones. used to determine network.direction - # default "untrust" - #var.external_zones: - diff --git a/salt/filebeat/modules/pensando.yml.disabled b/salt/filebeat/modules/pensando.yml.disabled deleted file mode 100644 index 66bd60d76..000000000 --- a/salt/filebeat/modules/pensando.yml.disabled +++ /dev/null @@ -1,13 +0,0 @@ -# Module: pensando -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-pensando.html - -- module: pensando -# Firewall logs - dfw: - enabled: true - var.syslog_host: 0.0.0.0 - var.syslog_port: 9001 - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - # var.paths: diff --git a/salt/filebeat/modules/postgresql.yml.disabled b/salt/filebeat/modules/postgresql.yml.disabled deleted file mode 100644 index 804b7f34f..000000000 --- a/salt/filebeat/modules/postgresql.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: postgresql -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-postgresql.html - -- module: postgresql - # All logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/proofpoint.yml.disabled b/salt/filebeat/modules/proofpoint.yml.disabled deleted file mode 100644 index 9aeebd5fe..000000000 --- a/salt/filebeat/modules/proofpoint.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: proofpoint -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-proofpoint.html - -- module: proofpoint - emailsecurity: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9531 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/rabbitmq.yml.disabled b/salt/filebeat/modules/rabbitmq.yml.disabled deleted file mode 100644 index e61a0a0c9..000000000 --- a/salt/filebeat/modules/rabbitmq.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: rabbitmq -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-rabbitmq.html - -- module: rabbitmq - # All logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: ["/var/log/rabbitmq/rabbit@localhost.log*"] diff --git a/salt/filebeat/modules/radware.yml.disabled b/salt/filebeat/modules/radware.yml.disabled deleted file mode 100644 index f9ab3e519..000000000 --- a/salt/filebeat/modules/radware.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: radware -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-radware.html - -- module: radware - defensepro: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9518 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/redis.yml.disabled b/salt/filebeat/modules/redis.yml.disabled deleted file mode 100644 index 9b621dc2d..000000000 --- a/salt/filebeat/modules/redis.yml.disabled +++ /dev/null @@ -1,21 +0,0 @@ -# Module: redis -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-redis.html - -- module: redis - # Main logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: ["/var/log/redis/redis-server.log*"] - - # Slow logs, retrieved via the Redis API (SLOWLOG) - slowlog: - enabled: true - - # The Redis hosts to connect to. - #var.hosts: ["localhost:6379"] - - # Optional, the password to use when connecting to Redis. - #var.password: diff --git a/salt/filebeat/modules/santa.yml.disabled b/salt/filebeat/modules/santa.yml.disabled deleted file mode 100644 index 1a7363547..000000000 --- a/salt/filebeat/modules/santa.yml.disabled +++ /dev/null @@ -1,9 +0,0 @@ -# Module: santa -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-santa.html - -- module: santa - log: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the the default path. - #var.paths: diff --git a/salt/filebeat/modules/snort.yml.disabled b/salt/filebeat/modules/snort.yml.disabled deleted file mode 100644 index 8c9bcc471..000000000 --- a/salt/filebeat/modules/snort.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: snort -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-snort.html - -- module: snort - log: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9532 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/snyk.yml.disabled b/salt/filebeat/modules/snyk.yml.disabled deleted file mode 100644 index 0b13f8155..000000000 --- a/salt/filebeat/modules/snyk.yml.disabled +++ /dev/null @@ -1,112 +0,0 @@ -# Module: snyk -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-snyk.html - -- module: snyk - audit: - enabled: true - - # Set which input to use between httpjson (default) or file. - #var.input: httpjson - # - # What audit type to collect, can be either "group" or "organization". - #var.audit_type: organization - # - # The ID related to the audit_type. If audit type is group, then this value should be - # the group ID and if it is organization it should be the organization ID to collect from. - #var.audit_id: 1235432-asdfdf-2341234-asdgjhg - - # How often the API should be polled, defaults to 1 hour. - #var.interval: 1h - # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). - #var.first_interval: 24h - - # The API token that is created for a specific user, found in the Snyk management dashboard. - #var.api_token: - - # Event filtering. - # All configuration items below is OPTIONAL and the default options will be overwritten - # for each entry that is not commented out. - - # Will return only logs for this specific project. - #var.project_id: "" - # User public ID. Will fetch only audit logs originated from this user's actions. - #var.user_id: "" - # Will return only logs for this specific event. - #var.event: "" - # User email address. Will fetch only audit logs originated from this user's actions. - #var.email_address: "" - - vulnerabilities: - enabled: true - - # Set which input to use between httpjson (default) or file. - #var.input: httpjson - - # How often the API should be polled. Data from the Snyk API is automatically updated - # once per day, so the default interval is 24 hours. - #var.interval: 24h - - # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). - #var.first_interval: 24h - - # The API token that is created for a specific user, found in the Snyk management dashboard. - #var.api_token: - - # The list of org IDs to filter the results by. - # One organization ID per line, starting with a - sign - #var.orgs: - # - 12354-asdfdf-123543-asdsdfg - # - 76554-jhggfd-654342-hgrfasd - - - # Event filtering. - # All configuration items below is OPTIONAL and the default options will be overwritten - # for each entry that is not commented out. - - # The severity levels of issues to filter the results by. - #var.included_severity: - # - high - # - medium - # - low - # - # The exploit maturity levels of issues to filter the results by. - #var.exploit_maturity: - # - mature - # - proof-of-concept - # - no-known-exploit - # - no-data - # - # The type of issues to filter the results by. - #var.types: - # - vuln - # - license - # - # The type of languages to filter the results by. - #var.languages: - # - javascript - # - ruby - # - java - # - scala - # - python - # - golang - # - php - # - dotnet - # - swift - # - docker - # - # Search term to filter issue name by, or an exact CVE or CWE. - #var.identifier: - # - "" - # - # If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. - #var.ignored: false - #var.patched: false - #var.fixable: false - #var.is_fixed: false - #var.is_patchable: false - #var.is_pinnable: false - # - # The priority score ranging between 0-1000 - #var.min_priority_score: 0 - #var.max_priority_score: 1000 - diff --git a/salt/filebeat/modules/sonicwall.yml.disabled b/salt/filebeat/modules/sonicwall.yml.disabled deleted file mode 100644 index de457109d..000000000 --- a/salt/filebeat/modules/sonicwall.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: sonicwall -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-sonicwall.html - -- module: sonicwall - firewall: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9519 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/sophos.yml.disabled b/salt/filebeat/modules/sophos.yml.disabled deleted file mode 100644 index 8fc346540..000000000 --- a/salt/filebeat/modules/sophos.yml.disabled +++ /dev/null @@ -1,46 +0,0 @@ -# Module: sophos -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-sophos.html - -- module: sophos - xg: - enabled: true - - # Set which input to use between tcp, udp (default) or file. - #var.input: udp - - # The interface to listen to syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The port to listen for syslog traffic. Defaults to 9004. - #var.syslog_port: 9005 - - # firewall default hostname - #var.default_host_name: firewall.localgroup.local - - # known firewalls - #var.known_devices: - #- serial_number: "1234567890123457" - # hostname: "a.host.local" - #- serial_number: "1234234590678557" - # hostname: "b.host.local" - - - utm: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9533 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/squid.yml.disabled b/salt/filebeat/modules/squid.yml.disabled deleted file mode 100644 index a47807253..000000000 --- a/salt/filebeat/modules/squid.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: squid -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-squid.html - -- module: squid - log: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9520 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/suricata.yml b/salt/filebeat/modules/suricata.yml deleted file mode 100644 index b7cc11e85..000000000 --- a/salt/filebeat/modules/suricata.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Module: suricata -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html - -- module: suricata - # All logs - eve: - enabled: true - var.paths: ["/nsm/suricata/eve*.json"] - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/system.yml.disabled b/salt/filebeat/modules/system.yml.disabled deleted file mode 100644 index d633bac04..000000000 --- a/salt/filebeat/modules/system.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: system -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-system.html - -- module: system - # Syslog - syslog: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Authorization logs - auth: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/threatintel.yml.disabled b/salt/filebeat/modules/threatintel.yml.disabled deleted file mode 100644 index b461d91e2..000000000 --- a/salt/filebeat/modules/threatintel.yml.disabled +++ /dev/null @@ -1,105 +0,0 @@ -# Module: threatintel -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-threatintel.html - -- module: threatintel - abuseurl: - enabled: true - - # Input used for ingesting threat intel data. - var.input: httpjson - - # The URL used for Threat Intel API calls. - var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/ - - # The interval to poll the API for updates. - var.interval: 10m - - abusemalware: - enabled: true - - # Input used for ingesting threat intel data. - var.input: httpjson - - # The URL used for Threat Intel API calls. - var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/ - - # The interval to poll the API for updates. - var.interval: 10m - - misp: - enabled: true - - # Input used for ingesting threat intel data, defaults to JSON. - var.input: httpjson - - # The URL of the MISP instance, should end with "/events/restSearch". - var.url: https://SERVER/events/restSearch - - # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI. - var.api_token: API_KEY - - # Configures the type of SSL verification done, if MISP is running on self signed certificates - # then the certificate would either need to be trusted, or verification_mode set to none. - #var.ssl.verification_mode: none - - # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context. - # For examples please reference the filebeat module documentation. - #var.filters: - # - threat_level: [4, 5] - # - to_ids: true - - # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer - # than the last event that was already ingested. - var.first_interval: 300h - - # The interval to poll the API for updates. - var.interval: 5m - - otx: - enabled: true - - # Input used for ingesting threat intel data - var.input: httpjson - - # The URL used for OTX Threat Intel API calls. - var.url: https://otx.alienvault.com/api/v1/indicators/export - - # The authentication token used to contact the OTX API, can be found on the OTX UI. - var.api_token: API_KEY - - # Optional filters that can be applied to retrieve only specific indicators. - #var.types: "domain,IPv4,hostname,url,FileHash-SHA256" - - # The timeout of the HTTP client connecting to the OTX API - #var.http_client_timeout: 120s - - # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module. - var.lookback_range: 1h - - # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 400h - - # The interval to poll the API for updates - var.interval: 5m - - anomali: - enabled: true - - # Input used for ingesting threat intel data - var.input: httpjson - - # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending - # on the type of threat intel source that is needed. - var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects - - # The Username used by anomali Limo, defaults to guest. - #var.username: guest - - # The password used by anomali Limo, defaults to guest. - #var.password: guest - - # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 400h - - # The interval to poll the API for updates - var.interval: 5m diff --git a/salt/filebeat/modules/tomcat.yml.disabled b/salt/filebeat/modules/tomcat.yml.disabled deleted file mode 100644 index 84f4619d5..000000000 --- a/salt/filebeat/modules/tomcat.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: tomcat -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-tomcat.html - -- module: tomcat - log: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9501 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/traefik.yml.disabled b/salt/filebeat/modules/traefik.yml.disabled deleted file mode 100644 index 657d5ccd9..000000000 --- a/salt/filebeat/modules/traefik.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: traefik -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-traefik.html - -- module: traefik - # Access logs - access: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/zeek.yml b/salt/filebeat/modules/zeek.yml deleted file mode 100644 index 9fd61c448..000000000 --- a/salt/filebeat/modules/zeek.yml +++ /dev/null @@ -1,122 +0,0 @@ -# Module: zeek -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html - -- module: zeek - capture_loss: - enabled: false - var.paths: ["/nsm/zeek/logs/current/capture_loss.log"] - connection: - enabled: true - var.paths: ["/nsm/zeek/logs/current/conn.log"] - dce_rpc: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dce_rpc.log"] - dhcp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dhcp.log"] - dnp3: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dnp3.log"] - dns: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dns.log"] - dpd: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dpd.log"] - files: - enabled: true - var.paths: ["/nsm/zeek/logs/current/files.log"] - ftp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/ftp.log"] - http: - enabled: true - var.paths: ["/nsm/zeek/logs/current/http.log"] - intel: - enabled: true - var.paths: ["/nsm/zeek/logs/current/intel.log"] - irc: - enabled: true - var.paths: ["/nsm/zeek/logs/current/irc.log"] - kerberos: - enabled: true - var.paths: ["/nsm/zeek/logs/current/kerberos.log"] - modbus: - enabled: true - var.paths: ["/nsm/zeek/logs/current/modbus.log"] - mysql: - enabled: true - var.paths: ["/nsm/zeek/logs/current/mysql.log"] - notice: - enabled: true - var.paths: ["/nsm/zeek/logs/current/notice.log"] - ntlm: - enabled: true - var.paths: ["/nsm/zeek/logs/current/ntlm.log"] - ocsp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/oscp.log"] - pe: - enabled: true - var.paths: ["/nsm/zeek/logs/current/pe.log"] - radius: - enabled: true - var.paths: ["/nsm/zeek/logs/current/radius.log"] - rdp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/rdp.log"] - rfb: - enabled: true - var.paths: ["/nsm/zeek/logs/current/rfb.log"] - signature: - enabled: true - var.paths: ["/nsm/zeek/logs/current/signature.log"] - sip: - enabled: true - var.paths: ["/nsm/zeek/logs/current/sip.log"] - smb_cmd: - enabled: true - var.paths: ["/nsm/zeek/logs/current/smb_cmd.log"] - smb_files: - enabled: true - var.paths: ["/nsm/zeek/logs/current/smb_files.log"] - smb_mapping: - enabled: true - var.paths: ["/nsm/zeek/logs/current/smb_mapping.log"] - smtp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/smtp.log"] - snmp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/snmp.log"] - socks: - enabled: true - var.paths: ["/nsm/zeek/logs/current/socks.log"] - ssh: - enabled: true - var.paths: ["/nsm/zeek/logs/current/ssh.log"] - ssl: - enabled: true - var.paths: ["/nsm/zeek/logs/current/ssl.log"] - stats: - enabled: false - var.paths: ["/nsm/zeek/logs/current/stats.log"] - syslog: - enabled: false - var.paths: ["/nsm/zeek/logs/current/syslog.log"] - traceroute: - enabled: false - var.paths: ["/nsm/zeek/logs/current/traceroute.log.log"] - tunnel: - enabled: true - var.paths: ["/nsm/zeek/logs/current/tunnel.log"] - weird: - enabled: true - var.paths: ["/nsm/zeek/logs/current/weird.log"] - x509: - enabled: true - var.paths: ["/nsm/zeek/logs/current/x509.log"] - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/zoom.yml.disabled b/salt/filebeat/modules/zoom.yml.disabled deleted file mode 100644 index 15fa9d4b2..000000000 --- a/salt/filebeat/modules/zoom.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: zoom -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zoom.html - -- module: zoom - webhook: - enabled: true - - # The type of input to use - #var.input: http_endpoint - - # The interface to listen for incoming HTTP requests. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.listen_address: localhost - - # The port to bind to - #var.listen_port: 80 - - # The header Zoom uses to send its secret token, defaults to "Authorization" - #secret.header: Authorization - - # The secret token value created by Zoom - #secret.value: ZOOMTOKEN diff --git a/salt/filebeat/modules/zscaler.yml.disabled b/salt/filebeat/modules/zscaler.yml.disabled deleted file mode 100644 index accdec9ea..000000000 --- a/salt/filebeat/modules/zscaler.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: zscaler -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zscaler.html - -- module: zscaler - zia: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9521 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 8bcecd618..19826a708 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -26,7 +26,7 @@ securityonion_filebeat: zeek: {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} {{ LOGNAME }}: - enabled: false + enabled: true var.paths: ["/nsm/zeek/logs/current/{{ LOGNAME }}.log"] {%- endfor %} {%- endif %} From dfaf40f58359ab0896c84a47dc85c4b5d5d27280 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 17:14:26 -0400 Subject: [PATCH 169/378] add zeekloglookup to translate zeeklogs to filebeat filesets --- salt/filebeat/init.sls | 1 - salt/filebeat/map.jinja | 4 ++++ salt/filebeat/securityoniondefaults.yaml | 11 +++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index b1a91b133..7850f4eb6 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -23,7 +23,6 @@ {% from 'filebeat/map.jinja' import THIRDPARTY with context %} {% from 'filebeat/map.jinja' import SO with context %} - filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index 6ae6e7cff..b5df8fea5 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -4,3 +4,7 @@ {% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} {% set SO = SODEFAULTS.securityonion_filebeat %} {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} + +{% set ZEEKLOGLOOKUP = { + 'conn': 'connection', +} %} diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 8bcecd618..300e7f42f 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -1,4 +1,6 @@ {%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %} +{% from 'filebeat/map.jinja' import ZEEKLOGLOOKUP with context %} + securityonion_filebeat: modules: elasticsearch: @@ -25,9 +27,14 @@ securityonion_filebeat: {%- if ZEEKVER != 'SURICATA' %} zeek: {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} - {{ LOGNAME }}: + {% if LOGNAME in ZEEKLOGLOOKUP.keys() %} + {% set FILESET = ZEEKLOGLOOKUP.get(LOGNAME) %} + {% else %} + {% set FILESET = LOGNAME %} + {% endif %} + {{ FILESET }}: enabled: false var.paths: ["/nsm/zeek/logs/current/{{ LOGNAME }}.log"] {%- endfor %} {%- endif %} - {%- endif %} \ No newline at end of file + {%- endif %} From ec2f8fe6c88527d0a4b1cc322b17115c930ba69d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 25 May 2021 17:16:05 -0400 Subject: [PATCH 170/378] Synchronize SOC passwords with Elastic --- salt/common/tools/sbin/so-user | 74 ++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index dbd755bc4..b516cf6ad 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -40,7 +40,8 @@ email=$2 kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434} databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite} bcryptRounds=${BCRYPT_ROUNDS:-12} -extractedHashFile=${EXTRACTED_HASH_FILE:-/opt/so/conf/elasticsearch/users} +elasticUsersFile=${ELASTIC_USERS_FILE:-/opt/so/conf/elasticsearch/users} +elasticRolesFile=${ELASTIC_ROLES_FILE:-/opt/so/conf/elasticsearch/users_roles} function fail() { msg=$1 @@ -93,6 +94,16 @@ function validateEmail() { fi } +function hashPassword() { + password=$1 + + passwordHash=$(echo "${password}" | htpasswd -niBC $bcryptRounds SOUSER) + passwordHash=$(echo "$passwordHash" | cut -c 11-) + passwordHash="\$2a${passwordHash}" # still waiting for https://github.com/elastic/elasticsearch/issues/51132 + echo "$passwordHash" +} + + function updatePassword() { identityId=$1 @@ -109,17 +120,54 @@ function updatePassword() { if [[ -n $identityId ]]; then # Generate password hash - passwordHash=$(echo "${password}" | htpasswd -niBC $bcryptRounds SOUSER) - passwordHash=$(echo "$passwordHash" | cut -c 11-) - passwordHash="\$2a${passwordHash}" + passwordHash=$(hashPassword "$password") # Update DB with new hash - echo "update identity_credentials set config=CAST('{\"hashed_password\":\"${passwordHash}\"}' as BLOB) where identity_id=${identityId};" | sqlite3 "$databasePath" + echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB) where identity_id=${identityId};" | sqlite3 "$databasePath" [[ $? != 0 ]] && fail "Unable to update password" fi } -function extractHashes() { - echo "select ici.identifier || ':' || json_extract(ic.config, '$.hashed_password') from identity_credential_identifiers ici, identity_credentials ic where ici.identity_credential_id=ic.id and json_extract(ic.config, '$.hashed_password') is not null order by ici.identifier" | sqlite3 "$databasePath" > "$extractedHashFile" +function createElasticTmpFile() { + filename=$1 + tmpFile=${filename}.tmp + truncate -s 0 "$tmpFile" + chmod 600 "$tmpFile" + chown elasticsearch:elasticsearch "$tmpFile" + echo "$tmpFile" +} + +function syncElastic() { + usersFileTmp=$(createElasticTmpFile "${elasticUsersFile}") + rolesFileTmp=$(createElasticTmpFile "${elasticRolesFile}") + + sysUser=$(lookup_pillar "auth:user" "elasticsearch") + sysPass=$(lookup_pillar "auth:pass" "elasticsearch") + sysHash=$(hashPassword "$sysPass") + + # Generate the new users file + echo "${sysUser}:${sysHash}" >> "$usersFileTmp" + echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \ + "from identity_credential_identifiers ici, identity_credentials ic " \ + "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ + "order by ici.identifier;" | \ + sqlite3 "$databasePath" | \ + jq -r '.user + ":" + .data.hashed_password' \ + >> "$usersFileTmp" + mv -f "$usersFileTmp" "$elasticUsersFile" + + # Generate the new users_roles file + echo "superuser:${sysUser}" >> "$rolesFileTmp" + echo "select 'superuser:' || ici.identifier " \ + "from identity_credential_identifiers ici, identity_credentials ic " \ + "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ + "order by ici.identifier;" | \ + sqlite3 "$databasePath" \ + >> "$rolesFileTmp" + mv -f "$rolesFileTmp" "$elasticRolesFile" +} + +function syncAll() { + syncElastic } function listUsers() { @@ -213,7 +261,7 @@ case "${operation}" in validateEmail "$email" updatePassword createUser "$email" - extractHashes + syncAll echo "Successfully added new user to SOC" check_container thehive && echo $password | so-thehive-user-add "$email" check_container fleet && echo $password | so-fleet-user-add "$email" @@ -229,7 +277,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateUser "$email" - extractHashes + syncAll echo "Successfully updated user" ;; @@ -238,7 +286,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateStatus "$email" 'active' - extractHashes + syncAll echo "Successfully enabled user" check_container thehive && so-thehive-user-enable "$email" true check_container fleet && so-fleet-user-enable "$email" true @@ -249,7 +297,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateStatus "$email" 'locked' - extractHashes + syncAll echo "Successfully disabled user" check_container thehive && so-thehive-user-enable "$email" false check_container fleet && so-fleet-user-enable "$email" false @@ -260,14 +308,14 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" deleteUser "$email" - extractHashes + syncAll echo "Successfully deleted user" check_container thehive && so-thehive-user-enable "$email" false check_container fleet && so-fleet-user-enable "$email" false ;; "sync") - extractHashes + syncAll echo "Synchronization complete" ;; From 525d4325c7530e4e137b956a954c523c868651d9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 17:18:58 -0400 Subject: [PATCH 171/378] define ZEEKLOGLOOKUP in the yaml --- salt/filebeat/map.jinja | 4 ---- salt/filebeat/securityoniondefaults.yaml | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index b5df8fea5..6ae6e7cff 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -4,7 +4,3 @@ {% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} {% set SO = SODEFAULTS.securityonion_filebeat %} {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} - -{% set ZEEKLOGLOOKUP = { - 'conn': 'connection', -} %} diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 58eef8361..0a1459d6b 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -1,5 +1,7 @@ {%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %} -{% from 'filebeat/map.jinja' import ZEEKLOGLOOKUP with context %} +{% set ZEEKLOGLOOKUP = { + 'conn': 'connection', +} %} securityonion_filebeat: modules: From 6da37966d928ef1af1d0a98dfbed7ee712e70b56 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 26 May 2021 09:32:25 -0400 Subject: [PATCH 172/378] Update wording for iso location prompt in soup --- salt/common/tools/sbin/soup | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d1261fe30..f7fa4e211 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -107,14 +107,15 @@ airgap_mounted() { echo "The ISO is already mounted" else echo "" - echo "Looks like we need access to the upgrade content" - echo "" - echo "If you just copied the .iso file over you can specify the path." - echo "If you burned the ISO to a disk the standard way you can specify the device." - echo "Example: /home/user/securityonion-2.X.0.iso" - echo "Example: /dev/sdx1" - echo "" - read -rp 'Enter the location of the iso: ' ISOLOC + cat << EOF +In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. + +For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. + +Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom. + +EOF + read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC if [[ -f $ISOLOC ]]; then # Mounting the ISO image mkdir -p /tmp/soagupdate From 0187c9d6df5aeb4a62c7c963231f4d0f7b2cefe6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 26 May 2021 09:51:37 -0400 Subject: [PATCH 173/378] Adds so-pcap-export --- salt/common/tools/sbin/so-pcap-export | 26 +++++++++++++++++++++ salt/common/tools/sbin/so-suricata-testrule | 5 ++++ 2 files changed, 31 insertions(+) create mode 100644 salt/common/tools/sbin/so-pcap-export diff --git a/salt/common/tools/sbin/so-pcap-export b/salt/common/tools/sbin/so-pcap-export new file mode 100644 index 000000000..076b4eae6 --- /dev/null +++ b/salt/common/tools/sbin/so-pcap-export @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +if [ $# -lt 2 ]; then + echo "Usage: $0 Output-Filename" + exit 1 +fi + +docker exec -it so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap + +echo "" +echo "If successful, the output was written to: /nsm/pcapout/$2.pcap" \ No newline at end of file diff --git a/salt/common/tools/sbin/so-suricata-testrule b/salt/common/tools/sbin/so-suricata-testrule index ac4b81d3c..f9f23e70f 100755 --- a/salt/common/tools/sbin/so-suricata-testrule +++ b/salt/common/tools/sbin/so-suricata-testrule @@ -23,6 +23,11 @@ TESTPCAP=$2 . /usr/sbin/so-common +if [ $# -lt 2 ]; then + echo "Usage: $0 " + exit 1 +fi + echo "" echo "===============" echo "Running all.rules and $TESTRULE against the following pcap: $TESTPCAP" From cb40a76247141f45ccb2886e3637e2386a3a1b76 Mon Sep 17 00:00:00 2001 From: John Bernal <59663876+j-bernal@users.noreply.github.com> Date: Wed, 26 May 2021 09:55:14 -0400 Subject: [PATCH 174/378] Update so-whiptail Updated Zeek capitalization when prompting for the number of processes. --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 5eca2d39a..b7fc50495 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -64,7 +64,7 @@ whiptail_basic_zeek() { [ -n "$TESTING" ] && return BASICZEEK=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter the number of zeek processes:" 10 75 "$1" 3>&1 1>&2 2>&3) + "Enter the number of Zeek processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 34d4eedf672cb523942c14f16c8a57ef1036fbe9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 May 2021 10:11:47 -0400 Subject: [PATCH 175/378] Remove old modules --- salt/common/tools/sbin/so-filebeat-module-setup | 8 +++++++- salt/common/tools/sbin/so-zeek-logs | 6 ------ salt/filebeat/etc/module_config.yml.jinja | 2 +- salt/filebeat/securityoniondefaults.yaml | 2 ++ salt/filebeat/thirdpartydefaults.yaml | 17 ----------------- salt/zeek/init.sls | 2 ++ setup/so-functions | 6 ------ setup/so-whiptail | 6 ------ 8 files changed, 12 insertions(+), 37 deletions(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 6616854eb..5aefe3ac2 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -49,6 +49,12 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Setting up ingest pipeline(s)" -docker exec -it so-filebeat filebeat setup modules -pipelines -modules activemq,apache,auditd,aws,azure,barracuda,bluecoat,cef,checkpoint,cisco,coredns,crowdstrike,cyberark,cylance,elasticsearch,envoyproxy,f5,fortinet,gcp,google_workspace,googlecloud,gsuite,haproxy,ibmmq,icinga,iis,imperva,infoblox,iptables,juniper,kafka,kibana,logstash,microsoft,misp,mondogb,mssql,mysql,mysqlenterprise,nats,netflow,netscout,nginx,o365,okta,osquery,panw,pensando,postgresql,rabbitmq,radware,redis,santa,snort,snyk,sonicwall,sophos,squid,suricata,system,threatintel,tomcat,traefik,zeek,zoom,zscaler -c $FB_MODULE_YML + +for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler +do + echo "Loading $MODULE" + docker exec -it so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML + sleep 2 +done diff --git a/salt/common/tools/sbin/so-zeek-logs b/salt/common/tools/sbin/so-zeek-logs index 551213580..521f7c9dc 100755 --- a/salt/common/tools/sbin/so-zeek-logs +++ b/salt/common/tools/sbin/so-zeek-logs @@ -14,7 +14,6 @@ whiptail_manager_adv_service_zeeklogs() { "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ - "dhcpv6" "DHCP IPv6 Logs" ON \ "dnp3" "DNP3 Logs" ON \ "dns" "DNS Logs" ON \ "dpd" "DPD Logs" ON \ @@ -25,25 +24,20 @@ whiptail_manager_adv_service_zeeklogs() { "irc" "IRC Chat Logs" ON \ "kerberos" "Kerberos Logs" ON \ "modbus" "MODBUS Logs" ON \ - "mqtt" "MQTT Logs" ON \ "notice" "Zeek Notice Logs" ON \ "ntlm" "NTLM Logs" ON \ - "openvpn" "OPENVPN Logs" ON \ "pe" "PE Logs" ON \ "radius" "Radius Logs" ON \ "rfb" "RFB Logs" ON \ "rdp" "RDP Logs" ON \ - "signatures" "Signatures Logs" ON \ "sip" "SIP Logs" ON \ "smb_files" "SMB Files Logs" ON \ "smb_mapping" "SMB Mapping Logs" ON \ "smtp" "SMTP Logs" ON \ "snmp" "SNMP Logs" ON \ - "software" "Software Logs" ON \ "ssh" "SSH Logs" ON \ "ssl" "SSL Logs" ON \ "syslog" "Syslog Logs" ON \ - "telnet" "Telnet Logs" ON \ "tunnel" "Tunnel Logs" ON \ "weird" "Zeek Weird Logs" ON \ "mysql" "MySQL Logs" ON \ diff --git a/salt/filebeat/etc/module_config.yml.jinja b/salt/filebeat/etc/module_config.yml.jinja index 7cd624895..8f4fbd7bf 100644 --- a/salt/filebeat/etc/module_config.yml.jinja +++ b/salt/filebeat/etc/module_config.yml.jinja @@ -3,7 +3,7 @@ - module: {{ module }} {%- for fileset in MODULES.modules[module] %} {{ fileset }}: - enabled: {{ MODULES.modules[module][fileset].enabled }} + enabled: {{ MODULES.modules[module][fileset].enabled|string|lower }} {#- only manage the settings if the fileset is enabled #} {%- if MODULES.modules[module][fileset].enabled %} {%- for var, value in MODULES.modules[module][fileset].items() %} diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 0a1459d6b..f503e5de1 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -21,6 +21,8 @@ securityonion_filebeat: log: enabled: true var.paths: ["/logs/redis.log"] + slowlog: + enabled: false suricata: eve: enabled: true diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml index cfb8d10d9..1b378f84b 100644 --- a/salt/filebeat/thirdpartydefaults.yaml +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -199,12 +199,6 @@ third_party_filebeat: okta: system: enabled: false - pesando: - dfw: - enabled: false - var.input: udp - var.syslog_host: 0.0.0.0 - var.syslog_port: 9001 proofpoint: emailsecurity: enabled: false @@ -251,17 +245,6 @@ third_party_filebeat: var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9520 - threatintel: - abuseurl: - enabled: false - abusemalware: - enabled: false - misp: - enabled: false - otx: - enabled: false - anomali: - enabled: false tomcat: log: enabled: false diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 4e597f597..ce0d6dccd 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -183,6 +183,8 @@ so-zeek: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }} - start: {{ START }} - privileged: True + - ulimits: + - core=0 - binds: - /nsm/zeek/logs:/nsm/zeek/logs:rw - /nsm/zeek/spool:/nsm/zeek/spool:rw diff --git a/setup/so-functions b/setup/so-functions index 5ce3d6dee..aefaa2cd5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2827,7 +2827,6 @@ zeek_logs_enabled() { " - conn"\ " - dce_rpc"\ " - dhcp"\ - " - dhcpv6"\ " - dnp3"\ " - dns"\ " - dpd"\ @@ -2838,25 +2837,20 @@ zeek_logs_enabled() { " - irc"\ " - kerberos"\ " - modbus"\ - " - mqtt"\ " - notice"\ " - ntlm"\ - " - openvpn"\ " - pe"\ " - radius"\ " - rfb"\ " - rdp"\ - " - signatures"\ " - sip"\ " - smb_files"\ " - smb_mapping"\ " - smtp"\ " - snmp"\ - " - software"\ " - ssh"\ " - ssl"\ " - syslog"\ - " - telnet"\ " - tunnel"\ " - weird"\ " - mysql"\ diff --git a/setup/so-whiptail b/setup/so-whiptail index 5eca2d39a..fdcc4acc9 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1154,7 +1154,6 @@ whiptail_manager_adv_service_zeeklogs() { "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ - "dhcpv6" "DHCP IPv6 Logs" ON \ "dnp3" "DNP3 Logs" ON \ "dns" "DNS Logs" ON \ "dpd" "DPD Logs" ON \ @@ -1165,25 +1164,20 @@ whiptail_manager_adv_service_zeeklogs() { "irc" "IRC Chat Logs" ON \ "kerberos" "Kerberos Logs" ON \ "modbus" "MODBUS Logs" ON \ - "mqtt" "MQTT Logs" ON \ "notice" "Zeek Notice Logs" ON \ "ntlm" "NTLM Logs" ON \ - "openvpn" "OPENVPN Logs" ON \ "pe" "PE Logs" ON \ "radius" "Radius Logs" ON \ "rfb" "RFB Logs" ON \ "rdp" "RDP Logs" ON \ - "signatures" "Signatures Logs" ON \ "sip" "SIP Logs" ON \ "smb_files" "SMB Files Logs" ON \ "smb_mapping" "SMB Mapping Logs" ON \ "smtp" "SMTP Logs" ON \ "snmp" "SNMP Logs" ON \ - "software" "Software Logs" ON \ "ssh" "SSH Logs" ON \ "ssl" "SSL Logs" ON \ "syslog" "Syslog Logs" ON \ - "telnet" "Telnet Logs" ON \ "tunnel" "Tunnel Logs" ON \ "weird" "Zeek Weird Logs" ON \ "mysql" "MySQL Logs" ON \ From 842aa97f7e9dd32ea9974e2d0db8bd8e04a08397 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 May 2021 11:00:18 -0400 Subject: [PATCH 176/378] load filebeat modules when es container starts and if fb container is running --- salt/filebeat/init.sls | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 372656038..312e815c9 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -22,6 +22,13 @@ {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} {% from 'filebeat/map.jinja' import SO with context %} +{% set ES_INCLUDED_NODES = ['so-standalone'] %} + +#only include elastic state for certain nodes +{% if grains.role in ES_INCLUDED_NODES %} +include: + - elasticsearch +{% endif %} filebeatetcdir: file.directory: @@ -126,6 +133,16 @@ so-filebeat: - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml +{% if grains.role in ES_INCLUDED_NODES %} +run_module_setup: + cmd.run: + - name: /usr/sbin/so-filebeat-module-setup + - require: + - docker_container: so-filebeat + - onchanges_in: + - docker_container: so-elasticsearch +{% endif %} + append_so-filebeat_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf From b525cfc787b6c662f861720e394ec2ca4d72c4b5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 May 2021 11:07:53 -0400 Subject: [PATCH 177/378] Remove old modules --- setup/so-functions | 6 ------ 1 file changed, 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index aefaa2cd5..bef4f384d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2862,7 +2862,6 @@ zeek_logs_enabled() { " - conn"\ " - dce_rpc"\ " - dhcp"\ - " - dhcpv6"\ " - dnp3"\ " - dns"\ " - dpd"\ @@ -2873,24 +2872,19 @@ zeek_logs_enabled() { " - irc"\ " - kerberos"\ " - modbus"\ - " - mqtt"\ " - notice"\ " - ntlm"\ - " - openvpn"\ " - pe"\ " - radius"\ " - rfb"\ " - rdp"\ - " - signatures"\ " - sip"\ " - smb_files"\ " - smb_mapping"\ " - smtp"\ " - snmp"\ - " - software"\ " - ssh"\ " - ssl"\ - " - telnet"\ " - tunnel"\ " - weird"\ " - mysql"\ From a6a4c03029567d8d008e82c60dbb44f2fb6d9049 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 26 May 2021 12:08:10 -0400 Subject: [PATCH 178/378] Improve error scenarios for user sync; Ensure user sync runs before Elastic container starts --- salt/common/tools/sbin/so-user | 5 +++++ salt/elasticsearch/init.sls | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index b516cf6ad..7362c94a2 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -142,6 +142,7 @@ function syncElastic() { sysUser=$(lookup_pillar "auth:user" "elasticsearch") sysPass=$(lookup_pillar "auth:pass" "elasticsearch") + [[ -z "$sysUser" || -z "$sysPass" ]] && fail "Elastic auth credentials for system user are missing" sysHash=$(hashPassword "$sysPass") # Generate the new users file @@ -153,7 +154,9 @@ function syncElastic() { sqlite3 "$databasePath" | \ jq -r '.user + ":" + .data.hashed_password' \ >> "$usersFileTmp" + [[ $? != 0 ]] && fail "Unable to read credential hashes from database" mv -f "$usersFileTmp" "$elasticUsersFile" + [[ $? != 0 ]] && fail "Unable to create users file: $elasticUsersFile" # Generate the new users_roles file echo "superuser:${sysUser}" >> "$rolesFileTmp" @@ -163,7 +166,9 @@ function syncElastic() { "order by ici.identifier;" | \ sqlite3 "$databasePath" \ >> "$rolesFileTmp" + [[ $? != 0 ]] && fail "Unable to read credential IDs from database" mv -f "$rolesFileTmp" "$elasticRolesFile" + [[ $? != 0 ]] && fail "Unable to create users file: $elasticRolesFile" } function syncAll() { diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index c6ac600ef..dd4af7696 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -169,6 +169,14 @@ eslogdir: - group: 939 - makedirs: True +# Must run before elasticsearch docker container is started! +syncesusers: + cmd.run: + - name: so-user sync + - creates: + - /opt/so/conf/elasticsearch/users + - /opt/so/conf/elasticsearch/users_roles + so-elasticsearch: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }} From c531ef0773df174d92dd2ce2ae5b9504442749bd Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 26 May 2021 13:44:30 -0400 Subject: [PATCH 179/378] Move user sync'd files to saltstack for grid propagation --- salt/common/tools/sbin/so-user | 4 ++-- salt/elasticsearch/init.sls | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 7362c94a2..23034a6e9 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -40,8 +40,8 @@ email=$2 kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434} databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite} bcryptRounds=${BCRYPT_ROUNDS:-12} -elasticUsersFile=${ELASTIC_USERS_FILE:-/opt/so/conf/elasticsearch/users} -elasticRolesFile=${ELASTIC_ROLES_FILE:-/opt/so/conf/elasticsearch/users_roles} +elasticUsersFile=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users} +elasticRolesFile=${ELASTIC_ROLES_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users_roles} function fail() { msg=$1 diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index dd4af7696..5ee936f1c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -174,8 +174,8 @@ syncesusers: cmd.run: - name: so-user sync - creates: - - /opt/so/conf/elasticsearch/users - - /opt/so/conf/elasticsearch/users_roles + - /opt/so/saltstack/local/salt/elasticsearch/files/users + - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles so-elasticsearch: docker_container.running: From b43e6c5d6bb0721bdbb0418e2cdeb93700e375c6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 26 May 2021 13:51:24 -0400 Subject: [PATCH 180/378] Salt will handle auto-sync --- salt/common/tools/sbin/so-user | 5 ----- 1 file changed, 5 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 23034a6e9..1ff637d23 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -266,7 +266,6 @@ case "${operation}" in validateEmail "$email" updatePassword createUser "$email" - syncAll echo "Successfully added new user to SOC" check_container thehive && echo $password | so-thehive-user-add "$email" check_container fleet && echo $password | so-fleet-user-add "$email" @@ -282,7 +281,6 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateUser "$email" - syncAll echo "Successfully updated user" ;; @@ -291,7 +289,6 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateStatus "$email" 'active' - syncAll echo "Successfully enabled user" check_container thehive && so-thehive-user-enable "$email" true check_container fleet && so-fleet-user-enable "$email" true @@ -302,7 +299,6 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateStatus "$email" 'locked' - syncAll echo "Successfully disabled user" check_container thehive && so-thehive-user-enable "$email" false check_container fleet && so-fleet-user-enable "$email" false @@ -313,7 +309,6 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" deleteUser "$email" - syncAll echo "Successfully deleted user" check_container thehive && so-thehive-user-enable "$email" false check_container fleet && so-fleet-user-enable "$email" false From ee675546ac751615ac9fa0fac00b5540bba4aaa5 Mon Sep 17 00:00:00 2001 From: doug Date: Wed, 26 May 2021 14:09:00 -0400 Subject: [PATCH 181/378] add menu.actions.json and update soc.json --- salt/soc/files/soc/menu.actions.json | 33 ++++++++++++++++++++++++++++ salt/soc/files/soc/soc.json | 8 +++++-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 salt/soc/files/soc/menu.actions.json diff --git a/salt/soc/files/soc/menu.actions.json b/salt/soc/files/soc/menu.actions.json new file mode 100644 index 000000000..558d10a36 --- /dev/null +++ b/salt/soc/files/soc/menu.actions.json @@ -0,0 +1,33 @@ +[ + { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", + "links": [ + "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" + ]}, + { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", + "links": [ + "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", + "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\") | groupby event.module event.dataset", + "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", + "/#/hunt?q=(\"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", + "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset", + "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset", + "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset" + ]}, + { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", + "links": [ + "/joblookup?esid={:soc_id}", + "/joblookup?ncid={:network.community_id}" + ]}, + { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", + "links": [ + "/cyberchef/#input={value|base64}" + ]}, + { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", + "links": [ + "https://www.google.com/search?q={value}" + ]}, + { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", + "links": [ + "https://www.virustotal.com/gui/search/{value}" + ]} +] diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 5dfd364d2..a3326ab4a 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -17,6 +17,7 @@ {%- import_json "soc/files/soc/hunt.queries.json" as hunt_queries %} {%- import_json "soc/files/soc/hunt.actions.json" as hunt_actions %} {%- import_json "soc/files/soc/hunt.eventfields.json" as hunt_eventfields %} +{%- import_json "soc/files/soc/menu.actions.json" as menu_actions %} {%- import_json "soc/files/soc/tools.json" as tools %} {%- set DNET = salt['pillar.get']('global:dockernet', '172.17.0.0') %} @@ -123,8 +124,11 @@ "queryBaseFilter": "", "queryToggleFilters": [], "queries": {{ hunt_queries | json }}, - "actions": {{ hunt_actions | json }} + "actions": {{ menu_actions | json }} }, + "job": { + "actions": {{ menu_actions | json }} + }, "alerts": { "advanced": false, "groupItemsPerPage": 50, @@ -143,7 +147,7 @@ { "name": "escalated", "filter": "event.escalated:true", "enabled": false, "exclusive": true, "enablesToggles":["acknowledged"] } ], "queries": {{ alerts_queries | json }}, - "actions": {{ alerts_actions | json }} + "actions": {{ menu_actions | json }} } } } From bfcde15a24ac0c39d649c2150a3546c1476228f8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 May 2021 14:22:14 -0400 Subject: [PATCH 182/378] elastic pipeline test --- .../config/so/9050_output_elasticsearch.jinja | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja new file mode 100644 index 000000000..eb0d8ef0c --- /dev/null +++ b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja @@ -0,0 +1,20 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +output { + if [event][module] == 'elasticsearch' { + elasticsearch { + id => "elastic_logs" + pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} From 4d991d3773f655ae510f7075f33bc3eee03dca7e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 May 2021 14:52:10 -0400 Subject: [PATCH 183/378] propogate users and users_roles --- salt/elasticsearch/init.sls | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 65a3d2bca..017c135d2 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -37,6 +37,8 @@ {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} +{% set ES_INCLUDED_NODES = ['so-standalone'] %} + vm.max_map_count: sysctl.present: - value: 262144 @@ -170,6 +172,35 @@ eslogdir: - group: 939 - makedirs: True +{% if grains.role in ES_INCLUDED_NODES %} +# Must run before elasticsearch docker container is started! +syncesusers: + cmd.run: + - name: so-user sync + - creates: + - /opt/so/saltstack/local/salt/elasticsearch/files/users + - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles +{% endif %} + +auth_users: + file.managed: + - name: /opt/so/conf/elasticsearch/users + - source: salt://elasticsearch/files/users + - require: +{% if grains.role in ES_INCLUDED_NODES %} + - cmd: syncesusers +{% endif %} + +auth_users_roles: + file.managed: + - name: /opt/so/conf/elasticsearch/users_roles + - source: salt://elasticsearch/files/users_roles +{% if grains.role in ES_INCLUDED_NODES %} + - require: + - cmd: syncesusers +{% endif %} + + so-elasticsearch: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }} From 7263e35a89273af973aefeb5cf91d87aa28da042 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 May 2021 14:52:59 -0400 Subject: [PATCH 184/378] happy little comment --- salt/elasticsearch/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 017c135d2..84de1fd0b 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -37,6 +37,7 @@ {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} +# used in this state to control who can run the so-users script {% set ES_INCLUDED_NODES = ['so-standalone'] %} vm.max_map_count: From aeea5701e490e69254b9fd6d1470a9e8db67da65 Mon Sep 17 00:00:00 2001 From: doug Date: Wed, 26 May 2021 16:34:05 -0400 Subject: [PATCH 185/378] completely disable both alerts.actions.json and hunt.actions.json --- salt/soc/files/soc/alerts.actions.json | 34 +------------------------- salt/soc/files/soc/hunt.actions.json | 34 +------------------------- salt/soc/files/soc/soc.json | 2 -- 3 files changed, 2 insertions(+), 68 deletions(-) diff --git a/salt/soc/files/soc/alerts.actions.json b/salt/soc/files/soc/alerts.actions.json index 558d10a36..1addf23c6 100644 --- a/salt/soc/files/soc/alerts.actions.json +++ b/salt/soc/files/soc/alerts.actions.json @@ -1,33 +1 @@ -[ - { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", - "links": [ - "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" - ]}, - { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", - "links": [ - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset", - "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset", - "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset" - ]}, - { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", - "links": [ - "/joblookup?esid={:soc_id}", - "/joblookup?ncid={:network.community_id}" - ]}, - { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", - "links": [ - "/cyberchef/#input={value|base64}" - ]}, - { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", - "links": [ - "https://www.google.com/search?q={value}" - ]}, - { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", - "links": [ - "https://www.virustotal.com/gui/search/{value}" - ]} -] +This file is no longer used. Please use menu.actions.json instead. diff --git a/salt/soc/files/soc/hunt.actions.json b/salt/soc/files/soc/hunt.actions.json index 558d10a36..1addf23c6 100644 --- a/salt/soc/files/soc/hunt.actions.json +++ b/salt/soc/files/soc/hunt.actions.json @@ -1,33 +1 @@ -[ - { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", - "links": [ - "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" - ]}, - { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", - "links": [ - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset", - "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset", - "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset" - ]}, - { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", - "links": [ - "/joblookup?esid={:soc_id}", - "/joblookup?ncid={:network.community_id}" - ]}, - { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", - "links": [ - "/cyberchef/#input={value|base64}" - ]}, - { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", - "links": [ - "https://www.google.com/search?q={value}" - ]}, - { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", - "links": [ - "https://www.virustotal.com/gui/search/{value}" - ]} -] +This file is no longer used. Please use menu.actions.json instead. diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index a3326ab4a..8048970f3 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -12,10 +12,8 @@ {%- set CACHE_EXPIRATION = salt['pillar.get']('sensoroni:cache_expiration_ms', 0) %} {%- set ES_FIELDCAPS_CACHE = salt['pillar.get']('sensoroni:es_fieldcaps_cache_ms', '300000') %} {%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %} -{%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %} {%- import_json "soc/files/soc/alerts.eventfields.json" as alerts_eventfields %} {%- import_json "soc/files/soc/hunt.queries.json" as hunt_queries %} -{%- import_json "soc/files/soc/hunt.actions.json" as hunt_actions %} {%- import_json "soc/files/soc/hunt.eventfields.json" as hunt_eventfields %} {%- import_json "soc/files/soc/menu.actions.json" as menu_actions %} {%- import_json "soc/files/soc/tools.json" as tools %} From dc8520df42c9b31ff6fc46f0329b45649def9342 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 May 2021 18:04:30 -0400 Subject: [PATCH 186/378] user curl.config for curl and elasticscripts --- salt/elasticsearch/auth.map.jinja | 2 +- salt/elasticsearch/init.sls | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/auth.map.jinja b/salt/elasticsearch/auth.map.jinja index 147b0157b..c6f099025 100644 --- a/salt/elasticsearch/auth.map.jinja +++ b/salt/elasticsearch/auth.map.jinja @@ -2,6 +2,6 @@ True: { 'user': salt['pillar.get']('elasticsearch:auth:user'), 'pass': salt['pillar.get']('elasticsearch:auth:pass'), - 'elasticcurl':'curl --user ' ~ salt['pillar.get']('elasticsearch:auth:user') ~ ':' ~ salt['pillar.get']('elasticsearch:auth:pass') }, + 'elasticcurl':'curl -K /opt/so/conf/elasticsearch/curl.config' }, False: {'elasticcurl': 'curl'}, }, pillar='elasticsearch:auth:enabled', default=False) %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 84de1fd0b..9c0372684 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -292,6 +292,7 @@ elastic_curl_config: file.managed: - name: /opt/so/conf/elasticsearch/curl.config - mode: 600 + - contents: user = "{{ salt['pillar.get']('elasticsearch:auth:user') }}:{{ salt['pillar.get']('elasticsearch:auth:pass') }}" - show_changes: False {% endif %} {# if grains['role'] != 'so-helix' #} From b23ce7462ea8b256eb1ccc9d23e0c055733a23d8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 27 May 2021 11:26:25 -0400 Subject: [PATCH 187/378] add depenency --- salt/filebeat/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 312e815c9..5cabaf828 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -138,6 +138,7 @@ run_module_setup: cmd.run: - name: /usr/sbin/so-filebeat-module-setup - require: + - file: filebeatmoduleconfsync - docker_container: so-filebeat - onchanges_in: - docker_container: so-elasticsearch From 423793ecf9807bf02fdbe73546c834447b77f447 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 27 May 2021 13:50:22 -0400 Subject: [PATCH 188/378] remove vault pg from testing --- salt/firewall/portgroups.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/firewall/portgroups.yaml b/salt/firewall/portgroups.yaml index 3a7316e98..1386267f5 100644 --- a/salt/firewall/portgroups.yaml +++ b/salt/firewall/portgroups.yaml @@ -89,9 +89,6 @@ firewall: - 514 udp: - 514 - vault: - tcp: - - 8200 wazuh_agent: tcp: - 1514 From ada8255af04aaf952c2c9a92f52507ce9ba5e757 Mon Sep 17 00:00:00 2001 From: doug Date: Fri, 28 May 2021 08:59:40 -0400 Subject: [PATCH 189/378] bump version to 7.13.0 --- salt/kibana/files/saved_objects.ndjson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/kibana/files/saved_objects.ndjson b/salt/kibana/files/saved_objects.ndjson index 863209d48..21ef5a94a 100644 --- a/salt/kibana/files/saved_objects.ndjson +++ b/salt/kibana/files/saved_objects.ndjson @@ -460,7 +460,7 @@ {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\",\"time_zone\":\"America/New_York\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":48,\"x\":0,\"y\":0,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":96,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":8,\"y\":20,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":28,\"y\":20,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":0,\"y\":72,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":48,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":24,\"x\":16,\"y\":72,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":28,\"y\":8,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":120,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"event_type\",\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"}]","timeRestore":false,"title":"z16.04 - Sysmon - Logs","version":1},"id":"6d189680-6d62-11e7-8ddb-e71eb260f4a3","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"8cfdeff0-6d6b-11e7-ad64-15aa071374a6","name":"panel_1","type":"visualization"},{"id":"0eb1fd80-6d70-11e7-b09b-f57b22df6524","name":"panel_2","type":"visualization"},{"id":"3072c750-6d71-11e7-b09b-f57b22df6524","name":"panel_3","type":"visualization"},{"id":"7bc74b40-6d71-11e7-b09b-f57b22df6524","name":"panel_4","type":"visualization"},{"id":"13ed0810-6d72-11e7-b09b-f57b22df6524","name":"panel_5","type":"visualization"},{"id":"3b6c92c0-6d72-11e7-b09b-f57b22df6524","name":"panel_6","type":"visualization"},{"id":"e09f6010-6d72-11e7-b09b-f57b22df6524","name":"panel_7","type":"visualization"},{"id":"29611940-6d75-11e7-b09b-f57b22df6524","name":"panel_8","type":"visualization"},{"id":"6b70b840-6d75-11e7-b09b-f57b22df6524","name":"panel_9","type":"visualization"},{"id":"248c1d20-6d6b-11e7-ad64-15aa071374a6","name":"panel_10","type":"search"},{"id":"AWDHHk1sxQT5EBNmq43Y","name":"panel_11","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQyLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} -{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.12.1","id":"7.12.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.13.0","id":"7.13.0","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":56,\"x\":0,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":8,\"y\":32,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":20,\"y\":32,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":56,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"columns\":[\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"uid\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":32,\"y\":32,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":40,\"h\":24,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"}]","timeRestore":false,"title":"z16.04 - Bro - Modbus","version":1},"id":"70c005f0-3583-11e7-a588-05992195c551","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"0d168a30-363f-11e7-a6f7-4f44d7bf1c33","name":"panel_1","type":"visualization"},{"id":"20eabd60-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_2","type":"visualization"},{"id":"3c65f500-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_3","type":"visualization"},{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"panel_4","type":"search"},{"id":"178209e0-6e1b-11e7-b553-7f80727663c1","name":"panel_5","type":"visualization"},{"id":"AWDG_9KpxQT5EBNmq4Oo","name":"panel_6","type":"visualization"},{"id":"453f8b90-4a58-11e8-9b0a-f1d33346f773","name":"panel_7","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ4LDRd"} @@ -730,4 +730,4 @@ {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - Syslog - Severity (Donut)","uiStateJSON":"{}","version":1,"visState":"{\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"syslog.severity.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Severity\"}}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"syslog.severity.keyword: Descending\",\"aggType\":\"terms\"}]}},\"title\":\"Security Onion - Syslog - Severity (Donut)\"}"},"id":"fc8d41a0-777b-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwNDExLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{}"},"savedSearchRefName":"search_0","title":"Security Onion - Connections - Top Source IPs","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - Connections - Top Source IPs\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"source.ip: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source IP\"}}]}"},"id":"fd8b4640-6e9f-11ea-9266-1fd14ca6af34","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"9b333020-6e9f-11ea-9266-1fd14ca6af34","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwNDEyLDRd"} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"event.module:strelka\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":7,\"i\":\"a2e0a619-a5c5-40d9-8593-e60f13ae22bf\"},\"panelIndex\":\"a2e0a619-a5c5-40d9-8593-e60f13ae22bf\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":8,\"y\":0,\"w\":21,\"h\":7,\"i\":\"566a9d04-f2dc-4868-9625-97a19d985703\"},\"panelIndex\":\"566a9d04-f2dc-4868-9625-97a19d985703\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":29,\"y\":0,\"w\":19,\"h\":7,\"i\":\"f247ec64-c278-4e05-ac4d-983bea9dfb7d\"},\"panelIndex\":\"f247ec64-c278-4e05-ac4d-983bea9dfb7d\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":7,\"w\":12,\"h\":20,\"i\":\"6e80a142-ab0e-4fd3-891c-e495b78a1625\"},\"panelIndex\":\"6e80a142-ab0e-4fd3-891c-e495b78a1625\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":12,\"y\":7,\"w\":11,\"h\":20,\"i\":\"292cc879-6bc0-4541-ba92-3b3c5f4e3368\"},\"panelIndex\":\"292cc879-6bc0-4541-ba92-3b3c5f4e3368\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":23,\"y\":7,\"w\":14,\"h\":20,\"i\":\"66979b2c-e7c1-4291-91ac-16537b7f9ec3\"},\"panelIndex\":\"66979b2c-e7c1-4291-91ac-16537b7f9ec3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":37,\"y\":7,\"w\":11,\"h\":20,\"i\":\"8bb1cf98-0401-4a2d-9dd8-deca08205a22\"},\"panelIndex\":\"8bb1cf98-0401-4a2d-9dd8-deca08205a22\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":27,\"w\":8,\"h\":20,\"i\":\"393f3cec-3ee0-4275-b319-f307e7a260c6\"},\"panelIndex\":\"393f3cec-3ee0-4275-b319-f307e7a260c6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":8,\"y\":27,\"w\":15,\"h\":20,\"i\":\"0e8800a9-a6f5-4a79-8370-61713f584886\"},\"panelIndex\":\"0e8800a9-a6f5-4a79-8370-61713f584886\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":23,\"y\":27,\"w\":25,\"h\":20,\"i\":\"be9a0a2a-d8c6-4d15-b5d7-d5599d0482a3\"},\"panelIndex\":\"be9a0a2a-d8c6-4d15-b5d7-d5599d0482a3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":47,\"w\":48,\"h\":27,\"i\":\"40296d2b-cb6f-423f-989c-3fdaa82d2aad\"},\"panelIndex\":\"40296d2b-cb6f-423f-989c-3fdaa82d2aad\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_10\"}]","timeRestore":false,"title":"Security Onion - Strelka","version":1},"id":"ff689c50-75f3-11ea-9565-7315f4ee5cac","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"8cfec8c0-6ec2-11ea-9266-1fd14ca6af34","name":"panel_0","type":"visualization"},{"id":"d04b5130-6e99-11ea-9266-1fd14ca6af34","name":"panel_1","type":"visualization"},{"id":"23ed13a0-6e9a-11ea-9266-1fd14ca6af34","name":"panel_2","type":"visualization"},{"id":"7a88adc0-75f0-11ea-9565-7315f4ee5cac","name":"panel_3","type":"visualization"},{"id":"49cfe850-772c-11ea-bee5-af7f7c7b8e05","name":"panel_4","type":"visualization"},{"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","name":"panel_5","type":"visualization"},{"id":"ce9e03f0-772c-11ea-bee5-af7f7c7b8e05","name":"panel_6","type":"visualization"},{"id":"a7ebb450-772c-11ea-bee5-af7f7c7b8e05","name":"panel_7","type":"visualization"},{"id":"08c0b770-772e-11ea-bee5-af7f7c7b8e05","name":"panel_8","type":"visualization"},{"id":"e087c7d0-772d-11ea-bee5-af7f7c7b8e05","name":"panel_9","type":"visualization"},{"id":"8b6f3150-72a2-11ea-8dd2-9d8795a1200b","name":"panel_10","type":"search"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwNDEzLDRd"} -{"exportedCount":732,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file +{"exportedCount":732,"missingRefCount":0,"missingReferences":[]} From ba011581ef0c1c2f35bd9925e4c1ddcfde1462c0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 28 May 2021 10:14:58 -0400 Subject: [PATCH 190/378] Add version to ending summary --- setup/so-whiptail | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index b7fc50495..d492d10cc 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -447,6 +447,7 @@ whiptail_end_settings() { # BASIC INFO (NETWORK, HOSTNAME, DESCRIPTION, ETC) read -r -d '' end_msg <<- EOM + Security Onion Version: $SOVERSION Node Type: $install_type Hostname: $HOSTNAME EOM From d55a9e6274ad381944504295af93a5905f0fbcfb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 28 May 2021 10:20:19 -0400 Subject: [PATCH 191/378] Add version to all whiptail titles --- setup/so-whiptail | 200 +++++++++++++++++++++++----------------------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index d492d10cc..09792459d 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -22,7 +22,7 @@ whiptail_airgap() { local node_str='node' [[ $is_manager || $is_import ]] && node_str='manager' - INTERWEBS=$(whiptail --title "Security Onion Setup" --menu \ + INTERWEBS=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu \ "How should this $node_str be installed?" 10 60 2 \ "Standard " "This $node_str has internet accesss" \ "Airgap " "This $node_str does not have internet access" 3>&1 1>&2 2>&3 ) @@ -42,7 +42,7 @@ whiptail_avoid_default_hostname() { You can choose to use this default hostname anyway, or change it to a new hostname. EOM - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --yesno "$message" 11 75 \ --yes-button "Use Anyway" --no-button "Change" --defaultno } @@ -51,7 +51,7 @@ whiptail_basic_suri() { [ -n "$TESTING" ] && return - BASICSURI=$(whiptail --title "Security Onion Setup" --inputbox \ + BASICSURI=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the number of Suricata processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -63,7 +63,7 @@ whiptail_basic_zeek() { [ -n "$TESTING" ] && return - BASICZEEK=$(whiptail --title "Security Onion Setup" --inputbox \ + BASICZEEK=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the number of Zeek processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -75,7 +75,7 @@ whiptail_bond_nics_mtu() { [ -n "$TESTING" ] && return # Set the MTU on the monitor interface - MTU=$(whiptail --title "Security Onion Setup" --inputbox \ + MTU=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the MTU for the monitor NICs:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -84,7 +84,7 @@ whiptail_bond_nics_mtu() { whiptail_cancel() { - whiptail --title "Security Onion Setup" --msgbox "Cancelling Setup." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Cancelling Setup." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -104,7 +104,7 @@ whiptail_check_exitstatus() { whiptail_cancel ;; 255) - whiptail --title "Security Onion Setup" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 exit ;; esac @@ -114,7 +114,7 @@ whiptail_components_adv_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 } whiptail_create_admin_user() { @@ -187,7 +187,7 @@ whiptail_create_soremote_user() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Set a password for the soremote user. This account is used for adding sensors remotely." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Set a password for the soremote user. This account is used for adding sensors remotely." 8 75 } @@ -258,11 +258,11 @@ whiptail_requirements_error() { [ -n "$TESTING" ] && return if [[ $(echo "$requirement_needed" | tr '[:upper:]' '[:lower:]') == 'nics' ]]; then - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --msgbox "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select OK to exit setup and reconfigure the machine." 10 75 # Same as whiptail_cancel, but changed the wording to exit instead of cancel. - whiptail --title "Security Onion Setup" --msgbox "Exiting Setup. No changes have been made." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Exiting Setup. No changes have been made." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -272,7 +272,7 @@ whiptail_requirements_error() { fi exit else - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select YES to continue anyway, or select NO to cancel." 10 75 local exitstatus=$? @@ -298,7 +298,7 @@ whiptail_storage_requirements() { EOM whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --yesno "$message" \ 14 75 @@ -310,7 +310,7 @@ whiptail_cur_close_days() { [ -n "$TESTING" ] && return - CURCLOSEDAYS=$(whiptail --title "Security Onion Setup" --inputbox \ + CURCLOSEDAYS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Please specify the threshold (in days) at which Elasticsearch indices will be closed:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -322,7 +322,7 @@ whiptail_dhcp_or_static() { [ -n "$TESTING" ] && return - address_type=$(whiptail --title "Security Onion Setup" --radiolist \ + address_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose how to set up your management interface:" 20 78 4 \ "STATIC" "Set a static IPv4 address" ON \ "DHCP" "Use DHCP to configure the Management Interface" OFF 3>&1 1>&2 2>&3 ) @@ -355,7 +355,7 @@ whiptail_dhcp_warn() { EOM whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --"$window_type" "$dhcp_message" \ 14 75 @@ -366,7 +366,7 @@ whiptail_dhcp_warn() { whiptail_dhcp_or_static ;; 255) - whiptail --title "Security Onion Setup" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 exit ;; esac @@ -380,7 +380,7 @@ whiptail_dockernet_check(){ [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno \ + whiptail --title "Security Onion Setup - $SOVERSION" --yesno \ "Do you want to keep the default Docker IP range?\n\nIf you are unsure, please accept the default option of Yes." 10 75 } @@ -389,7 +389,7 @@ whiptail_dockernet_net() { [ -n "$TESTING" ] && return - DOCKERNET=$(whiptail --title "Security Onion Setup" --inputbox \ + DOCKERNET=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "\nEnter a /24 size network range for docker to use WITHOUT the /24 suffix. This range will be used on ALL nodes." 11 65 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -409,7 +409,7 @@ whiptail_enable_components() { STRELKA=0 if [[ $is_eval ]]; then - COMPONENTS=$(whiptail --title "Security Onion Setup" --checklist \ + COMPONENTS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ "Select Components to install:" 20 75 8 \ GRAFANA "Enable Grafana for system monitoring" ON \ OSQUERY "Enable Fleet with osquery" ON \ @@ -418,7 +418,7 @@ if [[ $is_eval ]]; then PLAYBOOK "Enable Playbook" ON \ STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3) else - COMPONENTS=$(whiptail --title "Security Onion Setup" --checklist \ + COMPONENTS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ "Select Components to install:" 20 75 7 \ OSQUERY "Enable Fleet with osquery" ON \ WAZUH "Enable Wazuh" ON \ @@ -658,7 +658,7 @@ whiptail_eval_adv() { [ -n "$TESTING" ] && return - EVALADVANCED=$(whiptail --title "Security Onion Setup" --radiolist \ + EVALADVANCED=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose your eval install:" 20 75 4 \ "BASIC" "Install basic components for evaluation" ON \ "ADVANCED" "Choose additional components to be installed" OFF 3>&1 1>&2 2>&3 ) @@ -685,7 +685,7 @@ whiptail_gauge_post_setup() { else local msg=$1 - whiptail --title "Security Onion Setup" --gauge "$msg" 6 60 96 + whiptail --title "Security Onion Setup - $SOVERSION" --gauge "$msg" 6 60 96 fi } @@ -693,7 +693,7 @@ whiptail_helix_apikey() { [ -n "$TESTING" ] && return - HELIXAPIKEY=$(whiptail --title "Security Onion Setup" --inputbox \ + HELIXAPIKEY=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your Helix API Key: \n \nThis can be set later using so-helix-apikey" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -707,7 +707,7 @@ whiptail_homenet_manager() { [ -n "$TESTING" ] && return - HNMANAGER=$(whiptail --title "Security Onion Setup" --inputbox \ + HNMANAGER=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -719,13 +719,13 @@ whiptail_homenet_sensor_inherit() { [ -n "$TESTING" ] && return # Ask to inherit from manager - whiptail --title "Security Onion Setup" --yesno "Do you want to inherit the HOME_NET from the Manager?" 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to inherit the HOME_NET from the Manager?" 8 75 } whiptail_homenet_sensor() { [ -n "$TESTING" ] && return - HNSENSOR=$(whiptail --title "Security Onion Setup" --inputbox \ + HNSENSOR=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -738,7 +738,7 @@ whiptail_install_type() { [ -n "$TESTING" ] && return # What kind of install are we doing? - install_type=$(whiptail --title "Security Onion Setup" --radiolist \ + install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose install type:" 12 65 5 \ "EVAL" "Evaluation mode (not for production) " ON \ "STANDALONE" "Standalone production install " OFF \ @@ -764,7 +764,7 @@ whiptail_install_type_dist() { [ -n "$TESTING" ] && return - install_type=$(whiptail --title "Security Onion Setup" --radiolist \ + install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose distributed node type:" 13 60 6 \ "MANAGER" "Start a new grid " ON \ "SENSOR" "Create a forward only sensor " OFF \ @@ -792,14 +792,14 @@ whiptail_install_type_other() { # so-analyst-install will only work with a working network connection # so only show it on network installs for now if [[ $setup_type == 'network' ]]; then - install_type=$(whiptail --title "Security Onion Setup" --radiolist \ + install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose distributed node type:" 9 65 2 \ "ANALYST" "Quit setup and run so-analyst-install " ON \ "HELIXSENSOR" "Create a Helix sensor " OFF \ 3>&1 1>&2 2>&3 ) else - install_type=$(whiptail --title "Security Onion Setup" --radiolist \ + install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose distributed node type:" 8 65 1 \ "HELIXSENSOR" "Create a Helix sensor " ON \ 3>&1 1>&2 2>&3 @@ -815,7 +815,7 @@ whiptail_install_type_other() { whiptail_invalid_input() { # TODO: This should accept a list of arguments to specify what general pattern the input should follow [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox " Invalid input, please try again." 7 40 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox " Invalid input, please try again." 7 40 } @@ -829,13 +829,13 @@ whiptail_invalid_proxy() { Error was: ${proxy_test_err} EOM - whiptail --title "Security Onion Setup" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60 } whiptail_invalid_string() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 } @@ -843,21 +843,21 @@ whiptail_invalid_pass_characters_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 } whiptail_invalid_pass_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Please choose a more secure password." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please choose a more secure password." 8 75 } whiptail_invalid_user_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Please enter a valid email address." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please enter a valid email address." 8 75 } whiptail_invalid_hostname() { @@ -868,7 +868,7 @@ whiptail_invalid_hostname() { the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', \ and hyphen ('-')" | tr -d '\t') - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --msgbox "$error_message" 10 75 } @@ -891,7 +891,7 @@ whiptail_log_size_limit() { By default, this is set to ${percentage}% of the disk space allotted for /nsm. EOM - log_size_limit=$(whiptail --title "Security Onion Setup" --inputbox "$message" 11 75 "$1" 3>&1 1>&2 2>&3) + log_size_limit=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "$message" 11 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -901,7 +901,7 @@ whiptail_log_size_limit() { whiptail_first_menu_iso() { [ -n "$TESTING" ] && return - option=$(whiptail --title "Security Onion Setup" --menu "Select an option" 10 75 2 \ + option=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "Select an option" 10 75 2 \ "Install " "Run the standard Security Onion installation " \ "Configure Network " "Configure networking only " \ 3>&1 1>&2 2>&3 @@ -915,7 +915,7 @@ whiptail_make_changes() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -926,7 +926,7 @@ whiptail_management_interface_dns() { [ -n "$TESTING" ] && return - MDNS=$(whiptail --title "Security Onion Setup" --inputbox \ + MDNS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your DNS servers separated by commas:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -938,7 +938,7 @@ whiptail_management_interface_dns_search() { [ -n "$TESTING" ] && return - MSEARCH=$(whiptail --title "Security Onion Setup" --inputbox \ + MSEARCH=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your DNS search domain:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -949,7 +949,7 @@ whiptail_management_interface_gateway() { [ -n "$TESTING" ] && return - MGATEWAY=$(whiptail --title "Security Onion Setup" --inputbox \ + MGATEWAY=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your gateway's IPv4 address:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -959,7 +959,7 @@ whiptail_management_interface_gateway() { whiptail_management_interface_ip_mask() { [ -n "$TESTING" ] && return - manager_ip_mask=$(whiptail --title "Security Onion Setup" --inputbox \ + manager_ip_mask=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your IPv4 address with CIDR mask (e.g. 192.168.1.2/24):" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -970,7 +970,7 @@ whiptail_management_interface_ip() { [ -n "$TESTING" ] && return - MIP=$(whiptail --title "Security Onion Setup" --inputbox \ + MIP=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your IP address:" 10 60 X.X.X.X 3>&1 1>&2 2>&3) local exitstatus=$? @@ -981,7 +981,7 @@ whiptail_management_interface_mask() { [ -n "$TESTING" ] && return - MMASK=$(whiptail --title "Security Onion Setup" --inputbox \ + MMASK=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the bit mask for your subnet:" 10 60 24 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1060,7 +1060,7 @@ whiptail_net_method() { local option_count=$(( ${#options[@]} / 2 )) - network_traffic=$(whiptail --title "Security Onion Setup" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) + network_traffic=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1071,7 +1071,7 @@ whiptail_net_method() { whiptail_net_setup_complete() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --msgbox "Successfully set up networking, setup will now exit." 7 75 exit 0 } @@ -1085,7 +1085,7 @@ whiptail_network_init_notice() { Select OK to continue. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" 9 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 9 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } @@ -1094,7 +1094,7 @@ whiptail_management_server() { [ -n "$TESTING" ] && return - MSRV=$(whiptail --title "Security Onion Setup" --inputbox \ + MSRV=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your Manager Server hostname: \nIt is CASE SENSITIVE!" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1105,7 +1105,7 @@ whiptail_management_server() { whiptail_manager_ip() { [ -n "$TESTING" ] && return - MSRVIP=$(whiptail --title "Security Onion Setup" --inputbox \ + MSRVIP=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your Manager Server IP Address:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1117,7 +1117,7 @@ whiptail_manager_adv() { [ -n "$TESTING" ] && return - MANAGERADV=$(whiptail --title "Security Onion Setup" --radiolist \ + MANAGERADV=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose which type of manager to install:" 20 75 4 \ "BASIC" "Install manager with recommended settings" ON \ "ADVANCED" "Do additional configuration to the manager" OFF 3>&1 1>&2 2>&3 ) @@ -1132,7 +1132,7 @@ whiptail_manager_adv_escluster(){ [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno \ + whiptail --title "Security Onion Setup - $SOVERSION" --yesno \ "Do you want to set up a traditional ES cluster for using replicas and/or Hot-Warm indices? Recommended only for those who have experience with ES clustering! " 12 75 } @@ -1142,7 +1142,7 @@ whiptail_manager_adv_escluster_name(){ [ -n "$TESTING" ] && return - ESCLUSTERNAME=$(whiptail --title "Security Onion Setup" --inputbox \ + ESCLUSTERNAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter a name for your ES cluster!" 10 75 "$1" 3>&1 1>&2 2>&3) } @@ -1151,7 +1151,7 @@ whiptail_manager_adv_service_zeeklogs() { [ -n "$TESTING" ] && return - BLOGS=$(whiptail --title "Security Onion Setup" --checklist "Please select logs to send:" 24 75 12 \ + BLOGS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist "Please select logs to send:" 24 75 12 \ "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ @@ -1213,13 +1213,13 @@ whiptail_manager_error() { Would you like to continue anyway? EOM - whiptail --title "Security Onion Setup" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 } whiptail_manager_updates_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup"\ + whiptail --title "Security Onion Setup - $SOVERSION"\ --msgbox "Updating through the manager node requires the manager to have internet access, press ENTER to continue."\ 8 75 @@ -1230,7 +1230,7 @@ whiptail_manager_updates_warning() { whiptail_manager_unreachable() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Setup cannot determine if $1 is listening on port 22. Please check the address entered and try again." 7 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Setup cannot determine if $1 is listening on port 22. Please check the address entered and try again." 7 75 } whiptail_metadata_tool() { @@ -1248,7 +1248,7 @@ whiptail_metadata_tool() { EOM # Legacy variable naming - ZEEKVERSION=$(whiptail --title "Security Onion Setup" --menu "$message" 20 75 2 \ + ZEEKVERSION=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "$message" 20 75 2 \ "Zeek " "Use Zeek (Bro) for metadata and Suricata for NIDS alerts" \ "Suricata " "Use Suricata for both metadata and NIDS alerts" 3>&1 1>&2 2>&3) @@ -1262,7 +1262,7 @@ whiptail_nids() { [ -n "$TESTING" ] && return - NIDS=$(whiptail --title "Security Onion Setup" --radiolist \ + NIDS=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose which IDS to run: \n\n(Snort 3.0 support will be added once it is out of beta.)" 25 75 4 \ "Suricata" "Suricata" ON \ "Snort" "Placeholder for Snort 3.0 " OFF 3>&1 1>&2 2>&3 ) @@ -1276,7 +1276,7 @@ whiptail_network_notice() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1286,14 +1286,14 @@ whiptail_network_notice() { whiptail_net_reinit() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "The management interface has already been configured. Do you want to reconfigure it?" 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "The management interface has already been configured. Do you want to reconfigure it?" 8 75 } whiptail_node_advanced() { [ -n "$TESTING" ] && return - NODESETUP=$(whiptail --title "Security Onion Setup" --radiolist \ + NODESETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "What type of config would you like to use?:" 20 75 4 \ "NODEBASIC" "Install Search Node with recommended settings" ON \ "NODEADVANCED" "Advanced Node Setup" OFF 3>&1 1>&2 2>&3 ) @@ -1306,7 +1306,7 @@ whiptail_node_advanced() { whiptail_node_description() { [ -n "$TESTING" ] && return - NODE_DESCRIPTION=$(whiptail --title "Security Onion Setup" \ + NODE_DESCRIPTION=$(whiptail --title "Security Onion Setup - $SOVERSION" \ --inputbox "Enter a short description for the node or press ENTER to leave blank:" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1317,7 +1317,7 @@ whiptail_node_es_heap() { [ -n "$TESTING" ] && return - NODE_ES_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ + NODE_ES_HEAP_SIZE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter ES heap size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1329,7 +1329,7 @@ whiptail_node_ls_heap() { [ -n "$TESTING" ] && return - NODE_LS_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ + NODE_LS_HEAP_SIZE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter Logstash heap size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1341,7 +1341,7 @@ whiptail_node_ls_input_threads() { [ -n "$TESTING" ] && return - LSINPUTTHREADS=$(whiptail --title "Security Onion Setup" --inputbox \ + LSINPUTTHREADS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter number of Logstash input threads:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1354,7 +1354,7 @@ whiptail_node_ls_pipline_batchsize() { [ -n "$TESTING" ] && return - LSPIPELINEBATCH=$(whiptail --title "Security Onion Setup" --inputbox \ + LSPIPELINEBATCH=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter Logstash pipeline batch size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1366,7 +1366,7 @@ whiptail_node_ls_pipeline_worker() { [ -n "$TESTING" ] && return - LSPIPELINEWORKERS=$(whiptail --title "Security Onion Setup" --inputbox \ + LSPIPELINEWORKERS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter number of Logstash pipeline workers:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1377,13 +1377,13 @@ whiptail_node_ls_pipeline_worker() { whiptail_ntp_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Would you like to configure ntp servers?" 7 44 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Would you like to configure ntp servers?" 7 44 } whiptail_ntp_servers() { [ -n "$TESTING" ] && return - ntp_string=$(whiptail --title "Security Onion Setup" \ + ntp_string=$(whiptail --title "Security Onion Setup - $SOVERSION" \ --inputbox "Input the NTP server(s) you would like to use, separated by commas:" 8 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1394,7 +1394,7 @@ whiptail_oinkcode() { [ -n "$TESTING" ] && return - OINKCODE=$(whiptail --title "Security Onion Setup" --inputbox \ + OINKCODE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your ET Pro or oinkcode:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1407,7 +1407,7 @@ whiptail_oinkcode() { whiptail_passwords_dont_match() { - whiptail --title "Security Onion Setup" --msgbox "Passwords don't match. Please re-enter." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Passwords don't match. Please re-enter." 8 75 } @@ -1415,7 +1415,7 @@ whiptail_patch_name_new_schedule() { [ -n "$TESTING" ] && return - PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup" --inputbox \ + PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "What name do you want to give this OS patch schedule? This schedule needs to be named uniquely. Available schedules can be found on the manager under /opt/so/salt/patch/os/schedules/.yml" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1426,7 +1426,7 @@ whiptail_patch_schedule() { [ -n "$TESTING" ] && return - patch_schedule=$(whiptail --title "Security Onion Setup" --radiolist \ + patch_schedule=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose OS patch schedule.\n\nThis schedule will update the operating system packages but will NOT update Security Onion related tools such as Zeek, Elasticsearch, Kibana, SaltStack, etc." 20 75 5 \ "Automatic" "Updates installed every 8 hours if available" ON \ "Manual" "Updates will be installed manually" OFF \ @@ -1442,7 +1442,7 @@ whiptail_patch_schedule_import() { [ -n "$TESTING" ] && return unset PATCHSCHEDULENAME - PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup" --inputbox \ + PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the name of the OS patch schedule you want to inherit. \nAvailable schedules can be found on the manager under /opt/so/salt/patch/os/schedules/.yml" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1454,7 +1454,7 @@ whiptail_patch_schedule_select_days() { [ -n "$TESTING" ] && return # Select the days to patch - PATCHSCHEDULEDAYS=$(whiptail --title "Security Onion Setup" --checklist \ + PATCHSCHEDULEDAYS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ "Which days do you want to apply OS patches?" 15 75 8 \ Monday "" OFF \ Tuesday "" ON \ @@ -1478,7 +1478,7 @@ whiptail_patch_schedule_select_hours() { [ -n "$TESTING" ] && return # Select the hours to patch - PATCHSCHEDULEHOURS=$(whiptail --title "Security Onion Setup" --checklist \ + PATCHSCHEDULEHOURS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ "At which time, UTC, do you want to apply OS patches on the selected days?" 22 75 13 \ 00:00 "" OFF \ 01:00 "" OFF \ @@ -1519,7 +1519,7 @@ whiptail_proxy_ask() { local pkg_mngr if [[ $OS = 'centos' ]]; then pkg_mngr="yum"; else pkg_mngr='apt'; fi - whiptail --title "Security Onion Setup" --yesno "Do you want to proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment?" 9 65 --defaultno + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment?" 9 65 --defaultno } whiptail_proxy_addr() { @@ -1532,7 +1532,7 @@ whiptail_proxy_addr() { If your proxy requires a username and password do not include them in your input. Setup will ask for those values next. EOM - proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) + proxy_addr=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1541,13 +1541,13 @@ whiptail_proxy_addr() { whiptail_proxy_auth_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Does your proxy require authentication?" 7 60 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Does your proxy require authentication?" 7 60 } whiptail_proxy_auth_user() { [ -n "$TESTING" ] && return - proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) + proxy_user=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1558,7 +1558,7 @@ whiptail_proxy_auth_pass() { [ -n "$TESTING" ] && return - proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) + proxy_pass=$(whiptail --title "Security Onion Setup - $SOVERSION" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1572,7 +1572,7 @@ whiptail_requirements_error() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 10 75 local exitstatus=$? @@ -1584,7 +1584,7 @@ whiptail_rule_setup() { [ -n "$TESTING" ] && return # Get pulled pork info - RULESETUP=$(whiptail --title "Security Onion Setup" --radiolist \ + RULESETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Which IDS ruleset would you like to use?\n\nThis manager server is responsible for downloading the IDS ruleset from the Internet.\n\nSensors then pull a copy of this ruleset from the manager server.\n\nIf you select a commercial ruleset, it is your responsibility to purchase enough licenses for all of your sensors in compliance with your vendor's policies." 20 75 4 \ "ETOPEN" "Emerging Threats Open" ON \ "ETPRO" "Emerging Threats PRO" OFF \ @@ -1600,7 +1600,7 @@ whiptail_sensor_config() { [ -n "$TESTING" ] && return - NSMSETUP=$(whiptail --title "Security Onion Setup" --radiolist \ + NSMSETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "What type of configuration would you like to use?" 20 75 4 \ "BASIC" "Install NSM components with recommended settings" ON \ "ADVANCED" "Configure each component individually" OFF 3>&1 1>&2 2>&3 ) @@ -1642,7 +1642,7 @@ whiptail_sensor_nics() { for bond_nic in "${BNICS[@]}"; do if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ 8 75 exit @@ -1654,7 +1654,7 @@ whiptail_set_hostname() { [ -n "$TESTING" ] && return - HOSTNAME=$(whiptail --title "Security Onion Setup" --inputbox \ + HOSTNAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the hostname (not FQDN) you would like to set:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1665,7 +1665,7 @@ whiptail_set_redirect() { [ -n "$TESTING" ] && return - REDIRECTINFO=$(whiptail --title "Security Onion Setup" --radiolist \ + REDIRECTINFO=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "How would you like to access the web interface?\n\nSecurity Onion uses strict cookie enforcement, so whatever you choose here will be the only way that you can access the web interface.\n\nIf you choose something other than IP address, then you'll need to ensure that you can resolve the name via DNS or hosts entry. If you are unsure, please select IP." 20 75 4 \ "IP" "Use IP address to access the web interface" ON \ "HOSTNAME" "Use hostname to access the web interface" OFF \ @@ -1678,7 +1678,7 @@ whiptail_set_redirect_host() { [ -n "$TESTING" ] && return - REDIRECTHOST=$(whiptail --title "Security Onion Setup" --inputbox \ + REDIRECTHOST=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the Hostname, IP, or FQDN you would like to use for the web interface:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1706,7 +1706,7 @@ whiptail_setup_complete() { Press ENTER to reboot. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" 12 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 12 75 } whiptail_setup_failed() { @@ -1726,13 +1726,13 @@ whiptail_setup_failed() { Press Ok to exit. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" $height 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" $height 75 } whiptail_so_allow_yesno() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --yesno "Do you want to run so-allow to allow access to the web tools?" \ 8 75 } @@ -1741,7 +1741,7 @@ whiptail_so_allow() { [ -n "$TESTING" ] && return - ALLOW_CIDR=$(whiptail --title "Security Onion Setup" \ + ALLOW_CIDR=$(whiptail --title "Security Onion Setup - $SOVERSION" \ --inputbox "Enter a single IP address or an IP range, in CIDR notation, to allow:" \ 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1761,7 +1761,7 @@ whiptail_ssh_key_copy_notice() { Select OK to continue. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" 11 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 11 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } @@ -1800,7 +1800,7 @@ whiptail_storage_requirements() { EOM whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --yesno "$message" \ 14 75 @@ -1812,7 +1812,7 @@ whiptail_strelka_rules() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Do you want to enable the default YARA rules for Strelka?" 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to enable the default YARA rules for Strelka?" 8 75 local exitstatus=$? @@ -1852,7 +1852,7 @@ whiptail_node_updates() { [ -n "$TESTING" ] && return - NODEUPDATES=$(whiptail --title "Security Onion Setup" --radiolist \ + NODEUPDATES=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "How would you like to download OS package updates for your grid?" 20 75 4 \ "MANAGER" "Manager node is proxy for updates." ON \ "OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) @@ -1877,7 +1877,7 @@ whiptail_you_sure() { EOM whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --yesno "$you_sure_text" \ 20 75 From 18926009d326ef41b5f408b839f2488fef45e8c4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 28 May 2021 10:38:06 -0400 Subject: [PATCH 192/378] remove unneeded curl.config template --- salt/elasticsearch/files/curl.config | 1 - 1 file changed, 1 deletion(-) delete mode 100644 salt/elasticsearch/files/curl.config diff --git a/salt/elasticsearch/files/curl.config b/salt/elasticsearch/files/curl.config deleted file mode 100644 index e4633df6c..000000000 --- a/salt/elasticsearch/files/curl.config +++ /dev/null @@ -1 +0,0 @@ -user = "salt['pillar.get']('elasticsearch:auth:user'):salt['pillar.get']('elasticsearch:auth:pass')" From 63b31de2b841eb0212499a2d8b846425cb3fe405 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 28 May 2021 13:58:03 -0400 Subject: [PATCH 193/378] add additional users - manage file if user name isnt returned from grepping the file --- salt/elasticsearch/auth.sls | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index e8ab1e378..187922d6e 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -5,8 +5,27 @@ elastic_auth_pillar: elasticsearch: auth: enabled: False - user: so_elastic - pass: {{ salt['random.get_str'](20) }} + users: + so_elastic_user: + user: so_elastic + pass: {{ salt['random.get_str'](20) }} + so_kibana_user: + user: so_kibana + pass: {{ salt['random.get_str'](20) }} + so_logstash_user: + user: so_logstash + pass: {{ salt['random.get_str'](20) }} + so_beats_user: + user: so_beats + pass: {{ salt['random.get_str'](20) }} + so_monitor_user: + user: so_monitor + pass: {{ salt['random.get_str'](20) }} # since we are generating a random password, and we don't want that to happen everytime - # a highstate runs, we only manage the file if it doesn't exist - - unless: ls /opt/so/saltstack/local/pillar/elasticsearch/auth.sls + # a highstate runs, we only manage the file each user isn't present in the file. if the + # pillar file doesn't exists, then the default vault provided to pillar.get should not + # be within the file either, so it should then be created + - unless: + {% for so_app_user in salt['pillar.get']('elasticsearch:auth:users', {'so_noapp_user': {'user': 'r@NDumu53Rd0NtDOoP'}}) %} + - grep {{ so_app_user.user }} /opt/so/saltstack/local/pillar/elasticsearch/auth.sls + {% endfor%} From 68abaa5e3c6c3efa873a2db6fa617c470a5abcf2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 28 May 2021 14:03:21 -0400 Subject: [PATCH 194/378] update auth.map and curl.config to use new elasticsearch:auth pillar format --- salt/elasticsearch/auth.map.jinja | 4 ++-- salt/elasticsearch/init.sls | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/auth.map.jinja b/salt/elasticsearch/auth.map.jinja index c6f099025..3c3b42cdc 100644 --- a/salt/elasticsearch/auth.map.jinja +++ b/salt/elasticsearch/auth.map.jinja @@ -1,7 +1,7 @@ {% set ELASTICAUTH = salt['pillar.filter_by']({ True: { - 'user': salt['pillar.get']('elasticsearch:auth:user'), - 'pass': salt['pillar.get']('elasticsearch:auth:pass'), + 'user': salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user'), + 'pass': salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass'), 'elasticcurl':'curl -K /opt/so/conf/elasticsearch/curl.config' }, False: {'elasticcurl': 'curl'}, }, pillar='elasticsearch:auth:enabled', default=False) %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 7f287fa27..fb2927e99 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -293,7 +293,7 @@ elastic_curl_config: file.managed: - name: /opt/so/conf/elasticsearch/curl.config - mode: 600 - - contents: user = "{{ salt['pillar.get']('elasticsearch:auth:user') }}:{{ salt['pillar.get']('elasticsearch:auth:pass') }}" + - contents: user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}" - show_changes: False {% endif %} {# if grains['role'] != 'so-helix' #} From edf60f80f707b5c65b853d87f49b9652c6a5d117 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 28 May 2021 15:26:26 -0400 Subject: [PATCH 195/378] manager and common states now require elasticsearch.auth state --- salt/common/init.sls | 11 ++++++++++- salt/manager/init.sls | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index d61806a1a..33c1f28df 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -3,6 +3,12 @@ {% set role = grains.id.split('_') | last %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} +{% set ES_INCLUDED_NODES = ['so-standalone'] %} + +{% if grains.role in ES_INCLUDED_NODES %} +include: + - elasticsearch.auth +{% %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -179,7 +185,10 @@ utilsyncscripts: - source: salt://common/tools/sbin - defaults: ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} - +{% if grains.role in ES_INCLUDED_NODES %} + - require: + - file: elastic_auth_pillar +{% endif %} {% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %} # Add sensor cleanup diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 91635eb59..91be3136f 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -20,6 +20,9 @@ {% set MANAGER = salt['grains.get']('master') %} {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %} +include: + - elasticsearch.auth + socore_own_saltstack: file.directory: - name: /opt/so/saltstack From b8a10f2e8680792ca4c56eb2f650df0699d51eee Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 28 May 2021 14:49:43 -0400 Subject: [PATCH 196/378] Support multiple elastic system users --- salt/common/tools/sbin/so-common | 3 +- salt/common/tools/sbin/so-user | 51 ++++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index b48f84b90..a89f93eea 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -252,6 +252,7 @@ lookup_salt_value() { key=$1 group=$2 kind=$3 + output=${4:-newline_values_only} if [ -z "$kind" ]; then kind=pillar @@ -261,7 +262,7 @@ lookup_salt_value() { group=${group}: fi - salt-call --no-color ${kind}.get ${group}${key} --out=newline_values_only + salt-call --no-color ${kind}.get ${group}${key} --out=${output} } lookup_pillar() { diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 1ff637d23..d8d8fe34a 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -136,17 +136,56 @@ function createElasticTmpFile() { echo "$tmpFile" } +function syncElasticSystemUser() { + json=$1 + userid=$2 + usersFile=$3 + + user=$(echo "$json" | jq -r ".local.users.$userid.user") + pass=$(echo "$json" | jq -r ".local.users.$userid.pass") + + [[ -z "$user" || -z "$pass" ]] && fail "Elastic auth credentials for system user '$userid' are missing" + hash=$(hashPassword "$pass") + + echo "${user}:${hash}" >> "$usersFile" +} + +function syncElasticSystemRole() { + json=$1 + userid=$2 + role=$3 + rolesFile=$4 + + user=$(echo "$json" | jq -r ".local.users.$userid.user") + + [[ -z "$user" ]] && fail "Elastic auth credentials for system user '$userid' are missing" + + echo "${role}:${user}" >> "$rolesFile" +} + function syncElastic() { usersFileTmp=$(createElasticTmpFile "${elasticUsersFile}") rolesFileTmp=$(createElasticTmpFile "${elasticRolesFile}") - sysUser=$(lookup_pillar "auth:user" "elasticsearch") - sysPass=$(lookup_pillar "auth:pass" "elasticsearch") - [[ -z "$sysUser" || -z "$sysPass" ]] && fail "Elastic auth credentials for system user are missing" - sysHash=$(hashPassword "$sysPass") + authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") + + syncElasticSystemUser "$authPillarJson" "so_elastic_user" "$usersFileTmp" + syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesFileTmp" + + syncElasticSystemUser "$authPillarJson" "so_kibana_user" "$usersFileTmp" + syncElasticSystemRole "$authPillarJson" "so_kibana_user" "kibana_system" "$rolesFileTmp" + + syncElasticSystemUser "$authPillarJson" "so_logstash_user" "$usersFileTmp" + syncElasticSystemRole "$authPillarJson" "so_logstash_user" "logstash_system" "$rolesFileTmp" + + syncElasticSystemUser "$authPillarJson" "so_beats_user" "$usersFileTmp" + syncElasticSystemRole "$authPillarJson" "so_beats_user" "beats_system" "$rolesFileTmp" + + syncElasticSystemUser "$authPillarJson" "so_monitor_user" "$usersFileTmp" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesFileTmp" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesFileTmp" # Generate the new users file - echo "${sysUser}:${sysHash}" >> "$usersFileTmp" echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \ "from identity_credential_identifiers ici, identity_credentials ic " \ "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ @@ -159,7 +198,7 @@ function syncElastic() { [[ $? != 0 ]] && fail "Unable to create users file: $elasticUsersFile" # Generate the new users_roles file - echo "superuser:${sysUser}" >> "$rolesFileTmp" + echo "select 'superuser:' || ici.identifier " \ "from identity_credential_identifiers ici, identity_credentials ic " \ "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ From 3aad5a30e96f88830aad729baa200257d49092fb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 28 May 2021 18:28:53 -0400 Subject: [PATCH 197/378] fix logic on password created in pillar and fix how me manage --- salt/elasticsearch/auth.sls | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index 187922d6e..303383cfe 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -1,3 +1,9 @@ +{% set so_elastic_user_pass = salt['random.get_str'](20) %} +{% set so_kibana_user_pass = salt['random.get_str'](20) %} +{% set so_logstash_user_pass = salt['random.get_str'](20) %} +{% set so_beats_user_pass = salt['random.get_str'](20) %} +{% set so_monitor_user_pass = salt['random.get_str'](20) %} + elastic_auth_pillar: file.managed: - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls @@ -8,24 +14,26 @@ elastic_auth_pillar: users: so_elastic_user: user: so_elastic - pass: {{ salt['random.get_str'](20) }} + pass: {{ so_elastic_user_pass }} so_kibana_user: user: so_kibana - pass: {{ salt['random.get_str'](20) }} + pass: {{ so_kibana_user_pass }} so_logstash_user: user: so_logstash - pass: {{ salt['random.get_str'](20) }} + pass: {{ so_logstash_user_pass }} so_beats_user: user: so_beats - pass: {{ salt['random.get_str'](20) }} + pass: {{ so_beats_user_pass }} so_monitor_user: user: so_monitor - pass: {{ salt['random.get_str'](20) }} + pass: {{ so_monitor_user_pass }} # since we are generating a random password, and we don't want that to happen everytime # a highstate runs, we only manage the file each user isn't present in the file. if the # pillar file doesn't exists, then the default vault provided to pillar.get should not # be within the file either, so it should then be created +{% if salt['pillar.get']('elasticsearch:auth', False) %} - unless: - {% for so_app_user in salt['pillar.get']('elasticsearch:auth:users', {'so_noapp_user': {'user': 'r@NDumu53Rd0NtDOoP'}}) %} + {% for so_app_user in salt['pillar.get']('elasticsearch:auth:users') %} - grep {{ so_app_user.user }} /opt/so/saltstack/local/pillar/elasticsearch/auth.sls {% endfor%} +{% endif %} From 47b56e78b38377bb0284abbf983a1b999ada8afe Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 28 May 2021 20:07:51 -0400 Subject: [PATCH 198/378] Fix missing endif --- salt/common/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 33c1f28df..88b5c790b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -8,7 +8,7 @@ {% if grains.role in ES_INCLUDED_NODES %} include: - elasticsearch.auth -{% %} +{% endif %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: From a42a406f532fc26cc58ab0c38c24f25f22eb8275 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 29 May 2021 07:52:08 -0400 Subject: [PATCH 199/378] Remove extra users file mounts; disable elastic anon access when auth enabled --- salt/elasticsearch/files/elasticsearch.yml | 2 ++ salt/elasticsearch/init.sls | 2 -- salt/kibana/etc/kibana.yml | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index fed45bf79..de6b99000 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -30,11 +30,13 @@ xpack.security.http.ssl.client_authentication: none xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt +{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} xpack.security.authc: anonymous: username: anonymous_user roles: superuser authz_exception: true +{% endif %} node.name: {{ grains.host }} script.max_compilations_rate: 1000/1m {%- if TRUECLUSTER is sameas true %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index fb2927e99..b75833952 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -234,8 +234,6 @@ so-elasticsearch: - binds: - /opt/so/conf/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro - /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro - - /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro - - /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 856f87909..501d93c8a 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -14,7 +14,9 @@ elasticsearch.requestTimeout: 90000 logging.dest: /var/log/kibana/kibana.log telemetry.enabled: false security.showInsecureClusterWarning: false +{% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %} xpack.security.authc.providers: anonymous.anonymous1: order: 0 credentials: "elasticsearch_anonymous_user" +{% endif %} \ No newline at end of file From 1aa2852ed6c60e149aa79578d500c6a4cde2afae Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 1 Jun 2021 08:35:43 -0400 Subject: [PATCH 200/378] fix logic --- salt/elasticsearch/auth.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index 303383cfe..704d743ce 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -33,7 +33,7 @@ elastic_auth_pillar: # be within the file either, so it should then be created {% if salt['pillar.get']('elasticsearch:auth', False) %} - unless: - {% for so_app_user in salt['pillar.get']('elasticsearch:auth:users') %} - - grep {{ so_app_user.user }} /opt/so/saltstack/local/pillar/elasticsearch/auth.sls + {% for so_app_user, values in salt['pillar.get']('elasticsearch:auth:users', {'so_noapp_user': {'user': 'r@NDumu53Rd0NtDOoP'}}).items() %} + - grep {{ values.user }} /opt/so/saltstack/local/pillar/elasticsearch/auth.sls {% endfor%} {% endif %} From f87dce8ec1629a6254d4e7088c629bea79908e82 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 1 Jun 2021 09:38:07 -0400 Subject: [PATCH 201/378] generate pillar file if auth enabled or not --- salt/elasticsearch/auth.sls | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index 704d743ce..748e50d0b 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -31,9 +31,7 @@ elastic_auth_pillar: # a highstate runs, we only manage the file each user isn't present in the file. if the # pillar file doesn't exists, then the default vault provided to pillar.get should not # be within the file either, so it should then be created -{% if salt['pillar.get']('elasticsearch:auth', False) %} - unless: {% for so_app_user, values in salt['pillar.get']('elasticsearch:auth:users', {'so_noapp_user': {'user': 'r@NDumu53Rd0NtDOoP'}}).items() %} - grep {{ values.user }} /opt/so/saltstack/local/pillar/elasticsearch/auth.sls {% endfor%} -{% endif %} From ed8c85df2ba86a18166d21d1803d95db23a076f0 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 1 Jun 2021 10:26:33 -0400 Subject: [PATCH 202/378] Only sync web users if teh sqlite db exists --- salt/common/tools/sbin/so-user | 48 ++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index d8d8fe34a..cc3a0756c 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -185,29 +185,33 @@ function syncElastic() { syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesFileTmp" syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesFileTmp" - # Generate the new users file - echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \ - "from identity_credential_identifiers ici, identity_credentials ic " \ - "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ - "order by ici.identifier;" | \ - sqlite3 "$databasePath" | \ - jq -r '.user + ":" + .data.hashed_password' \ - >> "$usersFileTmp" - [[ $? != 0 ]] && fail "Unable to read credential hashes from database" - mv -f "$usersFileTmp" "$elasticUsersFile" - [[ $? != 0 ]] && fail "Unable to create users file: $elasticUsersFile" + if [[ -f "$databasePath" ]]; then + # Generate the new users file + echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \ + "from identity_credential_identifiers ici, identity_credentials ic " \ + "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ + "order by ici.identifier;" | \ + sqlite3 "$databasePath" | \ + jq -r '.user + ":" + .data.hashed_password' \ + >> "$usersFileTmp" + [[ $? != 0 ]] && fail "Unable to read credential hashes from database" + mv -f "$usersFileTmp" "$elasticUsersFile" + [[ $? != 0 ]] && fail "Unable to create users file: $elasticUsersFile" - # Generate the new users_roles file - - echo "select 'superuser:' || ici.identifier " \ - "from identity_credential_identifiers ici, identity_credentials ic " \ - "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ - "order by ici.identifier;" | \ - sqlite3 "$databasePath" \ - >> "$rolesFileTmp" - [[ $? != 0 ]] && fail "Unable to read credential IDs from database" - mv -f "$rolesFileTmp" "$elasticRolesFile" - [[ $? != 0 ]] && fail "Unable to create users file: $elasticRolesFile" + # Generate the new users_roles file + + echo "select 'superuser:' || ici.identifier " \ + "from identity_credential_identifiers ici, identity_credentials ic " \ + "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ + "order by ici.identifier;" | \ + sqlite3 "$databasePath" \ + >> "$rolesFileTmp" + [[ $? != 0 ]] && fail "Unable to read credential IDs from database" + mv -f "$rolesFileTmp" "$elasticRolesFile" + [[ $? != 0 ]] && fail "Unable to create users file: $elasticRolesFile" + else + info "Database file does not exist yet, skipping users export" + fi } function syncAll() { From d25a439bd4706432fbfc7f0f810d03284732b250 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 1 Jun 2021 10:53:58 -0400 Subject: [PATCH 203/378] more changes --- pillar/top.sls | 9 +++++++++ salt/common/init.sls | 17 +++++++---------- salt/elasticsearch/init.sls | 27 --------------------------- salt/manager/init.sls | 16 ++++++++++++++++ salt/top.sls | 3 +-- setup/so-setup | 1 + 6 files changed, 34 insertions(+), 39 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 8bd67c174..ff4cb5787 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -22,6 +22,9 @@ base: '*_manager or *_managersearch': - match: compound - data.* +{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} + - elasticsearch.auth +{% endif %} - secrets - global - minions.{{ grains.id }} @@ -38,6 +41,9 @@ base: - secrets - healthcheck.eval - elasticsearch.eval +{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} + - elasticsearch.auth +{% endif %} - global - minions.{{ grains.id }} @@ -91,5 +97,8 @@ base: - zeeklogs - secrets - elasticsearch.eval +{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} + - elasticsearch.auth +{% endif %} - global - minions.{{ grains.id }} diff --git a/salt/common/init.sls b/salt/common/init.sls index 33c1f28df..09a0db934 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -3,12 +3,6 @@ {% set role = grains.id.split('_') | last %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} -{% set ES_INCLUDED_NODES = ['so-standalone'] %} - -{% if grains.role in ES_INCLUDED_NODES %} -include: - - elasticsearch.auth -{% %} # Remove variables.txt from /tmp - This is temp rmvariablesfile: @@ -174,6 +168,13 @@ alwaysupdated: Etc/UTC: timezone.system +elastic_curl_config: + file.managed: + - name: /opt/so/conf/elasticsearch/curl.config + - source: salt://elasticsearch/curl.config + - mode: 600 + - show_changes: False + # Sync some Utilities utilsyncscripts: file.recurse: @@ -185,10 +186,6 @@ utilsyncscripts: - source: salt://common/tools/sbin - defaults: ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} -{% if grains.role in ES_INCLUDED_NODES %} - - require: - - file: elastic_auth_pillar -{% endif %} {% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %} # Add sensor cleanup diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index fb2927e99..4d5ce4921 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -37,8 +37,6 @@ {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} -# used in this state to control who can run the so-users script -{% set ES_INCLUDED_NODES = ['so-standalone'] %} vm.max_map_count: sysctl.present: @@ -173,33 +171,15 @@ eslogdir: - group: 939 - makedirs: True -{% if grains.role in ES_INCLUDED_NODES %} -# Must run before elasticsearch docker container is started! -syncesusers: - cmd.run: - - name: so-user sync - - creates: - - /opt/so/saltstack/local/salt/elasticsearch/files/users - - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles -{% endif %} - auth_users: file.managed: - name: /opt/so/conf/elasticsearch/users - source: salt://elasticsearch/files/users - - require: -{% if grains.role in ES_INCLUDED_NODES %} - - cmd: syncesusers -{% endif %} auth_users_roles: file.managed: - name: /opt/so/conf/elasticsearch/users_roles - source: salt://elasticsearch/files/users_roles -{% if grains.role in ES_INCLUDED_NODES %} - - require: - - cmd: syncesusers -{% endif %} so-elasticsearch: docker_container.running: @@ -289,13 +269,6 @@ so-elasticsearch-templates: - template: jinja {% endif %} -elastic_curl_config: - file.managed: - - name: /opt/so/conf/elasticsearch/curl.config - - mode: 600 - - contents: user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}" - - show_changes: False - {% endif %} {# if grains['role'] != 'so-helix' #} {% else %} diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 91be3136f..7ef489496 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -105,6 +105,22 @@ strelka_yara_update: - name: '/usr/sbin/so-yara-update >> /nsm/strelka/log/yara-update.log 2>&1' - hour: '7' - minute: '1' + +elastic_curl_config_distributed: + file.managed: + - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config + - mode: 600 + - contents: user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}" + - show_changes: False + +# Must run before elasticsearch docker container is started! +syncesusers: + cmd.run: + - name: so-user sync + - creates: + - /opt/so/saltstack/local/salt/elasticsearch/files/users + - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/top.sls b/salt/top.sls index f3f2958ae..a063c4279 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -238,7 +238,6 @@ base: {%- endif %} {%- if ELASTICSEARCH %} - elasticsearch - - elasticsearch.auth {%- endif %} {%- if LOGSTASH %} - logstash @@ -261,7 +260,7 @@ base: - filebeat {%- endif %} {%- if CURATOR %} - - curator + - curator0 {%- endif %} {%- if ELASTALERT %} - elastalert diff --git a/setup/so-setup b/setup/so-setup index 4c364ded6..6d2f6ac27 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -752,6 +752,7 @@ echo "1" > /root/accept_changes set_progress_str 60 "$(print_salt_state_apply 'manager')" salt-call state.apply -l info manager >> $setup_log 2>&1 + bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth fi set_progress_str 61 "$(print_salt_state_apply 'firewall')" From bfaffbc87e2b7b1ab7d52612178c0d632679d9c0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 1 Jun 2021 11:15:28 -0400 Subject: [PATCH 204/378] add reactor and beacon for sqlite db --- files/salt/master/master | 2 ++ salt/manager/files/beacons.conf | 5 +++++ salt/manager/init.sls | 5 +++++ salt/reactor/kratos.sls | 5 +++++ 4 files changed, 17 insertions(+) create mode 100644 salt/manager/files/beacons.conf create mode 100644 salt/reactor/kratos.sls diff --git a/files/salt/master/master b/files/salt/master/master index 93e8ff938..675556d14 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -67,3 +67,5 @@ peer: reactor: - 'so/fleet': - salt://reactor/fleet.sls + - salt/beacon/*/inotify//watch_sqlite.db: + - salt://reactor/kratos.sls diff --git a/salt/manager/files/beacons.conf b/salt/manager/files/beacons.conf new file mode 100644 index 000000000..b1cca51ba --- /dev/null +++ b/salt/manager/files/beacons.conf @@ -0,0 +1,5 @@ +beacons: + watch_sqlite.db: + - files: + /opt/so/conf/kratos/db/sqlite.db: {} + - beacon_module: inotify diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 7ef489496..10466d019 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -121,6 +121,11 @@ syncesusers: - /opt/so/saltstack/local/salt/elasticsearch/files/users - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles +beacons_config: + file.managed: + - name: /etc/salt/minion.d/beacons.conf + - source: salt://manager/files/beacons.conf + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/reactor/kratos.sls b/salt/reactor/kratos.sls new file mode 100644 index 000000000..a465f134c --- /dev/null +++ b/salt/reactor/kratos.sls @@ -0,0 +1,5 @@ +so_user_sync: + local.cmd.run: + - tgt: {{ data['data']['id'] }} + - arg: + - so-user sync From ef00695b07ef4d9cac79113928113ba32ab41940 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 1 Jun 2021 11:31:50 -0400 Subject: [PATCH 205/378] fix typo --- salt/top.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/top.sls b/salt/top.sls index a063c4279..8a12aaa26 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -260,7 +260,7 @@ base: - filebeat {%- endif %} {%- if CURATOR %} - - curator0 + - curator {%- endif %} {%- if ELASTALERT %} - elastalert From 73a0b313805fa116e6ad68d9195bb01d58916215 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Jun 2021 12:12:20 -0400 Subject: [PATCH 206/378] elastic pipeline enable --- salt/pipeline/load.sls | 4 ++++ salt/top.sls | 1 + 2 files changed, 5 insertions(+) create mode 100644 salt/pipeline/load.sls diff --git a/salt/pipeline/load.sls b/salt/pipeline/load.sls new file mode 100644 index 000000000..a43450d0a --- /dev/null +++ b/salt/pipeline/load.sls @@ -0,0 +1,4 @@ +load_elastic_pipelines: + cmd.run: + - name: /usr/sbin/so-filebeat-module-setup + \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index 8a12aaa26..340f83825 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -285,6 +285,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_searchnode and G@saltversion:{{saltversion}}': - match: compound From 7a59bee315589c75d74996e314ea3e4988e17f07 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 1 Jun 2021 12:48:53 -0400 Subject: [PATCH 207/378] Add so-elastic-auth script --- salt/common/tools/sbin/so-elastic-auth | 58 ++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 salt/common/tools/sbin/so-elastic-auth diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth new file mode 100644 index 000000000..b15b04c0f --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-auth @@ -0,0 +1,58 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +ES_AUTH_PILLAR=${ELASTIC_AUTH_PILLAR:-/opt/so/saltstack/local/pillar/elasticsearch/auth.sls} +ES_USERS_FILE=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users} + +authEnable=${1:-true} + +if ! grep -q "enabled: " "$ES_AUTH_PILLAR"; then + echo "Elastic auth pillar file is invalid. Unable to proceed." + exit 1 +fi + +if [[ "$authEnable" == "true" ]]; then + if grep -q "enabled: False" "$ES_AUTH_PILLAR"; then + sed -i 's/enabled: False/enabled: True/g' "$ES_AUTH_PILLAR" + echo "Applying highstate - this may take a few minutes..." + salt-call state.highstate queue=True + echo "Elastic auth is now enabled." + if grep -q "argon" "$ES_USERS_FILE"; then + echo "" + echo "IMPORTANT: The following users will need to change their password, after logging into SOC, in order to access Kibana:" + grep argon "$ES_USERS" | cut -d ":" -f 1 + fi + else + echo "Auth is already enabled." + fi +elif [[ "$authEnable" == "false" ]]; then + if grep -q "enabled: True" "$ES_AUTH_PILLAR"; then + sed -i 's/enabled: True/enabled: False/g' "$ES_AUTH_PILLAR" + echo "Applying highstate - this may take a few minutes..." + salt-call state.highstate queue=True + echo "Elastic auth is now disabled." + else + echo "Auth is already disabled." + fi +else + echo "Usage: $0 " + echo "" + echo "Enables Elastic authentication. Defaults to true." + echo "" +fi From 2a2247e1da55bc5b859c6d4bc6fdd9113a1e5946 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 1 Jun 2021 14:45:01 -0400 Subject: [PATCH 208/378] Additional so-user sync adjustments --- salt/common/tools/sbin/so-user | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index cc3a0756c..9ea6c0310 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -42,6 +42,8 @@ databasePath=${KRATOS_DB_PATH:-/opt/so/conf/kratos/db/db.sqlite} bcryptRounds=${BCRYPT_ROUNDS:-12} elasticUsersFile=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users} elasticRolesFile=${ELASTIC_ROLES_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users_roles} +esUID=${ELASTIC_UID:-930} +esGID=${ELASTIC_GID:-930} function fail() { msg=$1 @@ -132,7 +134,7 @@ function createElasticTmpFile() { tmpFile=${filename}.tmp truncate -s 0 "$tmpFile" chmod 600 "$tmpFile" - chown elasticsearch:elasticsearch "$tmpFile" + chown "${esUID}:${esGID}" "$tmpFile" echo "$tmpFile" } @@ -195,8 +197,6 @@ function syncElastic() { jq -r '.user + ":" + .data.hashed_password' \ >> "$usersFileTmp" [[ $? != 0 ]] && fail "Unable to read credential hashes from database" - mv -f "$usersFileTmp" "$elasticUsersFile" - [[ $? != 0 ]] && fail "Unable to create users file: $elasticUsersFile" # Generate the new users_roles file @@ -207,11 +207,16 @@ function syncElastic() { sqlite3 "$databasePath" \ >> "$rolesFileTmp" [[ $? != 0 ]] && fail "Unable to read credential IDs from database" - mv -f "$rolesFileTmp" "$elasticRolesFile" - [[ $? != 0 ]] && fail "Unable to create users file: $elasticRolesFile" else info "Database file does not exist yet, skipping users export" fi + + # Move the temp files over onto the final files + mv -f "$usersFileTmp" "$elasticUsersFile" + [[ $? != 0 ]] && fail "Unable to create users file: $elasticUsersFile" + + mv -f "$rolesFileTmp" "$elasticRolesFile" + [[ $? != 0 ]] && fail "Unable to create users file: $elasticRolesFile" } function syncAll() { From fd1de624c815071afb9d25d078ad0452c8bfbb51 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Jun 2021 14:50:21 -0400 Subject: [PATCH 209/378] Disable TTY for filebeat script --- salt/common/tools/sbin/so-filebeat-module-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 5aefe3ac2..21d94b44f 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -53,7 +53,7 @@ echo "Setting up ingest pipeline(s)" for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler do echo "Loading $MODULE" - docker exec -it so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML + docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML sleep 2 done From 867613669d6e8e556a0c78187e6001d1e23af727 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 1 Jun 2021 17:01:03 -0400 Subject: [PATCH 210/378] changes for syncing users --- files/salt/master/master | 4 +++- salt/common/init.sls | 1 + salt/manager/files/beacons.conf | 6 ++++-- salt/manager/init.sls | 3 +++ salt/reactor/kratos.sls | 2 +- salt/salt/helper-packages.sls | 9 ++++++++- salt/salt/map.jinja | 4 +++- 7 files changed, 23 insertions(+), 6 deletions(-) diff --git a/files/salt/master/master b/files/salt/master/master index 675556d14..e7c6030dd 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -67,5 +67,7 @@ peer: reactor: - 'so/fleet': - salt://reactor/fleet.sls - - salt/beacon/*/inotify//watch_sqlite.db: + - salt/beacon/*/watch_sqlite_db//opt/so/conf/kratos/db/sqlite.db - salt://reactor/kratos.sls + + diff --git a/salt/common/init.sls b/salt/common/init.sls index 09a0db934..9d20de62b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -174,6 +174,7 @@ elastic_curl_config: - source: salt://elasticsearch/curl.config - mode: 600 - show_changes: False + - makedirs: True # Sync some Utilities utilsyncscripts: diff --git a/salt/manager/files/beacons.conf b/salt/manager/files/beacons.conf index b1cca51ba..c40fe83cb 100644 --- a/salt/manager/files/beacons.conf +++ b/salt/manager/files/beacons.conf @@ -1,5 +1,7 @@ beacons: - watch_sqlite.db: + watch_sqlite_db: - files: - /opt/so/conf/kratos/db/sqlite.db: {} + /opt/so/conf/kratos/db/sqlite.db: + mask: + - modify - beacon_module: inotify diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 10466d019..9ae39e9a8 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -22,6 +22,7 @@ include: - elasticsearch.auth + - salt.minion socore_own_saltstack: file.directory: @@ -125,6 +126,8 @@ beacons_config: file.managed: - name: /etc/salt/minion.d/beacons.conf - source: salt://manager/files/beacons.conf + - onchanges_in: + - service: salt_minion_service {% else %} diff --git a/salt/reactor/kratos.sls b/salt/reactor/kratos.sls index a465f134c..fb90cc164 100644 --- a/salt/reactor/kratos.sls +++ b/salt/reactor/kratos.sls @@ -2,4 +2,4 @@ so_user_sync: local.cmd.run: - tgt: {{ data['data']['id'] }} - arg: - - so-user sync + - /usr/sbin/so-user sync diff --git a/salt/salt/helper-packages.sls b/salt/salt/helper-packages.sls index 93ad76a22..c26cdc7c0 100644 --- a/salt/salt/helper-packages.sls +++ b/salt/salt/helper-packages.sls @@ -1,3 +1,10 @@ +{% from 'salt/map.jinja' import PYINOTIFYPACKAGE with context%} +{% from 'salt/map.jinja' import PYTHONINSTALLER with context%} + patch_package: pkg.installed: - - name: patch \ No newline at end of file + - name: patch + +pyinotify: + {{PYTHONINSTALLER}}.installed: + - name: {{ PYINOTIFYPACKAGE }} diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index b0b9ffb2a..4b9577319 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -11,6 +11,7 @@ {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION %} {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} + {% set PYINOTIFYPACKAGE = 'pyinotify' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} @@ -21,6 +22,7 @@ {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} + {% set PYINOTIFYPACKAGE = 'securityonion-python3-pyinotify' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} @@ -33,4 +35,4 @@ {% endif %} {% else %} {% set UPGRADECOMMAND = 'echo Already running Salt Minion version ' ~ SALTVERSION %} -{% endif %} \ No newline at end of file +{% endif %} From 7b68c1bc9bb37fc8b6135ab502b98ea91cd01484 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 1 Jun 2021 17:45:52 -0400 Subject: [PATCH 211/378] fix typo --- files/salt/master/master | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/salt/master/master b/files/salt/master/master index e7c6030dd..5db41fb90 100644 --- a/files/salt/master/master +++ b/files/salt/master/master @@ -67,7 +67,7 @@ peer: reactor: - 'so/fleet': - salt://reactor/fleet.sls - - salt/beacon/*/watch_sqlite_db//opt/so/conf/kratos/db/sqlite.db + - 'salt/beacon/*/watch_sqlite_db//opt/so/conf/kratos/db/sqlite.db': - salt://reactor/kratos.sls From 5983eae3a8f7907d2bae84cd44dec295faea443f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Jun 2021 17:47:13 -0400 Subject: [PATCH 212/378] fix filebeat module syntax --- .../tools/sbin/so-filebeat-module-setup | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 21d94b44f..d7706366e 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -47,14 +47,21 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" echo fi +echo "Testing to see if the pipelines are already applied" +ESVER=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \") +PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) -echo "Setting up ingest pipeline(s)" +if [[ "$PIPELINES" -lt 5 ]]; then + echo "Setting up ingest pipeline(s)" -for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler -do - echo "Loading $MODULE" - docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML - sleep 2 -done + for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler + do + echo "Loading $MODULE" + docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML + sleep 2 + done +else + exit 0 +fi From 7aede4d058ed26b190d3f22d845b57f4f56fc38e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 09:01:16 -0400 Subject: [PATCH 213/378] Persist chown/chmod settings on users/roles files --- salt/common/tools/sbin/so-user | 4 ++-- salt/elasticsearch/init.sls | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 9ea6c0310..15a71edfa 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -208,7 +208,7 @@ function syncElastic() { >> "$rolesFileTmp" [[ $? != 0 ]] && fail "Unable to read credential IDs from database" else - info "Database file does not exist yet, skipping users export" + echo "Database file does not exist yet, skipping users export" fi # Move the temp files over onto the final files @@ -388,4 +388,4 @@ case "${operation}" in ;; esac -exit 0 \ No newline at end of file +exit 0 diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 86d7623d0..0bf442587 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -175,11 +175,17 @@ auth_users: file.managed: - name: /opt/so/conf/elasticsearch/users - source: salt://elasticsearch/files/users + - user: 930 + - group: 930 + - mode: 600 auth_users_roles: file.managed: - name: /opt/so/conf/elasticsearch/users_roles - source: salt://elasticsearch/files/users_roles + - user: 930 + - group: 930 + - mode: 600 so-elasticsearch: docker_container.running: From c757d21360ce261fc77e18b7249ff7d6076fbd24 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 09:38:59 -0400 Subject: [PATCH 214/378] Increase default SOC API and ES timeouts from 2m to 5m --- salt/nginx/etc/nginx.conf | 12 ++++++------ salt/soc/files/soc/soc.json | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index ea820442b..7a238fa54 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -175,8 +175,8 @@ http { auth_request_set $userid $upstream_http_x_kratos_authenticated_identity_id; proxy_set_header x-user-id $userid; proxy_pass http://{{ manager_ip }}:9822/; - proxy_read_timeout 90; - proxy_connect_timeout 90; + proxy_read_timeout 300; + proxy_connect_timeout 300; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -261,8 +261,8 @@ http { auth_request /auth/sessions/whoami; rewrite /kibana/(.*) /$1 break; proxy_pass http://{{ manager_ip }}:5601/; - proxy_read_timeout 90; - proxy_connect_timeout 90; + proxy_read_timeout 300; + proxy_connect_timeout 300; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -341,8 +341,8 @@ http { location /soctopus/ { proxy_pass http://{{ manager_ip }}:7000/; - proxy_read_timeout 90; - proxy_connect_timeout 90; + proxy_read_timeout 300; + proxy_connect_timeout 300; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 8048970f3..e275ec28b 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -50,7 +50,8 @@ "username": "", "password": "", "cacheMs": {{ ES_FIELDCAPS_CACHE }}, - "verifyCert": false + "verifyCert": false, + "timeoutMs": {{ API_TIMEOUT }} }, "influxdb": { {%- if grains['role'] in ['so-import'] or (grains['role'] == 'so-eval' and GRAFANA == 0) %} From e00fe0a732a111d89bf8eae4a243ebb25ae7f251 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 10:02:11 -0400 Subject: [PATCH 215/378] Enable for all modes --- salt/elasticsearch/files/elasticsearch.yml | 2 +- salt/top.sls | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index fed45bf79..a1d4c836b 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -36,7 +36,7 @@ xpack.security.authc: roles: superuser authz_exception: true node.name: {{ grains.host }} -script.max_compilations_rate: 1000/1m +script.max_compilations_rate: 2000/1m {%- if TRUECLUSTER is sameas true %} {%- if grains.role == 'so-manager' %} {%- if salt['pillar.get']('nodestab', {}) %} diff --git a/salt/top.sls b/salt/top.sls index 340f83825..99388fdcd 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -153,6 +153,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_manager and G@saltversion:{{saltversion}}': - match: compound @@ -213,6 +214,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_standalone and G@saltversion:{{saltversion}}': - match: compound @@ -314,6 +316,7 @@ base: {%- endif %} - schedule - docker_clean + - pipeline.load '*_managersearch and G@saltversion:{{saltversion}}': - match: compound @@ -378,6 +381,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_heavynode and G@saltversion:{{saltversion}}': - match: compound @@ -420,6 +424,7 @@ base: {%- endif %} - schedule - docker_clean + - pipeline.load '*_fleet and G@saltversion:{{saltversion}}': - match: compound @@ -463,3 +468,4 @@ base: - zeek - schedule - docker_clean + - pipeline.load From afbf7de9e38238baf4f275dfb60c9375a633c2de Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 2 Jun 2021 11:05:43 -0400 Subject: [PATCH 216/378] Remove empty lines in iso location prompt --- salt/common/tools/sbin/soup | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f7fa4e211..db0d26989 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -109,9 +109,7 @@ airgap_mounted() { echo "" cat << EOF In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. - For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. - Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom. EOF From 20e896cacf0616740f141fd8e2fb59090b778b55 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 12:17:15 -0400 Subject: [PATCH 217/378] Update all configs to pass user/pass to ES --- salt/curator/files/curator.yml | 4 ++++ salt/elastalert/defaults.yaml | 8 +++++--- salt/filebeat/etc/filebeat.yml | 4 ++++ salt/kibana/etc/kibana.yml | 8 +++++--- .../pipelines/config/so/9000_output_zeek.conf.jinja | 4 ++++ .../pipelines/config/so/9002_output_import.conf.jinja | 4 ++++ .../pipelines/config/so/9004_output_flow.conf.jinja | 4 ++++ .../pipelines/config/so/9033_output_snort.conf.jinja | 4 ++++ .../pipelines/config/so/9034_output_syslog.conf.jinja | 4 ++++ .../pipelines/config/so/9100_output_osquery.conf.jinja | 4 ++++ .../config/so/9101_output_osquery_livequery.conf.jinja | 4 ++++ .../pipelines/config/so/9200_output_firewall.conf.jinja | 4 ++++ .../pipelines/config/so/9400_output_suricata.conf.jinja | 4 ++++ .../pipelines/config/so/9500_output_beats.conf.jinja | 4 ++++ .../pipelines/config/so/9600_output_ossec.conf.jinja | 4 ++++ .../pipelines/config/so/9700_output_strelka.conf.jinja | 4 ++++ salt/soc/files/soc/soc.json | 6 ++++-- salt/soctopus/files/SOCtopus.conf | 7 ++++--- salt/telegraf/etc/telegraf.conf | 6 ++++++ 19 files changed, 80 insertions(+), 11 deletions(-) diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 7d86ccc04..a237416a1 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -3,6 +3,8 @@ {% elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %} {%- set elasticsearch = salt['pillar.get']('manager:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) --- # Remember, leave a key empty if there is no value. None will be a string, @@ -11,6 +13,8 @@ client: hosts: - {{elasticsearch}} port: 9200 + username: {{ ES_USER }} + password: {{ ES_PASS }} url_prefix: use_ssl: True certificate: diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index ad675b8ee..788d87f85 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -1,3 +1,5 @@ +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) elastalert: config: rules_folder: /opt/elastalert/rules/ @@ -19,8 +21,8 @@ elastalert: use_ssl: true verify_certs: false #es_send_get_body_as: GET - #es_username: someusername - #es_password: somepassword + es_username: {{ ES_USER }} + es_password: {{ ES_PASS }} writeback_index: elastalert_status alert_time_limit: days: 2 @@ -45,4 +47,4 @@ elastalert: level: INFO handlers: - file - propagate: false \ No newline at end of file + propagate: false diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 0f7c9c778..cd9c76de1 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -3,6 +3,8 @@ {%- else %} {%- set MANAGER = salt['grains.get']('master') %} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass', '') %}) {%- set HOSTNAME = salt['grains.get']('host', '') %} @@ -261,6 +263,8 @@ output.{{ type }}: output.elasticsearch: enabled: true hosts: ["https://{{ MANAGER }}:9200"] + username: "{{ ES_USER }}" + password: "{{ ES_PASS }}" ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"] pipelines: - pipeline: "%{[module]}.%{[dataset]}" diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 501d93c8a..94e170167 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -1,14 +1,16 @@ --- # Default Kibana configuration from kibana-docker. {%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass', '') %}) server.name: kibana server.host: "0" server.basePath: /kibana elasticsearch.hosts: [ "https://{{ ES }}:9200" ] elasticsearch.ssl.verificationMode: none #kibana.index: ".kibana" -#elasticsearch.username: elastic -#elasticsearch.password: changeme +elasticsearch.username: {{ ES_USER }} +elasticsearch.password: {{ ES_PASS }} #xpack.monitoring.ui.container.elasticsearch.enabled: true elasticsearch.requestTimeout: 90000 logging.dest: /var/log/kibana/kibana.log @@ -19,4 +21,4 @@ xpack.security.authc.providers: anonymous.anonymous1: order: 0 credentials: "elasticsearch_anonymous_user" -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index d17dc2b22..77f1b59ec 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,11 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-zeek" template_name => "so-zeek" template => "/templates/so-zeek-template.json" diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 4562dcee7..2453a25d8 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,11 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if "import" in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-import" template_name => "so-import" template => "/templates/so-import-template.json" diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index fb6eaee5d..010f01ee8 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,10 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if [event_type] == "sflow" { elasticsearch { hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-flow" template_name => "so-flow" template => "/templates/so-flow-template.json" diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 61aa21a82..2b42541dc 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,10 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-ids" template_name => "so-ids" template => "/templates/so-ids-template.json" diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 0afbf45ea..b2cd7107d 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,11 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if [module] =~ "syslog" { elasticsearch { pipeline => "%{module}" hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-syslog" template_name => "so-syslog" template => "/templates/so-syslog-template.json" diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index efa46c7af..d663e79b6 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,11 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_pass:pass', '') %}) output { if [module] =~ "osquery" and "live_query" not in [dataset] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-osquery" template_name => "so-osquery" template => "/templates/so-osquery-template.json" diff --git a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja index 6d7b71415..c8e70d85a 100644 --- a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja @@ -3,6 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) {% set FEATURES = salt['pillar.get']('elastic:features', False) %} filter { @@ -30,6 +32,8 @@ output { elasticsearch { pipeline => "osquery.live_query" hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-osquery" template_name => "so-osquery" template => "/templates/so-osquery-template.json" diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 764f597b9..2a8ab27bf 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,10 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if [dataset] =~ "firewall" { elasticsearch { hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-firewall" template_name => "so-firewall" template => "/templates/so-firewall-template.json" diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 5013bafc1..de93b6c81 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,11 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-ids" template_name => "so-ids" template => "/templates/so-ids-template.json" diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index 349c0ada1..95dce8160 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,11 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { pipeline => "beats.common" hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-beats" template_name => "so-beats" template => "/templates/so-beats-template.json" diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 1a4987a53..ce26d6093 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,11 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if [module] =~ "ossec" { elasticsearch { pipeline => "%{module}" hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-ossec" template_name => "so-ossec" template => "/templates/so-ossec-template.json" diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index d564486e4..ef804ea17 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,11 +3,15 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) output { if [module] =~ "strelka" { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" + username => "{{ ES_USER }}" + password => "{{ ES_PASS }}" index => "so-strelka" template_name => "so-strelka" template => "/templates/so-strelka-template.json" diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index e275ec28b..9f274e9f8 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -18,6 +18,8 @@ {%- import_json "soc/files/soc/menu.actions.json" as menu_actions %} {%- import_json "soc/files/soc/tools.json" as tools %} {%- set DNET = salt['pillar.get']('global:dockernet', '172.17.0.0') %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) { "logFilename": "/opt/sensoroni/logs/sensoroni-server.log", @@ -47,8 +49,8 @@ {%- endfor %} ], {%- endif %} - "username": "", - "password": "", + "username": "{{ ES_USER }}", + "password": "{{ ES_PASS }}", "cacheMs": {{ ES_FIELDCAPS_CACHE }}, "verifyCert": false, "timeoutMs": {{ API_TIMEOUT }} diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index b6ee45e74..424eb35cb 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -3,13 +3,14 @@ {%- set HIVEKEY = salt['pillar.get']('global:hivekey', '') %} {%- set CORTEXKEY = salt['pillar.get']('global:cortexorguserkey', '') %} {%- set PLAYBOOK_KEY = salt['pillar.get']('playbook:api_key', '') %} - +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) [es] es_url = https://{{MANAGER}}:9200 es_ip = {{MANAGER}} -es_user = -es_pass = +es_user = {{ ES_USER }} +es_pass = {{ ES_PASS }} es_index_pattern = so-* es_verifycert = no diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index af3474913..8c910b718 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -14,6 +14,8 @@ # for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR) {%- set MANAGER = salt['grains.get']('master') %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} @@ -620,10 +622,14 @@ {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} [[inputs.elasticsearch]] servers = ["https://{{ MANAGER }}:9200"] + username = "{{ ES_USER }}" + password = "{{ ES_PASS }}" insecure_skip_verify = true {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] + username = "{{ ES_USER }}" + password = "{{ ES_PASS }}" insecure_skip_verify = true {% endif %} From b3f2c60065a6fe80c15cd1af27bc3da05d378d1f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 2 Jun 2021 12:38:32 -0400 Subject: [PATCH 218/378] Whiptail title fixes - Use a variable for the title - Fix cases where the whiptail title wasn't changed previously --- salt/common/tools/sbin/so-common | 18 +-- salt/common/tools/sbin/so-zeek-logs | 6 +- salt/common/tools/sbin/soup | 1 + setup/so-functions | 5 +- setup/so-setup | 3 +- setup/so-variables | 3 + setup/so-whiptail | 200 ++++++++++++++-------------- 7 files changed, 121 insertions(+), 115 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index ec0ad390a..b770a5551 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -153,16 +153,16 @@ Do you agree to the terms of the Elastic License? If so, type AGREE to accept the Elastic License and continue. Otherwise, press Enter to exit this program without making any changes. EOM -AGREED=$(whiptail --title "Security Onion Setup" --inputbox \ -"$message" 20 75 3>&1 1>&2 2>&3) + AGREED=$(whiptail --title "$whiptail_title" --inputbox \ + "$message" 20 75 3>&1 1>&2 2>&3) -if [ "${AGREED^^}" = 'AGREE' ]; then - mkdir -p /opt/so/state - touch /opt/so/state/yeselastic.txt -else - echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion." - exit 1 -fi + if [ "${AGREED^^}" = 'AGREE' ]; then + mkdir -p /opt/so/state + touch /opt/so/state/yeselastic.txt + else + echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion." + exit 1 + fi } diff --git a/salt/common/tools/sbin/so-zeek-logs b/salt/common/tools/sbin/so-zeek-logs index 551213580..788312df5 100755 --- a/salt/common/tools/sbin/so-zeek-logs +++ b/salt/common/tools/sbin/so-zeek-logs @@ -10,7 +10,7 @@ zeek_logs_enabled() { } whiptail_manager_adv_service_zeeklogs() { - BLOGS=$(whiptail --title "Security Onion Setup" --checklist "Please Select Logs to Send:" 24 78 12 \ + BLOGS=$(whiptail --title "so-zeek-logs" --checklist "Please Select Logs to Send:" 24 78 12 \ "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ @@ -61,10 +61,10 @@ whiptail_manager_adv_service_zeeklogs return_code=$? case $return_code in 1) - whiptail --title "Security Onion Setup" --msgbox "Cancelling. No changes have been made." 8 75 + whiptail --title "so-zeek-logs" --msgbox "Cancelling. No changes have been made." 8 75 ;; 255) - whiptail --title "Security Onion Setup" --msgbox "Whiptail error occured, exiting." 8 75 + whiptail --title "so-zeek-logs" --msgbox "Whiptail error occured, exiting." 8 75 ;; *) zeek_logs_enabled diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d1261fe30..198761428 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -25,6 +25,7 @@ BATCHSIZE=5 SOUP_LOG=/root/soup.log INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup +whiptail_title='Security Onion UPdater' check_err() { local exit_code=$1 diff --git a/setup/so-functions b/setup/so-functions index 5ce3d6dee..19e0c804f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -896,7 +896,10 @@ compare_main_nic_ip() { This is not a supported configuration, please remediate and rerun setup. EOM - [[ -n $TESTING ]] || whiptail --title "Security Onion Setup" --msgbox "$message" 10 75 + + [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]] || local whiptail_title="so-monitor-add" + + [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 10 75 kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 fi else diff --git a/setup/so-setup b/setup/so-setup index 4c364ded6..8760a39de 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -108,11 +108,10 @@ catch() { automated=no progress() { - local title='Security Onion Setup' local msg=${1:-'Please wait while installing...'} if [ $automated == no ]; then - whiptail --title "$title" --gauge "$msg" 6 70 0 # append to text + whiptail --title "$whiptail_title" --gauge "$msg" 6 70 0 # append to text else cat >> $setup_log 2>&1 fi diff --git a/setup/so-variables b/setup/so-variables index 676cba4f0..a69ef9e1b 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -75,3 +75,6 @@ export net_init_file ntp_string="0.pool.ntp.org,1.pool.ntp.org" export ntp_string + +whiptail_title="Security Onion Setup - $SOVERSION" +export whiptail_title diff --git a/setup/so-whiptail b/setup/so-whiptail index 09792459d..458713591 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -22,7 +22,7 @@ whiptail_airgap() { local node_str='node' [[ $is_manager || $is_import ]] && node_str='manager' - INTERWEBS=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu \ + INTERWEBS=$(whiptail --title "$whiptail_title" --menu \ "How should this $node_str be installed?" 10 60 2 \ "Standard " "This $node_str has internet accesss" \ "Airgap " "This $node_str does not have internet access" 3>&1 1>&2 2>&3 ) @@ -42,7 +42,7 @@ whiptail_avoid_default_hostname() { You can choose to use this default hostname anyway, or change it to a new hostname. EOM - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --yesno "$message" 11 75 \ --yes-button "Use Anyway" --no-button "Change" --defaultno } @@ -51,7 +51,7 @@ whiptail_basic_suri() { [ -n "$TESTING" ] && return - BASICSURI=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + BASICSURI=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the number of Suricata processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -63,7 +63,7 @@ whiptail_basic_zeek() { [ -n "$TESTING" ] && return - BASICZEEK=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + BASICZEEK=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the number of Zeek processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -75,7 +75,7 @@ whiptail_bond_nics_mtu() { [ -n "$TESTING" ] && return # Set the MTU on the monitor interface - MTU=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MTU=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the MTU for the monitor NICs:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -84,7 +84,7 @@ whiptail_bond_nics_mtu() { whiptail_cancel() { - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Cancelling Setup." 8 75 + whiptail --title "$whiptail_title" --msgbox "Cancelling Setup." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -104,7 +104,7 @@ whiptail_check_exitstatus() { whiptail_cancel ;; 255) - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 + whiptail --title "$whiptail_title" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 exit ;; esac @@ -114,7 +114,7 @@ whiptail_components_adv_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 + whiptail --title "$whiptail_title" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 } whiptail_create_admin_user() { @@ -187,7 +187,7 @@ whiptail_create_soremote_user() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Set a password for the soremote user. This account is used for adding sensors remotely." 8 75 + whiptail --title "$whiptail_title" --msgbox "Set a password for the soremote user. This account is used for adding sensors remotely." 8 75 } @@ -258,11 +258,11 @@ whiptail_requirements_error() { [ -n "$TESTING" ] && return if [[ $(echo "$requirement_needed" | tr '[:upper:]' '[:lower:]') == 'nics' ]]; then - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --msgbox "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select OK to exit setup and reconfigure the machine." 10 75 # Same as whiptail_cancel, but changed the wording to exit instead of cancel. - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Exiting Setup. No changes have been made." 8 75 + whiptail --title "$whiptail_title" --msgbox "Exiting Setup. No changes have been made." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -272,7 +272,7 @@ whiptail_requirements_error() { fi exit else - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select YES to continue anyway, or select NO to cancel." 10 75 local exitstatus=$? @@ -298,7 +298,7 @@ whiptail_storage_requirements() { EOM whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --yesno "$message" \ 14 75 @@ -310,7 +310,7 @@ whiptail_cur_close_days() { [ -n "$TESTING" ] && return - CURCLOSEDAYS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + CURCLOSEDAYS=$(whiptail --title "$whiptail_title" --inputbox \ "Please specify the threshold (in days) at which Elasticsearch indices will be closed:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -322,7 +322,7 @@ whiptail_dhcp_or_static() { [ -n "$TESTING" ] && return - address_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + address_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose how to set up your management interface:" 20 78 4 \ "STATIC" "Set a static IPv4 address" ON \ "DHCP" "Use DHCP to configure the Management Interface" OFF 3>&1 1>&2 2>&3 ) @@ -355,7 +355,7 @@ whiptail_dhcp_warn() { EOM whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --"$window_type" "$dhcp_message" \ 14 75 @@ -366,7 +366,7 @@ whiptail_dhcp_warn() { whiptail_dhcp_or_static ;; 255) - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 + whiptail --title "$whiptail_title" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 exit ;; esac @@ -380,7 +380,7 @@ whiptail_dockernet_check(){ [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno \ + whiptail --title "$whiptail_title" --yesno \ "Do you want to keep the default Docker IP range?\n\nIf you are unsure, please accept the default option of Yes." 10 75 } @@ -389,7 +389,7 @@ whiptail_dockernet_net() { [ -n "$TESTING" ] && return - DOCKERNET=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + DOCKERNET=$(whiptail --title "$whiptail_title" --inputbox \ "\nEnter a /24 size network range for docker to use WITHOUT the /24 suffix. This range will be used on ALL nodes." 11 65 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -409,7 +409,7 @@ whiptail_enable_components() { STRELKA=0 if [[ $is_eval ]]; then - COMPONENTS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ + COMPONENTS=$(whiptail --title "$whiptail_title" --checklist \ "Select Components to install:" 20 75 8 \ GRAFANA "Enable Grafana for system monitoring" ON \ OSQUERY "Enable Fleet with osquery" ON \ @@ -418,7 +418,7 @@ if [[ $is_eval ]]; then PLAYBOOK "Enable Playbook" ON \ STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3) else - COMPONENTS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ + COMPONENTS=$(whiptail --title "$whiptail_title" --checklist \ "Select Components to install:" 20 75 7 \ OSQUERY "Enable Fleet with osquery" ON \ WAZUH "Enable Wazuh" ON \ @@ -658,7 +658,7 @@ whiptail_eval_adv() { [ -n "$TESTING" ] && return - EVALADVANCED=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + EVALADVANCED=$(whiptail --title "$whiptail_title" --radiolist \ "Choose your eval install:" 20 75 4 \ "BASIC" "Install basic components for evaluation" ON \ "ADVANCED" "Choose additional components to be installed" OFF 3>&1 1>&2 2>&3 ) @@ -685,7 +685,7 @@ whiptail_gauge_post_setup() { else local msg=$1 - whiptail --title "Security Onion Setup - $SOVERSION" --gauge "$msg" 6 60 96 + whiptail --title "$whiptail_title" --gauge "$msg" 6 60 96 fi } @@ -693,7 +693,7 @@ whiptail_helix_apikey() { [ -n "$TESTING" ] && return - HELIXAPIKEY=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + HELIXAPIKEY=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your Helix API Key: \n \nThis can be set later using so-helix-apikey" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -707,7 +707,7 @@ whiptail_homenet_manager() { [ -n "$TESTING" ] && return - HNMANAGER=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + HNMANAGER=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -719,13 +719,13 @@ whiptail_homenet_sensor_inherit() { [ -n "$TESTING" ] && return # Ask to inherit from manager - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to inherit the HOME_NET from the Manager?" 8 75 + whiptail --title "$whiptail_title" --yesno "Do you want to inherit the HOME_NET from the Manager?" 8 75 } whiptail_homenet_sensor() { [ -n "$TESTING" ] && return - HNSENSOR=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + HNSENSOR=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -738,7 +738,7 @@ whiptail_install_type() { [ -n "$TESTING" ] && return # What kind of install are we doing? - install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + install_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose install type:" 12 65 5 \ "EVAL" "Evaluation mode (not for production) " ON \ "STANDALONE" "Standalone production install " OFF \ @@ -764,7 +764,7 @@ whiptail_install_type_dist() { [ -n "$TESTING" ] && return - install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + install_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose distributed node type:" 13 60 6 \ "MANAGER" "Start a new grid " ON \ "SENSOR" "Create a forward only sensor " OFF \ @@ -792,14 +792,14 @@ whiptail_install_type_other() { # so-analyst-install will only work with a working network connection # so only show it on network installs for now if [[ $setup_type == 'network' ]]; then - install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + install_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose distributed node type:" 9 65 2 \ "ANALYST" "Quit setup and run so-analyst-install " ON \ "HELIXSENSOR" "Create a Helix sensor " OFF \ 3>&1 1>&2 2>&3 ) else - install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + install_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose distributed node type:" 8 65 1 \ "HELIXSENSOR" "Create a Helix sensor " ON \ 3>&1 1>&2 2>&3 @@ -815,7 +815,7 @@ whiptail_install_type_other() { whiptail_invalid_input() { # TODO: This should accept a list of arguments to specify what general pattern the input should follow [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox " Invalid input, please try again." 7 40 + whiptail --title "$whiptail_title" --msgbox " Invalid input, please try again." 7 40 } @@ -829,13 +829,13 @@ whiptail_invalid_proxy() { Error was: ${proxy_test_err} EOM - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60 + whiptail --title "$whiptail_title" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60 } whiptail_invalid_string() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 + whiptail --title "$whiptail_title" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 } @@ -843,21 +843,21 @@ whiptail_invalid_pass_characters_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 + whiptail --title "$whiptail_title" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 } whiptail_invalid_pass_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please choose a more secure password." 8 75 + whiptail --title "$whiptail_title" --msgbox "Please choose a more secure password." 8 75 } whiptail_invalid_user_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please enter a valid email address." 8 75 + whiptail --title "$whiptail_title" --msgbox "Please enter a valid email address." 8 75 } whiptail_invalid_hostname() { @@ -868,7 +868,7 @@ whiptail_invalid_hostname() { the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', \ and hyphen ('-')" | tr -d '\t') - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --msgbox "$error_message" 10 75 } @@ -891,7 +891,7 @@ whiptail_log_size_limit() { By default, this is set to ${percentage}% of the disk space allotted for /nsm. EOM - log_size_limit=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "$message" 11 75 "$1" 3>&1 1>&2 2>&3) + log_size_limit=$(whiptail --title "$whiptail_title" --inputbox "$message" 11 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -901,7 +901,7 @@ whiptail_log_size_limit() { whiptail_first_menu_iso() { [ -n "$TESTING" ] && return - option=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "Select an option" 10 75 2 \ + option=$(whiptail --title "$whiptail_title" --menu "Select an option" 10 75 2 \ "Install " "Run the standard Security Onion installation " \ "Configure Network " "Configure networking only " \ 3>&1 1>&2 2>&3 @@ -915,7 +915,7 @@ whiptail_make_changes() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 + whiptail --title "$whiptail_title" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -926,7 +926,7 @@ whiptail_management_interface_dns() { [ -n "$TESTING" ] && return - MDNS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MDNS=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your DNS servers separated by commas:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -938,7 +938,7 @@ whiptail_management_interface_dns_search() { [ -n "$TESTING" ] && return - MSEARCH=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MSEARCH=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your DNS search domain:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -949,7 +949,7 @@ whiptail_management_interface_gateway() { [ -n "$TESTING" ] && return - MGATEWAY=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MGATEWAY=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your gateway's IPv4 address:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -959,7 +959,7 @@ whiptail_management_interface_gateway() { whiptail_management_interface_ip_mask() { [ -n "$TESTING" ] && return - manager_ip_mask=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + manager_ip_mask=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your IPv4 address with CIDR mask (e.g. 192.168.1.2/24):" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -970,7 +970,7 @@ whiptail_management_interface_ip() { [ -n "$TESTING" ] && return - MIP=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MIP=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your IP address:" 10 60 X.X.X.X 3>&1 1>&2 2>&3) local exitstatus=$? @@ -981,7 +981,7 @@ whiptail_management_interface_mask() { [ -n "$TESTING" ] && return - MMASK=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MMASK=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the bit mask for your subnet:" 10 60 24 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1060,7 +1060,7 @@ whiptail_net_method() { local option_count=$(( ${#options[@]} / 2 )) - network_traffic=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) + network_traffic=$(whiptail --title "$whiptail_title" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1071,7 +1071,7 @@ whiptail_net_method() { whiptail_net_setup_complete() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --msgbox "Successfully set up networking, setup will now exit." 7 75 exit 0 } @@ -1085,7 +1085,7 @@ whiptail_network_init_notice() { Select OK to continue. EOM - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 9 75 + whiptail --title "$whiptail_title" --msgbox "$message" 9 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } @@ -1094,7 +1094,7 @@ whiptail_management_server() { [ -n "$TESTING" ] && return - MSRV=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MSRV=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your Manager Server hostname: \nIt is CASE SENSITIVE!" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1105,7 +1105,7 @@ whiptail_management_server() { whiptail_manager_ip() { [ -n "$TESTING" ] && return - MSRVIP=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MSRVIP=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your Manager Server IP Address:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1117,7 +1117,7 @@ whiptail_manager_adv() { [ -n "$TESTING" ] && return - MANAGERADV=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + MANAGERADV=$(whiptail --title "$whiptail_title" --radiolist \ "Choose which type of manager to install:" 20 75 4 \ "BASIC" "Install manager with recommended settings" ON \ "ADVANCED" "Do additional configuration to the manager" OFF 3>&1 1>&2 2>&3 ) @@ -1132,7 +1132,7 @@ whiptail_manager_adv_escluster(){ [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno \ + whiptail --title "$whiptail_title" --yesno \ "Do you want to set up a traditional ES cluster for using replicas and/or Hot-Warm indices? Recommended only for those who have experience with ES clustering! " 12 75 } @@ -1142,7 +1142,7 @@ whiptail_manager_adv_escluster_name(){ [ -n "$TESTING" ] && return - ESCLUSTERNAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + ESCLUSTERNAME=$(whiptail --title "$whiptail_title" --inputbox \ "Enter a name for your ES cluster!" 10 75 "$1" 3>&1 1>&2 2>&3) } @@ -1151,7 +1151,7 @@ whiptail_manager_adv_service_zeeklogs() { [ -n "$TESTING" ] && return - BLOGS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist "Please select logs to send:" 24 75 12 \ + BLOGS=$(whiptail --title "$whiptail_title" --checklist "Please select logs to send:" 24 75 12 \ "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ @@ -1213,13 +1213,13 @@ whiptail_manager_error() { Would you like to continue anyway? EOM - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 + whiptail --title "$whiptail_title" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 } whiptail_manager_updates_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION"\ + whiptail --title "$whiptail_title"\ --msgbox "Updating through the manager node requires the manager to have internet access, press ENTER to continue."\ 8 75 @@ -1230,7 +1230,7 @@ whiptail_manager_updates_warning() { whiptail_manager_unreachable() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Setup cannot determine if $1 is listening on port 22. Please check the address entered and try again." 7 75 + whiptail --title "$whiptail_title" --msgbox "Setup cannot determine if $1 is listening on port 22. Please check the address entered and try again." 7 75 } whiptail_metadata_tool() { @@ -1248,7 +1248,7 @@ whiptail_metadata_tool() { EOM # Legacy variable naming - ZEEKVERSION=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "$message" 20 75 2 \ + ZEEKVERSION=$(whiptail --title "$whiptail_title" --menu "$message" 20 75 2 \ "Zeek " "Use Zeek (Bro) for metadata and Suricata for NIDS alerts" \ "Suricata " "Use Suricata for both metadata and NIDS alerts" 3>&1 1>&2 2>&3) @@ -1262,7 +1262,7 @@ whiptail_nids() { [ -n "$TESTING" ] && return - NIDS=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + NIDS=$(whiptail --title "$whiptail_title" --radiolist \ "Choose which IDS to run: \n\n(Snort 3.0 support will be added once it is out of beta.)" 25 75 4 \ "Suricata" "Suricata" ON \ "Snort" "Placeholder for Snort 3.0 " OFF 3>&1 1>&2 2>&3 ) @@ -1276,7 +1276,7 @@ whiptail_network_notice() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75 + whiptail --title "$whiptail_title" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1286,14 +1286,14 @@ whiptail_network_notice() { whiptail_net_reinit() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "The management interface has already been configured. Do you want to reconfigure it?" 8 75 + whiptail --title "$whiptail_title" --yesno "The management interface has already been configured. Do you want to reconfigure it?" 8 75 } whiptail_node_advanced() { [ -n "$TESTING" ] && return - NODESETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + NODESETUP=$(whiptail --title "$whiptail_title" --radiolist \ "What type of config would you like to use?:" 20 75 4 \ "NODEBASIC" "Install Search Node with recommended settings" ON \ "NODEADVANCED" "Advanced Node Setup" OFF 3>&1 1>&2 2>&3 ) @@ -1306,7 +1306,7 @@ whiptail_node_advanced() { whiptail_node_description() { [ -n "$TESTING" ] && return - NODE_DESCRIPTION=$(whiptail --title "Security Onion Setup - $SOVERSION" \ + NODE_DESCRIPTION=$(whiptail --title "$whiptail_title" \ --inputbox "Enter a short description for the node or press ENTER to leave blank:" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1317,7 +1317,7 @@ whiptail_node_es_heap() { [ -n "$TESTING" ] && return - NODE_ES_HEAP_SIZE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + NODE_ES_HEAP_SIZE=$(whiptail --title "$whiptail_title" --inputbox \ "Enter ES heap size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1329,7 +1329,7 @@ whiptail_node_ls_heap() { [ -n "$TESTING" ] && return - NODE_LS_HEAP_SIZE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + NODE_LS_HEAP_SIZE=$(whiptail --title "$whiptail_title" --inputbox \ "Enter Logstash heap size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1341,7 +1341,7 @@ whiptail_node_ls_input_threads() { [ -n "$TESTING" ] && return - LSINPUTTHREADS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + LSINPUTTHREADS=$(whiptail --title "$whiptail_title" --inputbox \ "Enter number of Logstash input threads:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1354,7 +1354,7 @@ whiptail_node_ls_pipline_batchsize() { [ -n "$TESTING" ] && return - LSPIPELINEBATCH=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + LSPIPELINEBATCH=$(whiptail --title "$whiptail_title" --inputbox \ "Enter Logstash pipeline batch size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1366,7 +1366,7 @@ whiptail_node_ls_pipeline_worker() { [ -n "$TESTING" ] && return - LSPIPELINEWORKERS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + LSPIPELINEWORKERS=$(whiptail --title "$whiptail_title" --inputbox \ "Enter number of Logstash pipeline workers:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1377,13 +1377,13 @@ whiptail_node_ls_pipeline_worker() { whiptail_ntp_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Would you like to configure ntp servers?" 7 44 + whiptail --title "$whiptail_title" --yesno "Would you like to configure ntp servers?" 7 44 } whiptail_ntp_servers() { [ -n "$TESTING" ] && return - ntp_string=$(whiptail --title "Security Onion Setup - $SOVERSION" \ + ntp_string=$(whiptail --title "$whiptail_title" \ --inputbox "Input the NTP server(s) you would like to use, separated by commas:" 8 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1394,7 +1394,7 @@ whiptail_oinkcode() { [ -n "$TESTING" ] && return - OINKCODE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + OINKCODE=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your ET Pro or oinkcode:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1407,7 +1407,7 @@ whiptail_oinkcode() { whiptail_passwords_dont_match() { - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Passwords don't match. Please re-enter." 8 75 + whiptail --title "$whiptail_title" --msgbox "Passwords don't match. Please re-enter." 8 75 } @@ -1415,7 +1415,7 @@ whiptail_patch_name_new_schedule() { [ -n "$TESTING" ] && return - PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + PATCHSCHEDULENAME=$(whiptail --title "$whiptail_title" --inputbox \ "What name do you want to give this OS patch schedule? This schedule needs to be named uniquely. Available schedules can be found on the manager under /opt/so/salt/patch/os/schedules/.yml" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1426,7 +1426,7 @@ whiptail_patch_schedule() { [ -n "$TESTING" ] && return - patch_schedule=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + patch_schedule=$(whiptail --title "$whiptail_title" --radiolist \ "Choose OS patch schedule.\n\nThis schedule will update the operating system packages but will NOT update Security Onion related tools such as Zeek, Elasticsearch, Kibana, SaltStack, etc." 20 75 5 \ "Automatic" "Updates installed every 8 hours if available" ON \ "Manual" "Updates will be installed manually" OFF \ @@ -1442,7 +1442,7 @@ whiptail_patch_schedule_import() { [ -n "$TESTING" ] && return unset PATCHSCHEDULENAME - PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + PATCHSCHEDULENAME=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the name of the OS patch schedule you want to inherit. \nAvailable schedules can be found on the manager under /opt/so/salt/patch/os/schedules/.yml" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1454,7 +1454,7 @@ whiptail_patch_schedule_select_days() { [ -n "$TESTING" ] && return # Select the days to patch - PATCHSCHEDULEDAYS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ + PATCHSCHEDULEDAYS=$(whiptail --title "$whiptail_title" --checklist \ "Which days do you want to apply OS patches?" 15 75 8 \ Monday "" OFF \ Tuesday "" ON \ @@ -1478,7 +1478,7 @@ whiptail_patch_schedule_select_hours() { [ -n "$TESTING" ] && return # Select the hours to patch - PATCHSCHEDULEHOURS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ + PATCHSCHEDULEHOURS=$(whiptail --title "$whiptail_title" --checklist \ "At which time, UTC, do you want to apply OS patches on the selected days?" 22 75 13 \ 00:00 "" OFF \ 01:00 "" OFF \ @@ -1519,7 +1519,7 @@ whiptail_proxy_ask() { local pkg_mngr if [[ $OS = 'centos' ]]; then pkg_mngr="yum"; else pkg_mngr='apt'; fi - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment?" 9 65 --defaultno + whiptail --title "$whiptail_title" --yesno "Do you want to proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment?" 9 65 --defaultno } whiptail_proxy_addr() { @@ -1532,7 +1532,7 @@ whiptail_proxy_addr() { If your proxy requires a username and password do not include them in your input. Setup will ask for those values next. EOM - proxy_addr=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) + proxy_addr=$(whiptail --title "$whiptail_title" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1541,13 +1541,13 @@ whiptail_proxy_addr() { whiptail_proxy_auth_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Does your proxy require authentication?" 7 60 + whiptail --title "$whiptail_title" --yesno "Does your proxy require authentication?" 7 60 } whiptail_proxy_auth_user() { [ -n "$TESTING" ] && return - proxy_user=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) + proxy_user=$(whiptail --title "$whiptail_title" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1558,7 +1558,7 @@ whiptail_proxy_auth_pass() { [ -n "$TESTING" ] && return - proxy_pass=$(whiptail --title "Security Onion Setup - $SOVERSION" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) + proxy_pass=$(whiptail --title "$whiptail_title" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1572,7 +1572,7 @@ whiptail_requirements_error() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 10 75 local exitstatus=$? @@ -1584,7 +1584,7 @@ whiptail_rule_setup() { [ -n "$TESTING" ] && return # Get pulled pork info - RULESETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + RULESETUP=$(whiptail --title "$whiptail_title" --radiolist \ "Which IDS ruleset would you like to use?\n\nThis manager server is responsible for downloading the IDS ruleset from the Internet.\n\nSensors then pull a copy of this ruleset from the manager server.\n\nIf you select a commercial ruleset, it is your responsibility to purchase enough licenses for all of your sensors in compliance with your vendor's policies." 20 75 4 \ "ETOPEN" "Emerging Threats Open" ON \ "ETPRO" "Emerging Threats PRO" OFF \ @@ -1600,7 +1600,7 @@ whiptail_sensor_config() { [ -n "$TESTING" ] && return - NSMSETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + NSMSETUP=$(whiptail --title "$whiptail_title" --radiolist \ "What type of configuration would you like to use?" 20 75 4 \ "BASIC" "Install NSM components with recommended settings" ON \ "ADVANCED" "Configure each component individually" OFF 3>&1 1>&2 2>&3 ) @@ -1642,7 +1642,7 @@ whiptail_sensor_nics() { for bond_nic in "${BNICS[@]}"; do if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ 8 75 exit @@ -1654,7 +1654,7 @@ whiptail_set_hostname() { [ -n "$TESTING" ] && return - HOSTNAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + HOSTNAME=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the hostname (not FQDN) you would like to set:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1665,7 +1665,7 @@ whiptail_set_redirect() { [ -n "$TESTING" ] && return - REDIRECTINFO=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + REDIRECTINFO=$(whiptail --title "$whiptail_title" --radiolist \ "How would you like to access the web interface?\n\nSecurity Onion uses strict cookie enforcement, so whatever you choose here will be the only way that you can access the web interface.\n\nIf you choose something other than IP address, then you'll need to ensure that you can resolve the name via DNS or hosts entry. If you are unsure, please select IP." 20 75 4 \ "IP" "Use IP address to access the web interface" ON \ "HOSTNAME" "Use hostname to access the web interface" OFF \ @@ -1678,7 +1678,7 @@ whiptail_set_redirect_host() { [ -n "$TESTING" ] && return - REDIRECTHOST=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + REDIRECTHOST=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the Hostname, IP, or FQDN you would like to use for the web interface:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1706,7 +1706,7 @@ whiptail_setup_complete() { Press ENTER to reboot. EOM - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 12 75 + whiptail --title "$whiptail_title" --msgbox "$message" 12 75 } whiptail_setup_failed() { @@ -1726,13 +1726,13 @@ whiptail_setup_failed() { Press Ok to exit. EOM - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" $height 75 + whiptail --title "$whiptail_title" --msgbox "$message" $height 75 } whiptail_so_allow_yesno() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --yesno "Do you want to run so-allow to allow access to the web tools?" \ 8 75 } @@ -1741,7 +1741,7 @@ whiptail_so_allow() { [ -n "$TESTING" ] && return - ALLOW_CIDR=$(whiptail --title "Security Onion Setup - $SOVERSION" \ + ALLOW_CIDR=$(whiptail --title "$whiptail_title" \ --inputbox "Enter a single IP address or an IP range, in CIDR notation, to allow:" \ 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1761,7 +1761,7 @@ whiptail_ssh_key_copy_notice() { Select OK to continue. EOM - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 11 75 + whiptail --title "$whiptail_title" --msgbox "$message" 11 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } @@ -1800,7 +1800,7 @@ whiptail_storage_requirements() { EOM whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --yesno "$message" \ 14 75 @@ -1812,7 +1812,7 @@ whiptail_strelka_rules() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to enable the default YARA rules for Strelka?" 8 75 + whiptail --title "$whiptail_title" --yesno "Do you want to enable the default YARA rules for Strelka?" 8 75 local exitstatus=$? @@ -1852,7 +1852,7 @@ whiptail_node_updates() { [ -n "$TESTING" ] && return - NODEUPDATES=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + NODEUPDATES=$(whiptail --title "$whiptail_title" --radiolist \ "How would you like to download OS package updates for your grid?" 20 75 4 \ "MANAGER" "Manager node is proxy for updates." ON \ "OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) @@ -1877,7 +1877,7 @@ whiptail_you_sure() { EOM whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --yesno "$you_sure_text" \ 20 75 From 7b7111e12c15d107ad5b5ce68fa4e633d84c4210 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 13:53:39 -0400 Subject: [PATCH 219/378] Fix some hunt queries --- salt/elasticsearch/files/elasticsearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index a1d4c836b..af7cec1fa 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -36,7 +36,7 @@ xpack.security.authc: roles: superuser authz_exception: true node.name: {{ grains.host }} -script.max_compilations_rate: 2000/1m +script.max_compilations_rate: 20000/1m {%- if TRUECLUSTER is sameas true %} {%- if grains.role == 'so-manager' %} {%- if salt['pillar.get']('nodestab', {}) %} From e8cc88174f9918fd6575d20fe2a6feb723853556 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 13:55:05 -0400 Subject: [PATCH 220/378] Fix some hunt queries --- salt/soc/files/soc/hunt.queries.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index 93295364d..9d4cd85bd 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -2,8 +2,8 @@ { "name": "Default Query", "description": "Show all events grouped by the origin host", "query": "* | groupby observer.name"}, { "name": "Log Type", "description": "Show all events grouped by module and dataset", "query": "* | groupby event.module event.dataset"}, { "name": "Elastalerts", "description": "", "query": "_type:elastalert | groupby rule.name"}, - { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.dataset: alert | groupby event.module"}, - { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert", "query": "event.category: network AND event.dataset: alert | groupby rule.category rule.gid rule.uuid rule.name"}, + { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.kind: alert | groupby event.module"}, + { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert", "query": "event.category: network AND event.kind: alert | groupby rule.category rule.gid rule.uuid rule.name"}, { "name": "Wazuh/OSSEC Alerts", "description": "Show all Wazuh alerts at Level 5 or higher grouped by category", "query": "event.module:ossec AND event.dataset:alert AND rule.level:>4 | groupby rule.category rule.name"}, { "name": "Wazuh/OSSEC Alerts", "description": "Show all Wazuh alerts at Level 4 or lower grouped by category", "query": "event.module:ossec AND event.dataset:alert AND rule.level:<5 | groupby rule.category rule.name"}, { "name": "Wazuh/OSSEC Users and Commands", "description": "Show all Wazuh alerts grouped by username and command line", "query": "event.module:ossec AND event.dataset:alert | groupby user.escalated.keyword process.command_line"}, @@ -11,11 +11,11 @@ { "name": "Sysmon Events", "description": "Show all Sysmon logs grouped by event type", "query": "event.module:sysmon | groupby event.dataset"}, { "name": "Sysmon Usernames", "description": "Show all Sysmon logs grouped by username", "query": "event.module:sysmon | groupby event.dataset, user.name.keyword"}, { "name": "Strelka", "description": "Show all Strelka logs grouped by file type", "query": "event.module:strelka | groupby file.mime_type"}, - { "name": "Zeek Notice", "description": "Show notices from Zeek", "query": "event.dataset:notice | groupby notice.note notice.message"}, - { "name": "Connections", "description": "Connections grouped by IP and Port", "query": "event.dataset:conn | groupby source.ip destination.ip network.protocol destination.port"}, - { "name": "Connections", "description": "Connections grouped by Service", "query": "event.dataset:conn | groupby network.protocol destination.port"}, - { "name": "Connections", "description": "Connections grouped by destination country", "query": "event.dataset:conn | groupby destination.geo.country_name"}, - { "name": "Connections", "description": "Connections grouped by source country", "query": "event.dataset:conn | groupby source.geo.country_name"}, + { "name": "Zeek Notice", "description": "Show notices from Zeek", "query": "event.dataset:zeek.notice | groupby notice.note notice.message"}, + { "name": "Connections", "description": "Connections grouped by IP and Port", "query": "event.dataset:zeek.connection | groupby source.ip destination.ip network.protocol destination.port"}, + { "name": "Connections", "description": "Connections grouped by Service", "query": "event.dataset:zeek.connection | groupby network.protocol destination.port"}, + { "name": "Connections", "description": "Connections grouped by destination country", "query": "event.dataset:zeek.connection | groupby destination.geo.country_name"}, + { "name": "Connections", "description": "Connections grouped by source country", "query": "event.dataset:zeek.connection | groupby source.geo.country_name"}, { "name": "DCE_RPC", "description": "DCE_RPC grouped by operation", "query": "event.dataset:dce_rpc | groupby dce_rpc.operation"}, { "name": "DHCP", "description": "DHCP leases", "query": "event.dataset:dhcp | groupby host.hostname client.address"}, { "name": "DHCP", "description": "DHCP grouped by message type", "query": "event.dataset:dhcp | groupby dhcp.message_types"}, From e42db3cd2d1b4719749422dcf9911d3311de80bc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 14:05:02 -0400 Subject: [PATCH 221/378] Fix some hunt queries --- salt/common/tools/sbin/so-filebeat-module-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index d7706366e..7a6ae7446 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -49,7 +49,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Testing to see if the pipelines are already applied" ESVER=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \") -PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) +PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) if [[ "$PIPELINES" -lt 5 ]]; then echo "Setting up ingest pipeline(s)" From 588da4d7dcd6a41fd7975204ad7498991d88c3cc Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 14:34:21 -0400 Subject: [PATCH 222/378] Resolve salt pillar/state/jinja race condition --- salt/elasticsearch/auth.sls | 2 ++ salt/elasticsearch/files/curl.config.template | 1 + salt/manager/init.sls | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/files/curl.config.template diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index 748e50d0b..9bca61e01 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -7,6 +7,8 @@ elastic_auth_pillar: file.managed: - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls + - mode 600 + - reload_pillar: True - contents: | elasticsearch: auth: diff --git a/salt/elasticsearch/files/curl.config.template b/salt/elasticsearch/files/curl.config.template new file mode 100644 index 000000000..14f5a2a1d --- /dev/null +++ b/salt/elasticsearch/files/curl.config.template @@ -0,0 +1 @@ +user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}" \ No newline at end of file diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 9ae39e9a8..ca47af34f 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -110,8 +110,9 @@ strelka_yara_update: elastic_curl_config_distributed: file.managed: - name: /opt/so/saltstack/local/salt/elasticsearch/curl.config + - source: salt://elasticsearch/files/curl.config.template + - template: jinja - mode: 600 - - contents: user = "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user') }}:{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass') }}" - show_changes: False # Must run before elasticsearch docker container is started! From 9c9bcac61ba6899f1eb0afa3248c2d0269d6f5a4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 15:01:14 -0400 Subject: [PATCH 223/378] Update DNS queries --- salt/soc/files/soc/hunt.queries.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index 9d4cd85bd..c220060dd 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -20,11 +20,11 @@ { "name": "DHCP", "description": "DHCP leases", "query": "event.dataset:dhcp | groupby host.hostname client.address"}, { "name": "DHCP", "description": "DHCP grouped by message type", "query": "event.dataset:dhcp | groupby dhcp.message_types"}, { "name": "DNP3", "description": "DNP3 grouped by reply", "query": "event.dataset:dnp3 | groupby dnp3.fc_reply"}, - { "name": "DNS", "description": "DNS queries grouped by port", "query": "event.dataset:dns | groupby dns.query.name destination.port"}, - { "name": "DNS", "description": "DNS queries grouped by type", "query": "event.dataset:dns | groupby dns.query.type_name destination.port"}, - { "name": "DNS", "description": "DNS queries grouped by response code", "query": "event.dataset:dns | groupby dns.response.code_name destination.port"}, - { "name": "DNS", "description": "DNS highest registered domain", "query": "event.dataset:dns | groupby dns.highest_registered_domain.keyword destination.port"}, - { "name": "DNS", "description": "DNS grouped by parent domain", "query": "event.dataset:dns | groupby dns.parent_domain.keyword destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by port", "query": "_exists_:dns.id | groupby dns.question.name destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by type", "query": "_exists_:dns.id | groupby dns.question.type destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by response code", "query": "_exists_:dns.id | groupby dns.response_code destination.port"}, + { "name": "DNS", "description": "DNS highest registered domain", "query": "_exists_:dns.id | groupby dns.question.top_level_domain destination.port"}, + { "name": "DNS", "description": "DNS grouped by parent domain", "query": "_exists_:dns.id | groupby dns.question.registered_domain destination.port"}, { "name": "DPD", "description": "Dynamic Protocol Detection errors", "query": "event.dataset:dpd | groupby error.reason"}, { "name": "Files", "description": "Files grouped by mimetype", "query": "event.dataset:file | groupby file.mime_type source.ip"}, { "name": "Files", "description": "Files grouped by source", "query": "event.dataset:file | groupby file.source source.ip"}, From fc6b3726a47abe865f36eccfda8a793bb8e4fdca Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 15:23:16 -0400 Subject: [PATCH 224/378] Fix missing colon for mode --- salt/elasticsearch/auth.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/auth.sls b/salt/elasticsearch/auth.sls index 9bca61e01..373f2fbed 100644 --- a/salt/elasticsearch/auth.sls +++ b/salt/elasticsearch/auth.sls @@ -7,7 +7,7 @@ elastic_auth_pillar: file.managed: - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls - - mode 600 + - mode: 600 - reload_pillar: True - contents: | elasticsearch: From a1b34e7a8840b7730817c20e3f812fe5ef9e9c39 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 15:30:14 -0400 Subject: [PATCH 225/378] Fix Suricata index name --- .../pipelines/config/so/9400_output_suricata.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 258781f29..999b270cf 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -9,7 +9,7 @@ output { pipeline => "filebeat-%{[agent][version]}-suricata-%{[fileset][name]}-pipeline" hosts => "{{ ES }}" index => "so-ids" - template_name => "so-ids" + template_name => "so-ids-%{+YYYY.MM.dd}" template => "/templates/so-ids-template.json" ssl => true ssl_certificate_verification => false From db48c15f1d099be45e8178ac94c093159b7eb161 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 15:33:18 -0400 Subject: [PATCH 226/378] Create event.kind field and rename dataset to be module[dot]dataset --- salt/elasticsearch/files/ingest/ossec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/ossec b/salt/elasticsearch/files/ingest/ossec index 868de2798..1c5a0fd42 100644 --- a/salt/elasticsearch/files/ingest/ossec +++ b/salt/elasticsearch/files/ingest/ossec @@ -63,7 +63,8 @@ { "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { "set": { "if": "ctx.rule != null && ctx.rule.name != null", "field": "event.dataset", "value": "alert", "override": true } }, + { "set": { "if": "ctx.rule != null && ctx.rule.name != null", "field": "event.dataset", "value": "ossec.alert", "override": true } }, + { "set": { "if": "ctx.rule != null && ctx.rule.name != null", "field": "event.kind", "value": "alert", "override": true } }, { "pipeline": { "name": "common" } } ] } From 4c74e7f3089a57fd68cdd04c6a90c145ee5175b0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 15:35:26 -0400 Subject: [PATCH 227/378] Add event.kind and set name to module[dot]dataset --- salt/elasticsearch/files/ingest/strelka.file | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index e5e8560f8..cf2772305 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -53,7 +53,8 @@ { "set": { "if": "ctx.exiftool?.FileDirectory != null", "field": "file.directory", "value": "{{exiftool.FileDirectory}}", "ignore_failure": true }}, { "set": { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem", "value": "{{exiftool.Subsystem}}", "ignore_failure": true }}, { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "rule.name", "value": "{{scan.yara.matches.0}}" }}, - { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "dataset", "value": "alert", "override": true }}, + { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "dataset", "value": "strelka.alert", "override": true }}, + { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "event.kind", "value": "alert", "override": true }}, { "rename": { "field": "file.flavors.mime", "target_field": "file.mime_type", "ignore_missing": true }}, { "set": { "if": "ctx.rule?.name != null && ctx.rule?.score == null", "field": "event.severity", "value": 3, "override": true } }, { "convert" : { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}}, From 901242f7e97accf48b67a70c1c61fb5069473a5e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 16:23:45 -0400 Subject: [PATCH 228/378] remove extra parenthesis --- salt/curator/files/curator.yml | 4 ++-- salt/elastalert/defaults.yaml | 4 ++-- salt/filebeat/etc/filebeat.yml | 4 ++-- salt/kibana/etc/kibana.yml | 4 ++-- salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja | 4 ++-- .../pipelines/config/so/9002_output_import.conf.jinja | 4 ++-- salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja | 4 ++-- .../logstash/pipelines/config/so/9033_output_snort.conf.jinja | 4 ++-- .../pipelines/config/so/9034_output_syslog.conf.jinja | 4 ++-- .../pipelines/config/so/9100_output_osquery.conf.jinja | 4 ++-- .../config/so/9101_output_osquery_livequery.conf.jinja | 4 ++-- .../pipelines/config/so/9200_output_firewall.conf.jinja | 4 ++-- .../pipelines/config/so/9400_output_suricata.conf.jinja | 4 ++-- .../logstash/pipelines/config/so/9500_output_beats.conf.jinja | 4 ++-- .../logstash/pipelines/config/so/9600_output_ossec.conf.jinja | 4 ++-- .../pipelines/config/so/9700_output_strelka.conf.jinja | 4 ++-- salt/soc/files/soc/soc.json | 4 ++-- salt/soctopus/files/SOCtopus.conf | 4 ++-- salt/telegraf/etc/telegraf.conf | 4 ++-- 19 files changed, 38 insertions(+), 38 deletions(-) diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index a237416a1..0934aaf02 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -3,8 +3,8 @@ {% elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %} {%- set elasticsearch = salt['pillar.get']('manager:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} --- # Remember, leave a key empty if there is no value. None will be a string, diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index 788d87f85..accccaa3a 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -1,5 +1,5 @@ -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} elastalert: config: rules_folder: /opt/elastalert/rules/ diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index cd9c76de1..9ae408654 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -3,8 +3,8 @@ {%- else %} {%- set MANAGER = salt['grains.get']('master') %} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass', '') %} {%- set HOSTNAME = salt['grains.get']('host', '') %} diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 94e170167..a3f83a516 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -1,8 +1,8 @@ --- # Default Kibana configuration from kibana-docker. {%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass', '') %} server.name: kibana server.host: "0" server.basePath: /kibana diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index 77f1b59ec..2151b5f0c 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 2453a25d8..d8f888f40 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if "import" in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 010f01ee8..5e5c95f18 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if [event_type] == "sflow" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 2b42541dc..e7fc06f8c 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index b2cd7107d..a98c437ad 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if [module] =~ "syslog" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index d663e79b6..aab7318a2 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_pass:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_pass:pass', '') %} output { if [module] =~ "osquery" and "live_query" not in [dataset] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja index c8e70d85a..b6b84c771 100644 --- a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} filter { diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 2a8ab27bf..cda388f8c 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if [dataset] =~ "firewall" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index de93b6c81..ac71ec131 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index 95dce8160..21c307107 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index ce26d6093..b71718c8f 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if [module] =~ "ossec" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index ef804ea17..3581c061f 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,8 +3,8 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if [module] =~ "strelka" { elasticsearch { diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 9f274e9f8..2cd213276 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -18,8 +18,8 @@ {%- import_json "soc/files/soc/menu.actions.json" as menu_actions %} {%- import_json "soc/files/soc/tools.json" as tools %} {%- set DNET = salt['pillar.get']('global:dockernet', '172.17.0.0') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} { "logFilename": "/opt/sensoroni/logs/sensoroni-server.log", diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 424eb35cb..4b47c8b6a 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -3,8 +3,8 @@ {%- set HIVEKEY = salt['pillar.get']('global:hivekey', '') %} {%- set CORTEXKEY = salt['pillar.get']('global:cortexorguserkey', '') %} {%- set PLAYBOOK_KEY = salt['pillar.get']('playbook:api_key', '') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} [es] es_url = https://{{MANAGER}}:9200 diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 8c910b718..659a78398 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -14,8 +14,8 @@ # for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR) {%- set MANAGER = salt['grains.get']('master') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}) -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}) +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} From 4241bb08b85101673a08ffcdbecfc155d3ca7bc9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 16:37:43 -0400 Subject: [PATCH 229/378] Add suricata/zeek until we migrate templates --- .../templates/so/so-common-template.json | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 214e5c691..919cb521a 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -527,12 +527,20 @@ "version":{ "type":"long" } - } - }, + } + }, "x509":{ "type":"object", "dynamic": true - } + }, + "suricata":{ + "type":"object", + "dynamic": true + }, + "zeek":{ + "type":"object", + "dynamic": true + }, } } } From cba719b3a02b52d75d957612337070446b3ef66d Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 16:42:09 -0400 Subject: [PATCH 230/378] Remove extra comma --- salt/elasticsearch/templates/so/so-common-template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 919cb521a..2d1ef2a21 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -540,7 +540,7 @@ "zeek":{ "type":"object", "dynamic": true - }, + } } } } From fa6af06204a8a71e7607566f35399b2f81624201 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 17:13:59 -0400 Subject: [PATCH 231/378] Avoid running highstate during setup when flipping auth flag --- salt/common/tools/sbin/so-elastic-auth | 12 ++++++++---- setup/so-setup | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth index b15b04c0f..d4b8057a3 100644 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -30,8 +30,10 @@ fi if [[ "$authEnable" == "true" ]]; then if grep -q "enabled: False" "$ES_AUTH_PILLAR"; then sed -i 's/enabled: False/enabled: True/g' "$ES_AUTH_PILLAR" - echo "Applying highstate - this may take a few minutes..." - salt-call state.highstate queue=True + if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then + echo "Applying highstate - this may take a few minutes..." + salt-call state.highstate queue=True + fi echo "Elastic auth is now enabled." if grep -q "argon" "$ES_USERS_FILE"; then echo "" @@ -44,8 +46,10 @@ if [[ "$authEnable" == "true" ]]; then elif [[ "$authEnable" == "false" ]]; then if grep -q "enabled: True" "$ES_AUTH_PILLAR"; then sed -i 's/enabled: True/enabled: False/g' "$ES_AUTH_PILLAR" - echo "Applying highstate - this may take a few minutes..." - salt-call state.highstate queue=True + if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then + echo "Applying highstate - this may take a few minutes..." + salt-call state.highstate queue=True + fi echo "Elastic auth is now disabled." else echo "Auth is already disabled." diff --git a/setup/so-setup b/setup/so-setup index 6d2f6ac27..ed6899db8 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -752,7 +752,7 @@ echo "1" > /root/accept_changes set_progress_str 60 "$(print_salt_state_apply 'manager')" salt-call state.apply -l info manager >> $setup_log 2>&1 - bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth + ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth fi set_progress_str 61 "$(print_salt_state_apply 'firewall')" From 719d841353e927b5fa108ac153d4dee272d6a984 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 20:15:03 -0400 Subject: [PATCH 232/378] Update saved objects --- salt/kibana/files/saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/saved_objects.ndjson b/salt/kibana/files/saved_objects.ndjson index 21ef5a94a..639a38bfc 100644 --- a/salt/kibana/files/saved_objects.ndjson +++ b/salt/kibana/files/saved_objects.ndjson @@ -460,7 +460,7 @@ {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\",\"time_zone\":\"America/New_York\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":48,\"x\":0,\"y\":0,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":96,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":8,\"y\":20,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":28,\"y\":20,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":0,\"y\":72,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":48,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":24,\"x\":16,\"y\":72,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":28,\"y\":8,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":120,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"event_type\",\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"}]","timeRestore":false,"title":"z16.04 - Sysmon - Logs","version":1},"id":"6d189680-6d62-11e7-8ddb-e71eb260f4a3","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"8cfdeff0-6d6b-11e7-ad64-15aa071374a6","name":"panel_1","type":"visualization"},{"id":"0eb1fd80-6d70-11e7-b09b-f57b22df6524","name":"panel_2","type":"visualization"},{"id":"3072c750-6d71-11e7-b09b-f57b22df6524","name":"panel_3","type":"visualization"},{"id":"7bc74b40-6d71-11e7-b09b-f57b22df6524","name":"panel_4","type":"visualization"},{"id":"13ed0810-6d72-11e7-b09b-f57b22df6524","name":"panel_5","type":"visualization"},{"id":"3b6c92c0-6d72-11e7-b09b-f57b22df6524","name":"panel_6","type":"visualization"},{"id":"e09f6010-6d72-11e7-b09b-f57b22df6524","name":"panel_7","type":"visualization"},{"id":"29611940-6d75-11e7-b09b-f57b22df6524","name":"panel_8","type":"visualization"},{"id":"6b70b840-6d75-11e7-b09b-f57b22df6524","name":"panel_9","type":"visualization"},{"id":"248c1d20-6d6b-11e7-ad64-15aa071374a6","name":"panel_10","type":"search"},{"id":"AWDHHk1sxQT5EBNmq43Y","name":"panel_11","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQyLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} -{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.13.0","id":"7.13.0","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.13.1","id":"7.13.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":56,\"x\":0,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":8,\"y\":32,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":20,\"y\":32,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":56,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"columns\":[\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"uid\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":32,\"y\":32,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":40,\"h\":24,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"}]","timeRestore":false,"title":"z16.04 - Bro - Modbus","version":1},"id":"70c005f0-3583-11e7-a588-05992195c551","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"0d168a30-363f-11e7-a6f7-4f44d7bf1c33","name":"panel_1","type":"visualization"},{"id":"20eabd60-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_2","type":"visualization"},{"id":"3c65f500-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_3","type":"visualization"},{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"panel_4","type":"search"},{"id":"178209e0-6e1b-11e7-b553-7f80727663c1","name":"panel_5","type":"visualization"},{"id":"AWDG_9KpxQT5EBNmq4Oo","name":"panel_6","type":"visualization"},{"id":"453f8b90-4a58-11e8-9b0a-f1d33346f773","name":"panel_7","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ4LDRd"} From e6165f0046c3963a5fdd19741b91b0a4cda2b675 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 3 Jun 2021 07:47:32 -0400 Subject: [PATCH 233/378] Update kibana config load for auth changes --- salt/kibana/bin/so-kibana-config-load | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index 8f80b5c3d..fb50d0cf9 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -1,6 +1,4 @@ #!/bin/bash -# {%- set FLEET_MANAGER = salt['pillar.get']('global:fleet_manager', False) -%} -# {%- set FLEET_NODE = salt['pillar.get']('global:fleet_node', False) -%} # {%- set MANAGER = salt['pillar.get']('global:url_base', '') %} . /usr/sbin/so-common @@ -8,19 +6,10 @@ # Copy template file cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_objects.ndjson -# {% if FLEET_NODE or FLEET_MANAGER %} -# Fleet IP -#sed -i "s/FLEETPLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson -# {% endif %} - # SOCtopus and Manager sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson -#wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" -## This hackery will be removed if using Elastic Auth ## - -# Let's snag a cookie from Kibana -THECOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') +wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 {{ ELASTICCURL }} # Load saved objects -{{ ELASTICCURL }} -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log +curl -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log From 5c527b2c486686f090c3d73a5bdab27914cb7df1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 3 Jun 2021 07:51:43 -0400 Subject: [PATCH 234/378] Rename username param to user since logstash is 'unique' --- salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9002_output_import.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja | 2 +- .../logstash/pipelines/config/so/9100_output_osquery.conf.jinja | 2 +- .../config/so/9101_output_osquery_livequery.conf.jinja | 2 +- .../pipelines/config/so/9200_output_firewall.conf.jinja | 2 +- .../pipelines/config/so/9400_output_suricata.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja | 2 +- .../logstash/pipelines/config/so/9700_output_strelka.conf.jinja | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index 2151b5f0c..af3a9f93b 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -10,7 +10,7 @@ output { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-zeek" template_name => "so-zeek" diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index d8f888f40..feaddeded 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -10,7 +10,7 @@ output { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-import" template_name => "so-import" diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 5e5c95f18..e01792914 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -9,7 +9,7 @@ output { if [event_type] == "sflow" { elasticsearch { hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-flow" template_name => "so-flow" diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index e7fc06f8c..42e4dbee4 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -9,7 +9,7 @@ output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-ids" template_name => "so-ids" diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index a98c437ad..ca6308ada 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -10,7 +10,7 @@ output { elasticsearch { pipeline => "%{module}" hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-syslog" template_name => "so-syslog" diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index aab7318a2..7ed8c58e5 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -10,7 +10,7 @@ output { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-osquery" template_name => "so-osquery" diff --git a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja index b6b84c771..10c3cba9f 100644 --- a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja @@ -32,7 +32,7 @@ output { elasticsearch { pipeline => "osquery.live_query" hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-osquery" template_name => "so-osquery" diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index cda388f8c..6abe75f9d 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -9,7 +9,7 @@ output { if [dataset] =~ "firewall" { elasticsearch { hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-firewall" template_name => "so-firewall" diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index ac71ec131..62b713f0f 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -10,7 +10,7 @@ output { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-ids" template_name => "so-ids" diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index 21c307107..ffe30c8c5 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -10,7 +10,7 @@ output { elasticsearch { pipeline => "beats.common" hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-beats" template_name => "so-beats" diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index b71718c8f..7ef4bca1f 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -10,7 +10,7 @@ output { elasticsearch { pipeline => "%{module}" hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-ossec" template_name => "so-ossec" diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index 3581c061f..a26373397 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -10,7 +10,7 @@ output { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" - username => "{{ ES_USER }}" + user => "{{ ES_USER }}" password => "{{ ES_PASS }}" index => "so-strelka" template_name => "so-strelka" From 2af43d62ebda24ce4a992539c383cc76153280f6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 3 Jun 2021 08:53:59 -0400 Subject: [PATCH 235/378] Wrap curl param in quotes for function call --- salt/kibana/bin/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index fb50d0cf9..00dc7f9ff 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -9,7 +9,7 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o # SOCtopus and Manager sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson -wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 {{ ELASTICCURL }} +wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" # Load saved objects curl -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log From 58ae3479dcd76bf6ec46de4e89370de97fb5676a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 3 Jun 2021 10:11:10 -0400 Subject: [PATCH 236/378] Fix mispelled db filename; ensure ELASTICCURL is used for loading config objects --- salt/kibana/bin/so-kibana-config-load | 2 +- salt/manager/files/beacons.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index 00dc7f9ff..0bbcba375 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -12,4 +12,4 @@ sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" # Load saved objects -curl -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log +{{ ELASTICCURL }} -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log diff --git a/salt/manager/files/beacons.conf b/salt/manager/files/beacons.conf index c40fe83cb..fe40170ad 100644 --- a/salt/manager/files/beacons.conf +++ b/salt/manager/files/beacons.conf @@ -1,7 +1,7 @@ beacons: watch_sqlite_db: - files: - /opt/so/conf/kratos/db/sqlite.db: + /opt/so/conf/kratos/db/db.sqlite: mask: - modify - beacon_module: inotify From d1a185aaae34c650c0b9f251b29c87df909e47a6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Jun 2021 10:59:14 -0400 Subject: [PATCH 237/378] Further standardize whiptail titles --- setup/so-whiptail | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 458713591..258a7b0ac 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -121,7 +121,7 @@ whiptail_create_admin_user() { [ -n "$TESTING" ] && return - ADMINUSER=$(whiptail --title "Security Onion Install" --inputbox \ + ADMINUSER=$(whiptail --title "$whiptail_title" --inputbox \ "Please enter a username for a new system admin user: \nThe local onion account will be disabled during this install" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -132,7 +132,7 @@ whiptail_create_admin_user_password1() { [ -n "$TESTING" ] && return - ADMINPASS1=$(whiptail --title "Security Onion Install" --passwordbox \ + ADMINPASS1=$(whiptail --title "$whiptail_title" --passwordbox \ "Enter a password for $ADMINUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -143,7 +143,7 @@ whiptail_create_admin_user_password2() { [ -n "$TESTING" ] && return - ADMINPASS2=$(whiptail --title "Security Onion Install" --passwordbox \ + ADMINPASS2=$(whiptail --title "$whiptail_title" --passwordbox \ "Re-enter a password for $ADMINUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -155,7 +155,7 @@ whiptail_create_fleet_node_user() { [ -n "$TESTING" ] && return - FLEETNODEUSER=$(whiptail --title "Security Onion Install" --inputbox \ + FLEETNODEUSER=$(whiptail --title "$whiptail_title" --inputbox \ "Please enter an email for use as the username for the Fleet admin user:" 10 60 "$1" 3>&1 1>&2 2>&3) } @@ -164,7 +164,7 @@ whiptail_create_fleet_node_user_password1() { [ -n "$TESTING" ] && return - FLEETNODEPASSWD1=$(whiptail --title "Security Onion Install" --passwordbox \ + FLEETNODEPASSWD1=$(whiptail --title "$whiptail_title" --passwordbox \ "Enter a password for $FLEETNODEUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -175,7 +175,7 @@ whiptail_create_fleet_node_user_password2() { [ -n "$TESTING" ] && return - FLEETNODEPASSWD2=$(whiptail --title "Security Onion Install" --passwordbox \ + FLEETNODEPASSWD2=$(whiptail --title "$whiptail_title" --passwordbox \ "Re-enter a password for $FLEETNODEUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -195,7 +195,7 @@ whiptail_create_soremote_user_password1() { [ -n "$TESTING" ] && return - SOREMOTEPASS1=$(whiptail --title "Security Onion Install" --passwordbox \ + SOREMOTEPASS1=$(whiptail --title "$whiptail_title" --passwordbox \ "Enter a password for user soremote:" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -207,7 +207,7 @@ whiptail_create_soremote_user_password2() { [ -n "$TESTING" ] && return - SOREMOTEPASS2=$(whiptail --title "Security Onion Install" --passwordbox \ + SOREMOTEPASS2=$(whiptail --title "$whiptail_title" --passwordbox \ "Re-enter a password for user soremote:" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -219,7 +219,7 @@ whiptail_create_web_user() { [ -n "$TESTING" ] && return - WEBUSER=$(whiptail --title "Security Onion Install" --inputbox \ + WEBUSER=$(whiptail --title "$whiptail_title" --inputbox \ "Please enter an email address to create an administrator account for the web interface.\n\nThis will also be used for TheHive, Cortex, and Fleet." 12 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -230,7 +230,7 @@ whiptail_create_web_user_password1() { [ -n "$TESTING" ] && return - WEBPASSWD1=$(whiptail --title "Security Onion Install" --passwordbox \ + WEBPASSWD1=$(whiptail --title "$whiptail_title" --passwordbox \ "Enter a password for $WEBUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -241,7 +241,7 @@ whiptail_create_web_user_password2() { [ -n "$TESTING" ] && return - WEBPASSWD2=$(whiptail --title "Security Onion Install" --passwordbox \ + WEBPASSWD2=$(whiptail --title "$whiptail_title" --passwordbox \ "Re-enter a password for $WEBUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -671,7 +671,7 @@ whiptail_fleet_custom_hostname() { [ -n "$TESTING" ] && return - FLEETCUSTOMHOSTNAME=$(whiptail --title "Security Onion Install" --inputbox \ + FLEETCUSTOMHOSTNAME=$(whiptail --title "$whiptail_title" --inputbox \ "What FQDN should osquery clients use for connections to this Fleet node? Leave blank if the local system hostname will be used." 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -994,13 +994,13 @@ whiptail_management_nic() { filter_unused_nics - MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) + MNIC=$(whiptail --title "$whiptail_title" --radiolist "Please select your management NIC:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus while [ -z "$MNIC" ] do - MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) + MNIC=$(whiptail --title "$whiptail_title" --radiolist "Please select your management NIC:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus done @@ -1624,13 +1624,13 @@ whiptail_sensor_nics() { local list_type="checklist" fi - BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) + BNICS=$(whiptail --title "$whiptail_title" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus while [ -z "$BNICS" ] do - BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) + BNICS=$(whiptail --title "$whiptail_title" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus done @@ -1838,7 +1838,7 @@ whiptail_suricata_pins() { local PROCS=$lb_procs fi - SURIPINS=$(whiptail --noitem --title "Pin Suricata CPUS" --checklist "Please select $PROCS cores to pin Suricata to:" 20 75 12 "${filtered_core_str[@]}" 3>&1 1>&2 2>&3 ) + SURIPINS=$(whiptail --noitem --title "$whiptail_title" --checklist "Please select $PROCS cores to pin Suricata to:" 20 75 12 "${filtered_core_str[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1902,7 +1902,7 @@ whiptail_zeek_pins() { local PROCS=$lb_procs fi - ZEEKPINS=$(whiptail --noitem --title "Pin Zeek CPUS" --checklist "Please select $PROCS cores to pin Zeek to:" 20 75 12 "${cpu_core_list_whiptail[@]}" 3>&1 1>&2 2>&3 ) + ZEEKPINS=$(whiptail --noitem --title "$whiptail_title" --checklist "Please select $PROCS cores to pin Zeek to:" 20 75 12 "${cpu_core_list_whiptail[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 3aff3ac7e4757b44276f83d429c6ed8e028ca2d7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Jun 2021 11:00:20 -0400 Subject: [PATCH 238/378] Change logic to check for unmanaged nics Resolves issue mentioned in #4327 --- setup/so-whiptail | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 258a7b0ac..cbf74680f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1640,13 +1640,15 @@ whiptail_sensor_nics() { IFS=' ' read -ra BNICS <<< "$BNICS" for bond_nic in "${BNICS[@]}"; do - if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then - whiptail \ - --title "$whiptail_title" \ - --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ - 8 75 - exit - fi + for dev_status in "${nmcli_dev_status_list[@]}"; do + if [[ $dev_status == "${bond_nic}:unmanaged" ]]; then + whiptail \ + --title "$whiptail_title" \ + --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ + 8 75 + exit + fi + done done } From d1d09d4aab198b3781e62a0906d2bcfd3eb19969 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Jun 2021 14:20:52 -0400 Subject: [PATCH 239/378] Remove useless variable assignment --- setup/so-functions | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 19e0c804f..2a9554033 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -897,8 +897,6 @@ compare_main_nic_ip() { This is not a supported configuration, please remediate and rerun setup. EOM - [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]] || local whiptail_title="so-monitor-add" - [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 10 75 kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 fi From 316035910f02e361d0c679b42bca98896c443cff Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 3 Jun 2021 15:15:35 -0400 Subject: [PATCH 240/378] Remove inotify beacon due to it not functioning as documented; Add back so-user changes to sync upon so-user changes --- salt/common/tools/sbin/so-user | 5 +++++ salt/manager/files/beacons.conf | 7 ------- salt/manager/init.sls | 7 ------- 3 files changed, 5 insertions(+), 14 deletions(-) delete mode 100644 salt/manager/files/beacons.conf diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 15a71edfa..f1bfdc213 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -314,6 +314,7 @@ case "${operation}" in validateEmail "$email" updatePassword createUser "$email" + syncAll echo "Successfully added new user to SOC" check_container thehive && echo $password | so-thehive-user-add "$email" check_container fleet && echo $password | so-fleet-user-add "$email" @@ -329,6 +330,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateUser "$email" + syncAll echo "Successfully updated user" ;; @@ -337,6 +339,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateStatus "$email" 'active' + syncAll echo "Successfully enabled user" check_container thehive && so-thehive-user-enable "$email" true check_container fleet && so-fleet-user-enable "$email" true @@ -347,6 +350,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" updateStatus "$email" 'locked' + syncAll echo "Successfully disabled user" check_container thehive && so-thehive-user-enable "$email" false check_container fleet && so-fleet-user-enable "$email" false @@ -357,6 +361,7 @@ case "${operation}" in [[ "$email" == "" ]] && fail "Email address must be provided" deleteUser "$email" + syncAll echo "Successfully deleted user" check_container thehive && so-thehive-user-enable "$email" false check_container fleet && so-fleet-user-enable "$email" false diff --git a/salt/manager/files/beacons.conf b/salt/manager/files/beacons.conf deleted file mode 100644 index fe40170ad..000000000 --- a/salt/manager/files/beacons.conf +++ /dev/null @@ -1,7 +0,0 @@ -beacons: - watch_sqlite_db: - - files: - /opt/so/conf/kratos/db/db.sqlite: - mask: - - modify - - beacon_module: inotify diff --git a/salt/manager/init.sls b/salt/manager/init.sls index ca47af34f..e6e9c433b 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -123,13 +123,6 @@ syncesusers: - /opt/so/saltstack/local/salt/elasticsearch/files/users - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles -beacons_config: - file.managed: - - name: /etc/salt/minion.d/beacons.conf - - source: salt://manager/files/beacons.conf - - onchanges_in: - - service: salt_minion_service - {% else %} {{sls}}_state_not_allowed: From 416b38fc71714701e50b99ab28368cf494429d97 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 4 Jun 2021 11:24:58 -0400 Subject: [PATCH 241/378] Use cronjob to ensure user synchronization --- salt/common/tools/sbin/so-user | 10 ++++++++-- salt/soc/init.sls | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index f1bfdc213..4a7b6f0da 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -220,6 +220,13 @@ function syncElastic() { } function syncAll() { + if [[ -n "$STALE_MIN" ]]; then + staleCount=$(echo "select from identity_credentials where updated_at >= Datetime('now', '-${STALE_MIN} minutes');" \ + | sqlite3 "$databasePath") + if [[ "$staleCount" == "0" ]]; then + return 1 + fi + fi syncElastic } @@ -368,8 +375,7 @@ case "${operation}" in ;; "sync") - syncAll - echo "Synchronization complete" + syncAll && echo "Synchronization completed at $(date)" ;; "validate") diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 18fda41da..5fe403934 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -62,6 +62,11 @@ soccustom: - mode: 600 - template: jinja +sosyncusers: + cron.present: + - user: root + - name: 'STALE_MIN=1 /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' + so-soc: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-soc:{{ VERSION }} From e22421ec99f39ea5babbfac76a3bb53c35d19f79 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 4 Jun 2021 20:01:30 -0400 Subject: [PATCH 242/378] Refactor users/roles management via salt due to Salt's clobbering of the inode which breaks Docker mounts --- salt/common/tools/sbin/so-user | 47 +++++++++---------- salt/elasticsearch/init.sls | 20 +++++++- .../config/so/9100_output_osquery.conf.jinja | 2 +- 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 4a7b6f0da..877f245c8 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -129,13 +129,12 @@ function updatePassword() { fi } -function createElasticTmpFile() { +function createElasticFile() { filename=$1 - tmpFile=${filename}.tmp + tmpFile=${filename} truncate -s 0 "$tmpFile" chmod 600 "$tmpFile" chown "${esUID}:${esGID}" "$tmpFile" - echo "$tmpFile" } function syncElasticSystemUser() { @@ -166,26 +165,27 @@ function syncElasticSystemRole() { } function syncElastic() { - usersFileTmp=$(createElasticTmpFile "${elasticUsersFile}") - rolesFileTmp=$(createElasticTmpFile "${elasticRolesFile}") + createElasticFile "${elasticUsersFile}" + createElasticFile "${elasticRolesFile}" authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") - syncElasticSystemUser "$authPillarJson" "so_elastic_user" "$usersFileTmp" - syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesFileTmp" + syncElasticSystemUser "$authPillarJson" "so_elastic_user" "$elasticUsersFile" + syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$elasticRolesFile" - syncElasticSystemUser "$authPillarJson" "so_kibana_user" "$usersFileTmp" - syncElasticSystemRole "$authPillarJson" "so_kibana_user" "kibana_system" "$rolesFileTmp" + syncElasticSystemUser "$authPillarJson" "so_kibana_user" "$elasticUsersFile" + syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$elasticRolesFile" - syncElasticSystemUser "$authPillarJson" "so_logstash_user" "$usersFileTmp" - syncElasticSystemRole "$authPillarJson" "so_logstash_user" "logstash_system" "$rolesFileTmp" + syncElasticSystemUser "$authPillarJson" "so_logstash_user" "$elasticUsersFile" + syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$elasticRolesFile" - syncElasticSystemUser "$authPillarJson" "so_beats_user" "$usersFileTmp" - syncElasticSystemRole "$authPillarJson" "so_beats_user" "beats_system" "$rolesFileTmp" + syncElasticSystemUser "$authPillarJson" "so_beats_user" "$elasticUsersFile" + syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$elasticRolesFile" - syncElasticSystemUser "$authPillarJson" "so_monitor_user" "$usersFileTmp" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesFileTmp" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesFileTmp" + syncElasticSystemUser "$authPillarJson" "so_monitor_user" "$elasticUsersFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$elasticRolesFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$elasticRolesFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$elasticRolesFile" if [[ -f "$databasePath" ]]; then # Generate the new users file @@ -195,7 +195,7 @@ function syncElastic() { "order by ici.identifier;" | \ sqlite3 "$databasePath" | \ jq -r '.user + ":" + .data.hashed_password' \ - >> "$usersFileTmp" + >> "$elasticUsersFile" [[ $? != 0 ]] && fail "Unable to read credential hashes from database" # Generate the new users_roles file @@ -205,29 +205,26 @@ function syncElastic() { "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ "order by ici.identifier;" | \ sqlite3 "$databasePath" \ - >> "$rolesFileTmp" + >> "$elasticRolesFile" [[ $? != 0 ]] && fail "Unable to read credential IDs from database" else echo "Database file does not exist yet, skipping users export" fi - # Move the temp files over onto the final files - mv -f "$usersFileTmp" "$elasticUsersFile" - [[ $? != 0 ]] && fail "Unable to create users file: $elasticUsersFile" - - mv -f "$rolesFileTmp" "$elasticRolesFile" - [[ $? != 0 ]] && fail "Unable to create users file: $elasticRolesFile" + echo "Applying elastic state..." + salt-call state.apply elasticsearch queue=True } function syncAll() { if [[ -n "$STALE_MIN" ]]; then - staleCount=$(echo "select from identity_credentials where updated_at >= Datetime('now', '-${STALE_MIN} minutes');" \ + staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${STALE_MIN} minutes');" \ | sqlite3 "$databasePath") if [[ "$staleCount" == "0" ]]; then return 1 fi fi syncElastic + return 0 } function listUsers() { diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 0bf442587..05b58b79a 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -173,7 +173,7 @@ eslogdir: auth_users: file.managed: - - name: /opt/so/conf/elasticsearch/users + - name: /opt/so/conf/elasticsearch/users.tmp - source: salt://elasticsearch/files/users - user: 930 - group: 930 @@ -181,12 +181,28 @@ auth_users: auth_users_roles: file.managed: - - name: /opt/so/conf/elasticsearch/users_roles + - name: /opt/so/conf/elasticsearch/users_roles.tmp - source: salt://elasticsearch/files/users_roles - user: 930 - group: 930 - mode: 600 +auth_users_inode: + require: + - file: auth_users + cmd.run: + - name: cat /opt/so/conf/elasticsearch/users.tmp > /opt/so/conf/elasticsearch/users && chown 930:930 /opt/so/conf/elasticsearch/users && chmod 600 /opt/so/conf/elasticsearch/users + - onchanges: + - file: /opt/so/conf/elasticsearch/users.tmp + +auth_users_roles_inode: + require: + - file: auth_users_roles + cmd.run: + - name: cat /opt/so/conf/elasticsearch/users_roles.tmp > /opt/so/conf/elasticsearch/users_roles && chown 930:930 /opt/so/conf/elasticsearch/users_roles && chmod 600 /opt/so/conf/elasticsearch/users_roles + - onchanges: + - file: /opt/so/conf/elasticsearch/users_roles.tmp + so-elasticsearch: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }} diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 7ed8c58e5..43596c1cd 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -4,7 +4,7 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_pass:pass', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { if [module] =~ "osquery" and "live_query" not in [dataset] { elasticsearch { From ba29b5e0362c471ac5e00a89f89083bc9d72f09b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 4 Jun 2021 21:56:41 -0400 Subject: [PATCH 243/378] Do not apply salt state if already applying a state --- salt/common/tools/sbin/so-user | 6 ++++-- salt/manager/init.sls | 2 ++ salt/reactor/kratos.sls | 5 ----- 3 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 salt/reactor/kratos.sls diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 877f245c8..9873764d9 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -211,8 +211,10 @@ function syncElastic() { echo "Database file does not exist yet, skipping users export" fi - echo "Applying elastic state..." - salt-call state.apply elasticsearch queue=True + if [[ -z "$SKIP_STATE_APPLY" ]]; then + echo "Applying elastic state..." + salt-call state.apply elasticsearch queue=True + fi } function syncAll() { diff --git a/salt/manager/init.sls b/salt/manager/init.sls index e6e9c433b..0b54fbc2f 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -119,6 +119,8 @@ elastic_curl_config_distributed: syncesusers: cmd.run: - name: so-user sync + - env: + - SKIP_STATE_APPLY: 'true' - creates: - /opt/so/saltstack/local/salt/elasticsearch/files/users - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles diff --git a/salt/reactor/kratos.sls b/salt/reactor/kratos.sls deleted file mode 100644 index fb90cc164..000000000 --- a/salt/reactor/kratos.sls +++ /dev/null @@ -1,5 +0,0 @@ -so_user_sync: - local.cmd.run: - - tgt: {{ data['data']['id'] }} - - arg: - - /usr/sbin/so-user sync From 84141082abc86080356617dc02b6aa564a541132 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 5 Jun 2021 08:41:48 -0400 Subject: [PATCH 244/378] Avoid applying state when adding web user --- setup/so-functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 33e0514fc..426406c78 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -120,7 +120,9 @@ add_web_user() { wait_for_file /opt/so/conf/kratos/db/db.sqlite 30 5 { echo "Attempting to add administrator user for web interface..."; + export SKIP_STATE_APPLY=true echo "$WEBPASSWD1" | /usr/sbin/so-user add "$WEBUSER"; + unset SKIP_STATE_APPLY echo "Add user result: $?"; } >> "/root/so-user-add.log" 2>&1 } From 11b2b2a893e23b99eb227cef575026a23a0026a7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 6 Jun 2021 05:42:34 -0400 Subject: [PATCH 245/378] Switch ownership of curl config to socore --- salt/manager/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 0b54fbc2f..b45ffdf46 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -113,6 +113,8 @@ elastic_curl_config_distributed: - source: salt://elasticsearch/files/curl.config.template - template: jinja - mode: 600 + - user: 939 + - group: 939 - show_changes: False # Must run before elasticsearch docker container is started! From bebba7d280d39b91ccc177e3934aafffa6111d56 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 6 Jun 2021 07:43:53 -0400 Subject: [PATCH 246/378] Switch ownership of curl config to socore --- salt/common/init.sls | 2 ++ salt/manager/init.sls | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 9d20de62b..25e8a62cb 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -173,6 +173,8 @@ elastic_curl_config: - name: /opt/so/conf/elasticsearch/curl.config - source: salt://elasticsearch/curl.config - mode: 600 + - user: 939 + - group: 939 - show_changes: False - makedirs: True diff --git a/salt/manager/init.sls b/salt/manager/init.sls index b45ffdf46..0b54fbc2f 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -113,8 +113,6 @@ elastic_curl_config_distributed: - source: salt://elasticsearch/files/curl.config.template - template: jinja - mode: 600 - - user: 939 - - group: 939 - show_changes: False # Must run before elasticsearch docker container is started! From fdab17a3b9adc8ce16e86ee7965196248144af9c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 6 Jun 2021 16:36:35 -0400 Subject: [PATCH 247/378] Due to dir ownership restrictions need to run crossthestreams and eval as root --- salt/common/init.sls | 2 -- salt/utility/init.sls | 2 -- 2 files changed, 4 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 25e8a62cb..9d20de62b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -173,8 +173,6 @@ elastic_curl_config: - name: /opt/so/conf/elasticsearch/curl.config - source: salt://elasticsearch/curl.config - mode: 600 - - user: 939 - - group: 939 - show_changes: False - makedirs: True diff --git a/salt/utility/init.sls b/salt/utility/init.sls index 0b4f0f2c3..1ff69ae71 100644 --- a/salt/utility/init.sls +++ b/salt/utility/init.sls @@ -10,7 +10,6 @@ crossclusterson: cmd.script: - shell: /bin/bash - cwd: /opt/so - - runas: socore - source: salt://utility/bin/crossthestreams - template: jinja - defaults: @@ -22,7 +21,6 @@ fixsearch: cmd.script: - shell: /bin/bash - cwd: /opt/so - - runas: socore - source: salt://utility/bin/eval - template: jinja - defaults: From 14aa9805b443652a17bd747f0df8307379273575 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 6 Jun 2021 18:20:57 -0400 Subject: [PATCH 248/378] Stop failing an install because salt is already running when a highstate is applied at 95% --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index ed6899db8..f2db4d84e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -918,7 +918,7 @@ echo "1" > /root/accept_changes checkin_at_boot >> $setup_log 2>&1 set_progress_str 95 'Verifying setup' - salt-call -l info state.highstate >> $setup_log 2>&1 + salt-call -l info state.highstate queue=True >> $setup_log 2>&1 } | progress From e53f2217ecc26316d462685f301f36f760888c58 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 7 Jun 2021 17:24:22 -0400 Subject: [PATCH 249/378] Add so-elasticsearch-query tool --- salt/common/tools/sbin/so-elasticsearch-query | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 salt/common/tools/sbin/so-elasticsearch-query diff --git a/salt/common/tools/sbin/so-elasticsearch-query b/salt/common/tools/sbin/so-elasticsearch-query new file mode 100644 index 000000000..ffae93a0d --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-query @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +. /usr/sbin/so-common + +if [[ $# -lt 1 ]]; then + echo "Submit a cURL request to the local Security Onion Elasticsearch host." + echo "" + echo "Usage: $0 [ARGS,...]" + echo "" + echo "Where " + echo " PATH represents the elastic function being requested." + echo " ARGS is used to specify additional, optional curl parameters." + echo "" + echo "Examples:" + echo " $0 / | jq" + echo " $0 '*:so-*/_search' -d '{\"query\": {\"match_all\": {}},\"size\": 1}' | jq" + exit 1 +fi + +PATH=$1 +shift + +{{ ELASTICCURL }} -s -k -L -H "Content-Type: application/json" "https://localhost:9200/${PATH}" "$@" From 343c47d67abdc73cd1fd94b50d76d36c09bed36a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 7 Jun 2021 17:26:07 -0400 Subject: [PATCH 250/378] Add so-elasticsearch-query tool --- salt/common/tools/sbin/so-elasticsearch-query | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-query b/salt/common/tools/sbin/so-elasticsearch-query index ffae93a0d..c56f5671f 100644 --- a/salt/common/tools/sbin/so-elasticsearch-query +++ b/salt/common/tools/sbin/so-elasticsearch-query @@ -26,7 +26,7 @@ if [[ $# -lt 1 ]]; then echo " ARGS is used to specify additional, optional curl parameters." echo "" echo "Examples:" - echo " $0 / | jq" + echo " $0 /" echo " $0 '*:so-*/_search' -d '{\"query\": {\"match_all\": {}},\"size\": 1}' | jq" exit 1 fi From 56eb220ed6eedfd25e912beffeaf5a00c2f26f1f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 09:52:05 -0400 Subject: [PATCH 251/378] Revert to SO taxonomy for zeek and suricata --- salt/filebeat/etc/filebeat.yml | 77 ++++++++++++++++++++++++ salt/filebeat/securityoniondefaults.yaml | 20 +----- 2 files changed, 78 insertions(+), 19 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 46a59f772..f933cee2e 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -111,7 +111,84 @@ filebeat.inputs: fields_under_root: true {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} + {%- if ZEEKVER != 'SURICATA' %} + {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} +- type: log + paths: + - /nsm/zeek/logs/current/{{ LOGNAME }}.log + fields: + module: zeek + dataset: {{ LOGNAME }} + category: network + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + + fields_under_root: true + clean_removed: true + close_removed: false +- type: log + paths: + - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log + fields: + module: zeek + dataset: {{ LOGNAME }} + category: network + imported: true + processors: + - add_tags: + tags: ["import"] + - dissect: + tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}" + field: "log.file.path" + target_prefix: "" + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + + fields_under_root: true + clean_removed: false + close_removed: false + {%- endfor %} + {%- endif %} + +- type: log + paths: + - /nsm/suricata/eve*.json + fields: + module: suricata + dataset: common + category: network + + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + + fields_under_root: true + clean_removed: false + close_removed: false + +- type: log + paths: + - /nsm/import/*/suricata/eve*.json + fields: + module: suricata + dataset: common + category: network + imported: true + processors: + - add_tags: + tags: ["import"] + - dissect: + tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}" + field: "log.file.path" + target_prefix: "" + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + + fields_under_root: true + clean_removed: false + close_removed: false {%- if STRELKAENABLED == 1 %} - type: log paths: diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index f503e5de1..cd215e242 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -23,22 +23,4 @@ securityonion_filebeat: var.paths: ["/logs/redis.log"] slowlog: enabled: false - suricata: - eve: - enabled: true - var.paths: ["/nsm/suricata/eve*.json"] - {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} - {%- if ZEEKVER != 'SURICATA' %} - zeek: - {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} - {% if LOGNAME in ZEEKLOGLOOKUP.keys() %} - {% set FILESET = ZEEKLOGLOOKUP.get(LOGNAME) %} - {% else %} - {% set FILESET = LOGNAME %} - {% endif %} - {{ FILESET }}: - enabled: true - var.paths: ["/nsm/zeek/logs/current/{{ LOGNAME }}.log"] - {%- endfor %} - {%- endif %} - {%- endif %} + \ No newline at end of file From d2381b0209cf994f26431344dbb3dca5cbad9343 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 8 Jun 2021 11:03:56 -0400 Subject: [PATCH 252/378] Ensure empty/aborted users/roles files do not get copied onto final filenames --- salt/common/tools/sbin/so-user | 47 ++++++++++++++++++++-------------- salt/manager/init.sls | 1 + 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 9873764d9..d21ecc58c 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -165,27 +165,29 @@ function syncElasticSystemRole() { } function syncElastic() { - createElasticFile "${elasticUsersFile}" - createElasticFile "${elasticRolesFile}" + usersTmpFile="${elasticUsersFile}.tmp" + rolesTmpFile="${elasticRolesFile}.tmp" + createElasticFile "${usersTmpFile}" + createElasticFile "${rolesTmpFile}" authPillarJson=$(lookup_salt_value "auth" "elasticsearch" "pillar" "json") - syncElasticSystemUser "$authPillarJson" "so_elastic_user" "$elasticUsersFile" - syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$elasticRolesFile" + syncElasticSystemUser "$authPillarJson" "so_elastic_user" "$usersTmpFile" + syncElasticSystemRole "$authPillarJson" "so_elastic_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_kibana_user" "$elasticUsersFile" - syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$elasticRolesFile" + syncElasticSystemUser "$authPillarJson" "so_kibana_user" "$usersTmpFile" + syncElasticSystemRole "$authPillarJson" "so_kibana_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_logstash_user" "$elasticUsersFile" - syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$elasticRolesFile" + syncElasticSystemUser "$authPillarJson" "so_logstash_user" "$usersTmpFile" + syncElasticSystemRole "$authPillarJson" "so_logstash_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_beats_user" "$elasticUsersFile" - syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$elasticRolesFile" + syncElasticSystemUser "$authPillarJson" "so_beats_user" "$usersTmpFile" + syncElasticSystemRole "$authPillarJson" "so_beats_user" "superuser" "$rolesTmpFile" - syncElasticSystemUser "$authPillarJson" "so_monitor_user" "$elasticUsersFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$elasticRolesFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$elasticRolesFile" - syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$elasticRolesFile" + syncElasticSystemUser "$authPillarJson" "so_monitor_user" "$usersTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_collector" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "remote_monitoring_agent" "$rolesTmpFile" + syncElasticSystemRole "$authPillarJson" "so_monitor_user" "monitoring_user" "$rolesTmpFile" if [[ -f "$databasePath" ]]; then # Generate the new users file @@ -195,7 +197,7 @@ function syncElastic() { "order by ici.identifier;" | \ sqlite3 "$databasePath" | \ jq -r '.user + ":" + .data.hashed_password' \ - >> "$elasticUsersFile" + >> "$usersTmpFile" [[ $? != 0 ]] && fail "Unable to read credential hashes from database" # Generate the new users_roles file @@ -205,15 +207,22 @@ function syncElastic() { "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ "order by ici.identifier;" | \ sqlite3 "$databasePath" \ - >> "$elasticRolesFile" + >> "$rolesTmpFile" [[ $? != 0 ]] && fail "Unable to read credential IDs from database" else echo "Database file does not exist yet, skipping users export" fi - if [[ -z "$SKIP_STATE_APPLY" ]]; then - echo "Applying elastic state..." - salt-call state.apply elasticsearch queue=True + if [[ -s "${usersTmpFile}" ]]; then + mv "${usersTmpFile}" "${elasticUsersFile}" + mv "${rolesTmpFile}" "${elasticRolesFile}" + + if [[ -z "$SKIP_STATE_APPLY" ]]; then + echo "Applying elastic state..." + salt-call state.apply elasticsearch queue=True + fi + else + echo "Generated users/roles files are incomplete; aborting." fi } diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 0b54fbc2f..17b1ad9e0 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -124,6 +124,7 @@ syncesusers: - creates: - /opt/so/saltstack/local/salt/elasticsearch/files/users - /opt/so/saltstack/local/salt/elasticsearch/files/users_roles + - show_changes: False {% else %} From 3e138cbc6dda28b3de009239f61b0e6c63f6a169 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 13:14:46 -0400 Subject: [PATCH 253/378] Revert to SO taxonomy for zeek and suricata --- .../config/so/9000_output_zeek.conf.jinja | 25 +++++++++---------- .../config/so/9050_output_elasticsearch.jinja | 24 ++++++++---------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index 486d22bfe..d17dc2b22 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -4,17 +4,16 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [event][module] == 'zeek' { - elasticsearch { - id => "zeek_logs" - pipeline => "filebeat-%{[agent][version]}-zeek-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-zeek-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } + if [module] =~ "zeek" and "import" not in [tags] { + elasticsearch { + pipeline => "%{module}.%{dataset}" + hosts => "{{ ES }}" + index => "so-zeek" + template_name => "so-zeek" + template => "/templates/so-zeek-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } } diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja index eb0d8ef0c..5013bafc1 100644 --- a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja @@ -4,17 +4,15 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [event][module] == 'elasticsearch' { - elasticsearch { - id => "elastic_logs" - pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } + if [module] =~ "suricata" and "import" not in [tags] { + elasticsearch { + pipeline => "%{module}.%{dataset}" + hosts => "{{ ES }}" + index => "so-ids" + template_name => "so-ids" + template => "/templates/so-ids-template.json" + ssl => true + ssl_certificate_verification => false + } + } } From a959ec1eb1787b736df121c362863e48fb3f11a5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 13:23:31 -0400 Subject: [PATCH 254/378] Revert to SO taxonomy for zeek and suricata --- .../config/so/9050_output_elasticsearch.jinja | 24 ++++++++++--------- .../config/so/9400_output_suricata.conf.jinja | 8 +++---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja index 5013bafc1..eb0d8ef0c 100644 --- a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja @@ -4,15 +4,17 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [module] =~ "suricata" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ ES }}" - index => "so-ids" - template_name => "so-ids" - template => "/templates/so-ids-template.json" - ssl => true - ssl_certificate_verification => false - } - } + if [event][module] == 'elasticsearch' { + elasticsearch { + id => "elastic_logs" + pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 999b270cf..b56f35a29 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -4,15 +4,15 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [event][module] =~ "suricata" and "import" not in [tags] { + if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { - pipeline => "filebeat-%{[agent][version]}-suricata-%{[fileset][name]}-pipeline" + pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-ids" - template_name => "so-ids-%{+YYYY.MM.dd}" + template_name => "so-ids" template => "/templates/so-ids-template.json" ssl => true ssl_certificate_verification => false } } -} +} \ No newline at end of file From 88eea03f97cf8d87c5b18d9bf343c1b8bd15f810 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 13:36:50 -0400 Subject: [PATCH 255/378] Revert to SO taxonomy for zeek and suricata --- salt/soc/files/soc/hunt.queries.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index c220060dd..2aaef8e59 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -2,8 +2,8 @@ { "name": "Default Query", "description": "Show all events grouped by the origin host", "query": "* | groupby observer.name"}, { "name": "Log Type", "description": "Show all events grouped by module and dataset", "query": "* | groupby event.module event.dataset"}, { "name": "Elastalerts", "description": "", "query": "_type:elastalert | groupby rule.name"}, - { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.kind: alert | groupby event.module"}, - { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert", "query": "event.category: network AND event.kind: alert | groupby rule.category rule.gid rule.uuid rule.name"}, + { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.dataset: alert | groupby event.module"}, + { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert", "query": "event.category: network AND event.dataset: alert | groupby rule.category rule.gid rule.uuid rule.name"}, { "name": "Wazuh/OSSEC Alerts", "description": "Show all Wazuh alerts at Level 5 or higher grouped by category", "query": "event.module:ossec AND event.dataset:alert AND rule.level:>4 | groupby rule.category rule.name"}, { "name": "Wazuh/OSSEC Alerts", "description": "Show all Wazuh alerts at Level 4 or lower grouped by category", "query": "event.module:ossec AND event.dataset:alert AND rule.level:<5 | groupby rule.category rule.name"}, { "name": "Wazuh/OSSEC Users and Commands", "description": "Show all Wazuh alerts grouped by username and command line", "query": "event.module:ossec AND event.dataset:alert | groupby user.escalated.keyword process.command_line"}, @@ -11,20 +11,20 @@ { "name": "Sysmon Events", "description": "Show all Sysmon logs grouped by event type", "query": "event.module:sysmon | groupby event.dataset"}, { "name": "Sysmon Usernames", "description": "Show all Sysmon logs grouped by username", "query": "event.module:sysmon | groupby event.dataset, user.name.keyword"}, { "name": "Strelka", "description": "Show all Strelka logs grouped by file type", "query": "event.module:strelka | groupby file.mime_type"}, - { "name": "Zeek Notice", "description": "Show notices from Zeek", "query": "event.dataset:zeek.notice | groupby notice.note notice.message"}, - { "name": "Connections", "description": "Connections grouped by IP and Port", "query": "event.dataset:zeek.connection | groupby source.ip destination.ip network.protocol destination.port"}, - { "name": "Connections", "description": "Connections grouped by Service", "query": "event.dataset:zeek.connection | groupby network.protocol destination.port"}, - { "name": "Connections", "description": "Connections grouped by destination country", "query": "event.dataset:zeek.connection | groupby destination.geo.country_name"}, - { "name": "Connections", "description": "Connections grouped by source country", "query": "event.dataset:zeek.connection | groupby source.geo.country_name"}, + { "name": "Zeek Notice", "description": "Show notices from Zeek", "query": "event.dataset:notice | groupby notice.note notice.message"}, + { "name": "Connections", "description": "Connections grouped by IP and Port", "query": "event.dataset:conn | groupby source.ip destination.ip network.protocol destination.port"}, + { "name": "Connections", "description": "Connections grouped by Service", "query": "event.dataset:conn | groupby network.protocol destination.port"}, + { "name": "Connections", "description": "Connections grouped by destination country", "query": "event.dataset:conn | groupby destination.geo.country_name"}, + { "name": "Connections", "description": "Connections grouped by source country", "query": "event.dataset:conn | groupby source.geo.country_name"}, { "name": "DCE_RPC", "description": "DCE_RPC grouped by operation", "query": "event.dataset:dce_rpc | groupby dce_rpc.operation"}, { "name": "DHCP", "description": "DHCP leases", "query": "event.dataset:dhcp | groupby host.hostname client.address"}, { "name": "DHCP", "description": "DHCP grouped by message type", "query": "event.dataset:dhcp | groupby dhcp.message_types"}, { "name": "DNP3", "description": "DNP3 grouped by reply", "query": "event.dataset:dnp3 | groupby dnp3.fc_reply"}, - { "name": "DNS", "description": "DNS queries grouped by port", "query": "_exists_:dns.id | groupby dns.question.name destination.port"}, - { "name": "DNS", "description": "DNS queries grouped by type", "query": "_exists_:dns.id | groupby dns.question.type destination.port"}, - { "name": "DNS", "description": "DNS queries grouped by response code", "query": "_exists_:dns.id | groupby dns.response_code destination.port"}, - { "name": "DNS", "description": "DNS highest registered domain", "query": "_exists_:dns.id | groupby dns.question.top_level_domain destination.port"}, - { "name": "DNS", "description": "DNS grouped by parent domain", "query": "_exists_:dns.id | groupby dns.question.registered_domain destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by port", "query": "event.dataset:dns | groupby dns.query.name destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by type", "query": "event.dataset:dns | groupby dns.query.type_name destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by response code", "query": "event.dataset:dns | groupby dns.response.code_name destination.port"}, + { "name": "DNS", "description": "DNS highest registered domain", "query": "event.dataset:dns | groupby dns.highest_registered_domain.keyword destination.port"}, + { "name": "DNS", "description": "DNS grouped by parent domain", "query": "event.dataset:dns | groupby dns.parent_domain.keyword destination.port"}, { "name": "DPD", "description": "Dynamic Protocol Detection errors", "query": "event.dataset:dpd | groupby error.reason"}, { "name": "Files", "description": "Files grouped by mimetype", "query": "event.dataset:file | groupby file.mime_type source.ip"}, { "name": "Files", "description": "Files grouped by source", "query": "event.dataset:file | groupby file.source source.ip"}, @@ -63,4 +63,4 @@ { "name": "x509", "description": "x.509 grouped by name and issuer", "query": "event.dataset:x509 | groupby x509.san_dns x509.certificate.issuer"}, { "name": "x509", "description": "x.509 grouped by name and subject", "query": "event.dataset:x509 | groupby x509.san_dns x509.certificate.subject"}, { "name": "Firewall", "description": "Firewall events grouped by action", "query": "event.dataset:firewall | groupby rule.action"} - ] + ] \ No newline at end of file From 33db9023eb2334262a0bafaba9bc5780777fb6b4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 13:50:39 -0400 Subject: [PATCH 256/378] Revert to SO taxonomy for zeek and suricata --- pillar/logstash/search.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 2da8e6c59..8306ec58b 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -7,6 +7,7 @@ logstash: - so/9000_output_zeek.conf.jinja - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja + - so/9050_output_elasticsearch.jinja - so/9100_output_osquery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja From dd73ad544cbfa811cbc8dacf83a5fec1144310e0 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 9 Jun 2021 09:06:29 -0400 Subject: [PATCH 257/378] Rename PATH var to avoid collision with OS PATH var; wrapped password var in quotes to support spaces in Fleet/TheHive passwords --- salt/common/tools/sbin/so-elasticsearch-query | 4 ++-- salt/common/tools/sbin/so-user | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-query b/salt/common/tools/sbin/so-elasticsearch-query index c56f5671f..80dd6ee2e 100644 --- a/salt/common/tools/sbin/so-elasticsearch-query +++ b/salt/common/tools/sbin/so-elasticsearch-query @@ -31,7 +31,7 @@ if [[ $# -lt 1 ]]; then exit 1 fi -PATH=$1 +QUERYPATH=$1 shift -{{ ELASTICCURL }} -s -k -L -H "Content-Type: application/json" "https://localhost:9200/${PATH}" "$@" +{{ ELASTICCURL }} -s -k -L -H "Content-Type: application/json" "https://localhost:9200/${QUERYPATH}" "$@" diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index d21ecc58c..0f36deae2 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -165,6 +165,7 @@ function syncElasticSystemRole() { } function syncElastic() { + echo "Syncing users between SOC and Elastic..." usersTmpFile="${elasticUsersFile}.tmp" rolesTmpFile="${elasticRolesFile}.tmp" createElasticFile "${usersTmpFile}" @@ -331,8 +332,8 @@ case "${operation}" in createUser "$email" syncAll echo "Successfully added new user to SOC" - check_container thehive && echo $password | so-thehive-user-add "$email" - check_container fleet && echo $password | so-fleet-user-add "$email" + check_container thehive && echo "$password" | so-thehive-user-add "$email" + check_container fleet && echo "$password" | so-fleet-user-add "$email" ;; "list") From a0c65e2333887e7f75f5e8346deff1896c2e68ce Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 9 Jun 2021 09:38:50 -0400 Subject: [PATCH 258/378] Ensure elastic minions also update their auth files --- salt/common/tools/sbin/so-user | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 0f36deae2..ea278e941 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -221,6 +221,7 @@ function syncElastic() { if [[ -z "$SKIP_STATE_APPLY" ]]; then echo "Applying elastic state..." salt-call state.apply elasticsearch queue=True + salt -C 'G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True fi else echo "Generated users/roles files are incomplete; aborting." From 264080546c97d0826b6c5576678ef5d205775a43 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 11:37:27 -0400 Subject: [PATCH 259/378] Add log path --- salt/filebeat/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 5cabaf828..175213008 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -119,6 +119,7 @@ so-filebeat: - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro + - /opt/so/log:/logs:ro - port_bindings: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp From 579ff8c0b41291a369b067e2942a057dd1aa2781 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 11:40:17 -0400 Subject: [PATCH 260/378] Add verbosity to checkin --- salt/common/tools/sbin/so-checkin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-checkin b/salt/common/tools/sbin/so-checkin index 90eae6b1e..c70701b71 100755 --- a/salt/common/tools/sbin/so-checkin +++ b/salt/common/tools/sbin/so-checkin @@ -17,4 +17,4 @@ . /usr/sbin/so-common -salt-call state.highstate +salt-call state.highstate -linfo From 4c90a0ed7e02550ec68b48dc2eee479402256c8b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 12:04:32 -0400 Subject: [PATCH 261/378] Add templates for SO logs --- ...ja => 9050_output_elasticsearch_log.jinja} | 0 .../config/so/9050_output_kibana_log.jinja | 20 +++++++++++++++++++ .../config/so/9050_output_logstash_log.jinja | 20 +++++++++++++++++++ .../config/so/9050_output_redis_log.jinja | 20 +++++++++++++++++++ 4 files changed, 60 insertions(+) rename salt/logstash/pipelines/config/so/{9050_output_elasticsearch.jinja => 9050_output_elasticsearch_log.jinja} (100%) create mode 100644 salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja create mode 100644 salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja create mode 100644 salt/logstash/pipelines/config/so/9050_output_redis_log.jinja diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja similarity index 100% rename from salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja rename to salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja diff --git a/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja b/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja new file mode 100644 index 000000000..bf79e1073 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja @@ -0,0 +1,20 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +output { + if [event][module] == 'kibana' { + elasticsearch { + id => "kibana_logs" + pipeline => "filebeat-%{[agent][version]}-kibana-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} diff --git a/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja b/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja new file mode 100644 index 000000000..df6fba0e0 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja @@ -0,0 +1,20 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +output { + if [event][module] == 'logstash' { + elasticsearch { + id => "logstash_logs" + pipeline => "filebeat-%{[agent][version]}-logstash-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} diff --git a/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja b/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja new file mode 100644 index 000000000..9cc37de35 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja @@ -0,0 +1,20 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +output { + if [event][module] == 'redis' { + elasticsearch { + id => "redis_logs" + pipeline => "filebeat-%{[agent][version]}-redis-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} From 1c7741fdbe6055ec123c8bd4c28370bb2db5a2b2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 12:38:19 -0400 Subject: [PATCH 262/378] Add templates for SO logs --- salt/elasticsearch/templates/so/so-common-template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 2d1ef2a21..54e786cdc 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -1,5 +1,5 @@ { - "index_patterns": ["so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"], + "index_patterns": ["so-grid-*","so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"], "version":50001, "order":10, "settings":{ From 7fba904f750273d84ea53c7ef2b0c2eb5de8acae Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 15:32:39 -0400 Subject: [PATCH 263/378] Dynamix Pipelines take 1 --- pillar/logstash/search.sls | 2 +- .../pipelines/config/so/0009_input_beats.conf | 5 +++++ .../so/9050_output_elasticsearch_log.jinja | 20 ------------------- ...=> 9050_output_filebeatmodules.conf.jinja} | 10 +++++----- .../config/so/9050_output_kibana_log.jinja | 20 ------------------- .../config/so/9050_output_logstash_log.jinja | 20 ------------------- 6 files changed, 11 insertions(+), 66 deletions(-) delete mode 100644 salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja rename salt/logstash/pipelines/config/so/{9050_output_redis_log.jinja => 9050_output_filebeatmodules.conf.jinja} (70%) delete mode 100644 salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja delete mode 100644 salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 8306ec58b..10fab2ed1 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -7,7 +7,7 @@ logstash: - so/9000_output_zeek.conf.jinja - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja - - so/9050_output_elasticsearch.jinja + - so/9050_output_filebeatmodules.conf.jinja - so/9100_output_osquery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf index a5c1d491c..31ba798c9 100644 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ b/salt/logstash/pipelines/config/so/0009_input_beats.conf @@ -3,4 +3,9 @@ input { port => "5044" tags => [ "beat-ext" ] } +} +filter { + mutate { + rename => {“@metadata” => “metadata”} + } } \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja deleted file mode 100644 index eb0d8ef0c..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja +++ /dev/null @@ -1,20 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -output { - if [event][module] == 'elasticsearch' { - elasticsearch { - id => "elastic_logs" - pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja similarity index 70% rename from salt/logstash/pipelines/config/so/9050_output_redis_log.jinja rename to salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja index 9cc37de35..20e9f0c0a 100644 --- a/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja @@ -4,12 +4,12 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [event][module] == 'redis' { + if [metadata][pipeline] { elasticsearch { - id => "redis_logs" - pipeline => "filebeat-%{[agent][version]}-redis-%{[fileset][name]}-pipeline" + id => "filebeat_modules_metadata_pipeline" + pipeline => "%{[metadata][pipeline]}" hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" + index => "so-%{[event][module]}-%{+YYYY.MM.dd}" template_name => "so-common" template => "/templates/so-common-template.json" template_overwrite => true @@ -17,4 +17,4 @@ output { ssl_certificate_verification => false } } -} +} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja b/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja deleted file mode 100644 index bf79e1073..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja +++ /dev/null @@ -1,20 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -output { - if [event][module] == 'kibana' { - elasticsearch { - id => "kibana_logs" - pipeline => "filebeat-%{[agent][version]}-kibana-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja b/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja deleted file mode 100644 index df6fba0e0..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja +++ /dev/null @@ -1,20 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -output { - if [event][module] == 'logstash' { - elasticsearch { - id => "logstash_logs" - pipeline => "filebeat-%{[agent][version]}-logstash-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } -} From 89a02383b8705bc42c944aef694d31d3a94c7f00 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 9 Jun 2021 16:31:32 -0400 Subject: [PATCH 264/378] Correct cronjob path issue for sysctl; suppress diff outputs from users/roles files; suppress salt state output during user sync --- salt/common/tools/sbin/so-user | 11 +++++++---- salt/elasticsearch/init.sls | 2 ++ salt/soc/init.sls | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index ea278e941..54bcf7f71 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -219,12 +219,15 @@ function syncElastic() { mv "${rolesTmpFile}" "${elasticRolesFile}" if [[ -z "$SKIP_STATE_APPLY" ]]; then - echo "Applying elastic state..." - salt-call state.apply elasticsearch queue=True - salt -C 'G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True + echo "Applying elastic state locally; This can take a few minutes..." + echo "Applying elastic state locally at $(date)" >> /opt/so/log/soc/sync.log 2>&1 + salt-call state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 + echo "Applying elastic state to elastic minions; This can take a few minutes..." + echo "Applying elastic state to elastic minions at $(date)" >> /opt/so/log/soc/sync.log 2>&1 + salt -C 'G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 fi else - echo "Generated users/roles files are incomplete; aborting." + echo "Newly generated users/roles files are incomplete; aborting." fi } diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 05b58b79a..4045fa10f 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -178,6 +178,7 @@ auth_users: - user: 930 - group: 930 - mode: 600 + - show_changes: False auth_users_roles: file.managed: @@ -186,6 +187,7 @@ auth_users_roles: - user: 930 - group: 930 - mode: 600 + - show_changes: False auth_users_inode: require: diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 5fe403934..01b57c8ce 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -65,7 +65,7 @@ soccustom: sosyncusers: cron.present: - user: root - - name: 'STALE_MIN=1 /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' + - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin STALE_MIN=1 /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' so-soc: docker_container.running: From 1bef1d565235d24a21cd62394576616fcc421406 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 10 Jun 2021 08:16:00 -0400 Subject: [PATCH 265/378] Update to apply to any so-prefixed index --- salt/elasticsearch/templates/so/so-common-template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 54e786cdc..3e47fd780 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -1,5 +1,5 @@ { - "index_patterns": ["so-grid-*","so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"], + "index_patterns": ["so-*"], "version":50001, "order":10, "settings":{ From 12d4d4a4f757bef7b6287958bba4c0847eb25dd2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 10 Jun 2021 09:19:15 -0400 Subject: [PATCH 266/378] Dynamix Pipelines take 2 --- .../templates/so/so-common-template.json | 2 +- salt/filebeat/securityoniondefaults.yaml | 15 ++++++++++----- .../pipelines/config/so/0009_input_beats.conf | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 54e786cdc..3e47fd780 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -1,5 +1,5 @@ { - "index_patterns": ["so-grid-*","so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"], + "index_patterns": ["so-*"], "version":50001, "order":10, "settings":{ diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index cd215e242..be4f81bd1 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -2,25 +2,30 @@ {% set ZEEKLOGLOOKUP = { 'conn': 'connection', } %} - securityonion_filebeat: modules: + {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone','so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} elasticsearch: server: enabled: true var.paths: ["/logs/elasticsearch/*.log"] - kibana: - log: - enabled: true - var.paths: ["/logs/kibana/kibana.log"] logstash: log: enabled: true var.paths: ["/logs/logstash.log"] + {%- endif %} + {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} + kibana: + log: + enabled: true + var.paths: ["/logs/kibana/kibana.log"] + {%- endif %} + {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode'] %} redis: log: enabled: true var.paths: ["/logs/redis.log"] slowlog: enabled: false + {%- endif %} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf index 31ba798c9..9ca55b184 100644 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ b/salt/logstash/pipelines/config/so/0009_input_beats.conf @@ -6,6 +6,6 @@ input { } filter { mutate { - rename => {“@metadata” => “metadata”} + rename => {"@metadata" => "metadata"} } } \ No newline at end of file From ff807c9a6f52e31ee879aa9f5c7fb3c38bee36ce Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 10 Jun 2021 14:06:24 -0400 Subject: [PATCH 267/378] empty hotfix file for merge into dev --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index 12595f2e1..d3f5a12fa 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -SALTYSOUP + From 7205c5cb7bc47d0e161dde49d43b3d364bb2ce2a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 10 Jun 2021 15:21:03 -0400 Subject: [PATCH 268/378] Provide timestamp as arg to SOC PCAP pivots --- salt/soc/files/soc/menu.actions.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/files/soc/menu.actions.json b/salt/soc/files/soc/menu.actions.json index 558d10a36..665ca4c39 100644 --- a/salt/soc/files/soc/menu.actions.json +++ b/salt/soc/files/soc/menu.actions.json @@ -15,8 +15,8 @@ ]}, { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", "links": [ - "/joblookup?esid={:soc_id}", - "/joblookup?ncid={:network.community_id}" + "/joblookup?esid={:soc_id}&time={:@timestamp}", + "/joblookup?ncid={:network.community_id}&time={:@timestamp}" ]}, { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", "links": [ From f7600af89b42c068d809c43233c35ad4c03b7b79 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 11 Jun 2021 13:52:33 -0400 Subject: [PATCH 269/378] dont loop if modules arent defined for the node --- salt/filebeat/etc/module_config.yml.jinja | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/salt/filebeat/etc/module_config.yml.jinja b/salt/filebeat/etc/module_config.yml.jinja index 8f4fbd7bf..733d47c7e 100644 --- a/salt/filebeat/etc/module_config.yml.jinja +++ b/salt/filebeat/etc/module_config.yml.jinja @@ -1,16 +1,18 @@ # DO NOT EDIT THIS FILE -{% for module in MODULES.modules.keys() %} +{%- if MODULES.modules is iterable and MODULES.modules is not string and MODULES.modules|length > 0%} + {%- for module in MODULES.modules.keys() %} - module: {{ module }} - {%- for fileset in MODULES.modules[module] %} + {%- for fileset in MODULES.modules[module] %} {{ fileset }}: enabled: {{ MODULES.modules[module][fileset].enabled|string|lower }} - {#- only manage the settings if the fileset is enabled #} - {%- if MODULES.modules[module][fileset].enabled %} - {%- for var, value in MODULES.modules[module][fileset].items() %} - {%- if var|lower != 'enabled' %} + {#- only manage the settings if the fileset is enabled #} + {%- if MODULES.modules[module][fileset].enabled %} + {%- for var, value in MODULES.modules[module][fileset].items() %} + {%- if var|lower != 'enabled' %} {{ var }}: {{ value }} - {%- endif %} - {%- endfor %} - {%- endif %} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endfor %} {%- endfor %} -{% endfor %} +{% endif %} From 5941332d49cb8319490c031701c85fe51d666394 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 14 Jun 2021 08:51:29 -0400 Subject: [PATCH 270/378] fix two bugs --- salt/filebeat/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 175213008..f2a86cd5a 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -126,7 +126,7 @@ so-filebeat: - 0.0.0.0:5066:5066/tcp {% for module in THIRDPARTY.modules.keys() %} {% for submodule in THIRDPARTY.modules[module] %} - {% if THIRDPARTY.modules[module][submodule].enabled %} + {% if THIRDPARTY.modules[module][submodule].enabled and THIRDPARTY.modules[module][submodule]["var.syslog_port"] is defined %} - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/{{ THIRDPARTY.modules[module][submodule]["var.input"] }} {% endif %} {% endfor %} @@ -141,7 +141,7 @@ run_module_setup: - require: - file: filebeatmoduleconfsync - docker_container: so-filebeat - - onchanges_in: + - onchanges: - docker_container: so-elasticsearch {% endif %} From d2069dc5f2039108b6ba7fe8336e59e671e2d6a2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 14 Jun 2021 09:58:50 -0400 Subject: [PATCH 271/378] update roles that include es state --- salt/filebeat/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index f2a86cd5a..0b59fded2 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -22,7 +22,7 @@ {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} {% from 'filebeat/map.jinja' import SO with context %} -{% set ES_INCLUDED_NODES = ['so-standalone'] %} +{% set ES_INCLUDED_NODES = ['so-standalone', 'so-managersearch', 'so-node', 'so-import'] %} #only include elastic state for certain nodes {% if grains.role in ES_INCLUDED_NODES %} From fd5d540c7840fbcab6d88cbfc71b7ccb16e0a100 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 14 Jun 2021 10:00:19 -0400 Subject: [PATCH 272/378] update roles that include es state --- salt/filebeat/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 0b59fded2..c5d859307 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -22,7 +22,7 @@ {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} {% from 'filebeat/map.jinja' import SO with context %} -{% set ES_INCLUDED_NODES = ['so-standalone', 'so-managersearch', 'so-node', 'so-import'] %} +{% set ES_INCLUDED_NODES = ['so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %} #only include elastic state for certain nodes {% if grains.role in ES_INCLUDED_NODES %} From 37f4caf536f7d6bb19c84647e2c8a7064637686d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 14 Jun 2021 12:13:50 -0400 Subject: [PATCH 273/378] Make new ECS changes Elastic-auth compatible --- salt/common/tools/sbin/so-elastalert-create | 6 +++--- salt/common/tools/sbin/so-filebeat-module-setup | 6 +++--- salt/filebeat/etc/module-setup.yml | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-elastalert-create b/salt/common/tools/sbin/so-elastalert-create index 683b53ed1..56e1a5a25 100755 --- a/salt/common/tools/sbin/so-elastalert-create +++ b/salt/common/tools/sbin/so-elastalert-create @@ -145,9 +145,9 @@ EOF rulename=$(echo ${raw_rulename,,} | sed 's/ /_/g') cat << EOF >> "$rulename.yaml" -# Elasticsearch Host -es_host: elasticsearch -es_port: 9200 +# Elasticsearch Host Override (optional) +# es_host: elasticsearch +# es_port: 9200 # (Required) # Rule name, must be unique diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 7a6ae7446..4f9811ca7 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -31,7 +31,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + {{ ELASTICCURL }} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -48,8 +48,8 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then echo fi echo "Testing to see if the pipelines are already applied" -ESVER=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \") -PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) +ESVER=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \") +PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) if [[ "$PIPELINES" -lt 5 ]]; then echo "Setting up ingest pipeline(s)" diff --git a/salt/filebeat/etc/module-setup.yml b/salt/filebeat/etc/module-setup.yml index 431e432b3..35fbf5fbe 100644 --- a/salt/filebeat/etc/module-setup.yml +++ b/salt/filebeat/etc/module-setup.yml @@ -3,8 +3,12 @@ {%- else %} {%- set MANAGER = salt['grains.get']('master') %} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass', '') %} output.elasticsearch: enabled: true hosts: ["https://{{ MANAGER }}:9200"] + username: "{{ ES_USER }}" + password: "{{ ES_PASS }}" ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"] From 62187807f01ca156d8eab7163623259b1f81a45d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 14 Jun 2021 14:08:14 -0400 Subject: [PATCH 274/378] Specify elastic creds for playbook alert templates --- salt/elastalert/files/modules/so/playbook-es.py | 7 ++++++- salt/soctopus/files/templates/es-generic.template | 4 ++++ salt/soctopus/files/templates/generic.template | 5 +++++ salt/soctopus/files/templates/osquery.template | 5 +++++ salt/soctopus/init.sls | 1 + 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index ab2327ab7..5b1835bac 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -19,9 +19,14 @@ class PlaybookESAlerter(Alerter): today = strftime("%Y.%m.%d", gmtime()) timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime()) headers = {"Content-Type": "application/json"} + + creds = None + if 'elasticsearch_user' in self.rule and 'elasticsearch_pass' in self.rule: + creds = (self.rule['elasticsearch_user'], self.rule['elasticsearch_pass']) + payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp} url = f"https://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/" - requests.post(url, data=json.dumps(payload), headers=headers, verify=False) + requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) def get_info(self): return {'type': 'PlaybookESAlerter'} \ No newline at end of file diff --git a/salt/soctopus/files/templates/es-generic.template b/salt/soctopus/files/templates/es-generic.template index 8183a5af4..6e50a3f3e 100644 --- a/salt/soctopus/files/templates/es-generic.template +++ b/salt/soctopus/files/templates/es-generic.template @@ -1,7 +1,11 @@ {% set ES = salt['pillar.get']('global:managerip', '') %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} alert: modules.so.playbook-es.PlaybookESAlerter elasticsearch_host: "{{ ES }}:9200" +elasticsearch_user: "{{ ES_USER }}" +elasticsearch_pass: "{{ ES_PASS }}" play_title: "" play_url: "https://{{ ES }}/playbook/issues/6000" sigma_level: "" diff --git a/salt/soctopus/files/templates/generic.template b/salt/soctopus/files/templates/generic.template index f956eb8a6..33d8b7ea5 100644 --- a/salt/soctopus/files/templates/generic.template +++ b/salt/soctopus/files/templates/generic.template @@ -1,8 +1,13 @@ {% set es = salt['pillar.get']('global:url_base', '') %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} + alert: - "modules.so.playbook-es.PlaybookESAlerter" elasticsearch_host: "{{ es }}:9200" +elasticsearch_user: "{{ ES_USER }}" +elasticsearch_pass: "{{ ES_PASS }}" play_title: "" play_id: "" event.module: "playbook" diff --git a/salt/soctopus/files/templates/osquery.template b/salt/soctopus/files/templates/osquery.template index 0410cb288..22c29193a 100644 --- a/salt/soctopus/files/templates/osquery.template +++ b/salt/soctopus/files/templates/osquery.template @@ -1,8 +1,13 @@ {% set es = salt['pillar.get']('global:url_base', '') %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} + alert: - "modules.so.playbook-es.PlaybookESAlerter" elasticsearch_host: "{{ es }}:9200" +elasticsearch_user: "{{ ES_USER }}" +elasticsearch_pass: "{{ ES_PASS }}" play_title: "" event.module: "playbook" event.dataset: "alert" diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index c2c8dc1ac..984ff45a9 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -21,6 +21,7 @@ soctopus-sync: - source: salt://soctopus/files/templates - user: 939 - group: 939 + - mode: 600 - template: jinja soctopusconf: From 68e4c5e469138e140419a1aac2e7f53694b3439b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 14 Jun 2021 15:21:46 -0400 Subject: [PATCH 275/378] Add CONTRIBUTING.md draft, move markdown images to assets/images --- CONTRIBUTING.md | 41 ++++++++++++++++++ README.md | 4 +- .../images/screenshots}/alerts-1.png | Bin .../images/screenshots}/hunt-1.png | Bin assets/images/verified-commit-1.png | Bin 0 -> 24339 bytes 5 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 CONTRIBUTING.md rename {screenshots => assets/images/screenshots}/alerts-1.png (100%) rename {screenshots => assets/images/screenshots}/hunt-1.png (100%) create mode 100644 assets/images/verified-commit-1.png diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..801d3502d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,41 @@ +# Contributing to Security Onion - DRAFT + +### Questions, suggestions, and general comments +* Security Onion uses GitHub's [Discussions](https://github.com/Security-Onion-Solutions/securityonion/discussions) to provide a forum where the community and developers can interact as well as ask and answer questions. + +### Reporting a bug +* The primary place to report unexpected behavior or possible bugs is the repo's [Discussions forum](https://github.com/Security-Onion-Solutions/securityonion/discussions). + +* **If you are familiar with the current version of Security Onion and are confident you've discovered a bug**, first ensure there is not already an issue present by searching the open [issues](https://github.com/Security-Onion-Solutions/securityonion/issues). If there is, a thumbs up :+1: is a great way to show this bug is affecting you too. + +* If an issue doesn't exist, [open a new one](https://github.com/Security-Onion-Solutions/securityonion/issues/new), following the directions in the issue template. These boil down to including: + * **System information** and how Security Onion was installed + * **Log files** relevant to the bug report + * **Reproduction steps** + +### Contributing code + +* **All commits must be signed** with a valid key that has been added to your GitHub account. The commits should have all the "**Verified**" tag when viewed on GitHub as shown below: + + + +* If an issue does not already exist for the bug or feature for which you are submitting a pull request, [create one](https://github.com/Security-Onion-Solutions/securityonion/issues/new) with the relevant prefix. (**`FIX:`** for bug fixes, **`FEATURE:`** for new features.) + +* **Pull requests should be opened against the `dev` branch of this repo**, and should clearly describe the problem and solution. + +* Be sure you have tested your changes and are confident they will not break other parts of the product. + +* See this document's [code styling and conventions section](#code-style-and-conventions) below to be sure your PR fits our code requirements prior to submitting. + + + +### Code style and conventions +* **Keep code [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself)**. For example, Bash code used by multiple scripts will likely best be added to [`so-common`](salt/common/tools/sbin/so-common). + +* All new Bash code should pass [ShellCheck](https://www.shellcheck.net/) analysis. Where errors can be *safely* [ignored](https://github.com/koalaman/shellcheck/wiki/Ignore), the relevant disable directive should be accompanied by a brief explanation as to why the error is being ignored. + +* **Ensure all YAML (this includes Salt states and pillars) is properly formatted**. The spec for YAML v1.2 can be found [here](https://yaml.org/spec/1.2/spec.html), however there are numerous online resources with simpler descriptions of its formatting rules. + +* **Python** - TBD + +* **Golang** - TBD diff --git a/README.md b/README.md index 2bf903fa2..bc784dd22 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ Security Onion 2.3.52 is here! ## Screenshots Alerts -![Alerts](https://raw.githubusercontent.com/security-onion-solutions/securityonion/master/screenshots/alerts-1.png) +![Alerts](./assets/images/screenshots/alerts-1.png) Hunt -![Hunt](https://raw.githubusercontent.com/security-onion-solutions/securityonion/master/screenshots/hunt-1.png) +![Hunt](./assets/images/screenshots/hunt-1.png) ### Release Notes diff --git a/screenshots/alerts-1.png b/assets/images/screenshots/alerts-1.png similarity index 100% rename from screenshots/alerts-1.png rename to assets/images/screenshots/alerts-1.png diff --git a/screenshots/hunt-1.png b/assets/images/screenshots/hunt-1.png similarity index 100% rename from screenshots/hunt-1.png rename to assets/images/screenshots/hunt-1.png diff --git a/assets/images/verified-commit-1.png b/assets/images/verified-commit-1.png new file mode 100644 index 0000000000000000000000000000000000000000..62363b99b2824b9dc892c6631b3bde500bc61745 GIT binary patch literal 24339 zcmdpcgMTK;^Kg=lcVlgAb7R}KvB4AD#>Td7+uq&qiEZ09-`st5cfbGOozL`i&vaK; zS9NuvdqU-9#o%DEU_n4Y;3UL_6+u8grG1o*p&>t>Win@uARw^oWVwcQFkm4gqdtlEE+EW2>kTP)x99Kg zY6=^+eU>0O>;Wk32F_T&bp~C>(8Cx(`EiAB+!s@Sn{`FiQ|sq8(4 zfJh_{LJ*?@PT#Bx@@xs8%WQ(%aRX8bH`Z#*Umn+ezm-+T-Zj$RY0XOGT6w<23T%bG z2o+m|&750Hy5}T7z)1brPZ>|Gd}h8Tc|&p-LHFnz`xd;5OQDnAMq(O;^4KL^nk{Yo zfaB1|tM5P~iI8}fNO5H_@=c1lyPAYlux#HhRq?&GCyA@@=Qr!$RGM)5LoUzA7!YM4WdN9Ggui&73{GZ-_Vh0e__3%s>I)OW-PUyScK z9Z>Oz;F|=D!hywdQZm^_XCr6gsfGVf%ua@As{T-7qD^R2rEKTPcKc z0Ou!8Pe#v5f5i7S!&-Gf1=1YSadhM_GQ;Z{ts0No$^|-0KIgAoem$GYcE;>#+6+;F z!@X!*&fD-?vfI`hqPF;#jILH4aBCR&;TRii8^f0}4@wW%4<-*1KX88eZb(Cu*+O#! zSOstjl5b+^2QWw^kYgd~gl_g4ZMhpj)CRgnJ;Xeyii(P=^-CC+e$g||lnxxxPyNg& z#F#A(OQIBqK^a9tXhk|3-zI)9ZUZ!`BV6>yks2ehA!UmjkHe4W8&U+&0#fU678~nS z>a6P6jPw+sQ9r@@*Zqi-qAbQK)XwinARfdVas$@x%nhmlag(uQ(qoE;w1%OEi-9(= zHgVk)d}Y2$49XV@8x*J%GgKKWUyJfe+?DC6(%WJ+#dySU6?&<#C>hb>(ZbOXDDNnv zs3|EHDMhF%70VPkOPNa|6j_yxsfwv=lz8(z#U8K0Mj;{~Rv^+b8faQ-H~hoe!h&Jd z3dIXmcAJJwRMN_#&BM$?&D)QNSsKzxmlPW<8@P0h`o%wSnuEV5=jOA9M@a}z66iku`Zq|?hzi$>?^rgQ@O0=Es->&7Y_#>Xd}_VByzAe*KK%e) z^);y(&Y9ov&yCej*d*(5?ol8{BQ8>?QmCI4GOVj@w+ph1yre~`fLSH!GpYtUDJ{&b zC75Ue+(*B#IxuXfpRydY4Ww|Tejc$Hfua#d{}N6UZX8Z11sEn6hD$n4szyJekI+%C zt8v+6qxL9QF6S(VnuDDKox`r+s$5@4S*o!rv$S4}v+}Z%2P;`y7&!fU%}6spF=;%1&Rdna>n{DwtV}z(D}3tYJ_VFc)o%&`MZGg z!G41ULO(iDI-3nnl@-bfRSI1TdGwU%Q}?9(UJLdJviaK$?w zMHPt^q5HLy^T|F!^9x@QF**h|rU!wuLF#y2=CJ72$u{iv84a6em-%BKnmM(iT86fS z22j(D84m>?Z5{PJ5N)JPUxtJM<&GqxzH^~{fu0>NgK_NZe4^HnSJFY6dK!D$PFe%8 zS;YPr^?v7$$0hY+b^L4mJ-LrKjBKe~s$`_3l}t}@UokEQ1+*eK&!OgF`P6fu@o&M+ z??DdzV-a==IS0O9k?Ap9jHrlYrqDubcesG^Nm{BZa|0)XC;8<&YA~ugz;C0A8P_!P zZ1=_7j(A+vTzcd_UT+iT{YOVE6KMe~s}{F&=oPpc`^{DQJa%3~PvzJZtO?ex&EL!o zP5g|1wf)#Uf9bNIOQ1zXgSWI_YwMHgDn3xGRk3rsOr|}dP4=#N z%iRhttqj$AZsM?9ZpLlbsd<$R^^g!M^)$0Syol=KS{gD1hyc!=t5)19**A zRlmY=!lG86+MCbc6oKU(I=dRPZDNMwdp!7e_IGOw`?b}8?!C9I*KD7ich}~n>bA=M znxYPLt#p;=2D-Ycd-5l_NT?ZjRQv}``VxFgXsZJ=M@vh~DbuC7FKkk21MvO09o~b_ z@?zm4QcmUe7MJ?A>h=$hK|8ze-D|TZ_rt0(9udNis3dbQfPYEN>RIc%(d@ z#vroVZP)H!7Ij6>4?kC7$Z9)wxxOI3Dn0{(hpp63s&U%QeQwpWh4hwL1}t#rGpk(g z=#TiEdS3fe)7RL>*A3R%JUxJ?Z(K+&XYjpnU)FY4a66OT&u?EVp5qbbczU^M+)3_B zUTXWd)7GWEEpQ#ba&zqPIPOX;#6$I*Y+`hW`@AzMUL?FUyrkaM_ba4Nq8N4=&S%qm z@LmZ%7+e81`F7rG_Y*J6_1-QXwufAA9tJHYk4xFK*2=mneN67FJcx|8E}t9juyP7D@75-36VDkeJHqQmLrn1uU zMXEQniM75<{7l^9%GCU*@*7ACGDugFw2aK0tc+Ks$PX_Nezh@6IX+s=!Dk|DsEwdu z7ZmFs-NTmM@5MOpy#9yKCqLKseZbylu`(8Vl1e{3-Y|f=gt3ea2-Qa!8U!2^3k2e$ z1p1M9L9zc^76qjM`Sj;^Fc6SnGZ659w~_tG|F~j4(jPGY%0I;gfk1t{A%7&-9I*ei z{*;#U=|5!<`iH}^#&>jG=b}+SZ^j#qM!yAXO71wY80YNAIBY{dNl3aZB zKWC<_?x-#!&1q<3MW=6MV*sFYwX*%A9}sR=&X1xMz)_#T)ymS^fzy?T=+73MALTzj z(-RT=*~HO;he%yUoz>K9o(!P^<8PL9f<$c$$$D01~?eno7p;=*;o_& z(XYONjgun}5z!xr{(JpvoB&s||6#Ir_%~S}1k(ReL;sDAf&RaJe}HoT@s(5F%oSj% zA#7&#!Ltt>yx$lYx&Lhce`@}R@h_0-|AXXU{2Sz7YW@kS;sCH0va$L=>B#%P>H0U? zzgGSmkemJw%KyTNf3f+`uOEEoh2^II@0{_%c6Ukje@r8enXsJlNB$vXe_l)~eO1)~Y($aMsjkObZ3Tpf%%dsauu76Vo-6W%@b?2E@&^s3ee z)|nh+V|#aIJOaSM2tfXMs}jkB3rXBjvFb7dGx zyx%_nK%n_ex3ED#5&v=5P=P|rlyCJK{a^HmX%CS9>>mWYLJ~x9x{UoB=HCbO2f6^V zuK&|N2xvL7??eNYtTg05$s_n6!R8<1|9JiXg9O;fi4Lb@^w;zGddJP*B2+3z3T3*l z5>#qUG#u>g>ZgI{tLQ(ICOM$ zI|m0ThVMT%Vz(hDp#RCkN0RR@xvg%)&g6Tr)xUoJ5xf|v z0P{CQszfodsS;*pW(iqI0*Ai|;!}Zq_aSh6;!VWz1>#7#=???383Fpkz{S{%%uL{9 zSU;DPiw#?DDkX0hslYGPHfNnG53A)OmeCZ}#Fdp5`%{ah&7(#OX3rgpu!s)xrG!Fm z`v(TBiEf)L5vzIY>Z!-i7c?*bHS{ZpF*+XwSwMb%Ioy=R+Zs>>(g|Ld^XN?nNZ6FSU@^I) z-kz>XB{oJJt`5ZPN*yjf-HT+-(YNonT%sJNcRnj$*Y*St4i8Hua`YT^O*)K5(4?Tj&CLY?2F89yP)=nGlDLcvO45O_UGf|vN6+%I(#ob|;}P8{5%Z!^Iu)<{ zX$8Z2GBf!}k#wdgm$P@SuIqkNrE-@ihf0%Z{|wP$l@)dxn@Llj()+`-+>QIWciDSr z{^oD+Q$pH;c^hqv^HfPvDPE%JmoyJCJoOaU5)1*zznKVWXcAIVrI&RDoRt-A;PUWv z0aVyK37Fj_OhQHmQS%Au7wfY#H`-w|?$glM(nOWwwveu8eE4 zFgaP1D|8U(Y!!J#OS_O3T=>k-_ZO!51;_j98eMgd@>Ebr;>qR(Zdao8y>n0C6lf}0 zqD$ri*9@K!aA-Ql-{dBMmjOeA4>GTx&8(VpywS`C1SZBk~ zc^0>ZlGJExb4Dl=hPdL%6`R%C%vWjMm%@TY!t+u7V!wV0$?^MWC!OEa?kg7fAb{R| z9dWVd&^Z2Tm5Y*wv*0vyQ0usJ@a;Ca{MC_Gvz}_luQJt>KH*#_jjT(F!{N!pt z1f3|pb9hU(2$TK2j_P!b%p?C6$>3T*F&8NfC))m~0aJ9$=jG+a^mS?WMm?1g;VR19 zQd=8GWT01^TzGPF((&=ironj?lDORPLt~n4x5Y9?Go;DI#I9~>At?|*{n}j&DRjPq;4_aI z=OU)jlo^k>XFtFfCN0o(^E$$*lqoBe*8!PSTz{VO*+SRetJD5rS9DKi6sR?K-7e%) zxREPlSUnfyu^3HrRGHnE;)5qe<^Q@oT}@$(LVWVYL_~(~=~ZX238nGk=>3GUdwctV z&9~Rrgkn&^o}I0$fn**&nLH+VUN6AnBM5b)pekJnIpxy4Bg<5n&j=VPZsr!AEzIEp{eqh?xCQ>==iupro!|sz_O`UPK|_7h zMSutiu`oHfZ_%2mlKgD%WDmT{+*aJ8q@+!K;qeSabVzuuTIm!WAd(BM9R;GU+PjxW*`PgMvGdtJLJJ9(o!H zjYnd3Jz!Ajb78@t&!`^a0FZTG(~**r0_j*RW^&$MpHfb}H#P{}9Y)w^@Aj~X>u(xK z5gHj7|K=FReD)g|@o*$Rsyf*{R+84HW(q0Q4%%JFNJ$c=>1S_cZ<8l|zhcZ@tqzzG z4vXlbOJB{e3GCigDJZ6*b&dM{iGghIqNg-XY>p1`-sa+v{SBU@=@7QHf{Pw8 z<=MARj}qa_t+EkJURW;o6Rq9>lg0+4Nig>=5r;~O?70nnWST~F6v*_QhsIP3D)c{&X262X^mx@9UVpK!A`DldwWgd4Uhss^rcSO@_?8}roPkcw`1=0Gzq=`}NIqXnn zbQgCh?%QKNp>ydpalkMy8N_}jmqG(4^UPr0P&_>xd>&aOHqePqyFVIfp%X9KYjV`m|`)K_Xbu{Z0P)Hw2(f zvy8*Ti0iAi4k{WIhhx+Z##_2~9M2o^#B?mMXzcg)Y`ULV+OWKYSY=Fkf4I*AwP@TO z9QKr5i0=?6(a*_5P|hnW*?G(58j~6dB1XPwuvF;lncoY4raIdc6Itz`RLYpaX|e*V z-PrMpvcSs0H;?bA3^Wd7=ftLtn%JBFRH+lN2MmG6V%~XEP>}*Q`{d$|rt&B`oj)jI ze7K}#L!Oq2(FNv*3zsO&jtkJ0rZ+~l$uZxdCCh9~S4whG(?s4_oyIepeOay7^4ebdp*Zt`3j44RoY!T;hUfcU*2Z%n(o z1fUSS-P_vv9Ivs=Xi**~d56t8BQUnhpnF-V-N{=8e{zngZh!4qd&9dYZ-hUjyEhzL zDrMF5#nk#No&`S5mUnV^yNN{fVN zz}?jD@RND{<3(=pN}|BV_qeYPnV!&2MDpf`DE zH;$#ra}WrJ>grgClaxsSFyWpzv{uY>Ri)Pm%)rTL0Mv1xPx5G}1)Qz3?06)y1?KFw zXRlQSZU5SPvRf_WioKay>aveAM#ECPkgBgAF|n>VDR}NfS3sZaaGpg@9GvcB6yx&j zPONNyiHd&aCBsET3z&#N8ZhCI3~p{NwuNCrwf|bV~OgtTk9IO@(Y!ZCdE|RlUg62&l^nb2r@ZeGi@hv3RTLEP^n%``_^Kp9x;k=(GU9=HhhM8%7`mE9K zs6H*l0iSmLdwC{T(_%I_6I^Ds-S;koE9Yq8+9EPerBtg@_GT;IN`=Q{O9`b!;KB!h z%QcF*?#V8*!Xp*NQu96^?q3`@5NK3%u*q+B2}wz06{$_3nXQM99z^hW^68kAf`zpy%0ZO|2 zdBq7*5^b(~Z86N~#ky>WM-s{|b2ayDY*7;y^||&@=8LI?={Q>eKhBFKz2vZX@sb~8 zuGyKX)kZ8LED{k@d#_*`L?iEI=QuBZbO95a&zxF0cRiQWE0ji4;>3)BIVTs_XPxcJ zxh9h9XpFR^4w9$8#bF@O>Y@Zk*32JqVqCV(H+x6N21|K7_rqwI|4b*dfw6#u5wk_|0wJ#tBkE^17pe@W2vXx_U_ znPutbsE+0sGNgtg*oTYqnB~099sbE>62aJlT#JWeQPT8k&A76oGyV%h*vci}Isb;aSGrhNoG_w0r8)7;?!)80Fvea7LXz7-?y z)Uri5VFGbLktoDJSmP%skwXwQb`^ent~F-C7r3NmCE*_3knTa3pBAmE6ro;IYg^B3 zb&f(s>*b!S(dr7ZH0RN{NNGvPeEWOxB#$f+B`uoZj-niYzBvo}*8=m;g5nUnR78={ zWS7|64>4osOo!m2`p0hNmC(<*G2aHB!%8nt1P9~g&>86!lbs#%GuCGs-g$13@`Gqm?6g{(t^aO*igrtn!S72MVUp2 zg2wA4d4U5Vya}V{Gs)s;7P<`kr>XbC7^uG*_m4P_4OA_JfGqM_+n+aH;Ro^fE<WrQc`3d6}um3fyZnbbD??h1{_k&vKj=JnZCb2j;PB5PyT|exS;2r(u zq1@}yZI0%>m-0kH|dT%#0*hq_-Uyh~u&=dp}`%e240UrExO%FV+g zF1>uywYn*LB?~dF6-kimZ|j9y7@U!b=}=bLvCDvb4=SE_j3gR=p2qiS?$p|uDoSn9 z!2#se{rJbCbCnV3liRt<&hZ)X^MaP?71>^(m-w+P_V3C;nba2p(6%Nm0-o1wtg>Ca z)qH{M$b1wvxpGdgBHdI?$D=2Nx*=6D6P5w_H*emZjg1^V-uajJk7KR(_iZGcHV_c@ z-+!Dtoa#V*M^%~cJmet^AnSygDhTrp2?>MFTi}NP-9$n1$fuwpA|N8TTSG(p+@x1~ zFQ`&GWdVKI+5Dc-Ji^FjDJTvdYM&jHc~_K?xte)B*17+*vxAlp#wx;(BoB@X`q%7^ zFeWL9Fi=DU854<4K)`P{Y}lWVn^>;ev(GEShMYdREqC(FZC_YbplRgEF}GsjA#s3; zPmd4N>4k%_aD7*ioPDY@YWd#HC?(ZKuRF5CYq9>!=`QrVxjGJbF3l1y>ck^<#(qk6 ze`H0;1$g*uIQjytluZ~IZy)cBLJ=VqnX&>N8f#3K7#j8U&U4Q>?8V||$M4T~&^?Mk z;2l3TvI`W!b!aa2Hr%L>rDpurPJIM=K00o*;T`wi;!q~vx2Z$&x^-!ITXxRXd7e1n zEh{^b@pbbvGS-v)T-Q6AS&TLjxj)VquFsP{bPn?+%7LZ;`%m_B(vRQTI?D*KFE0N* zM_4pyekhzwKYTPKe&of{?5nOEmgo!y{)<|=mYrGX$!JJ&L56ZDY*dV~LS!KL<9FI> z3b`#tB(jP>~7?V?qG(OB2pjk+cIst`0+zfVtkM}8AquZer9AS@NiBtFEulfZr$tA zuR`ZB6C4tp0CG`oYMKdfKeif|l0cKHv8X7oO;gxn`VQh#ij7MLcRU~-PZn9IkRSBb z=~pnw*J$pB_M~k~poq>QR*U`2dcRj!K};A_n{kO8lF;)5(}loyk3(c*|q|YY8+l z4XOB#QToB;-&>xDviLD0kW-}&7-nl7sloxlekMO6l2D~!v~$4t)t|q7r5kYO&aG27 z%E?JiPS!Ik*5G1n3^Vm^coBJo5RR|wsJ0;pG4=)L zZyb@9Ni5*8i}Rbe?ESpIx9_K&B<`)ByF=*A-W+38|CN~b4H->Ijnm(<$!2|w-6$~u z^@k<;>9c>^VrBOYms0@lr%cs<=m1ndMnnq1)Oh6tF8$9!UAm^EIO4+#jdWEa&^rY9 zY}jtCmXj1Fg4QaDsum%8Q~T({NiseuHEOyE+1dCbpL<`JAeWykfh*X zeG-W3*2loOu!l`$_46WQBPmhUq#c31%K4n!1~ZQe$J0rfAO4?m!k#20>_C;L1ZP^1 zcC4L~e(gt9!&)z?{XH#? zUowZ7^gKL#1`G$?hJeBcroqISg^-nPB%n*c%nWBF2*V^Qh6Ze_NGqfM{{E`K-vsWK zMp|+nPc31_03!*E;eiRQBc>CWs-+MwAJHKbu-ark zuMrfHBCqn&(ET?};!9+!Oz?enZl|xT?s461_*BvuU}5Yol;0n^w=9(%*I3>qSVt); z8e@qCjVzIOc&;3$-Z3f|)1R>dR+8FEko-uH zky;0E>9Aa=m5R%;g$xcJ1{QybFw+QvzrRf~r14W08PHz)B@=IYxr!@>w;CLzM)#W{o-^;EGh z;$@R82KFIi;*ydh9MBK^q$nwY-mI5rkzhU))%qfG6S0|JxP9pr68L{D5j!;49fM}u zq$0!A6gd=VWm2UvO#Dd(&Nkh!5|{R7-Q94-x4hKUSSkk--Ei&^WW>?$Ld^b1$5tU^ zUwp$p!;-kUR5)6$3O+d>5_q>_ZpCL(iaZcc(tu5m&{8B>l2jV!;z*MDN{-VfH4x!1 zW~M>pr4n5ZuF`<$pDs0q<1AOvuEf{qwN3mmV)RV%0rmUE=slFl9xTp9$65Ioi}5cF z0wCb`C)m3|+e)N`rnE+l66bE5)dg2?A|YqPNaT)D4MF!6SKM1soqZTsMw|aS>G@hX z0GN%|z{B4cUwi*^5);VzO=}F4YyUOTfl^Ej-mlRe2CS zt!ZC!Pv3g0&GDrDzH)|h%LI(YnW7UXI8VVj6v(bV0X7SU(LhCVl8Ane+p?PTiinENOfk(& zEB;5S-O4cFciYIt16!mDM-f&yFte(1s?~9 zF7m-1mTg;SDK2V-;nt&%ELjAZ*F8@;9#7(33gNHl{QcFdz#l)j+b{_S3(YFbgbL8o z#8(n)b1{vM(LjZlk%(JKFanCw%23`0LRHd#@e3p6pw)@f2TyDh$5!`r?&a{~StYVSk5foAd?8)4Q-{t~Ny(D!0Yk z>%7-_ze?w7+2$A^hhrXWY$X2bep{fBFFMsMhaVbN_J+7#uw%5?Vi*5%Jq`&R9ErI* zUHT_M}838&I288Z1Y;p2(OC994@vD{A@5N8%QeOhCu6owle+#sM7Y_WuYyF%hj zhZ$I;<#HZgOF#vJO^$qOrRhKYC4MQ$tqmZ0V}}uLa4T1xI(*nBg&r6b;*n8{DhfBn zDngjU$J0=&N_T!IT-PS~`mwjox8uE^#iV`K`b-z?O@jX!gNI=K{UU$B{&{3DoG#|L zx)TznJy@?32$tKuJ*IKhm#nvlaF|sTd7F>GN5RwP;zG}*h{6&I*Uj+pyMO`R?EHWl zrz88B@9^7YaW>#8!D3~2{=KfB=QLxm&&$oAJAO4hx$7m*@oWXiQ;mWqwlUuQ7Au0l zW6jxMwu&5!%+|)m1f?&xqRX`YkRqz+<=k7s*XMlm!z+u;SoWBhzYMD7nKhOIkm*7Q&}5*Gkzq-j z0n*?hfT)6#McTJMktvFzEbs?NaY^?gk^*DF+K~jFP)g|3(PS|8A>-}b*f=lNCsaSv zpnyOLcjKf6(`n5mmjeQQq|La`ROTFGY1sI2j6x=t-4Wg9LCh9*6>J$p+RiN@Me!vG zZu}LdlXuNr88+=%&R;be$fYfvmyF&ceN%->Q1_Qj+qzj^=R{0C z`w{7dq`YincRtj-@y+Y6@-K*-S2u}qPuo`6s?w3Xs4FA4AGSGj11Yf)yVi&Zv< zIqn&(G1g_)V0&HqqNUL z?%EX7P-pm-&n`NFS(X3N;`~kow!NWy*izZa_sQKsh!eK)`IuN+2z8hN-QQt88{>`= zTvFa5`5+El#DT$xu8t^DNP;!ifsEB%dZ3V!es*GVfF-G((fdK1S70rjkU&7fN=b-x zrluiyHGqVw$M8V2W7|(4+{iH#1qsbJdMPq;)ip(iK1pMM!<4dTW}-_#p6xtFxG6nQ z`S=-LSunZtvB`rklglX~3iqB5#6cu^EHki-3|q)3_=}TFI>bXT$<@uYY&>hF7BH>x zc;}kF3nX@|Rux+%QZfbKU=2Ym-G0;wPm(HeI{xXF$$9qUa4xAh+Y4_#J+UI2~ zA>WR^q$qUBHxhZ&+})wvt@)iLuhr~YRpUpEayW@7Q5hjI3f+q7TFBlteK(mh@1i)F z1l8i#F~{}1V58?T9b@mg(`KTZrqWbioOg;zZXAj?t~>o$>0`_HmO~2*MZFJ0^Ji4- zN3192@T(*B>iLO9bn+u0(-$=95kzABoF@U8R(m0VQU$(=1WqA_rs)VwAjj0VIQ57E z2L?TPuNbFe;xW@xs6@PfDQ;m<#Oi~_C1(<9V5XAIM}8aRg|-v!lTLvENU&dFMGL?y z`5Q*XTIvOi#F;Ago5ax=d3K@a?HNButpaX()8U7$?A9QN`*y4iY?@`VsW76cv{+j9 z+OT()&%0dkU8c}bcGD=;TK72w1U+_D)P_G!Z&rtk*X-jXFuh7W);NCgVzXNJHlj%c zZ!e{%SnjiCkaL{7T=CzRrCWFC_D6S)@QlO6jp*-3*v79yT<-&8n4r?F!g66v4Dc-* zMB#(uM$n^ccntWbRvIKw5P-b6t@T>9V^ES)cBjiQQp>IuA6ia`_klnzkHhQ=_5RT7 zauF7W9~$hMO5-`PvIteCOjUXN>zwxFURxE-4g{Wi7ky7is)K&#z?~r@Dj8hv(Fieb z!bQZ<0{M73XNWN^LH{nBpJ$V!hRAow=TS<6@ouYdyv5k#@`pFnXN@{L~sPEIgXa z3J;4)m==6{Xs+cDC#}M4vCLJlpZ|vo$UjPK+lw}8L9R#k5i6I>BNI906JK7S`SMYu zxq^K5%fm!Rrdmwp*z&(3v%=~VuikDKe;LcH^O_a~_CSS&VFi#gf-W=SORMkvDdh}96b zm6h1j^46^mKz%ngDoJJ(6qMbDUnZ^5Zi?MqWC&Bs{_ghnVcHB=2F-PYh)W1-UIVN1 zX%AQB)w`9=>zDErt!DwcJb;CoT>G~Whf0}I>T7(l44+t!#bYVGq$p?>=O(mlPpaP< zbww@3-AZx$$HuPsa9?P{7sZ*n(vA>husNi5Yue!IJR{r z4AT*JD#INz2hu!py9vFZF@|I)OgOz=c0B!z4?H|~gK3uM(9AT zLyy+8GU|%Hz1wztpClK)2lWsh?CNeN225K;eE&#|gNNXPomsZEq*)6eyJ|w1?t=~o zUnzf~u}P9il3Zd+BU$pevw+{2j4!>u=PokEdfc z`t@Qu(ueVTrJ~ZKJxuy!rEz&>oy+CS*=H?8BlvD$=wX=783AwMzR%$qJiEE^A;Nn3 zOj2y))oJhW({6^o&<M3g_g0PTv0MKnLQIo zNN`DEQ}z&|D+-0F0<%>H%!z$rp#QYxOI4;~Ok@X9%d6#5n~?XrN4@1LE>wQDAt@)= z9IyezAdhrT>DO?dxGm(UZ`}n!fESy^wB`|;6-{g52yjJ=QG)rBU&&YD_T`6QOa_1J z{Vz3y;bmQgyw;P-X295r$Lhe##p|wXjOkvMi)_dJ8hn{& z&2Ag^kQyKpmRcng<*DWOQ44-t7ngOm9Y0>4)0h47Ow+kaEeuhSO+1q6r7p4PvpsEz zADt{g$73C%g}-gRWDJ$CL&*}$QeD=My??rF4MH7ZS7&|!TU_f53&RqklVBV+_uSwq zwXunwt>0Ut_Z+ld>`0v;P0YqNNoTlI)PQ9anO8F*_G{U%!o(A0`;cntq7~CID;#@f zxx;mgeH`lgJv!U0D37;Ra=e6SzE{CD&wa0Z#^>JJhpH62;;UZHpY8+m>}04vguJ*Zdi{tUHlr@ zsqSV=dF8QP=ed&^y*amS0(jFc<4)Vy-{&7!@&qSl!#sCNeJb$#%3Ug6Z@6Ri{B&zz zdy@VQe=6llILhou_+c{JtV}dPf{fO=I7< zlM2`nGokixTkcQzho4k(Et@QlIJ~g|5+EJ80mTVTMRvuMi&zt7Z2P?i`~gDTPB%#y zB57zt2{NUQ?D5cy0b+@|?@^@AZ203>qfF%F6f%KaBxH1TBB$R!z9`UAQ3$XXWuxYd z84GesX7B8V&|8QiWfw?icsz zIkf*b#5Ldov$(w(*X2?8ad&=w-BkDZ`Zr)z@&9Fz*b`62&>sjJ8Nae!G8Q zFbpkQ5N+9rmbDr#S2KcrABX9cOuU%qe%iW0X?j7vQsWXxzL)zLh@oe%*VCY7!%ETt z^V{z^duE)*g(?k)fuj7|z*tp+>Wov8evqfU$pZ0mt4TIs1~DkC`W4XGasBq58thtT zg&@>e4-J&6ri-uce*SfHINhH7g|o05$3a@?5vquR`FAc8dWo8sM`r}pBrz@jjlLjB{_X47~h9X&-*nq70>NgE8|XtfN|Qp@&Iq9aC~oC zuggG`PThu|*hp>hnro!OsOM&H|AmS-6hpz~wp`cGM{n~cH^)%wgoMT}8mH+q^W`>` z&#m8k7pFSRlomUj(mNU)Qx(`HjEvVgR)snX%bM*+IqTPXJ3WTXK0%51K;8!Kgh-Yp zY60Os%<1Tu<$Hn{v!2^4?KNIPvFa3Nl75w?gURZi-CI+StMpXgJzc;GG9NsNHY~)e zYrxtS)63{RACPVR^oaJkbH!g|*BtdW+Z4@PJ;( z_w=-MUl%M@>sUbe)c=b=WEM@n?fbV3pZPPMh535X%^f$sm!F>bznAX!Gn$lDymcjV zN$&b3B)%*~f$U@!iT@{FK$kUw6t3m{v7T$Ig#@72Eq`=FWK%U}NLW zg$7r;+rHq_uyZ{xW`}hyE7|9*^~TC&BvDQ)LtRwB3L5(*xj(j9{qj-#h-I1SxlLje zn1lp#bbpwNiUza@eK@~8h=SM#A;uu6L%wk|X7hQ0J0zreQO0F6zYwfDOWLls5vT#f zO*(N}A;BUPDOTxh2~SSHFE=*G-6-7RGUCq4azlyELTz*4gclUj=`u-~+u+4XEt7H$ zu0Vs`Qk_NW5jX3lvUux+uaiorcc;gB*=Yzn`}~<7vE|c@*&h zSC)r$#b8p!nZ|>UhWQ2G-^AGpvm553l!pMmGzWXREXxrz#xqj|G;E z5ZL6c&GGujy_v$(n!2&`VaPIN%%18dT<NL(d`LE>VTx?= z11__-N~6wq!LH@R7T?BN45J8*NfeoiPt|%(3v4$#U5IBoqh@7z-;cENjqE{%Ea?3u zjcnM)TZUt5taZ3N0&-RV5cg@$=$_boH!f|hL(^ptlFPH@;f8Fu?P*B5eE zynD!M4{qpPE>-Qjto0fL+e<)NHf<}XZ`EH;Gg$^i=2~(%L&$oJF?o+YK zX56Vk^;#S#6-<+D@|j|oxS0$aABj`qVBap`8j7)k|2lD)q5!U1Z*PWq#<^d$%zcJQ zwR%6IXG8?v(wS*mH+Lu$-P-k@xs$kT?`yIdC&pn0nL5Vf5t7QQ_)Y=jRun(B`}6Gb z;rve*VizMj68~NC<$;Kei5Y~$11ZQPHpd(Vw_Y?Qzd&cpo z9L`g{CVJqxt!mO+Rvmks7`<22)!vcz+l5q5jd|waeI9q!FxfVJ<|>UXOUJYa@=LgQFO*ZZgAQ29?YjRP=e zoy+$+S~aD#!tqDM!*=quo~U6F%}s!1=lJ{)KS->cmrr%I-}S22Ua!vtzgnz*YSVy? zcqPon)+mU{t#?^-4j1F&3eDGa#UX+`IgKUHV#&>QWGXB#oh>uWo)ux1aBQ(*waM+% zep``}+I_}XOY7h`<~*f3_|fW;8b`D;xArw`_(HYmh#+lRz!zoXw@%h1g2Z@%h4!b< z%4BkUvuK$5d~*9YxX{N1TXXedYOU^5;QQw6-uuG|*xH}4W%JyA_BTP?3B(N~(fEd8 z7|m(+CWW=}c+TOmTHu2}qL&?guI{jY(gio*e7I3UDAQh&-HsunolV)SzaO zGp*Bk?6xBdUTjtJc&o@XRo%I^f+KLnX12%Mw;iJf{=YV^GAycg?NSno;0z)lNDd4k z4MRwWFtjuX(%mUY3IozHlyrA@cSsEgNT*7tfaDP0yzlWn*LR(Be(fL6pS_=Z?R&51 zUTbZkG|SzNzahi_;R!!#RF%l|6C}(#gwhU3(Wf6w;NwXTzU)bsI7UG8wFYusMD5S&<9QpD3-v7L7@ zqjPy6sK0I;7S7_cg9Z6Q`(=78Po5xY9X4K~Fb4Uor;|?4uo4AQCO9xtoRV1L%Ssx> zq(e7##tjv+LbPU0oxXN7SVz}Gv?U_ETgfI-T_D0P8Tt5|kEfQ*$CF;345ssPrN#(0GxUzFrSNIaAGiBB!<7`TfJ#2^SCd4}n_q z(;2&03h!S{=tT={(?>VaN^h>Ze9|&u^Svu)#C$bTCksM^6)d}!&#IZsoq;p`G!6~p zW}}eLI@%6dn~(Nn*15j5rD=AjB*}4&NUF~={Yy@c|I`Hlr-W~=(D>bU5QxU3Up{i3 zp2V6k^&|o!zQ%B@%RC6{WVb!S49iz;vo800qvPX-a7x2#o$1L4 z>VvKX1||FM*04zZy~kg|Ea++SnDC-%tvqBcrwItjRMu#7cV(@Np2?-|kBq#8xe3WK z44I3_S|ACQ&2EU1(@7`(ujX39fQWv1N}$(Wrka^6rN!-0=g8;kq*fBqldnKBM`u5^ z0F!lk52{PvgnXG`7e4?Kk5 zF|DOj&cwzzM1ap7B58uAj+(xd)esScKRrKsPfMZCezvWvk>kmKp`$N#`Xy6qgqywX zhtu)~La52Hf@%5ACogrwg{xVSNTbZ5gTQ&no$589`n+QY#ttX zGiuz_$B=I}e8>|KK}P|>m8vH-r`i#ihd&PDdem@1#(`-)Tb=t`cn1`ZT@-ko=2;NT zAk(%383DM*6^0(*JQgv)&fxq(9iw6EhUzk3SVUs2xbUTmdxA#UGD41A@ZvA#1# z3|p{>0E3}ijWrJUx7iLLFObi9BFadMsU*DIzTLjz&|O@e?{#NL#|!WQ%oOOq~{iCjkqNgvTj+>_?(+=X+MxP(}vW~?i#o6&C7uIjN++vm6`M^fXEYcS5&l(!o3JCD}KK~=O;at zjBph5_%{A+@d)Ch9USbF770s@>a%sv5qRH?8BF0dvR8jD9`-^N&TUB zcoxK*L7sY7?^-hZ_xO0PO?F;@<7U?{aA%QvnD-4m9ay=+!d(9032V6A@z*UX>K#%} zW$fbNeB;nh;ThoH@hjl-0jB_jZ)DC;mCjA969;Jo9h3hr!(x%j-lfTQ1 zv#hw@@%o81Fx5%9zU*FSvLxVa7zmGY4-L=1| zs+41;rAj7(1f&J%2*RSuW)(8Fg&tI%f?6Ef&N3E#<`)I{Uo9-BQ0ej?p`B*oJfR(d zi@WCx`TWZG-ma!enct3zSy@hcCe_=IxI8p}p$mVzDaFm#!i9iRx5p}OL3F{jT(UM) z|A+@lJa|`id-q_n9VW+2x1h1f%Wm2fbso&v_d8kv6vK6qcczE#0C(5-vM$(`%em6qS~O_t2XecojL9{Qrs{4CwFdCYq3 z*4cAurJd_IjAi+YMH5@E`+P~brSmFp?KRTvcE?aL@Vl!&o|=to=d2!?DTGMRJYdg4@8&Q{Tt8-iQ0U=gYlV7EQGqHZL zIL)WP{e{{|2PTe90MbTOJfxIkJm((oq!q2uvpe4?&&XX~AM`zV>3KTlMU~RauET+| zY6qAp#!Z-P-D2h8ef!ser`dMQCtOi!eQgSRSZnppFeI(U7C`cDyJm z5$6Dxw8 zqEAi9-=pwC5w2*0jg>=&U$bq%tm;oOQWkFtU1^u}e&n2Q)ov-kz<`}??dx$VA18H_ zpJz50Rp!l@k`Kpom0ohI_5Y8miJf+w9`tSoMO8Hr^z)sj%wvwfRb{KffF=yY)klU#GG%(-*yPt=dS+1eG?PqVa$O)s*$*({{G&QXZ zBR3!drP)nMg%U`!)@I!i0lsD)eKZ{%5_fCc^e42}+g zZ{z)Blg>t(2=&P7e2VS!k8F0@A`pG#=>|Bo+y9=gUc^ShNs(6@8tQM@L6Kkxfq#H| zmFf%=ZH>cDcdUwbWH(}>a`CkkVSwoAl0X#7@srKBdC29z>V+YLOyd~DTmZmP!D?Ig zSQXT)@Swz}UNLD&lw{bbseKQ9N^_7+ zE26~_hL6RW-~2Tyy>*=?9#HcOC4AjwECr1-xk#4}Fk<;a<~^wL0`P0+Lm4z?7gdzL zv=h|7Zhe11iykt0wU_yLki>U-KXA*DaJKdiLFsIQ9Xu zRyoso7_IsJp)iBcWJlUm>1-MAaIQG$ho>$1>4U!r;Jeg( zS>4$ke8NphVTAcUVrRD%-_waDODbVnmO^_0gi%s3E&KdZ(y}LyRS===#~z9 z=R2SGgtL#>IJAAwrqq3b(zV!w zsrE_{h&?4)VyE)n9&UH7oa387r^hH*{=gmIc1T&;ae2bhd~B}uE|G6m$Ln%0kA-2F zSFPe?Th#Tk***aIjVKyQwLT^IR?#FbOclSfy+h&icnTbDtaS8y3VY~B5SwIFrd9Dcm-W^zbP$$% zr%j0TwzQwGX|D3dqv}YvgY=L!O12+A( zsXNt^74lf#^VN4qPZ7F+jT}ttiuIDtNY&wLr_Qi$oEP)ymLgbOwze7N_%uoZqOav5 zNPQD$#WPKpz{;W*VKO@e=gq&`T=INcAza9JT2dINbWlyg`f6YBvc|mj>mZ~%B-lpe ze%PkQec6GsR8~+Xx}^)h%d@O#ZPdcRo7Dkb>lGfIy*i+H?qp>@QfO?bK}e}2Oj+nO z;O3*(SK0uft=dl+Om-7V%p7lZutlSV&)PWDlkLXzxmD~7Vw2tPdefAGFHb1SemasH zD@eB-g|?^Z52tNe92gZPmA$a$-4vZ7PQ>JEW0&RM8g~zpd%LJz$rJv=9ba*4Yf6;{ ztkAJy>nxx?5{EpihVWIkSriul!6Vs8SQ!k4>R-}%ZG;82j0ec))!{mL)b-*2^1*bW z<9;JW5nY9(=30BHc7+zS;1rBy8r$cEDp27`)(d^bh1VGeEG(L#WiHE$6OFu)K$5NS(u!ZchLvy(s(Joag;r z1DzRo?I+S>791zy?CFky^XF);)|TkdZU~Hb0Nj__Mj2cQ=?M{iVVs_Aypc{j^oDjf zv_@J+#$>(SIEYNtra++9E8A|LvFZU6zFD^j7!OBccq+M=&6c(bo#* z$V~JVzrKW3Rt3Kz+1wVgPUZQsXygpJ_;~|W>6oc!-`IS#sVHGA+hcL;+uPgpQWha) ziFT5$(TiKdK#Y6;H6T&2p%?k@YQo7SVV4pfn}xj5)3r4jnhuAT0lJ&P8hS?HR5`9v z+yZy4f7vx1=~PUl;w-9mw<F`O@CwUES-rxKt#qU{Rt#|W5hTI3TY`(c==B#M`on` zTyx);w%3*lX!qQMc%Jv?GnDupDxSN{a{k(#yh55N00GNg~RXa zmI{g(OA1gmAmcmdYJQxZweTmpV@-uY<$J>v;RqQPC5I%(nRILz$43<7X*A_*&k@U? z(`4IX`T5T^wNqvf@SNsAyVQ{b?a`(Ij8K@vsK1fdAjO4aRWi2DdD&K+b}#3F(9> zWcQbd!!TJHG?Z(QuEswl$(3;`)3H#>np=d)o#H+sY6Ik~QjEZ7eHQ|FK}DFNUnLKR@z zVWM#=7Y!=FPcS1(U9NVbPGcH~s30C~G*)q!OZ~@O?Z`u|ViUs9Jtp;dR9WkmNq|uU z7;&9cc)ky!87zM(5w)qj>dUmJd1si|Xg5tUpG?o_1gHL3#HoqDHMi{89U<9tFgE7_ z>%W#o={JhXMIM{x`SR2~=8Mz@Ezq48D z0Cf5k4J?&bmj8$-{k`fzn`lrqQ`=$B&Nq$L*g%*G* z1q=V@PD*2;?4|&*gNpyPx)$mWR8$snRr+tJp*;CvxQr8JwEheF|7*32=;J+(!fUP} V)iopu>W_+OP-!Kp3JJr2{{x%~kDUMj literal 0 HcmV?d00001 From 7de02d541f3af4d287756c8ae1a0fa69d397649f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 14 Jun 2021 15:28:44 -0400 Subject: [PATCH 276/378] Increase width of verified commit screenshot --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 801d3502d..18362709a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,7 +17,7 @@ * **All commits must be signed** with a valid key that has been added to your GitHub account. The commits should have all the "**Verified**" tag when viewed on GitHub as shown below: - + * If an issue does not already exist for the bug or feature for which you are submitting a pull request, [create one](https://github.com/Security-Onion-Solutions/securityonion/issues/new) with the relevant prefix. (**`FIX:`** for bug fixes, **`FEATURE:`** for new features.) From 20437ef2c777864b154c35a8122919459584f40e Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 14 Jun 2021 15:42:18 -0400 Subject: [PATCH 277/378] Create SECURITY.md --- SECURITY.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..46ae8090c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 2.x.x | :white_check_mark: | +| 16.04.x | :x: | + +Security Onion 16.04 has reached End Of Life and is no longer supported. + +## Reporting a Vulnerability + +If you have any security concerns regarding Security Onion or believe you have uncovered a vulnerability, please follow these steps: + +- send an email to security@securityonion.net +- include a description of the issue and steps to reproduce +- please use plain text format (no Word documents or PDF files) +- please do not disclose publicly until we have had sufficient time to resolve the issue + +This security address should be used only for undisclosed vulnerabilities. Dealing with fixed issues or general questions on how to use Security Onion should be handled via the normal support channels. From 3891ca2929694dd2a9760f923eb6845bac7a88dc Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 14 Jun 2021 15:46:25 -0400 Subject: [PATCH 278/378] Use correct mode param to file.recurse --- salt/soctopus/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index 984ff45a9..1b1132f5a 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -21,7 +21,7 @@ soctopus-sync: - source: salt://soctopus/files/templates - user: 939 - group: 939 - - mode: 600 + - file_mode: 600 - template: jinja soctopusconf: From ca6f3807fc868d2b68230625167622da559560ee Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 14 Jun 2021 15:58:21 -0400 Subject: [PATCH 279/378] Don't use idioms, and remove TBD lines --- CONTRIBUTING.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18362709a..56e80801d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ * **If you are familiar with the current version of Security Onion and are confident you've discovered a bug**, first ensure there is not already an issue present by searching the open [issues](https://github.com/Security-Onion-Solutions/securityonion/issues). If there is, a thumbs up :+1: is a great way to show this bug is affecting you too. -* If an issue doesn't exist, [open a new one](https://github.com/Security-Onion-Solutions/securityonion/issues/new), following the directions in the issue template. These boil down to including: +* If an issue doesn't exist, [open a new one](https://github.com/Security-Onion-Solutions/securityonion/issues/new), following the directions in the issue template. This means including: * **System information** and how Security Onion was installed * **Log files** relevant to the bug report * **Reproduction steps** @@ -35,7 +35,3 @@ * All new Bash code should pass [ShellCheck](https://www.shellcheck.net/) analysis. Where errors can be *safely* [ignored](https://github.com/koalaman/shellcheck/wiki/Ignore), the relevant disable directive should be accompanied by a brief explanation as to why the error is being ignored. * **Ensure all YAML (this includes Salt states and pillars) is properly formatted**. The spec for YAML v1.2 can be found [here](https://yaml.org/spec/1.2/spec.html), however there are numerous online resources with simpler descriptions of its formatting rules. - -* **Python** - TBD - -* **Golang** - TBD From 71b079eb54b81cb3378270a2bef3083feb14a77c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 14 Jun 2021 16:04:22 -0400 Subject: [PATCH 280/378] Add bullet detailing linking pull request to issue --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56e80801d..b7a9731b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,8 @@ * If an issue does not already exist for the bug or feature for which you are submitting a pull request, [create one](https://github.com/Security-Onion-Solutions/securityonion/issues/new) with the relevant prefix. (**`FIX:`** for bug fixes, **`FEATURE:`** for new features.) +* Link the PR to the related issue, either using [keywords](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) in the PR description, or [manually](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-issues/linking-a-pull-request-to-an-issue#manually-linking-a-pull-request-to-an-issue). + * **Pull requests should be opened against the `dev` branch of this repo**, and should clearly describe the problem and solution. * Be sure you have tested your changes and are confident they will not break other parts of the product. From 2d5591a87ffa26ce2445273b1479f445d92f6ccd Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 14 Jun 2021 16:33:52 -0400 Subject: [PATCH 281/378] Remove draft label --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b7a9731b7..02ab437fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to Security Onion - DRAFT +# Contributing to Security Onion ### Questions, suggestions, and general comments * Security Onion uses GitHub's [Discussions](https://github.com/Security-Onion-Solutions/securityonion/discussions) to provide a forum where the community and developers can interact as well as ask and answer questions. From dd8eb29a1861205579c6c85aba8b5fc4057c2b44 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Jun 2021 09:11:58 -0400 Subject: [PATCH 282/378] Continue merge of ECS into Elastic Auth --- salt/curator/init.sls | 1 + salt/elastalert/init.sls | 1 + salt/filebeat/init.sls | 2 ++ salt/kibana/init.sls | 1 + salt/logstash/init.sls | 1 + .../so/9050_output_filebeatmodules.conf.jinja | 30 +++++++++++-------- salt/soctopus/init.sls | 1 + salt/telegraf/init.sls | 1 + 8 files changed, 25 insertions(+), 13 deletions(-) diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 966b0b857..48a10b4b8 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -49,6 +49,7 @@ curconf: - source: salt://curator/files/curator.yml - user: 934 - group: 939 + - mode: 660 - template: jinja curcloseddel: diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 8fcb46cda..205d6432e 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -99,6 +99,7 @@ elastaconf: elastalert_config: {{ elastalert_config.elastalert.config }} - user: 933 - group: 933 + - mode: 660 - template: jinja wait_for_elasticsearch: diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index c5d859307..1517226a3 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -1,3 +1,4 @@ + # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -83,6 +84,7 @@ filebeatmoduleconfsync: - source: salt://filebeat/etc/module-setup.yml - user: root - group: root + - mode: 660 - template: jinja sodefaults_module_conf: diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 61498cfeb..40ed8babc 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -35,6 +35,7 @@ synckibanaconfig: - source: salt://kibana/etc - user: 932 - group: 939 + - file_mode: 660 - template: jinja kibanalogdir: diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 2c2c89626..bfd08e4fe 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -78,6 +78,7 @@ ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: {% endif %} - user: 931 - group: 939 + - mode: 660 - makedirs: True {% endfor %} diff --git a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja index 20e9f0c0a..01d57c9d6 100644 --- a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja @@ -3,18 +3,22 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} output { - if [metadata][pipeline] { - elasticsearch { - id => "filebeat_modules_metadata_pipeline" - pipeline => "%{[metadata][pipeline]}" - hosts => "{{ ES }}" - index => "so-%{[event][module]}-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } + if [metadata][pipeline] { + elasticsearch { + id => "filebeat_modules_metadata_pipeline" + pipeline => "%{[metadata][pipeline]}" + hosts => "{{ ES }}" + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + index => "so-%{[event][module]}-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } } \ No newline at end of file diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index 1b1132f5a..b32c67487 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -44,6 +44,7 @@ playbookrulesdir: - name: /opt/so/rules/elastalert/playbook - user: 939 - group: 939 + - mode: 660 - makedirs: True playbookrulessync: diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index cea4d3f45..14373fe9d 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -38,6 +38,7 @@ tgrafconf: - name: /opt/so/conf/telegraf/etc/telegraf.conf - user: 939 - group: 939 + - mode: 660 - template: jinja - source: salt://telegraf/etc/telegraf.conf From b95437347e7d19b91f5bad9f66d7da1445d9f71b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Jun 2021 12:50:57 -0400 Subject: [PATCH 283/378] Upgrade ES to 7.13.2 --- salt/kibana/files/saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/saved_objects.ndjson b/salt/kibana/files/saved_objects.ndjson index 639a38bfc..ee2842b66 100644 --- a/salt/kibana/files/saved_objects.ndjson +++ b/salt/kibana/files/saved_objects.ndjson @@ -460,7 +460,7 @@ {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\",\"time_zone\":\"America/New_York\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":48,\"x\":0,\"y\":0,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":96,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":8,\"y\":20,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":28,\"y\":20,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":0,\"y\":72,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":48,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":24,\"x\":16,\"y\":72,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":28,\"y\":8,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":120,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"event_type\",\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"}]","timeRestore":false,"title":"z16.04 - Sysmon - Logs","version":1},"id":"6d189680-6d62-11e7-8ddb-e71eb260f4a3","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"8cfdeff0-6d6b-11e7-ad64-15aa071374a6","name":"panel_1","type":"visualization"},{"id":"0eb1fd80-6d70-11e7-b09b-f57b22df6524","name":"panel_2","type":"visualization"},{"id":"3072c750-6d71-11e7-b09b-f57b22df6524","name":"panel_3","type":"visualization"},{"id":"7bc74b40-6d71-11e7-b09b-f57b22df6524","name":"panel_4","type":"visualization"},{"id":"13ed0810-6d72-11e7-b09b-f57b22df6524","name":"panel_5","type":"visualization"},{"id":"3b6c92c0-6d72-11e7-b09b-f57b22df6524","name":"panel_6","type":"visualization"},{"id":"e09f6010-6d72-11e7-b09b-f57b22df6524","name":"panel_7","type":"visualization"},{"id":"29611940-6d75-11e7-b09b-f57b22df6524","name":"panel_8","type":"visualization"},{"id":"6b70b840-6d75-11e7-b09b-f57b22df6524","name":"panel_9","type":"visualization"},{"id":"248c1d20-6d6b-11e7-ad64-15aa071374a6","name":"panel_10","type":"search"},{"id":"AWDHHk1sxQT5EBNmq43Y","name":"panel_11","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQyLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} -{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.13.1","id":"7.13.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.13.2","id":"7.13.2","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":56,\"x\":0,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":8,\"y\":32,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":20,\"y\":32,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":56,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"columns\":[\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"uid\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":32,\"y\":32,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":40,\"h\":24,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"}]","timeRestore":false,"title":"z16.04 - Bro - Modbus","version":1},"id":"70c005f0-3583-11e7-a588-05992195c551","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"0d168a30-363f-11e7-a6f7-4f44d7bf1c33","name":"panel_1","type":"visualization"},{"id":"20eabd60-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_2","type":"visualization"},{"id":"3c65f500-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_3","type":"visualization"},{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"panel_4","type":"search"},{"id":"178209e0-6e1b-11e7-b553-7f80727663c1","name":"panel_5","type":"visualization"},{"id":"AWDG_9KpxQT5EBNmq4Oo","name":"panel_6","type":"visualization"},{"id":"453f8b90-4a58-11e8-9b0a-f1d33346f773","name":"panel_7","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ4LDRd"} From 989f9dce422fce5fc726ac7a9ea73d2029c08d87 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 15 Jun 2021 16:57:13 -0400 Subject: [PATCH 284/378] Ensure sqlite.db exists before querying it; Execute so-elastic-auth after common state has been applied and redirect output to setup log --- salt/common/tools/sbin/so-user | 2 +- setup/so-setup | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 54bcf7f71..800fca745 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -232,7 +232,7 @@ function syncElastic() { } function syncAll() { - if [[ -n "$STALE_MIN" ]]; then + if [[ -n "$STALE_MIN" && -f "$databasePath" ]]; then staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${STALE_MIN} minutes');" \ | sqlite3 "$databasePath") if [[ "$staleCount" == "0" ]]; then diff --git a/setup/so-setup b/setup/so-setup index 2c0dc934f..f5f32f3d4 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -751,7 +751,6 @@ echo "1" > /root/accept_changes set_progress_str 60 "$(print_salt_state_apply 'manager')" salt-call state.apply -l info manager >> $setup_log 2>&1 - ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth fi set_progress_str 61 "$(print_salt_state_apply 'firewall')" @@ -771,6 +770,10 @@ echo "1" > /root/accept_changes fi if [[ $is_manager || $is_helix || $is_import ]]; then + echo "Executing so-elastic-auth..." >> $setup_log 2>&1 + ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /usr/sbin/so-elastic-auth >> $setup_log 2>&1 + echo "Finished so-elastic-auth..." >> $setup_log 2>&1 + set_progress_str 63 "$(print_salt_state_apply 'idstools')" create_local_nids_rules >> $setup_log 2>&1 salt-call state.apply -l info idstools >> $setup_log 2>&1 From ad9441bb60d7df746e9ae1a05a2f60607513d08d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 16 Jun 2021 08:06:26 -0400 Subject: [PATCH 285/378] prevent suricata state from running on manager and managersearch https://github.com/Security-Onion-Solutions/securityonion/issues/2977 --- salt/suricata/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 58bbe2a05..8df1a5a00 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . {% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} +{% if sls in allowed_states and grains.role not in ['so-manager', 'so-managersearch'] %} {% set interface = salt['pillar.get']('sensor:interface', 'bond0') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} @@ -205,4 +205,4 @@ clean_suricata_eve_files: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From ebea9a7198d21f0e5d6e636a83a5626bf0235fb3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 16 Jun 2021 08:07:28 -0400 Subject: [PATCH 286/378] remove space --- salt/suricata/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 8df1a5a00..4cbda4bb4 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . {% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states and grains.role not in ['so-manager', 'so-managersearch'] %} +{% if sls in allowed_states and grains.role not in ['so-manager', 'so-managersearch'] %} {% set interface = salt['pillar.get']('sensor:interface', 'bond0') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} From 09fbb045a14e1f2675fab402b63f4cd2e325fa5d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 16 Jun 2021 09:59:57 -0400 Subject: [PATCH 287/378] If ES auth disabled ensure user/pass are blank --- salt/curator/files/curator.yml | 9 +++++++-- salt/elastalert/defaults.yaml | 9 +++++++-- salt/filebeat/etc/filebeat.yml | 10 +++++++--- salt/filebeat/etc/module-setup.yml | 9 +++++++-- salt/kibana/etc/kibana.yml | 9 +++++++-- .../pipelines/config/so/9000_output_zeek.conf.jinja | 9 +++++++-- .../pipelines/config/so/9002_output_import.conf.jinja | 9 +++++++-- .../pipelines/config/so/9004_output_flow.conf.jinja | 9 +++++++-- .../pipelines/config/so/9033_output_snort.conf.jinja | 9 +++++++-- .../pipelines/config/so/9034_output_syslog.conf.jinja | 9 +++++++-- .../config/so/9050_output_filebeatmodules.conf.jinja | 11 ++++++++--- .../config/so/9100_output_osquery.conf.jinja | 9 +++++++-- .../so/9101_output_osquery_livequery.conf.jinja | 11 ++++++++--- .../config/so/9200_output_firewall.conf.jinja | 9 +++++++-- .../config/so/9400_output_suricata.conf.jinja | 11 ++++++++--- .../pipelines/config/so/9500_output_beats.conf.jinja | 9 +++++++-- .../pipelines/config/so/9600_output_ossec.conf.jinja | 9 +++++++-- .../config/so/9700_output_strelka.conf.jinja | 9 +++++++-- salt/soc/files/soc/soc.json | 9 +++++++-- salt/soctopus/files/SOCtopus.conf | 9 +++++++-- salt/soctopus/files/templates/es-generic.template | 9 +++++++-- salt/soctopus/files/templates/generic.template | 11 ++++++++--- salt/soctopus/files/templates/osquery.template | 11 ++++++++--- salt/telegraf/etc/telegraf.conf | 9 +++++++-- 24 files changed, 173 insertions(+), 54 deletions(-) diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 0934aaf02..0215fbeeb 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -3,8 +3,13 @@ {% elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %} {%- set elasticsearch = salt['pillar.get']('manager:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} --- # Remember, leave a key empty if there is no value. None will be a string, diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index accccaa3a..b65d718ac 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -1,5 +1,10 @@ -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} elastalert: config: rules_folder: /opt/elastalert/rules/ diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 0297d2fe8..ba7bb0520 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -3,9 +3,13 @@ {%- else %} {%- set MANAGER = salt['grains.get']('master') %} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass', '') %} - +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} {%- set HOSTNAME = salt['grains.get']('host', '') %} {%- set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %} diff --git a/salt/filebeat/etc/module-setup.yml b/salt/filebeat/etc/module-setup.yml index 35fbf5fbe..50bda9cf4 100644 --- a/salt/filebeat/etc/module-setup.yml +++ b/salt/filebeat/etc/module-setup.yml @@ -3,8 +3,13 @@ {%- else %} {%- set MANAGER = salt['grains.get']('master') %} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output.elasticsearch: enabled: true diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index a3f83a516..efe62cc11 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -1,8 +1,13 @@ --- # Default Kibana configuration from kibana-docker. {%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} server.name: kibana server.host: "0" server.basePath: /kibana diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index af3a9f93b..c3e567645 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index feaddeded..709b1a0f9 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if "import" in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index e01792914..a98aaad26 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [event_type] == "sflow" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 42e4dbee4..d31f2f00c 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index ca6308ada..6527c7160 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [module] =~ "syslog" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja index 01d57c9d6..61aa4879a 100644 --- a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [metadata][pipeline] { elasticsearch { @@ -21,4 +26,4 @@ output { ssl_certificate_verification => false } } -} \ No newline at end of file +} diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 43596c1cd..8d6095ea3 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [module] =~ "osquery" and "live_query" not in [dataset] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja index 10c3cba9f..0045db734 100644 --- a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja @@ -3,9 +3,14 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} +{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} filter { if [type] =~ "live_query" { diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 6abe75f9d..f89ef7674 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [dataset] =~ "firewall" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index b2a2c15be..251aa5ff1 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { @@ -19,4 +24,4 @@ output { ssl_certificate_verification => false } } -} \ No newline at end of file +} diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index ffe30c8c5..cf8427f0b 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 7ef4bca1f..69a7e7309 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [module] =~ "ossec" { elasticsearch { diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index a26373397..bdcce2cae 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,8 +3,13 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} output { if [module] =~ "strelka" { elasticsearch { diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 2cd213276..36135b6e5 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -18,8 +18,13 @@ {%- import_json "soc/files/soc/menu.actions.json" as menu_actions %} {%- import_json "soc/files/soc/tools.json" as tools %} {%- set DNET = salt['pillar.get']('global:dockernet', '172.17.0.0') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} { "logFilename": "/opt/sensoroni/logs/sensoroni-server.log", diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 4b47c8b6a..bb95c2667 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -3,8 +3,13 @@ {%- set HIVEKEY = salt['pillar.get']('global:hivekey', '') %} {%- set CORTEXKEY = salt['pillar.get']('global:cortexorguserkey', '') %} {%- set PLAYBOOK_KEY = salt['pillar.get']('playbook:api_key', '') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} [es] es_url = https://{{MANAGER}}:9200 diff --git a/salt/soctopus/files/templates/es-generic.template b/salt/soctopus/files/templates/es-generic.template index 6e50a3f3e..08f1c796f 100644 --- a/salt/soctopus/files/templates/es-generic.template +++ b/salt/soctopus/files/templates/es-generic.template @@ -1,6 +1,11 @@ {% set ES = salt['pillar.get']('global:managerip', '') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} alert: modules.so.playbook-es.PlaybookESAlerter elasticsearch_host: "{{ ES }}:9200" diff --git a/salt/soctopus/files/templates/generic.template b/salt/soctopus/files/templates/generic.template index 33d8b7ea5..8fad2827d 100644 --- a/salt/soctopus/files/templates/generic.template +++ b/salt/soctopus/files/templates/generic.template @@ -1,6 +1,11 @@ {% set es = salt['pillar.get']('global:url_base', '') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} alert: - "modules.so.playbook-es.PlaybookESAlerter" @@ -17,4 +22,4 @@ rule.category: play_url: "https://{{ es }}/playbook/issues/6000" kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))" soc_pivot: "https://{{es}}/#/hunt" -sigma_level: "" \ No newline at end of file +sigma_level: "" diff --git a/salt/soctopus/files/templates/osquery.template b/salt/soctopus/files/templates/osquery.template index 22c29193a..ad55626c2 100644 --- a/salt/soctopus/files/templates/osquery.template +++ b/salt/soctopus/files/templates/osquery.template @@ -1,6 +1,11 @@ {% set es = salt['pillar.get']('global:url_base', '') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} alert: - "modules.so.playbook-es.PlaybookESAlerter" @@ -16,4 +21,4 @@ rule.category: play_url: "https://{{ es }}/playbook/issues/6000" kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))" soc_pivot: "https://{{es}}/#/hunt" -sigma_level: "" \ No newline at end of file +sigma_level: "" diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 659a78398..e93fa45fb 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -14,8 +14,13 @@ # for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR) {%- set MANAGER = salt['grains.get']('master') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} +{%- else %} +{%- set ES_USER = '' %} +{%- set ES_PASS = '' %} +{%- endif %} {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} From dd14235e31aac51203dd42aa933b893a3793c0e6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 16 Jun 2021 11:39:21 -0400 Subject: [PATCH 288/378] Accept either 200 or 401 instead of wasting 3 minutes waiting for this to timeout --- salt/elastalert/init.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 205d6432e..1487f2cf8 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -107,6 +107,10 @@ wait_for_elasticsearch: - http.wait_for_successful_query: - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*' - wait_for: 180 + - status: + - 200 + - 401 + - status_type: list - verify_ssl: False so-elastalert: From a323aeb8fad1a0d55272ffc45b60b875c465dfef Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 16 Jun 2021 14:23:58 -0400 Subject: [PATCH 289/378] Allow so-elastic-auth to run before common even though the script has dependency on a common-provided script (benign error). This is needed first since common will need to know if auth is enabled --- salt/common/tools/sbin/so-elastic-auth | 2 +- setup/so-setup | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth index d4b8057a3..9e2843b5c 100644 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -. /usr/sbin/so-common +. /usr/sbin/so-common 2> /dev/null ES_AUTH_PILLAR=${ELASTIC_AUTH_PILLAR:-/opt/so/saltstack/local/pillar/elasticsearch/auth.sls} ES_USERS_FILE=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users} diff --git a/setup/so-setup b/setup/so-setup index f5f32f3d4..34802fef9 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -751,6 +751,10 @@ echo "1" > /root/accept_changes set_progress_str 60 "$(print_salt_state_apply 'manager')" salt-call state.apply -l info manager >> $setup_log 2>&1 + + echo "Executing so-elastic-auth..." >> $setup_log 2>&1 + ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth >> $setup_log 2>&1 + echo "Finished so-elastic-auth..." >> $setup_log 2>&1 fi set_progress_str 61 "$(print_salt_state_apply 'firewall')" @@ -770,10 +774,6 @@ echo "1" > /root/accept_changes fi if [[ $is_manager || $is_helix || $is_import ]]; then - echo "Executing so-elastic-auth..." >> $setup_log 2>&1 - ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /usr/sbin/so-elastic-auth >> $setup_log 2>&1 - echo "Finished so-elastic-auth..." >> $setup_log 2>&1 - set_progress_str 63 "$(print_salt_state_apply 'idstools')" create_local_nids_rules >> $setup_log 2>&1 salt-call state.apply -l info idstools >> $setup_log 2>&1 From fefd2677fb7122997da835b7be883a4edaf352af Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 16 Jun 2021 14:26:26 -0400 Subject: [PATCH 290/378] Only include so-common if available. It only is used for requiring root, but since this script is needed before common is installed, we can safely assume that it's being run as root already (during the install) --- salt/common/tools/sbin/so-elastic-auth | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth index 9e2843b5c..f6c19d76f 100644 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -15,7 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -. /usr/sbin/so-common 2> /dev/null +if [ -f "/usr/sbin/so-common" ]; then + . /usr/sbin/so-common +fi ES_AUTH_PILLAR=${ELASTIC_AUTH_PILLAR:-/opt/so/saltstack/local/pillar/elasticsearch/auth.sls} ES_USERS_FILE=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users} From 703988b3761bcfa1610309de94002d23f58fb7ae Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 16 Jun 2021 14:28:20 -0400 Subject: [PATCH 291/378] Fix merge issue in soup --- salt/common/tools/sbin/soup | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b5229fca1..64c8de66b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -779,12 +779,6 @@ main() { verify_latest_update_script echo "" - echo "Stopping Salt Master service." - systemctl stop salt-master - echo "" - - upgrade_to_2.3.50_repo - echo "Generating new repo archive" generate_and_clean_tarballs if [ -f /usr/sbin/so-image-common ]; then @@ -837,7 +831,7 @@ main() { systemctl stop salt-master echo "" - preupgrade_changes_2.3.50_repo + upgrade_to_2.3.50_repo # Does salt need upgraded. If so update it. if [[ $UPGRADESALT -eq 1 ]]; then From 2d342082694b3547dfa14f50325bdffe11e33902 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 16 Jun 2021 17:52:22 -0400 Subject: [PATCH 292/378] Elastic auth: Fun with Salt --- salt/common/tools/sbin/so-airgap-hotfixapply | 0 salt/common/tools/sbin/so-airgap-hotfixdownload | 0 salt/common/tools/sbin/so-elastic-auth | 9 +++++++-- salt/common/tools/sbin/so-elasticsearch-query | 0 salt/common/tools/sbin/so-elasticsearch-wait | 5 +++++ salt/common/tools/sbin/so-influxdb-clean | 0 salt/common/tools/sbin/so-influxdb-migrate | 0 salt/common/tools/sbin/so-pcap-export | 0 salt/curator/files/curator.yml | 2 ++ salt/elastalert/defaults.yaml | 11 ++++------- salt/elastalert/init.sls | 11 ++--------- salt/filebeat/etc/filebeat.yml | 11 ++++------- salt/filebeat/etc/module-setup.yml | 11 ++++------- salt/kibana/etc/kibana.yml | 11 ++++------- .../pipelines/config/so/9000_output_zeek.conf.jinja | 11 ++++------- .../pipelines/config/so/9002_output_import.conf.jinja | 11 ++++------- .../pipelines/config/so/9004_output_flow.conf.jinja | 11 ++++------- .../pipelines/config/so/9033_output_snort.conf.jinja | 11 ++++------- .../pipelines/config/so/9034_output_syslog.conf.jinja | 11 ++++------- .../config/so/9050_output_filebeatmodules.conf.jinja | 11 ++++------- .../config/so/9100_output_osquery.conf.jinja | 11 ++++------- .../so/9101_output_osquery_livequery.conf.jinja | 11 ++++------- .../config/so/9200_output_firewall.conf.jinja | 11 ++++------- .../config/so/9400_output_suricata.conf.jinja | 11 ++++------- .../pipelines/config/so/9500_output_beats.conf.jinja | 11 ++++------- .../pipelines/config/so/9600_output_ossec.conf.jinja | 11 ++++------- .../config/so/9700_output_strelka.conf.jinja | 11 ++++------- salt/salt/helper-packages.sls | 7 ------- salt/salt/map.jinja | 2 -- salt/soctopus/files/templates/es-generic.template | 11 ++++------- salt/soctopus/files/templates/generic.template | 11 ++++------- salt/soctopus/files/templates/osquery.template | 11 ++++------- salt/telegraf/etc/telegraf.conf | 11 ++++------- 33 files changed, 100 insertions(+), 167 deletions(-) mode change 100644 => 100755 salt/common/tools/sbin/so-airgap-hotfixapply mode change 100644 => 100755 salt/common/tools/sbin/so-airgap-hotfixdownload mode change 100644 => 100755 salt/common/tools/sbin/so-elastic-auth mode change 100644 => 100755 salt/common/tools/sbin/so-elasticsearch-query create mode 100755 salt/common/tools/sbin/so-elasticsearch-wait mode change 100644 => 100755 salt/common/tools/sbin/so-influxdb-clean mode change 100644 => 100755 salt/common/tools/sbin/so-influxdb-migrate mode change 100644 => 100755 salt/common/tools/sbin/so-pcap-export diff --git a/salt/common/tools/sbin/so-airgap-hotfixapply b/salt/common/tools/sbin/so-airgap-hotfixapply old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-airgap-hotfixdownload b/salt/common/tools/sbin/so-airgap-hotfixdownload old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth old mode 100644 new mode 100755 index f6c19d76f..663dbb9f6 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -29,12 +29,17 @@ if ! grep -q "enabled: " "$ES_AUTH_PILLAR"; then exit 1 fi +function restart() { + so-elastic-stop + salt-call state.highstate queue=True +} + if [[ "$authEnable" == "true" ]]; then if grep -q "enabled: False" "$ES_AUTH_PILLAR"; then sed -i 's/enabled: False/enabled: True/g' "$ES_AUTH_PILLAR" if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then echo "Applying highstate - this may take a few minutes..." - salt-call state.highstate queue=True + restart fi echo "Elastic auth is now enabled." if grep -q "argon" "$ES_USERS_FILE"; then @@ -50,7 +55,7 @@ elif [[ "$authEnable" == "false" ]]; then sed -i 's/enabled: True/enabled: False/g' "$ES_AUTH_PILLAR" if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then echo "Applying highstate - this may take a few minutes..." - salt-call state.highstate queue=True + restart fi echo "Elastic auth is now disabled." else diff --git a/salt/common/tools/sbin/so-elasticsearch-query b/salt/common/tools/sbin/so-elasticsearch-query old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-elasticsearch-wait b/salt/common/tools/sbin/so-elasticsearch-wait new file mode 100755 index 000000000..f56aafcd3 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-wait @@ -0,0 +1,5 @@ +#!/bin/bash + +. /usr/sbin/so-common + +wait_for_web_response "https://localhost:9200/_cat/indices/.kibana*" "green open" 300 "{{ ELASTICCURL }}" diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-pcap-export b/salt/common/tools/sbin/so-pcap-export old mode 100644 new mode 100755 diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 0215fbeeb..bdde14fc1 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -18,8 +18,10 @@ client: hosts: - {{elasticsearch}} port: 9200 +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} username: {{ ES_USER }} password: {{ ES_PASS }} +{% endif %} url_prefix: use_ssl: True certificate: diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index b65d718ac..9bfb4f188 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -1,10 +1,5 @@ -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} elastalert: config: rules_folder: /opt/elastalert/rules/ @@ -26,8 +21,10 @@ elastalert: use_ssl: true verify_certs: false #es_send_get_body_as: GET +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} es_username: {{ ES_USER }} es_password: {{ ES_PASS }} +{%- endif %} writeback_index: elastalert_status alert_time_limit: days: 2 diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 1487f2cf8..5fc52eebf 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -103,15 +103,8 @@ elastaconf: - template: jinja wait_for_elasticsearch: - module.run: - - http.wait_for_successful_query: - - url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*' - - wait_for: 180 - - status: - - 200 - - 401 - - status_type: list - - verify_ssl: False + cmd.run: + - name: so-elasticsearch-wait so-elastalert: docker_container.running: diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index ba7bb0520..2a86b486f 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -3,13 +3,8 @@ {%- else %} {%- set MANAGER = salt['grains.get']('master') %} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} {%- set HOSTNAME = salt['grains.get']('host', '') %} {%- set ZEEKVER = salt['pillar.get']('global:mdengine', 'COMMUNITY') %} @@ -272,8 +267,10 @@ output.{{ type }}: output.elasticsearch: enabled: true hosts: ["https://{{ MANAGER }}:9200"] +{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} username: "{{ ES_USER }}" password: "{{ ES_PASS }}" +{%- endif %} ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"] pipelines: - pipeline: "%{[module]}.%{[dataset]}" diff --git a/salt/filebeat/etc/module-setup.yml b/salt/filebeat/etc/module-setup.yml index 50bda9cf4..6c2f91d18 100644 --- a/salt/filebeat/etc/module-setup.yml +++ b/salt/filebeat/etc/module-setup.yml @@ -3,17 +3,14 @@ {%- else %} {%- set MANAGER = salt['grains.get']('master') %} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output.elasticsearch: enabled: true hosts: ["https://{{ MANAGER }}:9200"] +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} username: "{{ ES_USER }}" password: "{{ ES_PASS }}" +{% endif %} ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"] diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index efe62cc11..6fcafe68f 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -1,21 +1,18 @@ --- # Default Kibana configuration from kibana-docker. {%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} server.name: kibana server.host: "0" server.basePath: /kibana elasticsearch.hosts: [ "https://{{ ES }}:9200" ] elasticsearch.ssl.verificationMode: none #kibana.index: ".kibana" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} elasticsearch.username: {{ ES_USER }} elasticsearch.password: {{ ES_PASS }} +{% endif %} #xpack.monitoring.ui.container.elasticsearch.enabled: true elasticsearch.requestTimeout: 90000 logging.dest: /var/log/kibana/kibana.log diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index c3e567645..670dcf49e 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,20 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-zeek" template_name => "so-zeek" template => "/templates/so-zeek-template.json" diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 709b1a0f9..1ebaa1082 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,20 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if "import" in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-import" template_name => "so-import" template => "/templates/so-import-template.json" diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index a98aaad26..affa32d1a 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,19 +3,16 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [event_type] == "sflow" { elasticsearch { hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-flow" template_name => "so-flow" template => "/templates/so-flow-template.json" diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index d31f2f00c..ea603b016 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,19 +3,16 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-ids" template_name => "so-ids" template => "/templates/so-ids-template.json" diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 6527c7160..ab8508bf3 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,20 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [module] =~ "syslog" { elasticsearch { pipeline => "%{module}" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-syslog" template_name => "so-syslog" template => "/templates/so-syslog-template.json" diff --git a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja index 61aa4879a..56c8a311b 100644 --- a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja @@ -3,21 +3,18 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [metadata][pipeline] { elasticsearch { id => "filebeat_modules_metadata_pipeline" pipeline => "%{[metadata][pipeline]}" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-%{[event][module]}-%{+YYYY.MM.dd}" template_name => "so-common" template => "/templates/so-common-template.json" diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 8d6095ea3..b997ea7be 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,20 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [module] =~ "osquery" and "live_query" not in [dataset] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-osquery" template_name => "so-osquery" template => "/templates/so-osquery-template.json" diff --git a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja index 0045db734..fce35b5a4 100644 --- a/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja @@ -4,13 +4,8 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} {%- set FEATURES = salt['pillar.get']('elastic:features', False) %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} filter { if [type] =~ "live_query" { @@ -37,8 +32,10 @@ output { elasticsearch { pipeline => "osquery.live_query" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-osquery" template_name => "so-osquery" template => "/templates/so-osquery-template.json" diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index f89ef7674..e82dbb4f8 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,19 +3,16 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [dataset] =~ "firewall" { elasticsearch { hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-firewall" template_name => "so-firewall" template => "/templates/so-firewall-template.json" diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 251aa5ff1..34e2bab7c 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,20 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-ids" template_name => "so-ids" template => "/templates/so-ids-template.json" diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index cf8427f0b..2ad403ab9 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,20 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { pipeline => "beats.common" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-beats" template_name => "so-beats" template => "/templates/so-beats-template.json" diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 69a7e7309..6e03d8c72 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,20 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [module] =~ "ossec" { elasticsearch { pipeline => "%{module}" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-ossec" template_name => "so-ossec" template => "/templates/so-ossec-template.json" diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index bdcce2cae..007f1370e 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,20 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { if [module] =~ "strelka" { elasticsearch { pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" password => "{{ ES_PASS }}" +{% endif %} index => "so-strelka" template_name => "so-strelka" template => "/templates/so-strelka-template.json" diff --git a/salt/salt/helper-packages.sls b/salt/salt/helper-packages.sls index c26cdc7c0..32480c163 100644 --- a/salt/salt/helper-packages.sls +++ b/salt/salt/helper-packages.sls @@ -1,10 +1,3 @@ -{% from 'salt/map.jinja' import PYINOTIFYPACKAGE with context%} -{% from 'salt/map.jinja' import PYTHONINSTALLER with context%} - patch_package: pkg.installed: - name: patch - -pyinotify: - {{PYTHONINSTALLER}}.installed: - - name: {{ PYINOTIFYPACKAGE }} diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 4b9577319..3ef334752 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -11,7 +11,6 @@ {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION %} {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} - {% set PYINOTIFYPACKAGE = 'pyinotify' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} @@ -22,7 +21,6 @@ {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} - {% set PYINOTIFYPACKAGE = 'securityonion-python3-pyinotify' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/soctopus/files/templates/es-generic.template b/salt/soctopus/files/templates/es-generic.template index 08f1c796f..9b5ace95a 100644 --- a/salt/soctopus/files/templates/es-generic.template +++ b/salt/soctopus/files/templates/es-generic.template @@ -1,16 +1,13 @@ {% set ES = salt['pillar.get']('global:managerip', '') %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} alert: modules.so.playbook-es.PlaybookESAlerter elasticsearch_host: "{{ ES }}:9200" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} elasticsearch_user: "{{ ES_USER }}" elasticsearch_pass: "{{ ES_PASS }}" +{% endif %} play_title: "" play_url: "https://{{ ES }}/playbook/issues/6000" sigma_level: "" diff --git a/salt/soctopus/files/templates/generic.template b/salt/soctopus/files/templates/generic.template index 8fad2827d..d3736f894 100644 --- a/salt/soctopus/files/templates/generic.template +++ b/salt/soctopus/files/templates/generic.template @@ -1,18 +1,15 @@ {% set es = salt['pillar.get']('global:url_base', '') %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} alert: - "modules.so.playbook-es.PlaybookESAlerter" elasticsearch_host: "{{ es }}:9200" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} elasticsearch_user: "{{ ES_USER }}" elasticsearch_pass: "{{ ES_PASS }}" +{% endif %} play_title: "" play_id: "" event.module: "playbook" diff --git a/salt/soctopus/files/templates/osquery.template b/salt/soctopus/files/templates/osquery.template index ad55626c2..328a7e275 100644 --- a/salt/soctopus/files/templates/osquery.template +++ b/salt/soctopus/files/templates/osquery.template @@ -1,18 +1,15 @@ {% set es = salt['pillar.get']('global:url_base', '') %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} alert: - "modules.so.playbook-es.PlaybookESAlerter" elasticsearch_host: "{{ es }}:9200" +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} elasticsearch_user: "{{ ES_USER }}" elasticsearch_pass: "{{ ES_PASS }}" +{% endif %} play_title: "" event.module: "playbook" event.dataset: "alert" diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index e93fa45fb..44e78ecda 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -14,13 +14,8 @@ # for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR) {%- set MANAGER = salt['grains.get']('master') %} -{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- else %} -{%- set ES_USER = '' %} -{%- set ES_PASS = '' %} -{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') %} {% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} {% set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} @@ -627,8 +622,10 @@ {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} [[inputs.elasticsearch]] servers = ["https://{{ MANAGER }}:9200"] +{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} username = "{{ ES_USER }}" password = "{{ ES_PASS }}" +{% endif %} insecure_skip_verify = true {% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} [[inputs.elasticsearch]] From 059b016c62058b820b59bbb3f7866e132d1af6b7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 16 Jun 2021 21:48:31 -0400 Subject: [PATCH 293/378] Fix require statement --- salt/elastalert/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 5fc52eebf..f3f4af3f9 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -121,7 +121,7 @@ so-elastalert: - extra_hosts: - {{MANAGER_URL}}:{{MANAGER_IP}} - require: - - module: wait_for_elasticsearch + - cmd: wait_for_elasticsearch - watch: - file: elastaconf From 2a5198cae41fb15ac179c82d015f6ff83fa86695 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 17 Jun 2021 08:49:21 -0400 Subject: [PATCH 294/378] change perms to resolve error about module-setup.yml being 660 --- salt/filebeat/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 1517226a3..5b84851a2 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -84,7 +84,7 @@ filebeatmoduleconfsync: - source: salt://filebeat/etc/module-setup.yml - user: root - group: root - - mode: 660 + - mode: 640 - template: jinja sodefaults_module_conf: From 8cd2bc7c13cce240cab39441edd99f4329b5bb1f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 17 Jun 2021 09:37:21 -0400 Subject: [PATCH 295/378] adding so-eval to ES_INCLUDED_NODES --- salt/filebeat/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 5b84851a2..0cbbf0594 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -23,7 +23,7 @@ {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} {% from 'filebeat/map.jinja' import SO with context %} -{% set ES_INCLUDED_NODES = ['so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %} +{% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %} #only include elastic state for certain nodes {% if grains.role in ES_INCLUDED_NODES %} From d587120613d4f8e630c38d416dab1365960c4765 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 17 Jun 2021 14:42:04 -0400 Subject: [PATCH 296/378] set ELASTICCUURL default as curl --- salt/common/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index 9d20de62b..c8f1c7f12 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -186,6 +186,8 @@ utilsyncscripts: - template: jinja - source: salt://common/tools/sbin - defaults: + ELASTICCURL: 'curl' + - context: ELASTICCURL: {{ ELASTICAUTH.elasticcurl }} {% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %} From 63b120e9e242674ae9fea12bf248d25e330b2de0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 17 Jun 2021 14:56:05 -0400 Subject: [PATCH 297/378] use just curl for elastic in soup --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 64c8de66b..32280f707 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -392,7 +392,7 @@ rc1_to_rc2() { local NAME=$(echo $p | awk '{print $1}') local IP=$(echo $p | awk '{print $2}') echo "Removing the old cross cluster config for $NAME" - {{ ELASTICCURL }} -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}' + curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}' done Date: Thu, 17 Jun 2021 15:19:27 -0400 Subject: [PATCH 298/378] Lock so-user to avoid two processes from overwriting eachother --- salt/common/tools/sbin/so-user | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 800fca745..0f44a3227 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -45,6 +45,13 @@ elasticRolesFile=${ELASTIC_ROLES_FILE:-/opt/so/saltstack/local/salt/elasticsearc esUID=${ELASTIC_UID:-930} esGID=${ELASTIC_GID:-930} +function lock() { + # Obtain file descriptor lock + exec 99>/var/tmp/so-user.lock || fail "Unable to create lock descriptor; if the system was not shutdown gracefully you may need to remove /var/tmp/so-user.lock manually." + flock -w 10 99 || fail "Another process is using so-user; if the system was not shutdown gracefully you may need to remove /var/tmp/so-user.lock manually." + trap 'rm -f /var/tmp/so-user.lock' EXIT +} + function fail() { msg=$1 echo "$1" @@ -219,12 +226,9 @@ function syncElastic() { mv "${rolesTmpFile}" "${elasticRolesFile}" if [[ -z "$SKIP_STATE_APPLY" ]]; then - echo "Applying elastic state locally; This can take a few minutes..." - echo "Applying elastic state locally at $(date)" >> /opt/so/log/soc/sync.log 2>&1 - salt-call state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 - echo "Applying elastic state to elastic minions; This can take a few minutes..." + echo "Elastic state will be re-applied to affected minions in the background." echo "Applying elastic state to elastic minions at $(date)" >> /opt/so/log/soc/sync.log 2>&1 - salt -C 'G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 + salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 & fi else echo "Newly generated users/roles files are incomplete; aborting." @@ -331,6 +335,7 @@ case "${operation}" in verifyEnvironment [[ "$email" == "" ]] && fail "Email address must be provided" + lock validateEmail "$email" updatePassword createUser "$email" @@ -349,6 +354,7 @@ case "${operation}" in verifyEnvironment [[ "$email" == "" ]] && fail "Email address must be provided" + lock updateUser "$email" syncAll echo "Successfully updated user" @@ -358,6 +364,7 @@ case "${operation}" in verifyEnvironment [[ "$email" == "" ]] && fail "Email address must be provided" + lock updateStatus "$email" 'active' syncAll echo "Successfully enabled user" @@ -369,6 +376,7 @@ case "${operation}" in verifyEnvironment [[ "$email" == "" ]] && fail "Email address must be provided" + lock updateStatus "$email" 'locked' syncAll echo "Successfully disabled user" @@ -380,6 +388,7 @@ case "${operation}" in verifyEnvironment [[ "$email" == "" ]] && fail "Email address must be provided" + lock deleteUser "$email" syncAll echo "Successfully deleted user" @@ -388,7 +397,8 @@ case "${operation}" in ;; "sync") - syncAll && echo "Synchronization completed at $(date)" + lock + syncAll ;; "validate") From afe7ddb48099a7a5685c681e0fcabfc49a59220d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Jun 2021 15:51:53 -0400 Subject: [PATCH 299/378] Remove passwords from soctopus templates since these are the basis for elastalert rules, which will use the user/pass at the elastalert global config level --- salt/soctopus/files/templates/es-generic.template | 6 ------ salt/soctopus/files/templates/generic.template | 6 ------ salt/soctopus/files/templates/osquery.template | 6 ------ 3 files changed, 18 deletions(-) diff --git a/salt/soctopus/files/templates/es-generic.template b/salt/soctopus/files/templates/es-generic.template index 9b5ace95a..8183a5af4 100644 --- a/salt/soctopus/files/templates/es-generic.template +++ b/salt/soctopus/files/templates/es-generic.template @@ -1,13 +1,7 @@ {% set ES = salt['pillar.get']('global:managerip', '') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} alert: modules.so.playbook-es.PlaybookESAlerter elasticsearch_host: "{{ ES }}:9200" -{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -elasticsearch_user: "{{ ES_USER }}" -elasticsearch_pass: "{{ ES_PASS }}" -{% endif %} play_title: "" play_url: "https://{{ ES }}/playbook/issues/6000" sigma_level: "" diff --git a/salt/soctopus/files/templates/generic.template b/salt/soctopus/files/templates/generic.template index d3736f894..c3733db2c 100644 --- a/salt/soctopus/files/templates/generic.template +++ b/salt/soctopus/files/templates/generic.template @@ -1,15 +1,9 @@ {% set es = salt['pillar.get']('global:url_base', '') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} alert: - "modules.so.playbook-es.PlaybookESAlerter" elasticsearch_host: "{{ es }}:9200" -{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -elasticsearch_user: "{{ ES_USER }}" -elasticsearch_pass: "{{ ES_PASS }}" -{% endif %} play_title: "" play_id: "" event.module: "playbook" diff --git a/salt/soctopus/files/templates/osquery.template b/salt/soctopus/files/templates/osquery.template index 328a7e275..f937de5ea 100644 --- a/salt/soctopus/files/templates/osquery.template +++ b/salt/soctopus/files/templates/osquery.template @@ -1,15 +1,9 @@ {% set es = salt['pillar.get']('global:url_base', '') %} -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} alert: - "modules.so.playbook-es.PlaybookESAlerter" elasticsearch_host: "{{ es }}:9200" -{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} -elasticsearch_user: "{{ ES_USER }}" -elasticsearch_pass: "{{ ES_PASS }}" -{% endif %} play_title: "" event.module: "playbook" event.dataset: "alert" From b294cee278ddbd5b620f78b5675a8c60f22071a9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Jun 2021 15:53:07 -0400 Subject: [PATCH 300/378] Remove passwords from soctopus templates since these are the basis for elastalert rules, which will use the user/pass at the elastalert global config level --- salt/soctopus/init.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index b32c67487..1c7c92434 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -21,7 +21,6 @@ soctopus-sync: - source: salt://soctopus/files/templates - user: 939 - group: 939 - - file_mode: 600 - template: jinja soctopusconf: From 0615d635ebafb7edc3bfebea6069e217999b7772 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 17 Jun 2021 16:12:39 -0400 Subject: [PATCH 301/378] let the first highstate pass --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 32280f707..ba32cc3ca 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -904,7 +904,9 @@ main() { echo "" echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." + set +e salt-call state.highstate -l info queue=True + set -e echo "" echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete." From 188b4424e4268a0650c00ec4a1e1b107c3b19bc9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Jun 2021 21:00:56 -0400 Subject: [PATCH 302/378] Fix wrong grep file --- salt/common/tools/sbin/so-elastic-auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth index 663dbb9f6..6631badf2 100755 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -45,7 +45,7 @@ if [[ "$authEnable" == "true" ]]; then if grep -q "argon" "$ES_USERS_FILE"; then echo "" echo "IMPORTANT: The following users will need to change their password, after logging into SOC, in order to access Kibana:" - grep argon "$ES_USERS" | cut -d ":" -f 1 + grep argon "$ES_USERS_FILE" | cut -d ":" -f 1 fi else echo "Auth is already enabled." From 1d4161ba3184ffbfeba71fb2296e0138d22dd0c2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 18 Jun 2021 08:36:36 -0400 Subject: [PATCH 303/378] Disable HaveIBeenPwned API (pwnedpasswords.com) --- salt/soc/files/kratos/kratos.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/soc/files/kratos/kratos.yaml b/salt/soc/files/kratos/kratos.yaml index a0a72b3ab..80340af4f 100644 --- a/salt/soc/files/kratos/kratos.yaml +++ b/salt/soc/files/kratos/kratos.yaml @@ -5,6 +5,8 @@ selfservice: methods: password: enabled: true + config: + haveibeenpwned_enabled: false flows: settings: From 101b835cf69e8e05b8374768985400320db186eb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 18 Jun 2021 14:34:42 -0400 Subject: [PATCH 304/378] Remove unused mode --- salt/soctopus/init.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index 1c7c92434..c2c8dc1ac 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -43,7 +43,6 @@ playbookrulesdir: - name: /opt/so/rules/elastalert/playbook - user: 939 - group: 939 - - mode: 660 - makedirs: True playbookrulessync: From 48c3cb4816306d1c022995f09a8fe8622a65c2cd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 18 Jun 2021 14:56:01 -0400 Subject: [PATCH 305/378] if the salt-minion service isnt running when the state is rendered, dont try to apply schedule - https://github.com/Security-Onion-Solutions/securityonion/issues/1333 --- salt/patch/os/schedule.sls | 2 +- salt/salt/minion-check.sls | 6 ++++-- salt/schedule.sls | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/patch/os/schedule.sls b/salt/patch/os/schedule.sls index a91e61dfe..a5445f3a9 100644 --- a/salt/patch/os/schedule.sls +++ b/salt/patch/os/schedule.sls @@ -1,4 +1,4 @@ -{% if salt['pillar.get']('patch:os:schedule_name') %} +{% if salt['pillar.get']('patch:os:schedule_name') and salt['service.status']('salt-minion', True) %} {% set patch_os_pillar = salt['pillar.get']('patch:os') %} {% set schedule_name = patch_os_pillar.schedule_name %} {% set splay = patch_os_pillar.get('splay', 300) %} diff --git a/salt/salt/minion-check.sls b/salt/salt/minion-check.sls index e8a0c2639..a9d2e8d8d 100644 --- a/salt/salt/minion-check.sls +++ b/salt/salt/minion-check.sls @@ -1,6 +1,7 @@ include: - salt.minion-state-apply-test - + +{% if salt['service.status']('salt-minion', True) %} state-apply-test: schedule.present: - name: salt-minion-state-apply-test @@ -16,4 +17,5 @@ state-apply-test: cron.present: - identifier: so-salt-minion-check - user: root - - minute: '*/5' \ No newline at end of file + - minute: '*/5' +{% endif %} diff --git a/salt/schedule.sls b/salt/schedule.sls index 12e1cd081..30b5f8608 100644 --- a/salt/schedule.sls +++ b/salt/schedule.sls @@ -1,5 +1,7 @@ +{% if salt['service.status']('salt-minion', True) %} schedule: schedule.present: - function: state.highstate - minutes: 15 - maxrunning: 1 +{% endif %} From c7a58816b68a940035c69a4d5bf84dcb62c34061 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 18 Jun 2021 15:30:51 -0400 Subject: [PATCH 306/378] move condition to avoid wrong notic about schedule not set in pillar --- salt/patch/os/schedule.sls | 40 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/salt/patch/os/schedule.sls b/salt/patch/os/schedule.sls index a5445f3a9..4e7820de1 100644 --- a/salt/patch/os/schedule.sls +++ b/salt/patch/os/schedule.sls @@ -1,12 +1,13 @@ -{% if salt['pillar.get']('patch:os:schedule_name') and salt['service.status']('salt-minion', True) %} - {% set patch_os_pillar = salt['pillar.get']('patch:os') %} - {% set schedule_name = patch_os_pillar.schedule_name %} - {% set splay = patch_os_pillar.get('splay', 300) %} +{% if salt['pillar.get']('patch:os:schedule_name') %} + {% if salt['service.status']('salt-minion', True) %} + {% set patch_os_pillar = salt['pillar.get']('patch:os') %} + {% set schedule_name = patch_os_pillar.schedule_name %} + {% set splay = patch_os_pillar.get('splay', 300) %} - {% if schedule_name != 'manual' and schedule_name != 'auto' %} - {% import_yaml "patch/os/schedules/"~schedule_name~".yml" as os_schedule %} + {% if schedule_name != 'manual' and schedule_name != 'auto' %} + {% import_yaml "patch/os/schedules/"~schedule_name~".yml" as os_schedule %} - {% if patch_os_pillar.enabled %} + {% if patch_os_pillar.enabled %} patch_os_schedule: schedule.present: @@ -14,28 +15,28 @@ patch_os_schedule: - job_args: - patch.os - when: - {% for days in os_schedule.patch.os.schedule %} - {% for day, times in days.items() %} - {% for time in times %} + {% for days in os_schedule.patch.os.schedule %} + {% for day, times in days.items() %} + {% for time in times %} - {{day}} {{time}} + {% endfor %} {% endfor %} {% endfor %} - {% endfor %} - splay: {{splay}} - return_job: True - {% else %} + {% else %} disable_patch_os_schedule: schedule.disabled: - name: patch_os_schedule - {% endif %} + {% endif %} - {% elif schedule_name == 'auto' %} + {% elif schedule_name == 'auto' %} - {% if patch_os_pillar.enabled %} + {% if patch_os_pillar.enabled %} patch_os_schedule: schedule.present: @@ -46,22 +47,23 @@ patch_os_schedule: - splay: {{splay}} - return_job: True - {% else %} + {% else %} disable_patch_os_schedule: schedule.disabled: - name: patch_os_schedule - {% endif %} + {% endif %} - {% elif schedule_name == 'manual' %} + {% elif schedule_name == 'manual' %} remove_patch_os_schedule: schedule.absent: - name: patch_os_schedule - {% endif %} + {% endif %} + {% endif %} {% else %} no_patch_os_schedule_name_set: From 60fd3c6bd3ad42fb28536067f6efe372cf120c22 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 18 Jun 2021 20:01:32 -0400 Subject: [PATCH 307/378] Ensure htpasswd exists earlier in the install process --- setup/so-functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 13438b1ba..ff019953e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2107,6 +2107,7 @@ saltify() { { if [[ ! $is_iso ]]; then yum -y install salt-minion-3003\ + httpd-tools\ python3\ python36-docker\ python36-dateutil\ @@ -2132,6 +2133,7 @@ saltify() { fi local pkg_arr=( + 'apache2-utils' 'ca-certificates' 'curl' 'software-properties-common' From 777bece2eb9648c47c2a57a28bc3b7c2bdce8789 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 20 Jun 2021 22:14:13 -0400 Subject: [PATCH 308/378] Fix intermittent 'like' failures; Ensure bash is on first line of load templates script --- salt/common/tools/sbin/so-elasticsearch-templates-load | 6 +++--- salt/common/tools/sbin/so-user | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load index 30ab66b48..fbb9e4e3a 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-load +++ b/salt/common/tools/sbin/so-elasticsearch-templates-load @@ -1,6 +1,3 @@ -{%- set mainint = salt['pillar.get']('host:mainint') %} -{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} - #!/bin/bash # Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC # @@ -17,6 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +{%- set mainint = salt['pillar.get']('host:mainint') %} +{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} + default_conf_dir=/opt/so/conf ELASTICSEARCH_HOST="{{ MYIP }}" ELASTICSEARCH_PORT=9200 diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 0f44a3227..fe0118438 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -201,7 +201,7 @@ function syncElastic() { # Generate the new users file echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \ "from identity_credential_identifiers ici, identity_credentials ic " \ - "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ + "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \ "order by ici.identifier;" | \ sqlite3 "$databasePath" | \ jq -r '.user + ":" + .data.hashed_password' \ @@ -212,7 +212,7 @@ function syncElastic() { echo "select 'superuser:' || ici.identifier " \ "from identity_credential_identifiers ici, identity_credentials ic " \ - "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ + "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \ "order by ici.identifier;" | \ sqlite3 "$databasePath" \ >> "$rolesTmpFile" From 5298cb8cfb2fb3c87e4ac30e4e4f77adee1e032f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 21 Jun 2021 07:06:49 -0400 Subject: [PATCH 309/378] Update copyrights --- salt/common/tools/sbin/so-elasticsearch-templates-load | 2 +- salt/common/tools/sbin/so-filebeat-module-setup | 2 +- salt/elasticsearch/files/scripts/so-catrust | 2 +- salt/elasticsearch/files/so-elasticsearch-pipelines | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load index fbb9e4e3a..c416a3ce2 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-load +++ b/salt/common/tools/sbin/so-elasticsearch-templates-load @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 4f9811ca7..ef35a9b17 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -2,7 +2,7 @@ {%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} #!/bin/bash -# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/salt/elasticsearch/files/scripts/so-catrust b/salt/elasticsearch/files/scripts/so-catrust index d49a29ce4..c157d9691 100644 --- a/salt/elasticsearch/files/scripts/so-catrust +++ b/salt/elasticsearch/files/scripts/so-catrust @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index 5d103963e..c1ff88397 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From ba5b5db2c4e8bb10b60cf8f08a4b54a3262edb93 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 21 Jun 2021 08:56:24 -0400 Subject: [PATCH 310/378] remove the salt-minion check for schedules --- salt/patch/os/schedule.sls | 2 -- salt/salt/minion-check.sls | 2 -- salt/schedule.sls | 2 -- 3 files changed, 6 deletions(-) diff --git a/salt/patch/os/schedule.sls b/salt/patch/os/schedule.sls index 4e7820de1..4ad9a454e 100644 --- a/salt/patch/os/schedule.sls +++ b/salt/patch/os/schedule.sls @@ -1,5 +1,4 @@ {% if salt['pillar.get']('patch:os:schedule_name') %} - {% if salt['service.status']('salt-minion', True) %} {% set patch_os_pillar = salt['pillar.get']('patch:os') %} {% set schedule_name = patch_os_pillar.schedule_name %} {% set splay = patch_os_pillar.get('splay', 300) %} @@ -63,7 +62,6 @@ remove_patch_os_schedule: {% endif %} - {% endif %} {% else %} no_patch_os_schedule_name_set: diff --git a/salt/salt/minion-check.sls b/salt/salt/minion-check.sls index a9d2e8d8d..66ab732e2 100644 --- a/salt/salt/minion-check.sls +++ b/salt/salt/minion-check.sls @@ -1,7 +1,6 @@ include: - salt.minion-state-apply-test -{% if salt['service.status']('salt-minion', True) %} state-apply-test: schedule.present: - name: salt-minion-state-apply-test @@ -18,4 +17,3 @@ state-apply-test: - identifier: so-salt-minion-check - user: root - minute: '*/5' -{% endif %} diff --git a/salt/schedule.sls b/salt/schedule.sls index 30b5f8608..12e1cd081 100644 --- a/salt/schedule.sls +++ b/salt/schedule.sls @@ -1,7 +1,5 @@ -{% if salt['service.status']('salt-minion', True) %} schedule: schedule.present: - function: state.highstate - minutes: 15 - maxrunning: 1 -{% endif %} From 591ef540a6dfcc175285f747c34466980feb2742 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 21 Jun 2021 10:50:09 -0400 Subject: [PATCH 311/378] esalerter ES creds fix --- salt/elastalert/files/modules/so/playbook-es.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index 5b1835bac..bae967001 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -12,7 +12,7 @@ class PlaybookESAlerter(Alerter): Use matched data to create alerts in elasticsearch """ - required_options = set(['play_title','play_url','sigma_level','elasticsearch_host']) + required_options = set(['play_title','play_url','sigma_level']) def alert(self, matches): for match in matches: @@ -21,11 +21,11 @@ class PlaybookESAlerter(Alerter): headers = {"Content-Type": "application/json"} creds = None - if 'elasticsearch_user' in self.rule and 'elasticsearch_pass' in self.rule: - creds = (self.rule['elasticsearch_user'], self.rule['elasticsearch_pass']) + if 'es_username' in self.rule and 'es_password' in self.rule: + creds = (self.rule['es_username'], self.rule['es_password']) payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp} - url = f"https://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/" + url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/so-playbook-alerts-{today}/_doc/" requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) def get_info(self): From 5e042bf4b8e4d19e2127a2f800638b78d0039770 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 21 Jun 2021 12:16:47 -0400 Subject: [PATCH 312/378] Improve algorithm for determining if a user sync is necessary; Apply salt state in foreground to avoid collisions with setup salt states. --- salt/common/tools/sbin/so-user | 9 +++++---- salt/soc/init.sls | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index fe0118438..9c2e1be50 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -226,9 +226,9 @@ function syncElastic() { mv "${rolesTmpFile}" "${elasticRolesFile}" if [[ -z "$SKIP_STATE_APPLY" ]]; then - echo "Elastic state will be re-applied to affected minions in the background." + echo "Elastic state will be re-applied to affected minions. This may take several minutes..." echo "Applying elastic state to elastic minions at $(date)" >> /opt/so/log/soc/sync.log 2>&1 - salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 & + salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 fi else echo "Newly generated users/roles files are incomplete; aborting." @@ -236,8 +236,9 @@ function syncElastic() { } function syncAll() { - if [[ -n "$STALE_MIN" && -f "$databasePath" ]]; then - staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${STALE_MIN} minutes');" \ + if [[ -f "$databasePath" && -f "$elasticUsersFile" ]]; then + usersFileAgeSecs=$((echo $(date +%s) - $(date +%s -r "$elasticUsersFile"))) + staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${usersFileAgeSecs} seconds');" \ | sqlite3 "$databasePath") if [[ "$staleCount" == "0" ]]; then return 1 diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 01b57c8ce..8e1943cb8 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -65,7 +65,7 @@ soccustom: sosyncusers: cron.present: - user: root - - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin STALE_MIN=1 /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' + - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' so-soc: docker_container.running: From f36ef86ccc95eeada67270770ce496f6ae556aa7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 21 Jun 2021 12:38:02 -0400 Subject: [PATCH 313/378] Improve algorithm for determining if a user sync is necessary; Apply salt state in foreground to avoid collisions with setup salt states. --- salt/common/tools/sbin/so-user | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 9c2e1be50..62ba737b1 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -236,12 +236,12 @@ function syncElastic() { } function syncAll() { - if [[ -f "$databasePath" && -f "$elasticUsersFile" ]]; then - usersFileAgeSecs=$((echo $(date +%s) - $(date +%s -r "$elasticUsersFile"))) + if [[ -z "$FORCE_SYNC" && -f "$databasePath" && -f "$elasticUsersFile" ]]; then + usersFileAgeSecs=$(echo $(($(date +%s) - $(date +%s -r "$elasticUsersFile")))) staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${usersFileAgeSecs} seconds');" \ | sqlite3 "$databasePath") if [[ "$staleCount" == "0" ]]; then - return 1 + fail "Users are already in sync. Run 'FORCE_SYNC=1 $0 sync' to force a full sync anyway." fi fi syncElastic From c9ee28ce0139a12124dff31bad8fb0e1888f85ea Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 21 Jun 2021 14:47:24 -0400 Subject: [PATCH 314/378] adding elasticsearch.auth to heavynode and searchnode --- pillar/top.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pillar/top.sls b/pillar/top.sls index ff4cb5787..cb170c0af 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -68,6 +68,7 @@ base: '*_heavynode': - zeeklogs + - elasticsearch.auth - global - minions.{{ grains.id }} @@ -89,6 +90,7 @@ base: - logstash - logstash.search - elasticsearch.search + - elasticsearch.auth - global - minions.{{ grains.id }} - data.nodestab From e4e3b199fc97a0ea11aa02e6caf616e1ee8dd93d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 22 Jun 2021 08:05:08 -0400 Subject: [PATCH 315/378] retry on so-dockerregistry --- salt/registry/init.sls | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/registry/init.sls b/salt/registry/init.sls index 1cec55fd2..b59465fb7 100644 --- a/salt/registry/init.sls +++ b/salt/registry/init.sls @@ -43,6 +43,10 @@ so-dockerregistry: - /nsm/docker-registry/docker:/var/lib/registry/docker:rw - /etc/pki/registry.crt:/etc/pki/registry.crt:ro - /etc/pki/registry.key:/etc/pki/registry.key:ro + - timeout: 180 + - retry: + attempts: 5 + interval: 30 append_so-dockerregistry_so-status.conf: file.append: @@ -55,4 +59,4 @@ append_so-dockerregistry_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From dcc9af946ad09b6c583ac8c929e68cd46d0beee2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 21 Jun 2021 21:52:02 -0400 Subject: [PATCH 316/378] Avoid logging when sync is unnecessary due to cronjob log output spam --- salt/common/tools/sbin/so-user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 62ba737b1..f0c064d03 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -241,7 +241,7 @@ function syncAll() { staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${usersFileAgeSecs} seconds');" \ | sqlite3 "$databasePath") if [[ "$staleCount" == "0" ]]; then - fail "Users are already in sync. Run 'FORCE_SYNC=1 $0 sync' to force a full sync anyway." + return 1 fi fi syncElastic From 8ddc99e91f7f5f53e3147865ecff0a733c032e94 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 22 Jun 2021 08:07:41 -0400 Subject: [PATCH 317/378] Allow for adjusting SOC session timeout --- salt/soc/files/kratos/kratos.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/soc/files/kratos/kratos.yaml b/salt/soc/files/kratos/kratos.yaml index 80340af4f..b1174af58 100644 --- a/salt/soc/files/kratos/kratos.yaml +++ b/salt/soc/files/kratos/kratos.yaml @@ -1,5 +1,9 @@ {%- set WEBACCESS = salt['pillar.get']('global:url_base', '') -%} {%- set KRATOSKEY = salt['pillar.get']('kratos:kratoskey', '') -%} +{%- set SESSIONTIMEOUT = salt['pillar.get']('kratos:sessiontimeout', '24h') -%} + +session: + lifespan: {{ SESSIONTIMEOUT }} selfservice: methods: From bc8659041166185d55f958fa808e63f1addf508b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 22 Jun 2021 08:23:16 -0400 Subject: [PATCH 318/378] only add sosyncuser cron if startup_states: highstate is set in minion config --- salt/soc/init.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 8e1943cb8..51751e9a6 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -62,10 +62,13 @@ soccustom: - mode: 600 - template: jinja +# we dont want this added to early in setup, so we add the onlyif to verify 'startup_states: highstate' +# is in the minion config. that is added before the final highstate during setup sosyncusers: cron.present: - user: root - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' + - onlyif: "grep 'startup_states: highstate' /etc/salt/minion" so-soc: docker_container.running: From c3deabae36fe509ee306d0a45ba8bf219a48b0ec Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 22 Jun 2021 08:30:54 -0400 Subject: [PATCH 319/378] Update init.sls --- salt/soc/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 51751e9a6..b8cdb09ba 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -62,8 +62,8 @@ soccustom: - mode: 600 - template: jinja -# we dont want this added to early in setup, so we add the onlyif to verify 'startup_states: highstate' -# is in the minion config. that is added before the final highstate during setup +# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' +# is in the minion config. That line is added before the final highstate during setup sosyncusers: cron.present: - user: root From 9fa83d1ceeee9585d7088e7616c8cad0ca5a0360 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 22 Jun 2021 09:54:25 -0400 Subject: [PATCH 320/378] change to client_timeout --- salt/registry/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/registry/init.sls b/salt/registry/init.sls index b59465fb7..eb0c2df0c 100644 --- a/salt/registry/init.sls +++ b/salt/registry/init.sls @@ -43,7 +43,7 @@ so-dockerregistry: - /nsm/docker-registry/docker:/var/lib/registry/docker:rw - /etc/pki/registry.crt:/etc/pki/registry.crt:ro - /etc/pki/registry.key:/etc/pki/registry.key:ro - - timeout: 180 + - client_timeout: 180 - retry: attempts: 5 interval: 30 From 85bb234cf94e1d7fd035fd02efdbb5c626d7b28b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 22 Jun 2021 10:14:33 -0400 Subject: [PATCH 321/378] Fix Retry Spam --- salt/common/tools/sbin/soup | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ba32cc3ca..e9313ad9e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -884,7 +884,8 @@ main() { # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." + echo "Waiting on the Salt Master service to be ready." + salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "" @@ -920,9 +921,9 @@ main() { echo "Starting Salt Master service." systemctl start salt-master - # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." + echo "Waiting on the Salt Master service to be ready." + salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "Running a highstate. This could take several minutes." @@ -997,4 +998,4 @@ EOF read -r input main "$@" | tee -a $SOUP_LOG - \ No newline at end of file + From 9fad0876c57cf2b6f3c9fd8a5fe11fe93a288bc2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 22 Jun 2021 11:31:31 -0400 Subject: [PATCH 322/378] only attempt to upgrade salt on minions if the minion count it > 1 --- salt/common/tools/sbin/soup | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index ba32cc3ca..1a8b7cef5 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -931,7 +931,8 @@ main() { [[ $is_airgap -eq 0 ]] && unmount_update thehive_maint - if [[ $UPGRADESALT -eq 1 ]]; then + NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) + if [[ $UPGRADESALT -eq 1 ]] && [[ $NUM_MINIONS -gt 1 ]]; then if [[ $is_airgap -eq 0 ]]; then echo "" echo "Cleaning repos on remote Security Onion nodes." @@ -958,8 +959,6 @@ main() { esac fi - NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) - if [[ $NUM_MINIONS -gt 1 ]]; then cat << EOF From 26bb6cc0110d1cb87612dbc8778fba64a28dd7ed Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 22 Jun 2021 15:29:36 -0400 Subject: [PATCH 323/378] Require either true | false in parameter to so-elastic-auth and ensure all minions are fully updated with the new auth setting --- salt/common/tools/sbin/so-elastic-auth | 22 ++++++++++------------ setup/so-setup | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth index 6631badf2..715f55fe4 100755 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -22,7 +22,7 @@ fi ES_AUTH_PILLAR=${ELASTIC_AUTH_PILLAR:-/opt/so/saltstack/local/pillar/elasticsearch/auth.sls} ES_USERS_FILE=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users} -authEnable=${1:-true} +authEnable=$1 if ! grep -q "enabled: " "$ES_AUTH_PILLAR"; then echo "Elastic auth pillar file is invalid. Unable to proceed." @@ -30,17 +30,18 @@ if ! grep -q "enabled: " "$ES_AUTH_PILLAR"; then fi function restart() { - so-elastic-stop - salt-call state.highstate queue=True + if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then + echo "Elasticsearch on all affected minions will now be stopped and then restarted..." + salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' cmd.run so-elastic-stop + echo "Applying highstate to all affected minions..." + salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.highstate queue=True + fi } if [[ "$authEnable" == "true" ]]; then if grep -q "enabled: False" "$ES_AUTH_PILLAR"; then sed -i 's/enabled: False/enabled: True/g' "$ES_AUTH_PILLAR" - if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then - echo "Applying highstate - this may take a few minutes..." - restart - fi + restart echo "Elastic auth is now enabled." if grep -q "argon" "$ES_USERS_FILE"; then echo "" @@ -53,10 +54,7 @@ if [[ "$authEnable" == "true" ]]; then elif [[ "$authEnable" == "false" ]]; then if grep -q "enabled: True" "$ES_AUTH_PILLAR"; then sed -i 's/enabled: True/enabled: False/g' "$ES_AUTH_PILLAR" - if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then - echo "Applying highstate - this may take a few minutes..." - restart - fi + restart echo "Elastic auth is now disabled." else echo "Auth is already disabled." @@ -64,6 +62,6 @@ elif [[ "$authEnable" == "false" ]]; then else echo "Usage: $0 " echo "" - echo "Enables Elastic authentication. Defaults to true." + echo "Toggles Elastic authentication. Elasticsearch will be restarted on each affected minion." echo "" fi diff --git a/setup/so-setup b/setup/so-setup index 34802fef9..68490657f 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -753,7 +753,7 @@ echo "1" > /root/accept_changes salt-call state.apply -l info manager >> $setup_log 2>&1 echo "Executing so-elastic-auth..." >> $setup_log 2>&1 - ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth >> $setup_log 2>&1 + ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth true >> $setup_log 2>&1 echo "Finished so-elastic-auth..." >> $setup_log 2>&1 fi From 3fbc850774c007b40399363f4fa4fba691b78b05 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 23 Jun 2021 09:17:37 -0400 Subject: [PATCH 324/378] Fix output so that it can be redirected to local file with appropriate syntax --- salt/common/tools/sbin/so-elasticsearch-pipeline-view | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/common/tools/sbin/so-elasticsearch-pipeline-view index 9f799c07f..3239aec0a 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-view +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-view @@ -21,5 +21,5 @@ if [ "$1" == "" ]; then {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . else - {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .[] fi From caa14e0cade92be5c481d65688178da37584af88 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 22 Jun 2021 10:14:33 -0400 Subject: [PATCH 325/378] Fix Retry Spam --- salt/common/tools/sbin/soup | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 1a8b7cef5..c1e25b208 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -884,7 +884,8 @@ main() { # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." + echo "Waiting on the Salt Master service to be ready." + salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "" @@ -920,9 +921,9 @@ main() { echo "Starting Salt Master service." systemctl start salt-master - # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." + echo "Waiting on the Salt Master service to be ready." + salt-call state.show_top -l error queue=True || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "Running a highstate. This could take several minutes." @@ -996,4 +997,4 @@ EOF read -r input main "$@" | tee -a $SOUP_LOG - \ No newline at end of file + From c5b81f2f4b57a67f577f632ea961251dd72b85b3 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 23 Jun 2021 09:17:37 -0400 Subject: [PATCH 326/378] Fix output so that it can be redirected to local file with appropriate syntax --- salt/common/tools/sbin/so-elasticsearch-pipeline-view | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/common/tools/sbin/so-elasticsearch-pipeline-view index 9f799c07f..3239aec0a 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-view +++ b/salt/common/tools/sbin/so-elasticsearch-pipeline-view @@ -21,5 +21,5 @@ if [ "$1" == "" ]; then {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . else - {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq . + {{ ELASTICCURL }} -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .[] fi From 243e888717a9a8bd5746e0f8e2196055c74b0e89 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 23 Jun 2021 14:41:34 -0400 Subject: [PATCH 327/378] Add queue=True -- needed for all salt commands, not just state changes --- salt/common/tools/sbin/so-elastic-auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth index 715f55fe4..6157cb4b4 100755 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -32,7 +32,7 @@ fi function restart() { if [[ -z "$ELASTIC_AUTH_SKIP_HIGHSTATE" ]]; then echo "Elasticsearch on all affected minions will now be stopped and then restarted..." - salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' cmd.run so-elastic-stop + salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' cmd.run so-elastic-stop queue=True echo "Applying highstate to all affected minions..." salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.highstate queue=True fi From 4af2f6d84ac630f5e3bbf0a1e1da152d57f8fbfe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 09:49:57 -0400 Subject: [PATCH 328/378] Fix airgap check --- salt/common/tools/sbin/so-common | 13 ------------- salt/common/tools/sbin/soup | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 7fb17092f..341f9a55a 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -88,19 +88,6 @@ add_interface_bond0() { fi } -check_airgap() { - # See if this is an airgap install - AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') - if [[ "$AIRGAP" == "True" ]]; then - is_airgap=0 - UPDATE_DIR=/tmp/soagupdate/SecurityOnion - AGDOCKER=/tmp/soagupdate/docker - AGREPO=/tmp/soagupdate/Packages - else - is_airgap=1 - fi -} - check_container() { docker ps | grep "$1:" > /dev/null 2>&1 return $? diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c1e25b208..b7104f759 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -168,6 +168,19 @@ update_registry() { salt-call state.apply registry queue=True } +check_airgap() { + # See if this is an airgap install + AIRGAP=$(cat /opt/so/saltstack/local/pillar/global.sls | grep airgap: | awk '{print $2}') + if [[ "$AIRGAP" == "True" ]]; then + is_airgap=0 + UPDATE_DIR=/tmp/soagupdate/SecurityOnion + AGDOCKER=/tmp/soagupdate/docker + AGREPO=/tmp/soagupdate/Packages + else + is_airgap=1 + fi +} + check_sudoers() { if grep -q "so-setup" /etc/sudoers; then echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." From 66e88cef42de09b182680951a5858f9f06534001 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 10:03:38 -0400 Subject: [PATCH 329/378] Move up script verification --- salt/common/tools/sbin/soup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b7104f759..cad6821e4 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -773,6 +773,10 @@ main() { check_airgap echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" + echo "" + echo "Verifying we have the latest soup script." + verify_latest_update_script + echo "" set_os set_palette check_elastic_license @@ -787,10 +791,6 @@ main() { clone_to_tmp fi check_os_updates - echo "" - echo "Verifying we have the latest soup script." - verify_latest_update_script - echo "" echo "Generating new repo archive" generate_and_clean_tarballs From 2681903c9396c7c9f1294c671c613109a015271e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 10:24:00 -0400 Subject: [PATCH 330/378] Move up script verification --- salt/common/tools/sbin/soup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index cad6821e4..c2d3a33e4 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -775,6 +775,11 @@ main() { echo "" echo "" echo "Verifying we have the latest soup script." + if [ -f /usr/sbin/so-image-common ]; then + . /usr/sbin/so-image-common + else + add_common + fi verify_latest_update_script echo "" set_os From 022f9ea76ed67303caefbfc26977000cb7949440 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 24 Jun 2021 10:45:12 -0400 Subject: [PATCH 331/378] Add Elasticsearch and Kibana to list of services that use webuser creds --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 031265065..afd691632 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -220,7 +220,7 @@ whiptail_create_web_user() { [ -n "$TESTING" ] && return WEBUSER=$(whiptail --title "$whiptail_title" --inputbox \ - "Please enter an email address to create an administrator account for the web interface.\n\nThis will also be used for TheHive, Cortex, and Fleet." 12 60 "$1" 3>&1 1>&2 2>&3) + "Please enter an email address to create an administrator account for the web interface.\n\nThis will also be used for Elasticsearch, Kibana, TheHive, Cortex, and Fleet." 12 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From a1ec40b54729b44cef64573d4aa73226e6ccb7a7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 10:52:27 -0400 Subject: [PATCH 332/378] Revert "Move up script verification" This reverts commit 2681903c9396c7c9f1294c671c613109a015271e. --- salt/common/tools/sbin/soup | 5 ----- 1 file changed, 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c2d3a33e4..cad6821e4 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -775,11 +775,6 @@ main() { echo "" echo "" echo "Verifying we have the latest soup script." - if [ -f /usr/sbin/so-image-common ]; then - . /usr/sbin/so-image-common - else - add_common - fi verify_latest_update_script echo "" set_os From 930d5b3627eccc39f77f47323b5c638e7c58e858 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 10:52:53 -0400 Subject: [PATCH 333/378] Revert "Move up script verification" This reverts commit 66e88cef42de09b182680951a5858f9f06534001. --- salt/common/tools/sbin/soup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index cad6821e4..b7104f759 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -773,10 +773,6 @@ main() { check_airgap echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" - echo "" - echo "Verifying we have the latest soup script." - verify_latest_update_script - echo "" set_os set_palette check_elastic_license @@ -791,6 +787,10 @@ main() { clone_to_tmp fi check_os_updates + echo "" + echo "Verifying we have the latest soup script." + verify_latest_update_script + echo "" echo "Generating new repo archive" generate_and_clean_tarballs From 3c85db1769670aeaa6dcfb4ce99f602f6d228f95 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 11:04:56 -0400 Subject: [PATCH 334/378] Fix regression --- salt/common/tools/sbin/soup | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b7104f759..fc9a4aa7e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -771,26 +771,28 @@ main() { echo "Checking to see if this is an airgap install" echo "" check_airgap + echo "Update dir is $UPDATEDIR" echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" - set_os - set_palette - check_elastic_license - echo "" if [[ $is_airgap -eq 0 ]]; then # Let's mount the ISO since this is airgap + echo "This is airgap. Ask for a location." airgap_mounted else echo "Cloning Security Onion github repo into $UPDATE_DIR." echo "Removing previous upgrade sources." rm -rf $UPDATE_DIR + echo "Cloning the Security Onion Repo." clone_to_tmp fi - check_os_updates - echo "" echo "Verifying we have the latest soup script." verify_latest_update_script echo "" + set_os + set_palette + check_elastic_license + echo "" + check_os_updates echo "Generating new repo archive" generate_and_clean_tarballs From 42a63f8ea597ae7212bf0406987ba8bd585be2a0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 11:15:16 -0400 Subject: [PATCH 335/378] remove a net check --- salt/common/tools/sbin/soup | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index fc9a4aa7e..fc57b7ac6 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -288,9 +288,7 @@ clone_to_tmp() { if [ -n "$BRANCH" ]; then SOUP_BRANCH="-b $BRANCH" fi - set +e - run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com" - set -e + git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git cd /tmp if [ ! -f $UPDATE_DIR/VERSION ]; then echo "Update was unable to pull from github. Please check your internet." From bdd1074be72eaf01736031cbe8cf956211741360 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 11:24:12 -0400 Subject: [PATCH 336/378] remove a net check --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index fc57b7ac6..a671bd9a5 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -18,6 +18,7 @@ . /usr/sbin/so-common UPDATE_DIR=/tmp/sogh/securityonion +DEFAULT_SALT_DIR=/opt/so/saltstack/default INSTALLEDVERSION=$(cat /etc/soversion) POSTVERSION=$INSTALLEDVERSION INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}') @@ -731,6 +732,7 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" + echo "Can I see $UPDATE_DIR" cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ From 81bb7c6534da887bdaef3763d3080ac8e4bfda69 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 11:32:01 -0400 Subject: [PATCH 337/378] remove a net check --- salt/common/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index a671bd9a5..47bff57e1 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -745,6 +745,7 @@ verify_latest_update_script() { main() { set -e + set +e trap 'check_err $?' EXIT echo "### Preparing soup at $(date) ###" @@ -808,7 +809,7 @@ main() { echo "Checking for Salt Master and Minion updates." upgrade_check_salt - + set -e if [ "$is_hotfix" == "true" ]; then echo "Applying $HOTFIXVERSION" From 3f8cb23cf6ef274a66c20551eec6e0d982b7f4de Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 13:29:16 -0400 Subject: [PATCH 338/378] remove some debug statements --- salt/common/tools/sbin/soup | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 47bff57e1..c2f5a7f08 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -732,7 +732,6 @@ verify_latest_update_script() { echo "This version of the soup script is up to date. Proceeding." else echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete" - echo "Can I see $UPDATE_DIR" cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ @@ -772,7 +771,6 @@ main() { echo "Checking to see if this is an airgap install" echo "" check_airgap - echo "Update dir is $UPDATEDIR" echo "Found that Security Onion $INSTALLEDVERSION is currently installed." echo "" if [[ $is_airgap -eq 0 ]]; then From ea50023ca5dcdf64de39b2f8505a4d20059e8f03 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 24 Jun 2021 15:53:14 -0400 Subject: [PATCH 339/378] Fix filebeat modules --- salt/filebeat/init.sls | 3 ++- salt/filebeat/thirdpartydefaults.yaml | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 0cbbf0594..f03d3dc1a 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -129,7 +129,8 @@ so-filebeat: {% for module in THIRDPARTY.modules.keys() %} {% for submodule in THIRDPARTY.modules[module] %} {% if THIRDPARTY.modules[module][submodule].enabled and THIRDPARTY.modules[module][submodule]["var.syslog_port"] is defined %} - - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/{{ THIRDPARTY.modules[module][submodule]["var.input"] }} + - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/tcp + - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/udp {% endif %} {% endfor %} {% endfor %} diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml index 1b378f84b..112ed6d6c 100644 --- a/salt/filebeat/thirdpartydefaults.yaml +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -42,39 +42,32 @@ third_party_filebeat: cef: log: enabled: false - var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9003 checkpoint: firewall: enabled: false - var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9505 cisco: asa: enabled: false - var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9001 ftd: enabled: false - var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9003 ios: enabled: false - var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9002 nexus: enabled: false - var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9506 meraki: enabled: false - var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9525 umbrella: From a3d7f4e35d9d6eabd750fe5d6234b5f279360f99 Mon Sep 17 00:00:00 2001 From: doug Date: Fri, 25 Jun 2021 11:16:26 -0400 Subject: [PATCH 340/378] fix grammar --- salt/common/tools/sbin/so-common | 2 +- salt/common/tools/sbin/soup | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 341f9a55a..c4f6aca30 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -277,7 +277,7 @@ lookup_role() { require_manager() { if is_manager_node; then - echo "This is a manager, We can proceed." + echo "This is a manager, so we can proceed." else echo "Please run this command on the manager; the manager controls the grid." exit 1 diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c2f5a7f08..bc95c5428 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -768,7 +768,7 @@ main() { echo "" require_manager set_minionid - echo "Checking to see if this is an airgap install" + echo "Checking to see if this is an airgap install." echo "" check_airgap echo "Found that Security Onion $INSTALLEDVERSION is currently installed." From d6aa672556e82dabcdf117f989a633179148a9f0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 25 Jun 2021 17:30:25 -0400 Subject: [PATCH 341/378] updating standalone dashboard --- .../dashboards/standalone/standalone.json | 1757 +++++++++++++++-- 1 file changed, 1589 insertions(+), 168 deletions(-) diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 3f9396f0f..e102917e6 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -17,7 +17,7 @@ "gnetId": 2381, "graphTooltip": 0, "id": 6, - "iteration": 1614088867725, + "iteration": 1624648744739, "links": [], "panels": [ { @@ -29,36 +29,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 60 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 - } - ] - }, "unit": "percent" }, "overrides": [] @@ -90,7 +60,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -154,6 +124,45 @@ "value": "cpu-total" } ] + }, + { + "groupBy": [], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_idle\" *-1+100 FROM \"so_long_term\".\"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_idle" + ], + "type": "field" + }, + { + "params": [ + "*-1+100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], "thresholds": [], @@ -201,7 +210,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "decimals": 2, "mappings": [], "thresholds": { @@ -236,9 +244,10 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto" }, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "targets": [ { "groupBy": [ @@ -297,7 +306,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -329,7 +337,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -373,7 +381,46 @@ }, { "params": [ - " / {{ CPUS }}" + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-elasticsearch') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" ], "type": "math" } @@ -443,7 +490,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -474,7 +520,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -517,7 +563,46 @@ }, { "params": [ - " / {{ CPUS }}" + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=", + "value": "so-kibana" + }, + { + "condition": "AND", + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"container_name\" = 'so-kibana' AND \"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" ], "type": "math" } @@ -589,7 +674,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -621,7 +705,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -678,6 +762,40 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "hide": false, + "measurement": "suridrop", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_drop\" * 100 FROM \"so_long_term\".\"suridrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_drop" + ], + "type": "field" + }, + { + "params": [ + " * 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -730,7 +848,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -762,7 +879,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -813,6 +930,34 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "hide": false, + "measurement": "stenodrop", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_drop\" FROM \"so_long_term\".\"stenodrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_drop" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -864,9 +1009,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -895,7 +1038,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -952,6 +1095,39 @@ "value": "/" } ] + }, + { + "groupBy": [], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_used_percent\" FROM \"so_long_term\".\"disk\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"path\" = '/') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] } ], "thresholds": [], @@ -1002,9 +1178,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1033,7 +1207,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1090,6 +1264,39 @@ "value": "/nsm" } ] + }, + { + "groupBy": [], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_used_percent\" FROM \"so_long_term\".\"disk\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"path\" = '/nsm') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] } ], "thresholds": [], @@ -1141,7 +1348,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1172,7 +1378,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1215,7 +1421,46 @@ }, { "params": [ - " / {{ CPUS }}" + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_percent\" / 8 FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + " / 8" ], "type": "math" } @@ -1286,7 +1531,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1317,7 +1561,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1360,7 +1604,46 @@ }, { "params": [ - " / {{ CPUS }}" + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" ], "type": "math" } @@ -1432,7 +1715,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1464,7 +1746,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1508,7 +1790,46 @@ }, { "params": [ - " / {{ CPUS }}" + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_percent\" / 8 FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + " / 8" ], "type": "math" } @@ -1579,7 +1900,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1611,7 +1931,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -1655,7 +1975,46 @@ }, { "params": [ - " / {{ CPUS }}" + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" ], "type": "math" } @@ -1736,7 +2095,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1770,7 +2128,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -2126,7 +2484,7 @@ "aliasColors": { "InBound": "#629E51", "OutBound": "#5195CE", - "net.non_negative_derivative": "#1F78C1" + "net.non_negative_derivative": "super-light-blue" }, "bars": false, "dashLength": 10, @@ -2136,7 +2494,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2170,7 +2527,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -2239,7 +2596,7 @@ "condition": "AND", "key": "interface", "operator": "=", - "value": "{{ MANINT }}" + "value": "eth0" } ] }, @@ -2303,7 +2660,97 @@ "condition": "AND", "key": "interface", "operator": "=", - "value": "{{ MANINT }}" + "value": "eth0" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(\"mean_bytes_recv\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = 'eth0') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "eth0" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(\"mean_bytes_sent\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = 'eth0') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_sent" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "eth0" } ] } @@ -2359,7 +2806,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2391,7 +2837,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2448,6 +2894,39 @@ "value": "so-suricata" } ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] } ], "thresholds": [], @@ -2500,7 +2979,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2532,7 +3010,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2589,6 +3067,39 @@ "value": "so-steno" } ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] } ], "thresholds": [], @@ -2648,7 +3159,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2682,7 +3192,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -2918,7 +3428,7 @@ "aliasColors": { "InBound": "#629E51", "OutBound": "#5195CE", - "net.non_negative_derivative": "#1F78C1" + "net.non_negative_derivative": "light-orange" }, "bars": false, "dashLength": 10, @@ -2928,7 +3438,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2962,7 +3471,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3031,7 +3540,52 @@ "condition": "AND", "key": "interface", "operator": "=", - "value": "{{ MONINT }}" + "value": "/{{ MONINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(\"mean_bytes_recv\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = '/{{ MONINT }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "/{{ MONINT }}" } ] } @@ -3082,7 +3636,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", @@ -3119,9 +3672,10 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto" }, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "targets": [ { "groupBy": [ @@ -3180,37 +3734,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, - "decimals": 2, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": null, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 259200 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 432000 - } - ] - }, "unit": "s" }, "overrides": [] @@ -3242,7 +3765,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3294,6 +3817,33 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "measurement": "pcapage", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_seconds\" FROM \"so_long_term\".\"pcapage\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_seconds" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -3351,7 +3901,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3385,7 +3934,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3575,7 +4124,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3607,7 +4155,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3737,6 +4285,100 @@ "value": "so-logstash" } ] + }, + { + "alias": "ds_Inbound", + "groupBy": [], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "alias": "ds_Outbound", + "groupBy": [], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] } ], "thresholds": [], @@ -3789,7 +4431,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3821,7 +4462,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -3865,7 +4506,46 @@ }, { "params": [ - " / {{ CPUS }}" + " / 8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" ], "type": "math" } @@ -3936,7 +4616,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3968,7 +4647,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4025,6 +4704,39 @@ "value": "so-zeek" } ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] } ], "thresholds": [], @@ -4078,7 +4790,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4112,7 +4823,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -4139,6 +4850,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -4163,6 +4875,34 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_total_threads\" FROM \"so_long_term\".\"processes\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_total_threads" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4216,7 +4956,6 @@ "description": "", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4248,7 +4987,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4378,6 +5117,100 @@ "value": "so-influxdb" } ] + }, + { + "alias": "ds_Inbound", + "groupBy": [], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "alias": "ds_Outbound", + "groupBy": [], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] } ], "thresholds": [], @@ -4429,7 +5262,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4460,7 +5292,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": true, "renderer": "flot", @@ -4510,6 +5342,34 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "hide": false, + "measurement": "zeekcaptureloss", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_loss\" FROM \"so_long_term\".\"zeekcaptureloss\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_loss" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4563,7 +5423,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4595,7 +5454,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4652,6 +5511,40 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "hide": false, + "measurement": "zeekdrop", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_drop\" * 100 FROM \"so_long_term\".\"zeekdrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_drop" + ], + "type": "field" + }, + { + "params": [ + " * 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4704,7 +5597,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4735,7 +5627,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4785,6 +5677,34 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_iowait\" FROM \"so_long_term\".\"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_iowait" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4837,7 +5757,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4869,7 +5788,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -4999,6 +5918,100 @@ "value": "so-aptcacherng" } ] + }, + { + "alias": "ds_Inbound", + "groupBy": [], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-aptcacherng') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] + }, + { + "alias": "ds_Outbound", + "groupBy": [], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-aptcacherng') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] } ], "thresholds": [], @@ -5050,7 +6063,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -5081,7 +6093,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5131,6 +6143,33 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "measurement": "influxsize", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_kbytes\" FROM \"so_long_term\".\"influxsize\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_kbytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -5183,7 +6222,6 @@ "description": "", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -5214,7 +6252,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5264,6 +6302,33 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "measurement": "redisqueue", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_unparsed\" FROM \"so_long_term\".\"redisqueue\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_unparsed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -5317,7 +6382,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -5350,7 +6414,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -5506,9 +6570,7 @@ "datasource": "InfluxDB", "description": "", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -5537,7 +6599,7 @@ "alertThreshold": false }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5589,6 +6651,33 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "measurement": "consumptioneps", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_eps\" FROM \"so_long_term\".\"consumptioneps\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_eps" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -5640,7 +6729,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -5671,7 +6759,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5727,6 +6815,39 @@ "value": "so-redis" } ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] } ], "thresholds": [], @@ -5779,7 +6900,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -5810,7 +6930,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -5837,6 +6957,8 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"usage_percent\") / 8 FROM \"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "A", "resultFormat": "time_series", "select": [ @@ -5853,7 +6975,46 @@ }, { "params": [ - "/ 16" + "/{{ CPU }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND time <= (now()-24h) and time <= now()", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/{{ CPU }}" ], "type": "math" } @@ -5932,7 +7093,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -5968,7 +7128,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -5995,7 +7155,10 @@ } ], "measurement": "mem", + "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"used\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, "refId": "A", "resultFormat": "time_series", "select": [ @@ -6038,7 +7201,10 @@ } ], "measurement": "mem", + "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"buffered\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -6081,7 +7247,10 @@ } ], "measurement": "mem", + "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"cached\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, "refId": "C", "resultFormat": "time_series", "select": [ @@ -6124,7 +7293,10 @@ } ], "measurement": "mem", + "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"free\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, "refId": "D", "resultFormat": "time_series", "select": [ @@ -6148,6 +7320,152 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "ds_Used", + "groupBy": [], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_used\" FROM \"so_long_term\".\"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", + "rawQuery": true, + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "ds_Buffered", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"mean_buffered\" FROM \"so_long_term\".\"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", + "rawQuery": true, + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "ds_Cached", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"mean_cached\" FROM \"so_long_term\".\"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", + "rawQuery": true, + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "alias": "ds_Free", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"mean_free\" FROM \"so_long_term\".\"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", + "rawQuery": true, + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] } ], "thresholds": [], @@ -6205,9 +7523,7 @@ "editable": true, "error": false, "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "format": "none", @@ -6326,7 +7642,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -6358,7 +7673,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6409,6 +7724,33 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_store_size_in_bytes\" FROM \"so_long_term\".\"elasticsearch_indices\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_store_size_in_bytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -6460,7 +7802,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -6491,7 +7832,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6541,6 +7882,33 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_threads_count\" FROM \"so_long_term\".\"elasticsearch_jvm\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_threads_count" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -6593,7 +7961,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -6624,7 +7991,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6674,6 +8041,33 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT \"mean_fielddata_memory_size_in_bytes\" FROM \"so_long_term\".\"elasticsearch_indices\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", + "rawQuery": true, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_fielddata_memory_size_in_bytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -6725,7 +8119,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -6756,7 +8149,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -6850,6 +8243,33 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "ds_Used", + "groupBy": [], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -6895,7 +8315,7 @@ } ], "refresh": "30s", - "schemaVersion": 26, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -6909,6 +8329,7 @@ "text": "10s", "value": "10s" }, + "description": null, "error": null, "hide": 0, "label": null, @@ -6983,7 +8404,7 @@ ] }, "time": { - "from": "now-1h", + "from": "now-3h", "to": "now" }, "timepicker": { @@ -7014,5 +8435,5 @@ "timezone": "browser", "title": "Standalone Mode - {{ SERVERNAME }} Overview", "uid": "so_overview", - "version": 1 -} \ No newline at end of file + "version": 5 +} From ce98f463312155df2d7ef53c5f6ae62fd807c837 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Jun 2021 08:49:02 -0400 Subject: [PATCH 342/378] update standalone dashboard for new influx --- .../dashboards/standalone/standalone.json | 50 +- .../dashboards/standalone/standalone.new.json | 7841 ----------------- 2 files changed, 25 insertions(+), 7866 deletions(-) delete mode 100644 salt/grafana/dashboards/standalone/standalone.new.json diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index e102917e6..4de98988c 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -381,7 +381,7 @@ }, { "params": [ - " / 8" + " /{{ CPUS }}" ], "type": "math" } @@ -406,7 +406,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-elasticsearch') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-elasticsearch') AND time <= (now()-24h) and time <= now()", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -420,7 +420,7 @@ }, { "params": [ - "/16" + "/{{ CPUS }}" ], "type": "math" } @@ -563,7 +563,7 @@ }, { "params": [ - " / 8" + " /{{ CPUS }}" ], "type": "math" } @@ -588,7 +588,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"container_name\" = 'so-kibana' AND \"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"container_name\" = 'so-kibana' AND \"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -602,7 +602,7 @@ }, { "params": [ - "/16" + "/{{ CPUS }}" ], "type": "math" } @@ -1421,7 +1421,7 @@ }, { "params": [ - " / 8" + " /{{ CPUS }}" ], "type": "math" } @@ -1446,7 +1446,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" / 8 FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND time <= (now()-24h) and time <= now()", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -1460,7 +1460,7 @@ }, { "params": [ - " / 8" + " /{{ CPUS }}" ], "type": "math" } @@ -1604,7 +1604,7 @@ }, { "params": [ - " / 8" + " /{{ CPUS }}" ], "type": "math" } @@ -1629,7 +1629,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND time <= (now()-24h) and time <= now()", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -1643,7 +1643,7 @@ }, { "params": [ - "/16" + "/{{ CPUS }}" ], "type": "math" } @@ -1790,7 +1790,7 @@ }, { "params": [ - " / 8" + " /{{ CPUS }}" ], "type": "math" } @@ -1815,7 +1815,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" / 8 FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND time <= (now()-24h) and time <= now()", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -1829,7 +1829,7 @@ }, { "params": [ - " / 8" + " /{{ CPUS }}" ], "type": "math" } @@ -1975,7 +1975,7 @@ }, { "params": [ - " / 8" + " /{{ CPUS }}" ], "type": "math" } @@ -2000,7 +2000,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND time <= (now()-24h) and time <= now()", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -2014,7 +2014,7 @@ }, { "params": [ - "/16" + "/{{ CPUS }}" ], "type": "math" } @@ -4506,7 +4506,7 @@ }, { "params": [ - " / 8" + " /{{ CPUS }}" ], "type": "math" } @@ -4531,7 +4531,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND time <= (now()-24h) and time <= now()", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -4545,7 +4545,7 @@ }, { "params": [ - "/16" + "/{{ CPUS }}" ], "type": "math" } @@ -6957,7 +6957,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"usage_percent\") / 8 FROM \"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_percent\") /{{ CPUS }} FROM \"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -6975,7 +6975,7 @@ }, { "params": [ - "/{{ CPU }}" + "/{{ CPUS }}" ], "type": "math" } @@ -7000,7 +7000,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPU }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND time <= (now()-24h) and time <= now()", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -7014,7 +7014,7 @@ }, { "params": [ - "/{{ CPU }}" + "/{{ CPUS }}" ], "type": "math" } diff --git a/salt/grafana/dashboards/standalone/standalone.new.json b/salt/grafana/dashboards/standalone/standalone.new.json deleted file mode 100644 index 41c4d3541..000000000 --- a/salt/grafana/dashboards/standalone/standalone.new.json +++ /dev/null @@ -1,7841 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "This Dashboard provides a general overview of Standalone Mode", - "editable": true, - "gnetId": 2381, - "graphTooltip": 0, - "id": 6, - "iteration": 1617106562807, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 60 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Usage", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_idle" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "* -1 + 100" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "cpu", - "operator": "=", - "value": "cpu-total" - } - ] - }, - { - "groupBy": [], - "measurement": "cpu", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage_idle" - ], - "type": "field" - }, - { - "params": [ - "*-1+100" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "cpu", - "operator": "=", - "value": "cpu-total" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 39, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.4", - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "system", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "uptime" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "{{ SERVERNAME }} - System Uptime", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 0 - }, - "hiddenSeries": false, - "id": 35, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - " / {{ CPUS }}" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-elasticsearch" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage_percent" - ], - "type": "field" - }, - { - "params": [ - "/16" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-elasticsearch" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - ES CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 43, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - " / {{ CPUS }}" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "container_name", - "operator": "=", - "value": "so-kibana" - }, - { - "condition": "AND", - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage_percent" - ], - "type": "field" - }, - { - "params": [ - "/16" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "container_name", - "operator": "=", - "value": "so-kibana" - }, - { - "condition": "AND", - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Kibana CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 2, - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 0 - }, - "hiddenSeries": false, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "suridrop", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "drop" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - }, - { - "params": [ - "* 100" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Suricata Packet Loss", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 0 - }, - "hiddenSeries": false, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "stenodrop", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "drop" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - PCAP Packet Loss", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 5 - }, - "hiddenSeries": false, - "id": 73, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Used", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "disk", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "used_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "path", - "operator": "=", - "value": "/" - } - ] - }, - { - "groupBy": [], - "measurement": "disk", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_used_percent" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "path", - "operator": "=", - "value": "/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Disk Used(/)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 5 - }, - "hiddenSeries": false, - "id": 74, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Used", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "disk", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "used_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "path", - "operator": "=", - "value": "/nsm" - } - ] - }, - { - "groupBy": [], - "measurement": "disk", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_used_percent" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "path", - "operator": "=", - "value": "/nsm" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Disk Used(/nsm)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 5 - }, - "hiddenSeries": false, - "id": 41, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - " / {{ CPUS }}" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-influxdb" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage_percent" - ], - "type": "field" - }, - { - "params": [ - "/16" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-influxdb" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - InfluxDB CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 2, - "format": "percent", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 5 - }, - "hiddenSeries": false, - "id": 45, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - " / {{ CPUS }}" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-logstash" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage_percent" - ], - "type": "field" - }, - { - "params": [ - "/16" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-logstash" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Logstash CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 2, - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 5 - }, - "hiddenSeries": false, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - " / {{ CPUS }}" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-suricata" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage_percent" - ], - "type": "field" - }, - { - "params": [ - " / {{ CPUS }}" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-suricata" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Suri CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 5 - }, - "hiddenSeries": false, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - " / {{ CPUS }}" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-steno" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage_percent" - ], - "type": "field" - }, - { - "params": [ - "/16" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-steno" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Steno CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "Interrupt": "#70DBED", - "Nice": "#629E51", - "SoftIRQ": "#EA6460", - "System": "#BF1B00", - "User": "#1F78C1", - "Wait": "#F2C96D", - "cpu.mean": "#629E51" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 4, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 10 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "System", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_system" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "cpu", - "operator": "=", - "value": "cpu-total" - } - ] - }, - { - "alias": "User", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_user" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "cpu", - "operator": "=", - "value": "cpu-total" - } - ] - }, - { - "alias": "Nice", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_nice" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "cpu", - "operator": "=", - "value": "cpu-total" - } - ] - }, - { - "alias": "Interrupt", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_irq" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "cpu", - "operator": "=", - "value": "cpu-total" - } - ] - }, - { - "alias": "Wait", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "E", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_iowait" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "cpu", - "operator": "=", - "value": "cpu-total" - } - ] - }, - { - "alias": "SoftIRQ", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "F", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_softirq" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "cpu", - "operator": "=", - "value": "cpu-total" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - CPU Usage", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": "Percent(%)", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "InBound": "#629E51", - "OutBound": "#5195CE", - "net.non_negative_derivative": "super-light-blue" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 10 - }, - "hiddenSeries": false, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Inbound", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "net", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "bytes_recv" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "interface", - "operator": "=", - "value": "{{ MANINT }}" - } - ] - }, - { - "alias": "Outbound", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "net", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", - "rawQuery": false, - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "bytes_sent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "interface", - "operator": "=", - "value": "{{ MANINT }}" - } - ] - }, - { - "groupBy": [], - "measurement": "net", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_bytes_recv" - ], - "type": "field" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "interface", - "operator": "=", - "value": "{{ MANINT }}" - } - ] - }, - { - "groupBy": [], - "measurement": "net", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_bytes_sent" - ], - "type": "field" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "interface", - "operator": "=", - "value": "{{ MANINT }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Management Traffic", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": "Bits/Sec", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 10 - }, - "hiddenSeries": false, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_mem", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-suricata" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_mem", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-suricata" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Suri Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 10 - }, - "hiddenSeries": false, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_mem", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-steno" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_mem", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-steno" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Steno Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "#cpu": "green", - "1 Minute Average": "#EAB839", - "15 Minute Average": "#BF1B00", - "5 Minute Average": "#E0752D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "#cpu", - "fill": 0 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "#cpu", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "system", - "orderByTime": "ASC", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "n_cpus" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "1 Minute Average", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "system", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "load1" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "5 Minute Average", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "system", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "load5" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "15 Minute Average", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "system", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "load15" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Load Average", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "InBound": "#629E51", - "OutBound": "#5195CE", - "net.non_negative_derivative": "light-orange" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 15 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Inbound", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "net", - "orderByTime": "ASC", - "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", - "rawQuery": false, - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "bytes_recv" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "interface", - "operator": "=", - "value": "{{ MONINT }}" - } - ] - }, - { - "groupBy": [], - "measurement": "net", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_bytes_recv" - ], - "type": "field" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "interface", - "operator": "=", - "value": "{{ MONINT }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Monitor Traffic", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": "Bits/Sec", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 15 - }, - "id": 37, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "7.3.4", - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "healthcheck", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "zeek_restart" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Zeek Restarts via Healthcheck", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 2, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": null, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 259200 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 432000 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 15 - }, - "hiddenSeries": false, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Oldest Pcap", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "pcapage", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "seconds" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "groupBy": [], - "measurement": "pcapage", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_seconds" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - PCAP Retention", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 2, - "format": "s", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "Blocked": "#BF1B00", - "Running": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 7, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 20 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "alias": "Blocked", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "hide": false, - "measurement": "processes", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "blocked" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "Running", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "processes", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "running" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "Sleep", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "processes", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "sleeping" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Processes", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 20 - }, - "hiddenSeries": false, - "id": 47, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Inbound", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_net", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "rx_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-logstash" - } - ] - }, - { - "alias": "Outbound", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_net", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "tx_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-logstash" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Logstash Traffic", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": "Bits/Sec", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 20 - }, - "hiddenSeries": false, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - " / {{ CPUS }}" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-zeek" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage_percent" - ], - "type": "field" - }, - { - "params": [ - "/16" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-zeek" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Zeek CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 20 - }, - "hiddenSeries": false, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_mem", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-zeek" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_mem", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-zeek" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Zeek Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Threads", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "processes", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "total_threads" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Total Threads", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 25 - }, - "hiddenSeries": false, - "id": 49, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Inbound", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_net", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "rx_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-influxdb" - } - ] - }, - { - "alias": "Outbound", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_net", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "tx_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-influxdb" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - InfluxDB Traffic", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": "Bits/Sec", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 25 - }, - "hiddenSeries": false, - "id": 71, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "zeekcaptureloss", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "loss" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Zeek Capture Loss", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "percent", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 25 - }, - "hiddenSeries": false, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "zeekdrop", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "drop" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - }, - { - "params": [ - "* 100" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Zeek Packet Loss", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 30 - }, - "hiddenSeries": false, - "id": 53, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_iowait" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - IO Wait", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 2, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 30 - }, - "hiddenSeries": false, - "id": 51, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Inbound", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_net", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "rx_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-aptcacherng" - } - ] - }, - { - "alias": "Outbound", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_net", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "tx_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "1s" - ], - "type": "non_negative_derivative" - }, - { - "params": [ - "*8" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-aptcacherng" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Proxy Traffic", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bps", - "label": "Bits/Sec", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 30 - }, - "hiddenSeries": false, - "id": 69, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "influxsize", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "kbytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "groupBy": [], - "measurement": "influxsize", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_kbytes" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - InfluxDB Size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "deckbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 30 - }, - "hiddenSeries": false, - "id": 55, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "redisqueue", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "unparsed" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "groupBy": [], - "measurement": "redisqueue", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_unparsed" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Redis Queue", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 35 - }, - "hiddenSeries": false, - "id": 13, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Read", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "diskio", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "read_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [], - "type": "non_negative_difference" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "Write", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "diskio", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "write_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [], - "type": "non_negative_difference" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Disk I/O", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "description": "", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 35 - }, - "hiddenSeries": false, - "id": 76, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": false - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "EPS", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "esteps", - "orderByTime": "ASC", - "policy": "default", - "queryType": "randomWalk", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "eps" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "groupBy": [], - "measurement": "esteps", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_eps" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Estimated EPS", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "EPS", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 35 - }, - "hiddenSeries": false, - "id": 61, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_mem", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-redis" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_mem", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-redis" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Redis Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 1, - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 35 - }, - "hiddenSeries": false, - "id": 59, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "usage_percent" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - }, - { - "params": [ - "/ 16" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-redis" - } - ] - }, - { - "groupBy": [], - "measurement": "docker_container_cpu", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_usage_percent" - ], - "type": "field" - }, - { - "params": [ - "/16" - ], - "type": "math" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - }, - { - "condition": "AND", - "key": "container_name", - "operator": "=", - "value": "so-redis" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Redis CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 2, - "format": "percent", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "Buffered": "#6ED0E0", - "Cached": "#F9934E", - "Free": "#629E51", - "Used": "#58140C" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "decimals": null, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 6, - "fillGradient": 0, - "grid": {}, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 40 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 0, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "alias": "Used", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "mem", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "used" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "Buffered", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "mem", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "buffered" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "Cached", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "mem", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "cached" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "Free", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "mem", - "policy": "default", - "refId": "D", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "free" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Memory", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "Bytes", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "InfluxDB", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 40 - }, - "id": 33, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "elasticsearch_indices", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "docs_count" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": "", - "title": "{{ SERVERNAME }} - ES Documents", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 40 - }, - "hiddenSeries": false, - "id": 34, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "elasticsearch_indices", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "store_size_in_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "groupBy": [], - "measurement": "elasticsearch_indices", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_store_size_in_bytes" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - ES Store Size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 40 - }, - "hiddenSeries": false, - "id": 65, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "elasticsearch_jvm", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "threads_count" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "groupBy": [], - "measurement": "elasticsearch_jvm", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_threads_count" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - ES Thread Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 40 - }, - "hiddenSeries": false, - "id": 63, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "elasticsearch_indices", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "fielddata_memory_size_in_bytes" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "groupBy": [], - "measurement": "elasticsearch_indices", - "orderByTime": "ASC", - "policy": "so_long_term", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "mean_fielddata_memory_size_in_bytes" - ], - "type": "field" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - ES Fielddata Cache Size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "InfluxDB", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 45 - }, - "hiddenSeries": false, - "id": 67, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "Total", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "mem", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "total" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - }, - { - "alias": "Used", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "mem", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "used" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=", - "value": "{{ SERVERNAME }}" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "{{ SERVERNAME }} - Memory(Used)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "30s", - "schemaVersion": 26, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "text": "10s", - "value": "10s" - }, - "error": null, - "hide": 0, - "label": null, - "name": "Interval", - "options": [ - { - "selected": false, - "text": "auto", - "value": "$__auto_interval_Interval" - }, - { - "selected": true, - "text": "10s", - "value": "10s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "10s, 1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Standalone Mode - {{ SERVERNAME }} Overview", - "uid": "so_overview", - "version": 1 - } \ No newline at end of file From 0ba685d0e2ee6da7bd9d83bd7a2f3b034a2e380e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Jun 2021 12:36:06 -0400 Subject: [PATCH 343/378] change time filter --- .../dashboards/standalone/standalone.json | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 4de98988c..4efe46009 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -130,7 +130,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_idle\" *-1+100 FROM \"so_long_term\".\"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_idle\" *-1+100 FROM \"so_long_term\".\"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -406,7 +406,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-elasticsearch') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-elasticsearch') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -588,7 +588,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"container_name\" = 'so-kibana' AND \"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"container_name\" = 'so-kibana' AND \"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -769,7 +769,7 @@ "measurement": "suridrop", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_drop\" * 100 FROM \"so_long_term\".\"suridrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_drop\" * 100 FROM \"so_long_term\".\"suridrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -937,7 +937,7 @@ "measurement": "stenodrop", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_drop\" FROM \"so_long_term\".\"stenodrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_drop\" FROM \"so_long_term\".\"stenodrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -1101,7 +1101,7 @@ "measurement": "disk", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_used_percent\" FROM \"so_long_term\".\"disk\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"path\" = '/') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_used_percent\" FROM \"so_long_term\".\"disk\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"path\" = '/') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -1270,7 +1270,7 @@ "measurement": "disk", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_used_percent\" FROM \"so_long_term\".\"disk\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"path\" = '/nsm') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_used_percent\" FROM \"so_long_term\".\"disk\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"path\" = '/nsm') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -1446,7 +1446,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -1629,7 +1629,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -1815,7 +1815,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -2000,7 +2000,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -2669,7 +2669,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_bytes_recv\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = 'eth0') AND time <= (now()-24h) and time <= now()", + "query": "SELECT non_negative_derivative(\"mean_bytes_recv\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = 'eth0') AND $timeFilter", "rawQuery": true, "refId": "C", "resultFormat": "time_series", @@ -2714,7 +2714,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_bytes_sent\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = 'eth0') AND time <= (now()-24h) and time <= now()", + "query": "SELECT non_negative_derivative(\"mean_bytes_sent\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = 'eth0') AND $timeFilter", "rawQuery": true, "refId": "D", "resultFormat": "time_series", @@ -2900,7 +2900,7 @@ "measurement": "docker_container_mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -3073,7 +3073,7 @@ "measurement": "docker_container_mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -3549,7 +3549,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_bytes_recv\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = '/{{ MONINT }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT non_negative_derivative(\"mean_bytes_recv\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = '/{{ MONINT }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -3823,7 +3823,7 @@ "measurement": "pcapage", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_seconds\" FROM \"so_long_term\".\"pcapage\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_seconds\" FROM \"so_long_term\".\"pcapage\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -4293,7 +4293,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND time <= (now()-24h) and time <= now()", + "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND $timeFilter", "rawQuery": true, "refId": "C", "resultFormat": "time_series", @@ -4340,7 +4340,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND time <= (now()-24h) and time <= now()", + "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND $timeFilter", "rawQuery": true, "refId": "D", "resultFormat": "time_series", @@ -4531,7 +4531,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -4710,7 +4710,7 @@ "measurement": "docker_container_mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -4882,7 +4882,7 @@ "measurement": "processes", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_total_threads\" FROM \"so_long_term\".\"processes\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_total_threads\" FROM \"so_long_term\".\"processes\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -5125,7 +5125,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND time <= (now()-24h) and time <= now()", + "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND $timeFilter", "rawQuery": true, "refId": "C", "resultFormat": "time_series", @@ -5172,7 +5172,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND time <= (now()-24h) and time <= now()", + "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND $timeFilter", "rawQuery": true, "refId": "D", "resultFormat": "time_series", @@ -5349,7 +5349,7 @@ "measurement": "zeekcaptureloss", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_loss\" FROM \"so_long_term\".\"zeekcaptureloss\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_loss\" FROM \"so_long_term\".\"zeekcaptureloss\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -5518,7 +5518,7 @@ "measurement": "zeekdrop", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_drop\" * 100 FROM \"so_long_term\".\"zeekdrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_drop\" * 100 FROM \"so_long_term\".\"zeekdrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -5684,7 +5684,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_iowait\" FROM \"so_long_term\".\"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_iowait\" FROM \"so_long_term\".\"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -5926,7 +5926,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-aptcacherng') AND time <= (now()-24h) and time <= now()", + "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-aptcacherng') AND $timeFilter", "rawQuery": true, "refId": "C", "resultFormat": "time_series", @@ -5973,7 +5973,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-aptcacherng') AND time <= (now()-24h) and time <= now()", + "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-aptcacherng') AND $timeFilter", "rawQuery": true, "refId": "D", "resultFormat": "time_series", @@ -6149,7 +6149,7 @@ "measurement": "influxsize", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_kbytes\" FROM \"so_long_term\".\"influxsize\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_kbytes\" FROM \"so_long_term\".\"influxsize\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -6308,7 +6308,7 @@ "measurement": "redisqueue", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_unparsed\" FROM \"so_long_term\".\"redisqueue\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_unparsed\" FROM \"so_long_term\".\"redisqueue\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -6657,7 +6657,7 @@ "measurement": "consumptioneps", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_eps\" FROM \"so_long_term\".\"consumptioneps\" WHERE (\"host\" = '{{ SERVERNAME }}') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_eps\" FROM \"so_long_term\".\"consumptioneps\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -6821,7 +6821,7 @@ "measurement": "docker_container_mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -7000,7 +7000,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND time <= (now()-24h) and time <= now()", + "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", From c72146587a07fe76375f8a15d54bc392f6730438 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Jun 2021 16:07:32 -0400 Subject: [PATCH 344/378] standalone dashboard --- .../dashboards/standalone/standalone.json | 2461 ++++++++++++++--- 1 file changed, 2077 insertions(+), 384 deletions(-) diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 4efe46009..ff0fd13b6 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -17,7 +17,7 @@ "gnetId": 2381, "graphTooltip": 0, "id": 6, - "iteration": 1624648744739, + "iteration": 1624906271265, "links": [], "panels": [ { @@ -64,13 +64,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Usage", + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -115,7 +123,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -126,12 +134,26 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_idle\" *-1+100 FROM \"so_long_term\".\"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -142,9 +164,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - "*-1+100" + "* -1 + 100" ], "type": "math" } @@ -154,7 +180,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -169,7 +195,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - CPU", + "title": "jppce2350sa-2 - CPU", "tooltip": { "shared": true, "sort": 0, @@ -287,14 +313,14 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } ], "timeFrom": null, "timeShift": null, - "title": "{{ SERVERNAME }} - System Uptime", + "title": "jppce2350sa-2 - System Uptime", "type": "stat" }, { @@ -341,12 +367,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -381,7 +416,7 @@ }, { "params": [ - " /{{ CPUS }}" + " /8" ], "type": "math" } @@ -391,7 +426,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -402,12 +437,26 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-elasticsearch') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -418,9 +467,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - "/{{ CPUS }}" + " /8" ], "type": "math" } @@ -430,7 +483,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -445,7 +498,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - ES CPU Usage", + "title": "jppce2350sa-2 - ES CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -524,12 +577,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -563,7 +625,7 @@ }, { "params": [ - " /{{ CPUS }}" + " /8" ], "type": "math" } @@ -579,17 +641,30 @@ "condition": "AND", "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"container_name\" = 'so-kibana' AND \"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -600,9 +675,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - "/{{ CPUS }}" + " /8" ], "type": "math" } @@ -618,7 +697,7 @@ "condition": "AND", "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -627,7 +706,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Kibana CPU", + "title": "jppce2350sa-2 - Kibana CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -709,12 +788,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "dsType": "influxdb", "groupBy": [ { @@ -759,18 +847,31 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Loss Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "suridrop", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_drop\" * 100 FROM \"so_long_term\".\"suridrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -781,9 +882,13 @@ ], "type": "field" }, + { + "params": [], + "type": "last" + }, { "params": [ - " * 100" + "* 100" ], "type": "math" } @@ -793,7 +898,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -802,7 +907,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Suricata Packet Loss", + "title": "jppce2350sa-2 - Suricata Packet Loss", "tooltip": { "shared": true, "sort": 0, @@ -883,12 +988,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "dsType": "influxdb", "groupBy": [ { @@ -927,18 +1041,31 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Loss Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "stenodrop", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_drop\" FROM \"so_long_term\".\"stenodrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -948,6 +1075,10 @@ "mean_drop" ], "type": "field" + }, + { + "params": [], + "type": "last" } ] ], @@ -955,7 +1086,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -964,7 +1095,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - PCAP Packet Loss", + "title": "jppce2350sa-2 - PCAP Packet Loss", "tooltip": { "shared": true, "sort": 0, @@ -1042,13 +1173,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -1086,7 +1225,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1097,12 +1236,25 @@ ] }, { - "groupBy": [], + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "disk", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_used_percent\" FROM \"so_long_term\".\"disk\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"path\" = '/') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -1112,6 +1264,10 @@ "mean_used_percent" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -1119,7 +1275,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1134,7 +1290,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Disk Used(/)", + "title": "jppce2350sa-2 - Disk Used(/)", "tooltip": { "shared": true, "sort": 0, @@ -1211,13 +1367,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -1255,7 +1419,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1266,12 +1430,25 @@ ] }, { - "groupBy": [], + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "disk", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_used_percent\" FROM \"so_long_term\".\"disk\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"path\" = '/nsm') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -1281,6 +1458,10 @@ "mean_used_percent" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -1288,7 +1469,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1303,7 +1484,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Disk Used(/nsm)", + "title": "jppce2350sa-2 - Disk Used(/nsm)", "tooltip": { "shared": true, "sort": 0, @@ -1382,12 +1563,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -1421,7 +1611,7 @@ }, { "params": [ - " /{{ CPUS }}" + " /8" ], "type": "math" } @@ -1431,7 +1621,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1442,12 +1632,25 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -1458,9 +1661,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - " /{{ CPUS }}" + " /8" ], "type": "math" } @@ -1470,7 +1677,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1485,7 +1692,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - InfluxDB CPU Usage", + "title": "jppce2350sa-2 - InfluxDB CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -1565,12 +1772,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -1604,7 +1820,7 @@ }, { "params": [ - " /{{ CPUS }}" + " /8" ], "type": "math" } @@ -1614,7 +1830,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1625,12 +1841,25 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -1641,9 +1870,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - "/{{ CPUS }}" + " /8" ], "type": "math" } @@ -1653,7 +1886,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1668,7 +1901,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Logstash CPU Usage", + "title": "jppce2350sa-2 - Logstash CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -1750,12 +1983,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1790,7 +2032,7 @@ }, { "params": [ - " /{{ CPUS }}" + " /8" ], "type": "math" } @@ -1800,7 +2042,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1811,12 +2053,26 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -1827,9 +2083,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - " /{{ CPUS }}" + " /8" ], "type": "math" } @@ -1839,7 +2099,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1854,7 +2114,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Suri CPU Usage", + "title": "jppce2350sa-2 - Suri CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -1935,12 +2195,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1975,7 +2244,7 @@ }, { "params": [ - " /{{ CPUS }}" + " /8" ], "type": "math" } @@ -1985,7 +2254,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -1996,12 +2265,26 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -2012,9 +2295,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - "/{{ CPUS }}" + " /8" ], "type": "math" } @@ -2024,7 +2311,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2039,7 +2326,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Steno CPU Usage", + "title": "jppce2350sa-2 - Steno CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -2132,13 +2419,21 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "System", + "alias": "System Current", "dsType": "influxdb", "groupBy": [ { @@ -2157,6 +2452,8 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"usage_system\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "A", "resultFormat": "time_series", "select": [ @@ -2177,7 +2474,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2188,7 +2485,7 @@ ] }, { - "alias": "User", + "alias": "User Current", "dsType": "influxdb", "groupBy": [ { @@ -2207,6 +2504,8 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"usage_user\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "B", "resultFormat": "time_series", "select": [ @@ -2227,7 +2526,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2238,7 +2537,7 @@ ] }, { - "alias": "Nice", + "alias": "Nice Current", "dsType": "influxdb", "groupBy": [ { @@ -2257,6 +2556,8 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"usage_nice\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "C", "resultFormat": "time_series", "select": [ @@ -2277,7 +2578,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2288,7 +2589,7 @@ ] }, { - "alias": "Interrupt", + "alias": "Interrupt Current", "dsType": "influxdb", "groupBy": [ { @@ -2307,6 +2608,8 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"usage_irq\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, "refId": "D", "resultFormat": "time_series", "select": [ @@ -2327,7 +2630,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2338,7 +2641,7 @@ ] }, { - "alias": "Wait", + "alias": "Wait Current", "dsType": "influxdb", "groupBy": [ { @@ -2357,6 +2660,8 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"usage_iowait\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, "refId": "E", "resultFormat": "time_series", "select": [ @@ -2377,7 +2682,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2388,7 +2693,7 @@ ] }, { - "alias": "SoftIRQ", + "alias": "SoftIRQ Current", "dsType": "influxdb", "groupBy": [ { @@ -2407,6 +2712,8 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", + "query": "SELECT mean(\"usage_softirq\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, "refId": "F", "resultFormat": "time_series", "select": [ @@ -2427,7 +2734,325 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "System Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT mean(\"usage_system\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_system" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "User Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT mean(\"usage_user\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_user" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Nice Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT mean(\"usage_nice\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "I", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_nice" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Interrupt Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT mean(\"usage_irq\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "J", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_irq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Wait Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT mean(\"usage_iowait\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "K", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "SoftIRQ Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT mean(\"usage_softirq\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "L", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_softirq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2442,7 +3067,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - CPU Usage", + "title": "jppce2350sa-2 - CPU Usage", "tooltip": { "msResolution": true, "shared": true, @@ -2531,13 +3156,21 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2590,7 +3223,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2601,7 +3234,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2654,7 +3287,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2665,12 +3298,28 @@ ] }, { - "groupBy": [], + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_bytes_recv\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = 'eth0') AND $timeFilter", - "rawQuery": true, + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, "refId": "C", "resultFormat": "time_series", "select": [ @@ -2681,6 +3330,10 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ "1s" @@ -2699,7 +3352,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2710,12 +3363,28 @@ ] }, { - "groupBy": [], + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_bytes_sent\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = 'eth0') AND $timeFilter", - "rawQuery": true, + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, "refId": "D", "resultFormat": "time_series", "select": [ @@ -2726,6 +3395,10 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ "1s" @@ -2744,7 +3417,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2759,7 +3432,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Management Traffic", + "title": "jppce2350sa-2 - Management Traffic", "tooltip": { "msResolution": true, "shared": true, @@ -2841,12 +3514,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -2885,7 +3567,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2896,12 +3578,26 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-suricata') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -2911,6 +3607,10 @@ "mean_usage" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -2918,7 +3618,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -2933,7 +3633,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Suri Memory Usage", + "title": "jppce2350sa-2 - Suri Memory Usage", "tooltip": { "shared": true, "sort": 0, @@ -3014,12 +3714,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -3058,7 +3767,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -3069,12 +3778,26 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-steno') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -3084,6 +3807,10 @@ "mean_usage" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -3091,7 +3818,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -3106,7 +3833,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Steno Memory Usage", + "title": "jppce2350sa-2 - Steno Memory Usage", "tooltip": { "shared": true, "sort": 0, @@ -3198,8 +3925,11 @@ "renderer": "flot", "seriesOverrides": [ { - "alias": "#cpu", - "fill": 0 + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 } ], "spaceLength": 10, @@ -3207,7 +3937,7 @@ "steppedLine": false, "targets": [ { - "alias": "#cpu", + "alias": "#cpu Current", "groupBy": [ { "params": [ @@ -3245,12 +3975,12 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "1 Minute Average", + "alias": "1 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -3289,12 +4019,12 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "5 Minute Average", + "alias": "5 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -3333,12 +4063,12 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "15 Minute Average", + "alias": "15 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -3377,7 +4107,186 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" + } + ] + }, + { + "alias": "#cpu Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_n_cpus" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + } + ] + }, + { + "alias": "1 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + } + ] + }, + { + "alias": "5 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load5" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + } + ] + }, + { + "alias": "15 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load15" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" } ] } @@ -3386,7 +4295,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Load Average", + "title": "jppce2350sa-2 - Load Average", "tooltip": { "msResolution": true, "shared": true, @@ -3475,13 +4384,21 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -3534,23 +4451,39 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", "key": "interface", "operator": "=", - "value": "/{{ MONINT }}" + "value": "bond0" } ] }, { - "groupBy": [], + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_bytes_recv\", 1s) *8 FROM \"so_long_term\".\"net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"interface\" = '/{{ MONINT }}') AND $timeFilter", - "rawQuery": true, + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, "refId": "B", "resultFormat": "time_series", "select": [ @@ -3561,6 +4494,10 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ "1s" @@ -3579,13 +4516,13 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", "key": "interface", "operator": "=", - "value": "/{{ MONINT }}" + "value": "bond0" } ] } @@ -3594,7 +4531,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Monitor Traffic", + "title": "jppce2350sa-2 - Monitor Traffic", "tooltip": { "msResolution": true, "shared": true, @@ -3715,7 +4652,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -3769,13 +4706,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Oldest Pcap", + "alias": "Oldest Pcap Current", "dsType": "influxdb", "groupBy": [ { @@ -3814,17 +4759,31 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Oldest Pcap Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "pcapage", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_seconds\" FROM \"so_long_term\".\"pcapage\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -3834,6 +4793,10 @@ "mean_seconds" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -3841,7 +4804,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -3850,7 +4813,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - PCAP Retention", + "title": "jppce2350sa-2 - PCAP Retention", "tooltip": { "shared": true, "sort": 0, @@ -3938,13 +4901,21 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { - "alias": "Blocked", + "alias": "Blocked Current", "dsType": "influxdb", "groupBy": [ { @@ -3962,6 +4933,7 @@ ], "hide": false, "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -3983,12 +4955,12 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "Running", + "alias": "Running Current", "dsType": "influxdb", "groupBy": [ { @@ -4005,6 +4977,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "B", "resultFormat": "time_series", @@ -4026,12 +4999,12 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "Sleep", + "alias": "Sleep Current", "dsType": "influxdb", "groupBy": [ { @@ -4048,6 +5021,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "C", "resultFormat": "time_series", @@ -4069,7 +5043,142 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" + } + ] + }, + { + "alias": "Blocked Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_blocked" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + } + ] + }, + { + "alias": "Running Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_running" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + } + ] + }, + { + "alias": "Sleep Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_sleeping" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" } ] } @@ -4078,7 +5187,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Processes", + "title": "jppce2350sa-2 - Processes", "tooltip": { "msResolution": true, "shared": true, @@ -4159,13 +5268,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "groupBy": [ { "params": [ @@ -4215,7 +5332,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -4226,7 +5343,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "groupBy": [ { "params": [ @@ -4276,7 +5393,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -4287,14 +5404,25 @@ ] }, { - "alias": "ds_Inbound", - "groupBy": [], + "alias": "Inbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND $timeFilter", - "rawQuery": true, "refId": "C", "resultFormat": "time_series", "select": [ @@ -4305,6 +5433,10 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ "1s" @@ -4323,7 +5455,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -4334,14 +5466,25 @@ ] }, { - "alias": "ds_Outbound", - "groupBy": [], + "alias": "Outbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-logstash') AND $timeFilter", - "rawQuery": true, "refId": "D", "resultFormat": "time_series", "select": [ @@ -4352,9 +5495,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - "10s" + "1s" ], "type": "non_negative_derivative" }, @@ -4370,7 +5517,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -4385,7 +5532,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Logstash Traffic", + "title": "jppce2350sa-2 - Logstash Traffic", "tooltip": { "shared": true, "sort": 0, @@ -4466,12 +5613,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -4506,7 +5662,7 @@ }, { "params": [ - " /{{ CPUS }}" + " /8" ], "type": "math" } @@ -4516,7 +5672,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -4527,12 +5683,26 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -4543,9 +5713,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - "/{{ CPUS }}" + " /8" ], "type": "math" } @@ -4555,7 +5729,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -4570,7 +5744,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Zeek CPU Usage", + "title": "jppce2350sa-2 - Zeek CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -4651,12 +5825,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -4695,7 +5878,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -4706,12 +5889,26 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-zeek') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -4721,6 +5918,10 @@ "mean_usage" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -4728,7 +5929,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -4743,7 +5944,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Zeek Memory Usage", + "title": "jppce2350sa-2 - Zeek Memory Usage", "tooltip": { "shared": true, "sort": 0, @@ -4827,13 +6028,21 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Threads", + "alias": "Threads Current", "dsType": "influxdb", "groupBy": [ { @@ -4872,18 +6081,31 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Threads Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "processes", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_total_threads\" FROM \"so_long_term\".\"processes\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -4893,6 +6115,10 @@ "mean_total_threads" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -4900,7 +6126,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -4909,7 +6135,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Total Threads", + "title": "jppce2350sa-2 - Total Threads", "tooltip": { "msResolution": true, "shared": true, @@ -4991,13 +6217,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "groupBy": [ { "params": [ @@ -5047,7 +6281,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -5058,7 +6292,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "groupBy": [ { "params": [ @@ -5108,7 +6342,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -5119,14 +6353,25 @@ ] }, { - "alias": "ds_Inbound", - "groupBy": [], + "alias": "Inbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND $timeFilter", - "rawQuery": true, "refId": "C", "resultFormat": "time_series", "select": [ @@ -5137,6 +6382,10 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ "1s" @@ -5155,7 +6404,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -5166,14 +6415,25 @@ ] }, { - "alias": "ds_Outbound", - "groupBy": [], + "alias": "Outbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND $timeFilter", - "rawQuery": true, "refId": "D", "resultFormat": "time_series", "select": [ @@ -5184,6 +6444,10 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ "1s" @@ -5202,7 +6466,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -5217,7 +6481,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - InfluxDB Traffic", + "title": "jppce2350sa-2 - InfluxDB Traffic", "tooltip": { "shared": true, "sort": 0, @@ -5296,12 +6560,21 @@ "pointradius": 2, "points": true, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "groupBy": [ { "params": [ @@ -5339,18 +6612,30 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Loss Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "zeekcaptureloss", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_loss\" FROM \"so_long_term\".\"zeekcaptureloss\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -5360,6 +6645,10 @@ "mean_loss" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -5367,7 +6656,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -5376,7 +6665,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Zeek Capture Loss", + "title": "jppce2350sa-2 - Zeek Capture Loss", "tooltip": { "shared": true, "sort": 0, @@ -5458,12 +6747,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "dsType": "influxdb", "groupBy": [ { @@ -5508,18 +6806,31 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Loss Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "zeekdrop", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_drop\" * 100 FROM \"so_long_term\".\"zeekdrop\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -5530,9 +6841,13 @@ ], "type": "field" }, + { + "params": [], + "type": "last" + }, { "params": [ - " * 100" + "* 100" ], "type": "math" } @@ -5542,7 +6857,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -5551,7 +6866,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Zeek Packet Loss", + "title": "jppce2350sa-2 - Zeek Packet Loss", "tooltip": { "shared": true, "sort": 0, @@ -5631,12 +6946,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "IO Wait Current", "groupBy": [ { "params": [ @@ -5674,18 +6998,30 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "IO Wait Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_iowait\" FROM \"so_long_term\".\"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -5695,6 +7031,10 @@ "mean_usage_iowait" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -5702,7 +7042,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -5711,7 +7051,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - IO Wait", + "title": "jppce2350sa-2 - IO Wait", "tooltip": { "shared": true, "sort": 0, @@ -5792,13 +7132,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "groupBy": [ { "params": [ @@ -5848,7 +7196,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -5859,7 +7207,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "groupBy": [ { "params": [ @@ -5909,7 +7257,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -5920,14 +7268,25 @@ ] }, { - "alias": "ds_Inbound", - "groupBy": [], + "alias": "Inbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_rx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-aptcacherng') AND $timeFilter", - "rawQuery": true, "refId": "C", "resultFormat": "time_series", "select": [ @@ -5938,6 +7297,10 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ "1s" @@ -5956,7 +7319,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -5967,14 +7330,25 @@ ] }, { - "alias": "ds_Outbound", - "groupBy": [], + "alias": "Outbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT non_negative_derivative(\"mean_tx_bytes\", 1s) *8 FROM \"so_long_term\".\"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-aptcacherng') AND $timeFilter", - "rawQuery": true, "refId": "D", "resultFormat": "time_series", "select": [ @@ -5985,6 +7359,10 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ "1s" @@ -6003,7 +7381,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -6018,7 +7396,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Proxy Traffic", + "title": "jppce2350sa-2 - Proxy Traffic", "tooltip": { "shared": true, "sort": 0, @@ -6097,12 +7475,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Size Current", "groupBy": [ { "params": [ @@ -6140,17 +7527,30 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Size Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "influxsize", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_kbytes\" FROM \"so_long_term\".\"influxsize\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -6160,6 +7560,10 @@ "mean_kbytes" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -6167,7 +7571,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -6176,7 +7580,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - InfluxDB Size", + "title": "jppce2350sa-2 - InfluxDB Size", "tooltip": { "shared": true, "sort": 0, @@ -6256,12 +7660,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Queue Current", "groupBy": [ { "params": [ @@ -6299,17 +7712,30 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Queue Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "redisqueue", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_unparsed\" FROM \"so_long_term\".\"redisqueue\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -6319,6 +7745,10 @@ "mean_unparsed" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -6326,7 +7756,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -6335,7 +7765,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Redis Queue", + "title": "jppce2350sa-2 - Redis Queue", "tooltip": { "shared": true, "sort": 0, @@ -6418,13 +7848,21 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Read", + "alias": "Read Current", "dsType": "influxdb", "groupBy": [ { @@ -6467,12 +7905,12 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "Write", + "alias": "Write Current", "dsType": "influxdb", "groupBy": [ { @@ -6515,7 +7953,105 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" + } + ] + }, + { + "alias": "Read Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_read_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + } + ] + }, + { + "alias": "Write Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_write_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" } ] } @@ -6524,7 +8060,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Disk I/O", + "title": "jppce2350sa-2 - Disk I/O", "tooltip": { "msResolution": true, "shared": true, @@ -6603,13 +8139,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "EPS", + "alias": "EPS Current", "groupBy": [ { "params": [ @@ -6648,17 +8192,31 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "EPS Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "consumptioneps", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_eps\" FROM \"so_long_term\".\"consumptioneps\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, + "queryType": "randomWalk", "refId": "B", "resultFormat": "time_series", "select": [ @@ -6668,6 +8226,10 @@ "mean_eps" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -6675,7 +8237,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -6684,7 +8246,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Estimated EPS", + "title": "jppce2350sa-2 - Estimated EPS", "tooltip": { "shared": true, "sort": 0, @@ -6763,12 +8325,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -6806,7 +8377,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -6817,12 +8388,25 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage\" FROM \"so_long_term\".\"docker_container_mem\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -6832,6 +8416,10 @@ "mean_usage" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -6839,7 +8427,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -6854,7 +8442,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Redis Memory Usage", + "title": "jppce2350sa-2 - Redis Memory Usage", "tooltip": { "shared": true, "sort": 0, @@ -6934,12 +8522,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -6957,7 +8554,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"usage_percent\") /{{ CPUS }} FROM \"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_percent\") /8 FROM \"docker_container_cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -6975,7 +8572,7 @@ }, { "params": [ - "/{{ CPUS }}" + "/8" ], "type": "math" } @@ -6985,7 +8582,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -6996,12 +8593,27 @@ ] }, { - "groupBy": [], + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_usage_percent\" /{{ CPUS }} FROM \"so_long_term\".\"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter", - "rawQuery": true, + "query": "SELECT mean(\"usage_percent\") /8 FROM \"docker_container_cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "B", "resultFormat": "time_series", "select": [ @@ -7012,9 +8624,13 @@ ], "type": "field" }, + { + "params": [], + "type": "mean" + }, { "params": [ - "/{{ CPUS }}" + "/8" ], "type": "math" } @@ -7024,7 +8640,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" }, { "condition": "AND", @@ -7039,7 +8655,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Redis CPU Usage", + "title": "jppce2350sa-2 - Redis CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -7132,13 +8748,21 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "dsType": "influxdb", "groupBy": [ { @@ -7157,8 +8781,8 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"used\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", - "rawQuery": true, + "query": "SELECT mean(\"used\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "A", "resultFormat": "time_series", "select": [ @@ -7179,12 +8803,12 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "Buffered", + "alias": "Buffered Current", "dsType": "influxdb", "groupBy": [ { @@ -7203,8 +8827,8 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"buffered\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", - "rawQuery": true, + "query": "SELECT mean(\"buffered\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "B", "resultFormat": "time_series", "select": [ @@ -7225,12 +8849,12 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "Cached", + "alias": "Cached Current", "dsType": "influxdb", "groupBy": [ { @@ -7249,8 +8873,8 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"cached\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", - "rawQuery": true, + "query": "SELECT mean(\"cached\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "C", "resultFormat": "time_series", "select": [ @@ -7271,12 +8895,12 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "Free", + "alias": "Free Current", "dsType": "influxdb", "groupBy": [ { @@ -7295,8 +8919,8 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"free\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", - "rawQuery": true, + "query": "SELECT mean(\"free\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "D", "resultFormat": "time_series", "select": [ @@ -7317,19 +8941,33 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "ds_Used", - "groupBy": [], + "alias": "Used Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_used\" FROM \"so_long_term\".\"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, + "query": "SELECT mean(\"used\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "E", "resultFormat": "time_series", "select": [ @@ -7339,6 +8977,10 @@ "mean_used" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -7346,16 +8988,17 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "alias": "ds_Buffered", + "alias": "Buffered Trend", + "dsType": "influxdb", "groupBy": [ { "params": [ - "$__interval" + "$interval" ], "type": "time" }, @@ -7367,17 +9010,18 @@ } ], "hide": false, + "measurement": "mem", "orderByTime": "ASC", - "policy": "default", - "query": "SELECT \"mean_buffered\" FROM \"so_long_term\".\"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, + "policy": "so_long_term", + "query": "SELECT mean(\"buffered\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "F", "resultFormat": "time_series", "select": [ [ { "params": [ - "value" + "mean_buffered" ], "type": "field" }, @@ -7387,14 +9031,21 @@ } ] ], - "tags": [] + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + } + ] }, { - "alias": "ds_Cached", + "alias": "Cached Trend", + "dsType": "influxdb", "groupBy": [ { "params": [ - "$__interval" + "$interval" ], "type": "time" }, @@ -7406,17 +9057,18 @@ } ], "hide": false, + "measurement": "mem", "orderByTime": "ASC", - "policy": "default", - "query": "SELECT \"mean_cached\" FROM \"so_long_term\".\"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, + "policy": "so_long_term", + "query": "SELECT mean(\"cached\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "G", "resultFormat": "time_series", "select": [ [ { "params": [ - "value" + "mean_cached" ], "type": "field" }, @@ -7426,14 +9078,21 @@ } ] ], - "tags": [] + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + } + ] }, { - "alias": "ds_Free", + "alias": "Free Trend", + "dsType": "influxdb", "groupBy": [ { "params": [ - "$__interval" + "$interval" ], "type": "time" }, @@ -7445,17 +9104,18 @@ } ], "hide": false, + "measurement": "mem", "orderByTime": "ASC", - "policy": "default", - "query": "SELECT \"mean_free\" FROM \"so_long_term\".\"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, + "policy": "so_long_term", + "query": "SELECT mean(\"free\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, "refId": "H", "resultFormat": "time_series", "select": [ [ { "params": [ - "value" + "mean_free" ], "type": "field" }, @@ -7465,14 +9125,20 @@ } ] ], - "tags": [] + "tags": [ + { + "key": "host", + "operator": "=", + "value": "jppce2350sa-2" + } + ] } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Memory", + "title": "jppce2350sa-2 - Memory", "tooltip": { "msResolution": true, "shared": true, @@ -7615,13 +9281,13 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } ], "thresholds": "", - "title": "{{ SERVERNAME }} - ES Documents", + "title": "jppce2350sa-2 - ES Documents", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -7677,12 +9343,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Size Current", "dsType": "influxdb", "groupBy": [ { @@ -7721,17 +9396,31 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, { - "groupBy": [], + "alias": "Size Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "elasticsearch_indices", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_store_size_in_bytes\" FROM \"so_long_term\".\"elasticsearch_indices\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -7741,6 +9430,10 @@ "mean_store_size_in_bytes" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -7748,7 +9441,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -7757,7 +9450,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - ES Store Size", + "title": "jppce2350sa-2 - ES Store Size", "tooltip": { "shared": true, "sort": 0, @@ -7879,7 +9572,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, @@ -7888,7 +9581,7 @@ "measurement": "elasticsearch_jvm", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_threads_count\" FROM \"so_long_term\".\"elasticsearch_jvm\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", + "query": "SELECT \"mean_threads_count\" FROM \"so_long_term\".\"elasticsearch_jvm\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -7906,7 +9599,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -7915,7 +9608,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - ES Thread Count", + "title": "jppce2350sa-2 - ES Thread Count", "tooltip": { "shared": true, "sort": 0, @@ -8038,7 +9731,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, @@ -8047,7 +9740,7 @@ "measurement": "elasticsearch_indices", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_fielddata_memory_size_in_bytes\" FROM \"so_long_term\".\"elasticsearch_indices\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter", + "query": "SELECT \"mean_fielddata_memory_size_in_bytes\" FROM \"so_long_term\".\"elasticsearch_indices\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter", "rawQuery": true, "refId": "B", "resultFormat": "time_series", @@ -8065,7 +9758,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -8074,7 +9767,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - ES Fielddata Cache Size", + "title": "jppce2350sa-2 - ES Fielddata Cache Size", "tooltip": { "shared": true, "sort": 0, @@ -8197,7 +9890,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, @@ -8240,7 +9933,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] }, @@ -8267,7 +9960,7 @@ { "key": "host", "operator": "=", - "value": "{{ SERVERNAME }}" + "value": "jppce2350sa-2" } ] } @@ -8276,7 +9969,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Memory(Used)", + "title": "jppce2350sa-2 - Memory(Used)", "tooltip": { "shared": true, "sort": 0, @@ -8433,7 +10126,7 @@ ] }, "timezone": "browser", - "title": "Standalone Mode - {{ SERVERNAME }} Overview", + "title": "Standalone Mode - jppce2350sa-2 Overview", "uid": "so_overview", - "version": 5 + "version": 9 } From 21c9388ee63e46dee1102b9190c47b7bfbb9389b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Jun 2021 16:12:36 -0400 Subject: [PATCH 345/378] generate measurement list and cq for each --- salt/influxdb/defaults.yaml | 6 +++++- salt/influxdb/init.sls | 9 +++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 7ba83dd6d..1c7faf022 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -10,4 +10,8 @@ influxdb: shard_duration: 7d downsample: so_long_term: - resolution: 5m \ No newline at end of file + resolution: 5m + measurements: +{% for measurement in salt['cmd.run']('docker exec -t so-influxdb /bin/bash -c "influx -format json -ssl -unsafeSsl -database telegraf -execute \"show measurements\"" | jq -r ".results[0].series[0].values[][0]"') %} + - {{ measurement }} +{% endfor %} diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 346d971fd..fb84f8bc8 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -113,11 +113,12 @@ telegraf_database: {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} -so_downsample_cq: + {% for measurement in influxdb.downsample[dest_rp].measurements %} +so_downsample_{{measurement}}_cq: influxdb_continuous_query.present: - - name: so_downsample_cq + - name: so_downsample_{{measurement}}_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] @@ -126,7 +127,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - - sls: salt.python3-influxdb + {% endfor %} {% endfor %} {% endif %} From 1c516daa96cbd9e7cbe6f88f062591b103b655c8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Jun 2021 17:05:32 -0400 Subject: [PATCH 346/378] fix measurement list --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 1c7faf022..651722315 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -12,6 +12,6 @@ influxdb: so_long_term: resolution: 5m measurements: -{% for measurement in salt['cmd.run']('docker exec -t so-influxdb /bin/bash -c "influx -format json -ssl -unsafeSsl -database telegraf -execute \"show measurements\"" | jq -r ".results[0].series[0].values[][0]"') %} +{% for measurement in salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" | jq -r .results[0].series[0].values[][0]').splitlines() %} - {{ measurement }} {% endfor %} From 6b68a39cbe87384628172a2e5bb6b04401a2c6bf Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 07:46:25 -0400 Subject: [PATCH 347/378] handle senario where there are no measurements --- salt/influxdb/defaults.yaml | 8 ++++++-- salt/influxdb/init.sls | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 651722315..bfcee7927 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -1,3 +1,5 @@ +{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" | jq -r .results[0].series[0].values[][0]') %} + influxdb: retention_policies: so_short_term: @@ -11,7 +13,9 @@ influxdb: downsample: so_long_term: resolution: 5m +{% if measurements is defined %} measurements: -{% for measurement in salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" | jq -r .results[0].series[0].values[][0]').splitlines() %} + {% for measurement in measurements.splitlines() %} - {{ measurement }} -{% endfor %} + {% endfor %} +{% endif %} diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index fb84f8bc8..37ebe39a5 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -113,7 +113,7 @@ telegraf_database: {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} - {% for measurement in influxdb.downsample[dest_rp].measurements %} + {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) %} so_downsample_{{measurement}}_cq: influxdb_continuous_query.present: - name: so_downsample_{{measurement}}_cq From bf8bba7b84659463aece0b668db6256382c8b09f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 08:57:51 -0400 Subject: [PATCH 348/378] only set measurements if conditions are met --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index bfcee7927..dae7f1cb4 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -13,7 +13,7 @@ influxdb: downsample: so_long_term: resolution: 5m -{% if measurements is defined %} +{% if 'jq: error' not in measurements and 'Error response from daemon' not in measurements and 'parse error:' not in measurements and measurements|length > 0 %} measurements: {% for measurement in measurements.splitlines() %} - {{ measurement }} From ca152ab04cddf3b3649d121346f2c63f27da1733 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 09:54:17 -0400 Subject: [PATCH 349/378] redefine measurements --- salt/influxdb/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index dae7f1cb4..205c2ba67 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -1,4 +1,4 @@ -{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" | jq -r .results[0].series[0].values[][0]') %} +{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" 2> /root/measurement_query.log | jq -r .results[0].series[0].values[]?[0] 2>> /root/measurement_query.log') %} influxdb: retention_policies: @@ -13,7 +13,7 @@ influxdb: downsample: so_long_term: resolution: 5m -{% if 'jq: error' not in measurements and 'Error response from daemon' not in measurements and 'parse error:' not in measurements and measurements|length > 0 %} +{% if measurements|length > 0 %} measurements: {% for measurement in measurements.splitlines() %} - {{ measurement }} From f18b64faafe78ee88f49762b13097246832844e7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 11:11:23 -0400 Subject: [PATCH 350/378] new standalone dashboard --- .../dashboards/standalone/standalone.json | 611 +++++++++++------- 1 file changed, 362 insertions(+), 249 deletions(-) diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index ff0fd13b6..fbf0b8cf5 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -16,8 +16,8 @@ "editable": true, "gnetId": 2381, "graphTooltip": 0, - "id": 6, - "iteration": 1624906271265, + "id": 7, + "iteration": 1624977014309, "links": [], "panels": [ { @@ -123,7 +123,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -180,7 +180,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -195,7 +195,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - CPU", + "title": "{{ SERVERNAME }} - CPU", "tooltip": { "shared": true, "sort": 0, @@ -313,14 +313,14 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } ], "timeFrom": null, "timeShift": null, - "title": "jppce2350sa-2 - System Uptime", + "title": "{{ SERVERNAME }} - System Uptime", "type": "stat" }, { @@ -416,7 +416,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -426,7 +426,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -473,7 +473,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -483,7 +483,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -498,7 +498,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - ES CPU Usage", + "title": "{{ SERVERNAME }} - ES CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -625,7 +625,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -641,7 +641,7 @@ "condition": "AND", "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -681,7 +681,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -697,7 +697,7 @@ "condition": "AND", "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -706,7 +706,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Kibana CPU Usage", + "title": "{{ SERVERNAME }} - Kibana CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -847,7 +847,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -898,7 +898,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -907,7 +907,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Suricata Packet Loss", + "title": "{{ SERVERNAME }} - Suricata Packet Loss", "tooltip": { "shared": true, "sort": 0, @@ -1041,7 +1041,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -1086,7 +1086,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -1095,7 +1095,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - PCAP Packet Loss", + "title": "{{ SERVERNAME }} - PCAP Packet Loss", "tooltip": { "shared": true, "sort": 0, @@ -1225,7 +1225,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -1275,7 +1275,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -1290,7 +1290,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Disk Used(/)", + "title": "{{ SERVERNAME }} - Disk Used(/)", "tooltip": { "shared": true, "sort": 0, @@ -1419,7 +1419,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -1469,7 +1469,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -1484,7 +1484,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Disk Used(/nsm)", + "title": "{{ SERVERNAME }} - Disk Used(/nsm)", "tooltip": { "shared": true, "sort": 0, @@ -1611,7 +1611,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -1621,7 +1621,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -1667,7 +1667,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -1677,7 +1677,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -1692,7 +1692,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - InfluxDB CPU Usage", + "title": "{{ SERVERNAME }} - InfluxDB CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -1820,7 +1820,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -1830,7 +1830,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -1876,7 +1876,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -1886,7 +1886,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -1901,7 +1901,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Logstash CPU Usage", + "title": "{{ SERVERNAME }} - Logstash CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -2032,7 +2032,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -2042,7 +2042,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2089,7 +2089,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -2099,7 +2099,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2114,7 +2114,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Suri CPU Usage", + "title": "{{ SERVERNAME }} - Suri CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -2244,7 +2244,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -2254,7 +2254,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2301,7 +2301,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -2311,7 +2311,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2326,7 +2326,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Steno CPU Usage", + "title": "{{ SERVERNAME }} - Steno CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -2452,7 +2452,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"usage_system\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_system\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -2474,7 +2474,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2504,7 +2504,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"usage_user\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_user\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -2526,7 +2526,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2556,7 +2556,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"usage_nice\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_nice\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "C", "resultFormat": "time_series", @@ -2578,7 +2578,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2608,7 +2608,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"usage_irq\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_irq\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": true, "refId": "D", "resultFormat": "time_series", @@ -2630,7 +2630,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2660,7 +2660,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"usage_iowait\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_iowait\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": true, "refId": "E", "resultFormat": "time_series", @@ -2682,7 +2682,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2712,7 +2712,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"usage_softirq\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_softirq\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": true, "refId": "F", "resultFormat": "time_series", @@ -2734,7 +2734,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2765,7 +2765,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"usage_system\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_system\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "G", "resultFormat": "time_series", @@ -2787,7 +2787,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2818,7 +2818,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"usage_user\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_user\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "H", "resultFormat": "time_series", @@ -2840,7 +2840,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2871,7 +2871,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"usage_nice\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_nice\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "I", "resultFormat": "time_series", @@ -2893,7 +2893,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2924,7 +2924,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"usage_irq\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_irq\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "J", "resultFormat": "time_series", @@ -2946,7 +2946,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -2977,7 +2977,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"usage_iowait\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_iowait\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "K", "resultFormat": "time_series", @@ -2999,7 +2999,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -3030,7 +3030,7 @@ "measurement": "cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"usage_softirq\") FROM \"cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_softirq\") FROM \"cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "L", "resultFormat": "time_series", @@ -3052,7 +3052,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -3067,7 +3067,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - CPU Usage", + "title": "{{ SERVERNAME }} - CPU Usage", "tooltip": { "msResolution": true, "shared": true, @@ -3189,7 +3189,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -3223,13 +3223,13 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", "key": "interface", "operator": "=", - "value": "eth0" + "value": "{{ MANINT }}" } ] }, @@ -3253,7 +3253,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -3287,13 +3287,13 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", "key": "interface", "operator": "=", - "value": "eth0" + "value": "{{ MANINT }}" } ] }, @@ -3318,7 +3318,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "C", "resultFormat": "time_series", @@ -3352,13 +3352,13 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", "key": "interface", "operator": "=", - "value": "eth0" + "value": "{{ MANINT }}" } ] }, @@ -3383,7 +3383,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "D", "resultFormat": "time_series", @@ -3417,13 +3417,13 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", "key": "interface", "operator": "=", - "value": "eth0" + "value": "{{ MANINT }}" } ] } @@ -3432,7 +3432,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Management Traffic", + "title": "{{ SERVERNAME }} - Management Traffic", "tooltip": { "msResolution": true, "shared": true, @@ -3567,7 +3567,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -3618,7 +3618,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -3633,7 +3633,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Suri Memory Usage", + "title": "{{ SERVERNAME }} - Suri Memory Usage", "tooltip": { "shared": true, "sort": 0, @@ -3767,7 +3767,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -3818,7 +3818,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -3833,7 +3833,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Steno Memory Usage", + "title": "{{ SERVERNAME }} - Steno Memory Usage", "tooltip": { "shared": true, "sort": 0, @@ -3975,7 +3975,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -4019,7 +4019,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -4063,7 +4063,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -4107,7 +4107,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -4151,7 +4151,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -4196,7 +4196,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -4241,7 +4241,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -4286,7 +4286,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -4295,7 +4295,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Load Average", + "title": "{{ SERVERNAME }} - Load Average", "tooltip": { "msResolution": true, "shared": true, @@ -4417,7 +4417,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -4451,13 +4451,13 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", "key": "interface", "operator": "=", - "value": "bond0" + "value": "{{ MONINT }}" } ] }, @@ -4482,7 +4482,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -4516,13 +4516,13 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", "key": "interface", "operator": "=", - "value": "bond0" + "value": "{{ MONINT }}" } ] } @@ -4531,7 +4531,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Monitor Traffic", + "title": "{{ SERVERNAME }} - Monitor Traffic", "tooltip": { "msResolution": true, "shared": true, @@ -4652,7 +4652,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -4759,7 +4759,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -4804,7 +4804,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -4813,7 +4813,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - PCAP Retention", + "title": "{{ SERVERNAME }} - PCAP Retention", "tooltip": { "shared": true, "sort": 0, @@ -4955,7 +4955,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -4999,7 +4999,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -5043,7 +5043,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -5088,7 +5088,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -5133,7 +5133,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -5178,7 +5178,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -5187,7 +5187,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Processes", + "title": "{{ SERVERNAME }} - Processes", "tooltip": { "msResolution": true, "shared": true, @@ -5332,7 +5332,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -5393,7 +5393,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -5455,7 +5455,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -5517,7 +5517,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -5532,7 +5532,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Logstash Traffic", + "title": "{{ SERVERNAME }} - Logstash Traffic", "tooltip": { "shared": true, "sort": 0, @@ -5662,7 +5662,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -5672,7 +5672,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -5719,7 +5719,7 @@ }, { "params": [ - " /8" + " /{{ CPUS }}" ], "type": "math" } @@ -5729,7 +5729,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -5744,7 +5744,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Zeek CPU Usage", + "title": "{{ SERVERNAME }} - Zeek CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -5878,7 +5878,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -5929,7 +5929,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -5944,7 +5944,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Zeek Memory Usage", + "title": "{{ SERVERNAME }} - Zeek Memory Usage", "tooltip": { "shared": true, "sort": 0, @@ -6081,7 +6081,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -6126,7 +6126,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -6135,7 +6135,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Total Threads", + "title": "{{ SERVERNAME }} - Total Threads", "tooltip": { "msResolution": true, "shared": true, @@ -6281,7 +6281,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -6342,7 +6342,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -6404,7 +6404,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -6466,7 +6466,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -6481,7 +6481,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - InfluxDB Traffic", + "title": "{{ SERVERNAME }} - InfluxDB Traffic", "tooltip": { "shared": true, "sort": 0, @@ -6520,7 +6520,7 @@ }, { "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "InfluxDB", @@ -6549,16 +6549,16 @@ "total": false, "values": false }, - "lines": false, + "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.4", "pointradius": 2, - "points": true, + "points": false, "renderer": "flot", "seriesOverrides": [ { @@ -6612,7 +6612,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -6656,7 +6656,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -6665,7 +6665,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Zeek Capture Loss", + "title": "{{ SERVERNAME }} - Zeek Capture Loss", "tooltip": { "shared": true, "sort": 0, @@ -6806,7 +6806,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -6857,7 +6857,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -6866,7 +6866,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Zeek Packet Loss", + "title": "{{ SERVERNAME }} - Zeek Packet Loss", "tooltip": { "shared": true, "sort": 0, @@ -6960,7 +6960,7 @@ "steppedLine": false, "targets": [ { - "alias": "IO Wait Current", + "alias": "Wait Current", "groupBy": [ { "params": [ @@ -6998,12 +6998,12 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, { - "alias": "IO Wait Trend", + "alias": "Wait Trend", "groupBy": [ { "params": [ @@ -7042,7 +7042,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -7051,7 +7051,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - IO Wait", + "title": "{{ SERVERNAME }} - IO Wait", "tooltip": { "shared": true, "sort": 0, @@ -7196,7 +7196,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -7257,7 +7257,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -7319,7 +7319,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -7381,7 +7381,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -7396,7 +7396,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Proxy Traffic", + "title": "{{ SERVERNAME }} - Proxy Traffic", "tooltip": { "shared": true, "sort": 0, @@ -7527,7 +7527,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -7571,7 +7571,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -7580,7 +7580,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - InfluxDB Size", + "title": "{{ SERVERNAME }} - InfluxDB Size", "tooltip": { "shared": true, "sort": 0, @@ -7712,7 +7712,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -7756,7 +7756,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -7765,7 +7765,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Redis Queue", + "title": "{{ SERVERNAME }} - Redis Queue", "tooltip": { "shared": true, "sort": 0, @@ -7905,7 +7905,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -7953,7 +7953,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -8002,7 +8002,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -8051,7 +8051,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -8060,7 +8060,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Disk I/O", + "title": "{{ SERVERNAME }} - Disk I/O", "tooltip": { "msResolution": true, "shared": true, @@ -8192,7 +8192,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -8237,7 +8237,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -8246,7 +8246,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Estimated EPS", + "title": "{{ SERVERNAME }} - Estimated EPS", "tooltip": { "shared": true, "sort": 0, @@ -8377,7 +8377,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -8427,7 +8427,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -8442,7 +8442,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Redis Memory Usage", + "title": "{{ SERVERNAME }} - Redis Memory Usage", "tooltip": { "shared": true, "sort": 0, @@ -8554,7 +8554,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"usage_percent\") /8 FROM \"docker_container_cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_percent\") /{{ CPUS }} FROM \"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -8572,7 +8572,7 @@ }, { "params": [ - "/8" + "/{{ CPUS }}" ], "type": "math" } @@ -8582,7 +8582,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -8612,7 +8612,7 @@ "measurement": "docker_container_cpu", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"usage_percent\") /8 FROM \"docker_container_cpu\" WHERE (\"host\" = 'jppce2350sa-2' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"usage_percent\") /{{ CPUS }} FROM \"docker_container_cpu\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-redis') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -8630,7 +8630,7 @@ }, { "params": [ - "/8" + "/{{ CPUS }}" ], "type": "math" } @@ -8640,7 +8640,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" }, { "condition": "AND", @@ -8655,7 +8655,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Redis CPU Usage", + "title": "{{ SERVERNAME }} - Redis CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -8748,15 +8748,7 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/Trend/", - "dashLength": 4, - "dashes": true, - "fill": 0, - "linewidth": 4 - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, @@ -8781,7 +8773,7 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"used\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"used\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -8803,7 +8795,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -8827,7 +8819,7 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"buffered\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"buffered\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -8849,7 +8841,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -8873,7 +8865,7 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"cached\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"cached\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "C", "resultFormat": "time_series", @@ -8895,7 +8887,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -8919,7 +8911,7 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "default", - "query": "SELECT mean(\"free\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"free\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "D", "resultFormat": "time_series", @@ -8941,7 +8933,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -8951,7 +8943,7 @@ "groupBy": [ { "params": [ - "$interval" + "$__interval" ], "type": "time" }, @@ -8966,7 +8958,7 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"used\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"used\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "E", "resultFormat": "time_series", @@ -8988,7 +8980,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -8998,7 +8990,7 @@ "groupBy": [ { "params": [ - "$interval" + "$__interval" ], "type": "time" }, @@ -9013,7 +9005,7 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"buffered\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"buffered\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "F", "resultFormat": "time_series", @@ -9035,7 +9027,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -9045,7 +9037,7 @@ "groupBy": [ { "params": [ - "$interval" + "$__interval" ], "type": "time" }, @@ -9060,7 +9052,7 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"cached\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"cached\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "G", "resultFormat": "time_series", @@ -9082,7 +9074,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -9092,7 +9084,7 @@ "groupBy": [ { "params": [ - "$interval" + "$__interval" ], "type": "time" }, @@ -9107,7 +9099,7 @@ "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT mean(\"free\") FROM \"mem\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT mean(\"free\") FROM \"mem\" WHERE (\"host\" = '{{ SERVERNAME }}') AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "H", "resultFormat": "time_series", @@ -9129,7 +9121,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -9138,7 +9130,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Memory", + "title": "{{ SERVERNAME }} - Memory", "tooltip": { "msResolution": true, "shared": true, @@ -9281,13 +9273,13 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } ], "thresholds": "", - "title": "jppce2350sa-2 - ES Documents", + "title": "{{ SERVERNAME }} - ES Documents", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -9396,7 +9388,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, @@ -9441,7 +9433,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -9450,7 +9442,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - ES Store Size", + "title": "{{ SERVERNAME }} - ES Store Size", "tooltip": { "shared": true, "sort": 0, @@ -9529,12 +9521,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Count Current", "groupBy": [ { "params": [ @@ -9572,17 +9573,30 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, { - "groupBy": [], + "alias": "Count Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "elasticsearch_jvm", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_threads_count\" FROM \"so_long_term\".\"elasticsearch_jvm\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -9592,6 +9606,10 @@ "mean_threads_count" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -9599,7 +9617,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -9608,7 +9626,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - ES Thread Count", + "title": "{{ SERVERNAME }} - ES Thread Count", "tooltip": { "shared": true, "sort": 0, @@ -9688,12 +9706,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Size Current", "groupBy": [ { "params": [ @@ -9731,17 +9758,30 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, { - "groupBy": [], + "alias": "Size Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, "measurement": "elasticsearch_indices", "orderByTime": "ASC", "policy": "so_long_term", - "query": "SELECT \"mean_fielddata_memory_size_in_bytes\" FROM \"so_long_term\".\"elasticsearch_indices\" WHERE (\"host\" = 'jppce2350sa-2') AND $timeFilter", - "rawQuery": true, "refId": "B", "resultFormat": "time_series", "select": [ @@ -9751,6 +9791,10 @@ "mean_fielddata_memory_size_in_bytes" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -9758,7 +9802,7 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] } @@ -9767,7 +9811,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - ES Fielddata Cache Size", + "title": "{{ SERVERNAME }} - ES Fielddata Cache Size", "tooltip": { "shared": true, "sort": 0, @@ -9846,13 +9890,21 @@ "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Total", + "alias": "Total Current", "groupBy": [ { "params": [ @@ -9890,12 +9942,12 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -9933,13 +9985,26 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" } ] }, { - "alias": "ds_Used", - "groupBy": [], + "alias": "Total Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], "hide": false, "measurement": "mem", "orderByTime": "ASC", @@ -9950,9 +10015,13 @@ [ { "params": [ - "mean_used" + "mean_total" ], "type": "field" + }, + { + "params": [], + "type": "mean" } ] ], @@ -9960,7 +10029,51 @@ { "key": "host", "operator": "=", - "value": "jppce2350sa-2" + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" } ] } @@ -9969,7 +10082,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "jppce2350sa-2 - Memory(Used)", + "title": "{{ SERVERNAME }} - Memory(Used)", "tooltip": { "shared": true, "sort": 0, @@ -10090,14 +10203,14 @@ } ], "query": "10s, 1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, + "refresh": 3, "skipUrlSync": false, "type": "interval" } ] }, "time": { - "from": "now-3h", + "from": "now-1h", "to": "now" }, "timepicker": { @@ -10126,7 +10239,7 @@ ] }, "timezone": "browser", - "title": "Standalone Mode - jppce2350sa-2 Overview", - "uid": "so_overview", - "version": 9 + "title": "Standalone Mode - {{ SERVERNAME }} Overview", + "uid": "{{ UID }}", + "version": 1 } From 3d8cbe942718427dc6899d570e69b6c518388451 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 11:22:14 -0400 Subject: [PATCH 351/378] add trend lines --- salt/grafana/dashboards/standalone/standalone.json | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index fbf0b8cf5..5f315e3ff 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -17,7 +17,7 @@ "gnetId": 2381, "graphTooltip": 0, "id": 7, - "iteration": 1624977014309, + "iteration": 1624979732000, "links": [], "panels": [ { @@ -8748,7 +8748,15 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": true, "steppedLine": false, @@ -10241,5 +10249,5 @@ "timezone": "browser", "title": "Standalone Mode - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", - "version": 1 + "version": 2 } From 822165f1682c6cca0fc7fac52589060f3f7ac981 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 29 Jun 2021 13:32:02 -0400 Subject: [PATCH 352/378] Move salt lines after shebang --- salt/common/tools/sbin/so-filebeat-module-setup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index ef35a9b17..401f54289 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -1,6 +1,3 @@ -{%- set mainint = salt['pillar.get']('host:mainint') %} -{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} - #!/bin/bash # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # @@ -17,6 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +{%- set mainint = salt['pillar.get']('host:mainint') %} +{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} + default_conf_dir=/opt/so/conf ELASTICSEARCH_HOST="{{ MYIP }}" ELASTICSEARCH_PORT=9200 From fa3e5eebe2e56f3ab231385f4d12db83657b314e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 15:11:31 -0400 Subject: [PATCH 353/378] update manager dashboard --- salt/grafana/dashboards/manager/manager.json | 2554 +++++++++++++++++- 1 file changed, 2412 insertions(+), 142 deletions(-) diff --git a/salt/grafana/dashboards/manager/manager.json b/salt/grafana/dashboards/manager/manager.json index 2c21a160e..81f459152 100644 --- a/salt/grafana/dashboards/manager/manager.json +++ b/salt/grafana/dashboards/manager/manager.json @@ -17,7 +17,7 @@ "gnetId": 2381, "graphTooltip": 0, "id": 6, - "iteration": 1614092189289, + "iteration": 1624989522072, "links": [], "panels": [ { @@ -29,36 +29,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 60 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 - } - ] - }, "unit": "percent" }, "overrides": [] @@ -90,17 +60,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Usage", + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -154,6 +132,63 @@ "value": "cpu-total" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* -1 + 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], "thresholds": [], @@ -201,7 +236,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "decimals": 2, "mappings": [], "thresholds": { @@ -236,9 +270,10 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto" }, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "targets": [ { "groupBy": [ @@ -297,7 +332,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -329,16 +363,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -392,6 +435,63 @@ "value": "so-elasticsearch" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] } ], "thresholds": [], @@ -445,7 +545,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -477,16 +576,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -540,13 +648,70 @@ "value": "so-kibana" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-kibana" + } + ] } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Kibana CPU", + "title": "{{ SERVERNAME }} - Kibana CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -593,7 +758,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -625,16 +789,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Queue Current", "dsType": "influxdb", "groupBy": [ { @@ -676,6 +849,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Queue Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "redisqueue", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_unparsed" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -729,7 +947,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -761,16 +978,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -818,6 +1044,57 @@ "value": "so-redis" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] } ], "thresholds": [], @@ -869,9 +1146,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -900,17 +1175,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -957,6 +1240,56 @@ "value": "/" } ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] } ], "thresholds": [], @@ -1007,9 +1340,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1038,17 +1369,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -1095,6 +1434,56 @@ "value": "/nsm" } ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] } ], "thresholds": [], @@ -1147,7 +1536,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1179,16 +1567,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1242,6 +1639,63 @@ "value": "so-influxdb" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] } ], "thresholds": [], @@ -1295,7 +1749,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1327,16 +1780,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1390,6 +1852,63 @@ "value": "so-logstash" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] } ], "thresholds": [], @@ -1443,7 +1962,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1475,16 +1993,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1538,6 +2065,63 @@ "value": "so-redis" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] } ], "thresholds": [], @@ -1590,7 +2174,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1621,16 +2204,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Size Current", "groupBy": [ { "params": [ @@ -1671,6 +2263,50 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Size Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "influxsize", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_kbytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -1732,7 +2368,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1766,17 +2401,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "System", + "alias": "System Current", "dsType": "influxdb", "groupBy": [ { @@ -1826,7 +2469,7 @@ ] }, { - "alias": "User", + "alias": "User Current", "dsType": "influxdb", "groupBy": [ { @@ -1876,7 +2519,7 @@ ] }, { - "alias": "Nice", + "alias": "Nice Current", "dsType": "influxdb", "groupBy": [ { @@ -1926,7 +2569,7 @@ ] }, { - "alias": "Interrupt", + "alias": "Interrupt Current", "dsType": "influxdb", "groupBy": [ { @@ -1976,7 +2619,7 @@ ] }, { - "alias": "Wait", + "alias": "Wait Current", "dsType": "influxdb", "groupBy": [ { @@ -2026,7 +2669,7 @@ ] }, { - "alias": "SoftIRQ", + "alias": "SoftIRQ Current", "dsType": "influxdb", "groupBy": [ { @@ -2074,6 +2717,312 @@ "value": "cpu-total" } ] + }, + { + "alias": "System Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_system" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "User Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_user" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Nice Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "I", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_nice" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Interrupt Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "J", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_irq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Wait Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "K", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "SoftIRQ Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "L", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_softirq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], "thresholds": [], @@ -2132,7 +3081,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2166,17 +3114,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2195,7 +3151,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -2240,7 +3196,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2259,7 +3215,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -2302,6 +3258,136 @@ "value": "{{ MANINT }}" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] } ], "thresholds": [], @@ -2355,7 +3441,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2388,17 +3473,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Total", + "alias": "Total Current", "groupBy": [ { "params": [ @@ -2441,7 +3534,7 @@ ] }, { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -2482,6 +3575,94 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Total Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_total" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -2540,7 +3721,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2574,14 +3754,17 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { - "alias": "#cpu", - "fill": 0 + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 } ], "spaceLength": 10, @@ -2589,7 +3772,7 @@ "steppedLine": false, "targets": [ { - "alias": "#cpu", + "alias": "#cpu Current", "groupBy": [ { "params": [ @@ -2632,7 +3815,7 @@ ] }, { - "alias": "1 Minute Average", + "alias": "1 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -2676,7 +3859,7 @@ ] }, { - "alias": "5 Minute Average", + "alias": "5 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -2720,7 +3903,7 @@ ] }, { - "alias": "15 Minute Average", + "alias": "15 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -2762,6 +3945,185 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "#cpu Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_n_cpus" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "1 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "5 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load5" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "15 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load15" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -2820,7 +4182,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2854,17 +4215,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2883,7 +4252,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -2928,7 +4297,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2947,7 +4316,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -2990,6 +4359,136 @@ "value": "so-logstash" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] } ], "thresholds": [], @@ -3050,7 +4549,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3086,11 +4584,19 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": true, "steppedLine": false, @@ -3113,6 +4619,7 @@ } ], "measurement": "mem", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -3156,6 +4663,7 @@ } ], "measurement": "mem", + "orderByTime": "ASC", "policy": "default", "refId": "B", "resultFormat": "time_series", @@ -3199,6 +4707,7 @@ } ], "measurement": "mem", + "orderByTime": "ASC", "policy": "default", "refId": "C", "resultFormat": "time_series", @@ -3242,6 +4751,7 @@ } ], "measurement": "mem", + "orderByTime": "ASC", "policy": "default", "refId": "D", "resultFormat": "time_series", @@ -3266,6 +4776,186 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Used Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Buffered Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_buffered" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Cached Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_cached" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Free Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_free" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -3323,7 +5013,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3357,17 +5046,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { - "alias": "Blocked", + "alias": "Blocked Current", "dsType": "influxdb", "groupBy": [ { @@ -3385,6 +5082,7 @@ ], "hide": false, "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -3411,7 +5109,7 @@ ] }, { - "alias": "Running", + "alias": "Running Current", "dsType": "influxdb", "groupBy": [ { @@ -3428,6 +5126,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "B", "resultFormat": "time_series", @@ -3454,7 +5153,7 @@ ] }, { - "alias": "Sleep", + "alias": "Sleep Current", "dsType": "influxdb", "groupBy": [ { @@ -3471,6 +5170,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "C", "resultFormat": "time_series", @@ -3495,6 +5195,141 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Blocked Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_blocked" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Running Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_running" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Sleep Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_sleeping" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -3553,7 +5388,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3587,11 +5421,19 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, @@ -3616,7 +5458,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -3680,7 +5522,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -3723,6 +5565,136 @@ "value": "so-influxdb" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] } ], "thresholds": [], @@ -3777,7 +5749,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3810,17 +5781,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Read", + "alias": "Read Current", "dsType": "influxdb", "groupBy": [ { @@ -3837,6 +5816,7 @@ } ], "measurement": "diskio", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -3867,7 +5847,7 @@ ] }, { - "alias": "Write", + "alias": "Write Current", "dsType": "influxdb", "groupBy": [ { @@ -3884,6 +5864,7 @@ } ], "measurement": "diskio", + "orderByTime": "ASC", "policy": "default", "refId": "B", "resultFormat": "time_series", @@ -3912,6 +5893,104 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Read Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_read_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Write Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_write_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -3966,7 +6045,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4000,17 +6078,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Threads", + "alias": "Threads Current", "dsType": "influxdb", "groupBy": [ { @@ -4027,6 +6113,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -4051,6 +6138,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Threads Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_total_threads" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4109,7 +6241,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4143,17 +6274,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -4172,7 +6311,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -4217,7 +6356,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "dsType": "influxdb", "groupBy": [ { @@ -4236,7 +6375,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -4279,6 +6418,136 @@ "value": "so-aptcacherng" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] + }, + { + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] } ], "thresholds": [], @@ -4325,7 +6594,7 @@ } ], "refresh": false, - "schemaVersion": 26, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -4339,6 +6608,7 @@ "text": "10s", "value": "10s" }, + "description": null, "error": null, "hide": 0, "label": null, @@ -4443,6 +6713,6 @@ }, "timezone": "browser", "title": "Manager Node - {{ SERVERNAME }} Overview", - "uid": "so_overview", + "uid": "{{ UID }}", "version": 3 -} \ No newline at end of file +} From dac19d224f405b24d1165a124070a86b97a39a95 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 17:15:00 -0400 Subject: [PATCH 354/378] update cq --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 37ebe39a5..f270c9f73 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -118,7 +118,7 @@ so_downsample_{{measurement}}_cq: influxdb_continuous_query.present: - name: so_downsample_{{measurement}}_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) + - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] From cad4efdded6a8f0241a07c637c0923517a4b7bef Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 29 Jun 2021 17:51:04 -0400 Subject: [PATCH 355/378] Fixed PCAP files are readable by root only, which prevents Suricata from being able to scan the file during import --- salt/common/tools/sbin/so-import-pcap | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin/so-import-pcap index 32121d8cc..c525849ef 100755 --- a/salt/common/tools/sbin/so-import-pcap +++ b/salt/common/tools/sbin/so-import-pcap @@ -132,6 +132,8 @@ for PCAP in "$@"; do PCAP_FIXED=`mktemp /tmp/so-import-pcap-XXXXXXXXXX.pcap` echo "- attempting to recover corrupted PCAP file" pcapfix "${PCAP}" "${PCAP_FIXED}" + # Make fixed file world readable since the Suricata docker container will runas a non-root user + chmod a+r "${PCAP_FIXED}" PCAP="${PCAP_FIXED}" TEMP_PCAPS+=(${PCAP_FIXED}) fi From ebb45a866bb5e077c4d6cef2078b51badb992b1f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 19:20:29 -0400 Subject: [PATCH 356/378] remove queries from standalone dashboard --- .../dashboards/standalone/standalone.json | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 5f315e3ff..6b7d5caff 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -16,8 +16,8 @@ "editable": true, "gnetId": 2381, "graphTooltip": 0, - "id": 7, - "iteration": 1624979732000, + "id": 6, + "iteration": 1625007678418, "links": [], "panels": [ { @@ -5064,7 +5064,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "processes", "orderByTime": "ASC", "policy": "so_long_term", @@ -5109,7 +5109,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "processes", "orderByTime": "ASC", "policy": "so_long_term", @@ -5154,7 +5154,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "processes", "orderByTime": "ASC", "policy": "so_long_term", @@ -7974,7 +7974,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "diskio", "orderByTime": "ASC", "policy": "so_long_term", @@ -8023,7 +8023,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "diskio", "orderByTime": "ASC", "policy": "so_long_term", @@ -8962,7 +8962,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", @@ -9009,7 +9009,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", @@ -9056,7 +9056,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", @@ -9103,7 +9103,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", @@ -10249,5 +10249,5 @@ "timezone": "browser", "title": "Standalone Mode - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", - "version": 2 + "version": 3 } From 3a47563b27cfbeb8036cd55907c86887b9155cef Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 19:26:40 -0400 Subject: [PATCH 357/378] remove queries manager dashboard --- salt/grafana/dashboards/manager/manager.json | 46 ++++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/salt/grafana/dashboards/manager/manager.json b/salt/grafana/dashboards/manager/manager.json index 81f459152..dc67052d7 100644 --- a/salt/grafana/dashboards/manager/manager.json +++ b/salt/grafana/dashboards/manager/manager.json @@ -17,7 +17,7 @@ "gnetId": 2381, "graphTooltip": 0, "id": 6, - "iteration": 1624989522072, + "iteration": 1625008764846, "links": [], "panels": [ { @@ -416,7 +416,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -473,7 +473,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -629,7 +629,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -686,7 +686,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -1620,7 +1620,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -1677,7 +1677,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -1833,7 +1833,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -1890,7 +1890,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -2046,7 +2046,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -2103,7 +2103,7 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" ], "type": "math" } @@ -4794,7 +4794,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", @@ -4839,7 +4839,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", @@ -4884,7 +4884,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", @@ -4929,7 +4929,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "mem", "orderByTime": "ASC", "policy": "so_long_term", @@ -5213,7 +5213,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "processes", "orderByTime": "ASC", "policy": "so_long_term", @@ -5258,7 +5258,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "processes", "orderByTime": "ASC", "policy": "so_long_term", @@ -5303,7 +5303,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "processes", "orderByTime": "ASC", "policy": "so_long_term", @@ -5911,7 +5911,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "diskio", "orderByTime": "ASC", "policy": "so_long_term", @@ -5960,7 +5960,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "diskio", "orderByTime": "ASC", "policy": "so_long_term", @@ -6593,7 +6593,7 @@ } } ], - "refresh": false, + "refresh": "30s", "schemaVersion": 27, "style": "dark", "tags": [], @@ -6676,7 +6676,7 @@ } ], "query": "10s, 1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, + "refresh": 3, "skipUrlSync": false, "type": "interval" } @@ -6714,5 +6714,5 @@ "timezone": "browser", "title": "Manager Node - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", - "version": 3 + "version": 4 } From 88275cd96894fd7f158bea53dbb9a476406dc18a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 19:50:53 -0400 Subject: [PATCH 358/378] remove trend zeek capture loss, turn on line and points for capture loss standalone graf --- salt/grafana/dashboards/standalone/standalone.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 6b7d5caff..7dddcb16e 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -6558,7 +6558,7 @@ "percentage": false, "pluginVersion": "7.5.4", "pointradius": 2, - "points": false, + "points": true, "renderer": "flot", "seriesOverrides": [ { @@ -6632,7 +6632,7 @@ "type": "fill" } ], - "hide": false, + "hide": true, "measurement": "zeekcaptureloss", "orderByTime": "ASC", "policy": "so_long_term", From 813ef7d81ad031a95f18b0d45d40d6bf13b8893c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 20:23:27 -0400 Subject: [PATCH 359/378] new eval dashboard with trends --- salt/grafana/dashboards/eval/eval.json | 1900 ++++++++++++++++++++++-- 1 file changed, 1758 insertions(+), 142 deletions(-) diff --git a/salt/grafana/dashboards/eval/eval.json b/salt/grafana/dashboards/eval/eval.json index 1b67267a5..aa1588a07 100644 --- a/salt/grafana/dashboards/eval/eval.json +++ b/salt/grafana/dashboards/eval/eval.json @@ -16,14 +16,13 @@ "editable": true, "gnetId": 2381, "graphTooltip": 0, - "id": 3, + "id": 6, "links": [], "panels": [ { "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "decimals": 2, "mappings": [], "thresholds": { @@ -58,9 +57,10 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto" }, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "targets": [ { "groupBy": [ @@ -119,37 +119,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [], - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 60 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 - } - ] - }, "unit": "percent" }, "overrides": [] @@ -181,16 +151,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -244,6 +223,63 @@ "value": "cpu-total" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* -1 + 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], "thresholds": [], @@ -289,13 +325,12 @@ }, { "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -319,23 +354,32 @@ "total": false, "values": false }, - "lines": false, + "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": true, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "groupBy": [ { "params": [ @@ -376,6 +420,50 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Loss Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": true, + "measurement": "zeekcaptureloss", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_loss" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -429,7 +517,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -461,16 +548,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "dsType": "influxdb", "groupBy": [ { @@ -518,6 +614,57 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Loss Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "zeekdrop", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -571,7 +718,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -603,16 +749,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "dsType": "influxdb", "groupBy": [ { @@ -660,6 +815,57 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Loss Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "suridrop", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -712,7 +918,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -744,16 +949,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "dsType": "influxdb", "groupBy": [ { @@ -795,6 +1009,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Loss Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "stenodrop", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -846,9 +1105,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -877,17 +1134,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Usage", + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -934,6 +1199,56 @@ "value": "/" } ] + }, + { + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] } ], "thresholds": [], @@ -985,9 +1300,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1016,17 +1329,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Usage", + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -1074,6 +1395,57 @@ "value": "/nsm" } ] + }, + { + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "queryType": "randomWalk", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] } ], "thresholds": [], @@ -1127,7 +1499,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1159,16 +1530,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1203,7 +1583,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1274,7 +1711,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1306,16 +1742,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1350,7 +1795,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1421,7 +1923,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1453,16 +1954,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1497,7 +2007,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1568,7 +2135,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1600,16 +2166,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1644,7 +2219,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1725,7 +2357,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1759,17 +2390,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashes": true, + "fill": 0, + "linewidth": 4, + "spaceLength": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "System", + "alias": "System Current", "dsType": "influxdb", "groupBy": [ { @@ -1819,7 +2458,7 @@ ] }, { - "alias": "User", + "alias": "User Current", "dsType": "influxdb", "groupBy": [ { @@ -1869,7 +2508,7 @@ ] }, { - "alias": "Nice", + "alias": "Nice Current", "dsType": "influxdb", "groupBy": [ { @@ -1919,7 +2558,7 @@ ] }, { - "alias": "Interrupt", + "alias": "Interrupt Current", "dsType": "influxdb", "groupBy": [ { @@ -1969,7 +2608,7 @@ ] }, { - "alias": "Wait", + "alias": "Wait Current", "dsType": "influxdb", "groupBy": [ { @@ -2067,6 +2706,312 @@ "value": "cpu-total" } ] + }, + { + "alias": "System Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_system" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "User Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_user" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Nice Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "I", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_nice" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Interrupt Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "J", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_irq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Wait Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "K", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "SoftIRQ Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "L", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_softirq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], "thresholds": [], @@ -2119,7 +3064,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2150,16 +3094,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -2206,6 +3159,56 @@ "value": "so-elasticsearch" } ] + }, + { + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] } ], "thresholds": [], @@ -2259,7 +3262,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2291,16 +3293,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -2348,6 +3359,57 @@ "value": "so-zeek" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] } ], "thresholds": [], @@ -2400,7 +3462,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2432,16 +3493,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -2489,6 +3559,57 @@ "value": "so-suricata" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] } ], "thresholds": [], @@ -2541,7 +3662,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2573,16 +3693,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -2630,6 +3759,57 @@ "value": "so-steno" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] } ], "thresholds": [], @@ -2687,9 +3867,7 @@ "editable": true, "error": false, "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "format": "none", @@ -2808,7 +3986,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2840,7 +4017,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -2938,7 +4115,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", @@ -2975,9 +4151,10 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto" }, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "targets": [ { "groupBy": [ @@ -3032,7 +4209,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "decimals": 2, "mappings": [ { @@ -3045,7 +4221,6 @@ ], "max": 1209600, "min": 0, - "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ @@ -3085,9 +4260,10 @@ "values": false }, "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "text": {} }, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "targets": [ { "dsType": "influxdb", @@ -3153,7 +4329,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3187,14 +4362,17 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { - "alias": "#cpu", - "fill": 0 + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 } ], "spaceLength": 10, @@ -3202,7 +4380,7 @@ "steppedLine": false, "targets": [ { - "alias": "#cpu", + "alias": "#cpu Current", "groupBy": [ { "params": [ @@ -3245,7 +4423,7 @@ ] }, { - "alias": "1 Minute Average", + "alias": "1 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -3289,7 +4467,7 @@ ] }, { - "alias": "5 Minute Average", + "alias": "5 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -3333,7 +4511,7 @@ ] }, { - "alias": "15 Minute Average", + "alias": "15 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -3375,6 +4553,185 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "#cpu Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_n_cpus" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "1 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "5 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load5" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "15 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load15" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -3435,7 +4792,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3471,7 +4827,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3709,7 +5065,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3743,17 +5098,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -3772,7 +5135,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -3815,6 +5178,71 @@ "value": "{{ MONINT }}" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MONINT }}" + } + ] } ], "thresholds": [], @@ -3872,7 +5300,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3906,7 +5333,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -4098,7 +5525,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4131,7 +5557,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -4293,7 +5719,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4327,17 +5752,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -4356,7 +5789,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -4401,7 +5834,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "dsType": "influxdb", "groupBy": [ { @@ -4420,7 +5853,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($__interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($__interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -4463,6 +5896,136 @@ "value": "{{ MANINT }}" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] } ], "thresholds": [], @@ -4517,7 +6080,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4551,17 +6113,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Threads", + "alias": "Threads Current", "dsType": "influxdb", "groupBy": [ { @@ -4578,6 +6148,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -4602,6 +6173,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Threads Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_total_threads" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4648,7 +6264,7 @@ } ], "refresh": "30s", - "schemaVersion": 26, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -4685,5 +6301,5 @@ "timezone": "browser", "title": "Evaluation Mode - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", - "version": 1 -} \ No newline at end of file + "version": 10 +} From b22cd2d27c418dba670a2b4a8763b32ab1bd66cb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 21:07:02 -0400 Subject: [PATCH 360/378] managersearch dash with trends --- .../managersearch/managersearch.json | 2503 +++++++++++++++-- 1 file changed, 2343 insertions(+), 160 deletions(-) diff --git a/salt/grafana/dashboards/managersearch/managersearch.json b/salt/grafana/dashboards/managersearch/managersearch.json index 8b4e57495..eae0cda10 100644 --- a/salt/grafana/dashboards/managersearch/managersearch.json +++ b/salt/grafana/dashboards/managersearch/managersearch.json @@ -17,7 +17,7 @@ "gnetId": 2381, "graphTooltip": 0, "id": 6, - "iteration": 1614096099337, + "iteration": 1625012701746, "links": [], "panels": [ { @@ -29,36 +29,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 100, - "min": 0, - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 60 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 - } - ] - }, "unit": "percent" }, "overrides": [] @@ -90,17 +60,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Usage", + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -154,6 +132,63 @@ "value": "cpu-total" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* -1 + 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], "thresholds": [], @@ -201,7 +236,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "decimals": 2, "mappings": [], "thresholds": { @@ -236,9 +270,10 @@ "fields": "", "values": false }, + "text": {}, "textMode": "auto" }, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "targets": [ { "groupBy": [ @@ -297,7 +332,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -329,16 +363,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -373,7 +416,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -444,7 +544,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -475,16 +574,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -518,7 +626,63 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=", + "value": "so-kibana" + }, + { + "condition": "AND", + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -543,7 +707,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - Kibana CPU", + "title": "{{ SERVERNAME }} - Kibana CPU Usage", "tooltip": { "shared": true, "sort": 0, @@ -589,7 +753,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -620,16 +783,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Queue Current", "groupBy": [ { "params": [ @@ -670,6 +842,50 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Queue Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "redisqueue", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_unparsed" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -721,7 +937,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -752,16 +967,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -808,6 +1032,56 @@ "value": "so-redis" } ] + }, + { + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] } ], "thresholds": [], @@ -859,9 +1133,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -890,17 +1162,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -947,6 +1227,56 @@ "value": "/" } ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] } ], "thresholds": [], @@ -997,9 +1327,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1028,17 +1356,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -1085,6 +1421,56 @@ "value": "/nsm" } ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] } ], "thresholds": [], @@ -1136,7 +1522,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1167,16 +1552,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -1210,7 +1604,63 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1282,7 +1732,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1314,16 +1763,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1358,7 +1816,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1429,7 +1944,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1460,16 +1974,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "groupBy": [ { "params": [ @@ -1503,7 +2026,63 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + }, + { + "alias": "Usage Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1574,7 +2153,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1605,16 +2183,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Size Current", "groupBy": [ { "params": [ @@ -1655,6 +2242,50 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Size Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "influxsize", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_kbytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -1716,7 +2347,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -1750,17 +2380,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "System", + "alias": "System Current", "dsType": "influxdb", "groupBy": [ { @@ -1810,7 +2448,7 @@ ] }, { - "alias": "User", + "alias": "User Current", "dsType": "influxdb", "groupBy": [ { @@ -1860,7 +2498,7 @@ ] }, { - "alias": "Nice", + "alias": "Nice Current", "dsType": "influxdb", "groupBy": [ { @@ -1910,7 +2548,7 @@ ] }, { - "alias": "Interrupt", + "alias": "Interrupt Current", "dsType": "influxdb", "groupBy": [ { @@ -1960,7 +2598,7 @@ ] }, { - "alias": "Wait", + "alias": "Wait Current", "dsType": "influxdb", "groupBy": [ { @@ -2010,7 +2648,7 @@ ] }, { - "alias": "SoftIRQ", + "alias": "SoftIRQ Current", "dsType": "influxdb", "groupBy": [ { @@ -2058,6 +2696,312 @@ "value": "cpu-total" } ] + }, + { + "alias": "System Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_system" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "User Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_user" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Nice Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "I", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_nice" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Interrupt Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "J", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_irq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Wait Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "K", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "SoftIRQ Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "L", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_softirq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], "thresholds": [], @@ -2116,7 +3060,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2150,17 +3093,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashes": true, + "fill": 0, + "linewidth": 4, + "spaceLength": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2179,7 +3130,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -2224,7 +3175,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2243,7 +3194,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -2286,6 +3237,136 @@ "value": "{{ MANINT }}" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] } ], "thresholds": [], @@ -2339,7 +3420,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2371,16 +3451,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Count Current", "dsType": "influxdb", "groupBy": [ { @@ -2422,13 +3511,58 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Count Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_docs_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - ES Documents", + "title": "{{ SERVERNAME }} - ES Documents Count", "tooltip": { "shared": true, "sort": 0, @@ -2475,7 +3609,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2507,16 +3640,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Size Current", "dsType": "influxdb", "groupBy": [ { @@ -2558,6 +3700,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Size Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_fielddata_memory_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -2615,7 +3802,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2649,14 +3835,21 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { - "alias": "#cpu", + "alias": "#cpu Current", "fill": 0 + }, + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 } ], "spaceLength": 10, @@ -2664,7 +3857,7 @@ "steppedLine": false, "targets": [ { - "alias": "#cpu", + "alias": "#cpu Current", "groupBy": [ { "params": [ @@ -2707,7 +3900,7 @@ ] }, { - "alias": "1 Minute Average", + "alias": "1 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -2751,7 +3944,7 @@ ] }, { - "alias": "5 Minute Average", + "alias": "5 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -2795,7 +3988,7 @@ ] }, { - "alias": "15 Minute Average", + "alias": "15 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -2837,6 +4030,185 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "#cpu Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_n_cpus" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "1 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "5 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load5" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "15 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load15" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -2895,7 +4267,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -2929,17 +4300,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2958,7 +4337,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -3003,7 +4382,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "dsType": "influxdb", "groupBy": [ { @@ -3022,7 +4401,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -3065,6 +4444,136 @@ "value": "so-logstash" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] } ], "thresholds": [], @@ -3118,7 +4627,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3150,16 +4658,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Count Current", "dsType": "influxdb", "groupBy": [ { @@ -3201,6 +4718,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Count Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_threads_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -3254,7 +4816,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3286,16 +4847,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Size Current", "dsType": "influxdb", "groupBy": [ { @@ -3337,6 +4907,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Size Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_store_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -3394,7 +5009,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3428,7 +5042,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3618,7 +5232,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3650,17 +5263,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "groupBy": [ { "params": [ @@ -3723,7 +5344,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "groupBy": [ { "params": [ @@ -3782,6 +5403,132 @@ "value": "so-influxdb" } ] + }, + { + "alias": "Inbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT non_negative_derivative(mean(\"rx_bytes\"), 1s) *8 FROM \"docker_container_net\" WHERE (\"host\" = '{{ SERVERNAME }}' AND \"container_name\" = 'so-influxdb') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + " *8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "alias": "Outbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] } ], "thresholds": [], @@ -3834,7 +5581,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3866,16 +5612,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -3917,6 +5672,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_mem_heap_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -3971,7 +5771,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4005,17 +5804,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Threads", + "alias": "Threads Current", "dsType": "influxdb", "groupBy": [ { @@ -4032,6 +5839,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -4056,6 +5864,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Threads Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_total_threads" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4108,7 +5961,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4140,11 +5992,19 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, @@ -4270,6 +6130,130 @@ "value": "so-aptcacherng" } ] + }, + { + "alias": "Inbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] + }, + { + "alias": "Outbound Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] } ], "thresholds": [], @@ -4322,7 +6306,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4354,17 +6337,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Total", + "alias": "Total Current", "groupBy": [ { "params": [ @@ -4407,7 +6398,7 @@ ] }, { - "alias": "Used", + "alias": "Used Current", "dsType": "influxdb", "groupBy": [ { @@ -4449,6 +6440,95 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Total Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_total" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Used Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4502,7 +6582,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4535,7 +6614,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -4690,7 +6769,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4722,16 +6800,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Wait Current", "dsType": "influxdb", "groupBy": [ { @@ -4773,6 +6860,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Wait Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4833,7 +6965,6 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -4869,7 +7000,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -5101,9 +7232,7 @@ "datasource": "InfluxDB", "description": "", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -5132,17 +7261,25 @@ "alertThreshold": false }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "EPS", + "alias": "EPS Current", "groupBy": [ { "params": [ @@ -5184,6 +7321,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "EPS Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "consumptioneps", + "orderByTime": "ASC", + "policy": "so_long_term", + "queryType": "randomWalk", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_eps" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -5229,7 +7411,7 @@ } ], "refresh": "30s", - "schemaVersion": 26, + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -5243,6 +7425,7 @@ "text": "10s", "value": "10s" }, + "description": null, "error": null, "hide": 0, "label": null, @@ -5347,6 +7530,6 @@ }, "timezone": "browser", "title": "ManagerSearch Node - {{ SERVERNAME }} Overview", - "uid": "so_overview", - "version": 6 -} \ No newline at end of file + "uid": "{{ UID }}", + "version": 11 +} From 45e90750a0eb31c13dba4063b4c601c694a9cbe6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 21:37:20 -0400 Subject: [PATCH 361/378] add trends for searchnode grafs --- .../dashboards/search_nodes/searchnode.json | 2033 +++++++++++++++-- 1 file changed, 1842 insertions(+), 191 deletions(-) diff --git a/salt/grafana/dashboards/search_nodes/searchnode.json b/salt/grafana/dashboards/search_nodes/searchnode.json index 72ebe768a..dd2430b00 100644 --- a/salt/grafana/dashboards/search_nodes/searchnode.json +++ b/salt/grafana/dashboards/search_nodes/searchnode.json @@ -16,61 +16,69 @@ "editable": true, "gnetId": 2381, "graphTooltip": 0, - "iteration": 1586957065151, + "id": 6, + "iteration": 1625015408259, "links": [], "panels": [ { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 60 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 - } - ] - }, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected" + "unit": "percent" }, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 5, "w": 4, "x": 0, "y": 0 }, + "hiddenSeries": false, "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, "links": [], + "nullPointMode": "connected", "options": { "alertThreshold": true }, - "pluginVersion": "7.3.4", + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -123,83 +131,126 @@ "operator": "=", "value": "cpu-total" } - ], - "alias": "Usage" - } - ], - "title": "{{ SERVERNAME }} - CPU", - "type": "graph", - "cacheTimeout": null, - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "percent", - "$$hashKey": "object:395" + ] }, { - "label": null, - "show": false, - "logBase": 1, - "min": null, - "max": null, - "format": "short", - "$$hashKey": "object:396" + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* -1 + 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", "xaxis": { - "show": true, + "buckets": null, "mode": "time", "name": null, - "values": [], - "buckets": null + "show": true, + "values": [] }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], "yaxis": { "align": false, "alignLevel": null - }, - "lines": true, - "fill": 1, - "fillGradient": 0, - "linewidth": 1, - "dashes": false, - "hiddenSeries": false, - "dashLength": 10, - "spaceLength": 10, - "points": false, - "pointradius": 2, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": false, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "individual", - "shared": true, - "sort": 0 - }, - "timeFrom": null, - "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "thresholds": [], - "timeRegions": [] + } }, { "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 4, @@ -209,32 +260,20 @@ "id": 39, "options": { "colorMode": "value", - "fieldOptions": { + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": [ "lastNotNull" ], - "defaults": { - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(255, 255, 255)", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [], + "fields": "", "values": false }, - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto" + "text": {}, + "textMode": "auto" }, - "pluginVersion": "6.6.2", + "pluginVersion": "7.5.4", "targets": [ { "groupBy": [ @@ -291,6 +330,12 @@ "dashLength": 10, "dashes": false, "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -315,18 +360,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -361,7 +416,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -431,6 +543,12 @@ "dashLength": 10, "dashes": false, "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -455,18 +573,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -501,7 +629,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -571,6 +756,12 @@ "dashLength": 10, "dashes": false, "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -595,18 +786,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Count Current", "dsType": "influxdb", "groupBy": [ { @@ -648,13 +849,58 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Count Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_docs_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "{{ SERVERNAME }} - ES Documents", + "title": "{{ SERVERNAME }} - ES Documents Count", "tooltip": { "shared": true, "sort": 0, @@ -699,6 +945,12 @@ "dashLength": 10, "dashes": false, "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -723,18 +975,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Size Current", "dsType": "influxdb", "groupBy": [ { @@ -776,6 +1038,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Size Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_fielddata_memory_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -826,9 +1133,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -857,17 +1162,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -914,6 +1227,56 @@ "value": "/" } ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] } ], "thresholds": [], @@ -936,7 +1299,6 @@ }, "yaxes": [ { - "$$hashKey": "object:708", "format": "percent", "label": null, "logBase": 1, @@ -945,7 +1307,6 @@ "show": true }, { - "$$hashKey": "object:709", "format": "short", "label": null, "logBase": 1, @@ -966,9 +1327,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -997,17 +1356,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -1054,6 +1421,56 @@ "value": "/nsm" } ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] } ], "thresholds": [], @@ -1076,7 +1493,6 @@ }, "yaxes": [ { - "$$hashKey": "object:708", "format": "percent", "label": null, "logBase": 1, @@ -1085,7 +1501,6 @@ "show": true }, { - "$$hashKey": "object:709", "format": "short", "label": null, "logBase": 1, @@ -1106,6 +1521,12 @@ "dashLength": 10, "dashes": false, "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1130,18 +1551,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1183,6 +1614,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_mem_heap_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -1234,6 +1710,12 @@ "dashLength": 10, "dashes": false, "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1258,18 +1740,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Size Current", "dsType": "influxdb", "groupBy": [ { @@ -1311,6 +1803,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Size Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_store_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -1362,6 +1899,12 @@ "dashLength": 10, "dashes": false, "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1386,18 +1929,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Count Current", "dsType": "influxdb", "groupBy": [ { @@ -1439,6 +1992,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Count Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_threads_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -1490,6 +2088,12 @@ "dashLength": 10, "dashes": false, "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -1514,18 +2118,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Wait Current", "dsType": "influxdb", "groupBy": [ { @@ -1567,6 +2181,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Wait Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -1627,6 +2286,12 @@ "datasource": "InfluxDB", "editable": true, "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 4, "fillGradient": 0, "grid": {}, @@ -1653,19 +2318,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "System", + "alias": "System Current", "dsType": "influxdb", "groupBy": [ { @@ -1715,7 +2389,7 @@ ] }, { - "alias": "User", + "alias": "User Current", "dsType": "influxdb", "groupBy": [ { @@ -1765,7 +2439,7 @@ ] }, { - "alias": "Nice", + "alias": "Nice Current", "dsType": "influxdb", "groupBy": [ { @@ -1815,7 +2489,7 @@ ] }, { - "alias": "Interrupt", + "alias": "Interrupt Current", "dsType": "influxdb", "groupBy": [ { @@ -1865,7 +2539,7 @@ ] }, { - "alias": "Wait", + "alias": "Wait Current", "dsType": "influxdb", "groupBy": [ { @@ -1915,7 +2589,7 @@ ] }, { - "alias": "SoftIRQ", + "alias": "SoftIRQ Current", "dsType": "influxdb", "groupBy": [ { @@ -1963,6 +2637,312 @@ "value": "cpu-total" } ] + }, + { + "alias": "System Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_system" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "User Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_user" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Nice Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "I", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_nice" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Interrupt Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "J", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_irq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Wait Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "K", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "SoftIRQ Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "L", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_softirq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], "thresholds": [], @@ -2014,6 +2994,12 @@ "dashLength": 10, "dashes": false, "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { @@ -2038,19 +3024,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Total", + "alias": "Total Current", "groupBy": [ { "params": [ @@ -2093,7 +3088,7 @@ ] }, { - "alias": "Used", + "alias": "Used Current", "dsType": "influxdb", "groupBy": [ { @@ -2135,6 +3130,95 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Total Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_total" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Used Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -2186,6 +3270,12 @@ "datasource": "InfluxDB", "editable": true, "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "grid": {}, @@ -2211,9 +3301,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -2371,6 +3462,12 @@ "datasource": "InfluxDB", "editable": true, "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "grid": {}, @@ -2397,16 +3494,24 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { - "alias": "#cpu", + "alias": "#cpu Current", "fill": 0 + }, + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 } ], "spaceLength": 10, @@ -2414,7 +3519,7 @@ "steppedLine": false, "targets": [ { - "alias": "#cpu", + "alias": "#cpu Current", "groupBy": [ { "params": [ @@ -2457,7 +3562,7 @@ ] }, { - "alias": "1 Minute Average", + "alias": "1 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -2501,7 +3606,7 @@ ] }, { - "alias": "5 Minute Average", + "alias": "5 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -2545,7 +3650,7 @@ ] }, { - "alias": "15 Minute Average", + "alias": "15 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -2587,6 +3692,185 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "#cpu Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_n_cpus" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "1 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "5 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load5" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "15 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load15" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -2645,6 +3929,12 @@ "decimals": null, "editable": true, "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 6, "fillGradient": 0, "grid": {}, @@ -2673,9 +3963,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -2911,6 +4202,12 @@ "datasource": "InfluxDB", "editable": true, "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "grid": {}, @@ -2937,19 +4234,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -2968,7 +4274,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -3013,7 +4319,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "dsType": "influxdb", "groupBy": [ { @@ -3032,7 +4338,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -3075,6 +4381,136 @@ "value": "{{ MANINT }}" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] } ], "thresholds": [], @@ -3130,6 +4566,12 @@ "datasource": "InfluxDB", "editable": true, "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 7, "fillGradient": 0, "grid": {}, @@ -3156,9 +4598,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3348,6 +4791,12 @@ "datasource": "InfluxDB", "editable": true, "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "grid": {}, @@ -3374,19 +4823,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Threads", + "alias": "Threads Current", "dsType": "influxdb", "groupBy": [ { @@ -3403,6 +4861,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -3427,6 +4886,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Threads Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_total_threads" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -3483,6 +4987,12 @@ "datasource": "InfluxDB", "editable": true, "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "grid": {}, @@ -3509,19 +5019,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Inbound", + "alias": "Inbound Current", "dsType": "influxdb", "groupBy": [ { @@ -3540,7 +5059,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -3585,7 +5104,7 @@ ] }, { - "alias": "Outbound", + "alias": "Outbound Current", "dsType": "influxdb", "groupBy": [ { @@ -3604,7 +5123,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -3647,6 +5166,136 @@ "value": "so-logstash" } ] + }, + { + "alias": "Inbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "alias": "Outbound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] } ], "thresholds": [], @@ -3692,8 +5341,8 @@ } } ], - "refresh": false, - "schemaVersion": 22, + "refresh": "30s", + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -3707,6 +5356,8 @@ "text": "10s", "value": "10s" }, + "description": null, + "error": null, "hide": 0, "label": null, "name": "Interval", @@ -3811,5 +5462,5 @@ "timezone": "browser", "title": "Search Node - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", - "version": 1 -} \ No newline at end of file + "version": 13 +} From 649e539ca6169f07077dff00085ab500ce721ec3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 22:08:29 -0400 Subject: [PATCH 362/378] add trends to sensor dash --- .../dashboards/sensor_nodes/sensor.json | 2354 ++++++++++++++--- 1 file changed, 2046 insertions(+), 308 deletions(-) diff --git a/salt/grafana/dashboards/sensor_nodes/sensor.json b/salt/grafana/dashboards/sensor_nodes/sensor.json index f9dbc9f9c..2bdb0ff12 100644 --- a/salt/grafana/dashboards/sensor_nodes/sensor.json +++ b/salt/grafana/dashboards/sensor_nodes/sensor.json @@ -16,14 +16,15 @@ "editable": true, "gnetId": 2381, "graphTooltip": 0, - "iteration": 1602105838173, + "id": 6, + "iteration": 1625017205779, "links": [], "panels": [ { "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", @@ -34,8 +35,7 @@ } ] }, - "unit": "s", - "decimals": 2 + "unit": "s" }, "overrides": [] }, @@ -57,9 +57,11 @@ ], "fields": "", "values": false - } + }, + "text": {}, + "textMode": "auto" }, - "pluginVersion": "7.0.5", + "pluginVersion": "7.5.4", "targets": [ { "groupBy": [ @@ -110,57 +112,64 @@ "type": "stat" }, { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 60 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 80 - } - ] - }, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected" + "unit": "percent" }, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 5, "w": 4, "x": 4, "y": 0 }, + "hiddenSeries": false, "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, "links": [], + "nullPointMode": "connected", "options": { "alertThreshold": true }, - "pluginVersion": "7.3.4", + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -213,90 +222,116 @@ "operator": "=", "value": "cpu-total" } - ], - "alias": "Usage" - } - ], - "title": "{{ SERVERNAME }} - CPU", - "type": "graph", - "cacheTimeout": null, - "renderer": "flot", - "yaxes": [ - { - "label": null, - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "percent", - "$$hashKey": "object:395" + ] }, { - "label": null, - "show": false, - "logBase": 1, - "min": null, - "max": null, - "format": "short", - "$$hashKey": "object:396" + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* -1 + 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", "xaxis": { - "show": true, + "buckets": null, "mode": "time", "name": null, - "values": [], - "buckets": null + "show": true, + "values": [] }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], "yaxis": { "align": false, "alignLevel": null - }, - "lines": true, - "fill": 1, - "fillGradient": 0, - "linewidth": 1, - "dashes": false, - "hiddenSeries": false, - "dashLength": 10, - "spaceLength": 10, - "points": false, - "pointradius": 2, - "bars": false, - "stack": false, - "percentage": false, - "legend": { - "show": false, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "steppedLine": false, - "tooltip": { - "value_type": "individual", - "shared": true, - "sort": 0 - }, - "timeFrom": null, - "timeShift": null, - "aliasColors": {}, - "seriesOverrides": [], - "thresholds": [], - "timeRegions": [] + } }, { "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -319,13 +354,14 @@ "total": false, "values": false }, - "lines": false, + "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": true, "renderer": "flot", @@ -335,6 +371,7 @@ "steppedLine": false, "targets": [ { + "alias": "Loss Current", "groupBy": [ { "params": [ @@ -375,6 +412,50 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Loss Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": true, + "measurement": "zeekcaptureloss", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_loss" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -397,7 +478,6 @@ }, "yaxes": [ { - "$$hashKey": "object:139", "decimals": 1, "format": "percent", "label": "", @@ -407,7 +487,6 @@ "show": true }, { - "$$hashKey": "object:140", "format": "short", "label": null, "logBase": 1, @@ -430,7 +509,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -458,18 +537,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "dsType": "influxdb", "groupBy": [ { @@ -517,6 +606,57 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Loss Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "zeekdrop", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_drop" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -569,7 +709,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -597,18 +737,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "dsType": "influxdb", "groupBy": [ { @@ -656,6 +806,57 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Loss Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "suridrop", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_drop" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -708,7 +909,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -736,18 +937,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Loss Current", "dsType": "influxdb", "groupBy": [ { @@ -789,6 +1000,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Loss Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "stenodrop", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_drop" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -839,9 +1095,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -870,17 +1124,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -927,6 +1189,56 @@ "value": "/" } ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] } ], "thresholds": [], @@ -949,7 +1261,6 @@ }, "yaxes": [ { - "$$hashKey": "object:708", "format": "percent", "label": null, "logBase": 1, @@ -958,7 +1269,6 @@ "show": true }, { - "$$hashKey": "object:709", "format": "short", "label": null, "logBase": 1, @@ -979,9 +1289,7 @@ "dashes": false, "datasource": "InfluxDB", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1010,17 +1318,25 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.3.4", + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Used", + "alias": "Used Current", "groupBy": [ { "params": [ @@ -1067,6 +1383,56 @@ "value": "/nsm" } ] + }, + { + "alias": "Used Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] } ], "thresholds": [], @@ -1089,7 +1455,6 @@ }, "yaxes": [ { - "$$hashKey": "object:708", "format": "percent", "label": null, "logBase": 1, @@ -1098,7 +1463,6 @@ "show": true }, { - "$$hashKey": "object:709", "format": "short", "label": null, "logBase": 1, @@ -1113,58 +1477,64 @@ } }, { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, - "unit": "s", - "min": 0, - "max": null, - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(245, 54, 54, 0.9)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 259200 - }, - { - "color": "rgba(50, 172, 45, 0.97)", - "value": 432000 - } - ] - }, - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "nullValueMode": "connected" + "unit": "s" }, "overrides": [] }, + "fill": 1, + "fillGradient": 0, "gridPos": { "h": 5, "w": 4, "x": 8, "y": 5 }, + "hiddenSeries": false, "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, "links": [], + "nullPointMode": "connected", "options": { "alertThreshold": true }, - "pluginVersion": "7.3.4", + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { + "alias": "Oldest Pcap Current", "dsType": "influxdb", "groupBy": [ { @@ -1205,82 +1575,96 @@ "operator": "=", "value": "{{ SERVERNAME }}" } - ], - "alias": "Oldest Pcap" - } - ], - "title": "{{ SERVERNAME }} - PCAP Retention", - "type": "graph", - "renderer": "flot", - "yaxes": [ - { - "label": "", - "show": true, - "logBase": 1, - "min": null, - "max": null, - "format": "s", - "$$hashKey": "object:643", - "decimals": 2 + ] }, { - "label": null, - "show": false, - "logBase": 1, - "min": null, - "max": null, - "format": "short", - "$$hashKey": "object:644" + "alias": "Oldest Pcap Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "pcapage", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_seconds" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - PCAP Retention", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", "xaxis": { - "show": true, + "buckets": null, "mode": "time", "name": null, - "values": [], - "buckets": null + "show": true, + "values": [] }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], "yaxis": { "align": false, "alignLevel": null - }, - "lines": true, - "fill": 1, - "linewidth": 1, - "dashLength": 10, - "spaceLength": 10, - "pointradius": 2, - "legend": { - "show": true, - "values": false, - "min": false, - "max": false, - "current": false, - "total": false, - "avg": false - }, - "nullPointMode": "connected", - "tooltip": { - "value_type": "individual", - "shared": true, - "sort": 0 - }, - "aliasColors": {}, - "seriesOverrides": [], - "thresholds": [], - "timeRegions": [], - "cacheTimeout": null, - "timeFrom": null, - "timeShift": null, - "fillGradient": 0, - "dashes": false, - "hiddenSeries": false, - "points": false, - "bars": false, - "stack": false, - "percentage": false, - "steppedLine": false - }, + } + }, { "aliasColors": {}, "bars": false, @@ -1290,7 +1674,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -1318,18 +1702,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1364,7 +1758,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1435,7 +1886,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -1463,18 +1914,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1509,7 +1970,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1580,7 +2098,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -1608,18 +2126,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -1654,7 +2182,64 @@ }, { "params": [ - " / {{ CPUS }}" + " /{{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " /{{ CPUS }}" ], "type": "math" } @@ -1734,7 +2319,7 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -1764,19 +2349,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "System", + "alias": "System Current", "dsType": "influxdb", "groupBy": [ { @@ -1826,7 +2420,7 @@ ] }, { - "alias": "User", + "alias": "User Current", "dsType": "influxdb", "groupBy": [ { @@ -1876,7 +2470,7 @@ ] }, { - "alias": "Nice", + "alias": "Nice Current", "dsType": "influxdb", "groupBy": [ { @@ -1926,7 +2520,7 @@ ] }, { - "alias": "Interrupt", + "alias": "Interrupt Current", "dsType": "influxdb", "groupBy": [ { @@ -1976,7 +2570,7 @@ ] }, { - "alias": "Wait", + "alias": "Wait Current", "dsType": "influxdb", "groupBy": [ { @@ -2026,7 +2620,7 @@ ] }, { - "alias": "SoftIRQ", + "alias": "SoftIRQ Current", "dsType": "influxdb", "groupBy": [ { @@ -2074,6 +2668,312 @@ "value": "cpu-total" } ] + }, + { + "alias": "System Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_system" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "User Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_user" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Nice Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "I", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_nice" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Interrupt Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "J", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_irq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Wait Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "K", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "SoftIRQ Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "L", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_softirq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] } ], "thresholds": [], @@ -2127,7 +3027,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -2155,18 +3055,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -2208,6 +3118,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_active" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -2260,7 +3215,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -2288,18 +3243,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -2347,6 +3312,57 @@ "value": "so-zeek" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] } ], "thresholds": [], @@ -2399,7 +3415,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -2427,18 +3443,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -2486,6 +3512,57 @@ "value": "so-suricata" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] } ], "thresholds": [], @@ -2538,7 +3615,7 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -2566,18 +3643,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 2, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "alias": "Usage Current", "dsType": "influxdb", "groupBy": [ { @@ -2625,6 +3712,57 @@ "value": "so-steno" } ] + }, + { + "alias": "Usage Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] } ], "thresholds": [], @@ -2678,7 +3816,7 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -2708,19 +3846,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "Threads", + "alias": "Threads Current", "dsType": "influxdb", "groupBy": [ { @@ -2737,6 +3884,7 @@ } ], "measurement": "processes", + "orderByTime": "ASC", "policy": "default", "refId": "A", "resultFormat": "time_series", @@ -2761,6 +3909,51 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "Threads Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_total_threads" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -2809,7 +4002,6 @@ "datasource": "InfluxDB", "fieldConfig": { "defaults": { - "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", @@ -2845,9 +4037,11 @@ ], "fields": "", "values": false - } + }, + "text": {}, + "textMode": "auto" }, - "pluginVersion": "7.0.5", + "pluginVersion": "7.5.4", "targets": [ { "groupBy": [ @@ -2911,7 +4105,7 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -2941,19 +4135,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "InBound", + "alias": "InBound Current", "dsType": "influxdb", "groupBy": [ { @@ -2972,7 +4175,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -3017,7 +4220,7 @@ ] }, { - "alias": "OutBound", + "alias": "OutBound Current", "dsType": "influxdb", "groupBy": [ { @@ -3036,7 +4239,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -3079,6 +4282,136 @@ "value": "{{ MANINT }}" } ] + }, + { + "alias": "InBound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "alias": "OutBound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] } ], "thresholds": [], @@ -3139,7 +4472,7 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -3171,9 +4504,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3407,7 +4741,7 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -3436,9 +4770,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -3600,7 +4935,7 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -3630,19 +4965,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "InBound", + "alias": "InBound Current", "dsType": "influxdb", "groupBy": [ { @@ -3661,7 +5005,7 @@ "measurement": "net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -3704,6 +5048,71 @@ "value": "{{ MONINT }}" } ] + }, + { + "alias": "InBound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MONINT }}" + } + ] } ], "thresholds": [], @@ -3762,7 +5171,7 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -3792,16 +5201,24 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { - "alias": "#cpu", + "alias": "#cpu Current", "fill": 0 + }, + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 } ], "spaceLength": 10, @@ -3809,7 +5226,7 @@ "steppedLine": false, "targets": [ { - "alias": "#cpu", + "alias": "#cpu Current", "groupBy": [ { "params": [ @@ -3852,7 +5269,7 @@ ] }, { - "alias": "1 Minute Average", + "alias": "1 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -3896,7 +5313,7 @@ ] }, { - "alias": "5 Minute Average", + "alias": "5 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -3940,7 +5357,7 @@ ] }, { - "alias": "15 Minute Average", + "alias": "15 Minute Average Current", "dsType": "influxdb", "groupBy": [ { @@ -3982,6 +5399,185 @@ "value": "{{ SERVERNAME }}" } ] + }, + { + "alias": "#cpu Trend", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_n_cpus" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "1 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "5 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load5" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "15 Minute Average Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "system", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_load15" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] } ], "thresholds": [], @@ -4039,7 +5635,7 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -4069,9 +5665,10 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", @@ -4267,7 +5864,7 @@ "error": false, "fieldConfig": { "defaults": { - "custom": {} + "links": [] }, "overrides": [] }, @@ -4297,19 +5894,28 @@ "links": [], "nullPointMode": "connected", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.5.4", "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/Trend/", + "dashLength": 4, + "dashes": true, + "fill": 0, + "linewidth": 4 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "alias": "InBound", + "alias": "InBound Current", "dsType": "influxdb", "groupBy": [ { @@ -4328,7 +5934,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "A", "resultFormat": "time_series", @@ -4373,7 +5979,7 @@ ] }, { - "alias": "OutBound", + "alias": "OutBound Current", "dsType": "influxdb", "groupBy": [ { @@ -4392,7 +5998,7 @@ "measurement": "docker_container_net", "orderByTime": "ASC", "policy": "default", - "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", "rawQuery": false, "refId": "B", "resultFormat": "time_series", @@ -4435,6 +6041,136 @@ "value": "so-filebeat" } ] + }, + { + "alias": "InBound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-filebeat" + } + ] + }, + { + "alias": "OutBound Trend", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "so_long_term", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = '{{ MANINT }}' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-filebeat" + } + ] } ], "thresholds": [], @@ -4480,8 +6216,8 @@ } } ], - "refresh": false, - "schemaVersion": 25, + "refresh": "30s", + "schemaVersion": 27, "style": "dark", "tags": [], "templating": { @@ -4495,6 +6231,8 @@ "text": "10s", "value": "10s" }, + "description": null, + "error": null, "hide": 0, "label": null, "name": "Interval", @@ -4561,7 +6299,7 @@ } ], "query": "10s, 1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, + "refresh": 3, "skipUrlSync": false, "type": "interval" } @@ -4599,5 +6337,5 @@ "timezone": "browser", "title": "Sensor Node - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", - "version": 1 -} \ No newline at end of file + "version": 15 +} From 8d3ae65e040d6039b60cb3c347b0405c39315ddd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 22:13:51 -0400 Subject: [PATCH 363/378] fix load graf standalone --- salt/grafana/dashboards/standalone/standalone.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 7dddcb16e..5f70b922d 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -17,7 +17,7 @@ "gnetId": 2381, "graphTooltip": 0, "id": 6, - "iteration": 1625007678418, + "iteration": 1625018989654, "links": [], "panels": [ { @@ -3930,6 +3930,10 @@ "dashes": true, "fill": 0, "linewidth": 4 + }, + { + "alias": "#cpu Current", + "fill": 0 } ], "spaceLength": 10, @@ -10249,5 +10253,5 @@ "timezone": "browser", "title": "Standalone Mode - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", - "version": 3 + "version": 17 } From 37d6529ae0d85c6e3fcc0f7908d6e141ce7235bd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 22:18:17 -0400 Subject: [PATCH 364/378] fix load panel for manager graf --- salt/grafana/dashboards/manager/manager.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/grafana/dashboards/manager/manager.json b/salt/grafana/dashboards/manager/manager.json index dc67052d7..7585706c3 100644 --- a/salt/grafana/dashboards/manager/manager.json +++ b/salt/grafana/dashboards/manager/manager.json @@ -17,7 +17,7 @@ "gnetId": 2381, "graphTooltip": 0, "id": 6, - "iteration": 1625008764846, + "iteration": 1625019296449, "links": [], "panels": [ { @@ -3765,6 +3765,10 @@ "dashes": true, "fill": 0, "linewidth": 4 + }, + { + "alias": "#cpu Current", + "fill": 0 } ], "spaceLength": 10, @@ -6714,5 +6718,5 @@ "timezone": "browser", "title": "Manager Node - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", - "version": 4 + "version": 19 } From aa47a726565f840289b251d17539c3331e404993 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 07:25:51 -0400 Subject: [PATCH 365/378] source common to require root --- salt/common/tools/sbin/so-influxdb-clean | 2 ++ salt/common/tools/sbin/so-influxdb-migrate | 12 +++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 635ac6a02..7b586f03b 100755 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +. /usr/sbin/so-common + wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index 4d65cba95..7ab5378f7 100755 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -15,19 +15,21 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +. /usr/sbin/so-common + echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy." -echo -e "\nInfluxDB will now be migrated. This could take a few hours depending on how large the database is and hardware resources available." +echo -e "\nInfluxDB will now be downsampled. This could take a few hours depending on how large the database is and hardware resources available." read -r -p "Are you sure you want to continue? [y/N] " yorn if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then - echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be moved." + echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." day=0 startdate=`date` while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` - echo "Migration of $migrationdate started at $startdate and completed at `date`." + echo "Downsampling of $migrationdate started at $startdate and completed at `date`." newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s") today=$(date +"%s") @@ -36,11 +38,11 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then else ((day=day+1)) startdate=`date` - echo -e "\nMigrating the next day's worth of data." + echo -e "\nDownsampling the next day's worth of data." fi done - echo -e "\nInfluxDb data migration complete." + echo -e "\nInfluxDb data downsampling complete." else echo -e "\nExiting as requested." From ada729087d1ea1743b1b7f7ca7458275da81e44d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 08:14:52 -0400 Subject: [PATCH 366/378] add script to drop autogen, rename so-influxdb-migrate to so-influxdb-downsample --- ...nfluxdb-migrate => so-influxdb-downsample} | 0 .../tools/sbin/so-influxdb-drop-autogen | 34 +++++++++++++++++++ 2 files changed, 34 insertions(+) rename salt/common/tools/sbin/{so-influxdb-migrate => so-influxdb-downsample} (100%) create mode 100644 salt/common/tools/sbin/so-influxdb-drop-autogen diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-downsample similarity index 100% rename from salt/common/tools/sbin/so-influxdb-migrate rename to salt/common/tools/sbin/so-influxdb-downsample diff --git a/salt/common/tools/sbin/so-influxdb-drop-autogen b/salt/common/tools/sbin/so-influxdb-drop-autogen new file mode 100644 index 000000000..56c00234e --- /dev/null +++ b/salt/common/tools/sbin/so-influxdb-drop-autogen @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +echo -e "\nThis script is used to reduce the size of InfluxDB by dropping the autogen retention policy." +echo "If you want to retain historical data prior to 2.3.60, then this should only be run after you have downsampled your data using so-influxdb-downsample." + +echo -e "\nThe autogen retention policy will now be dropped from InfluxDB." +read -r -p "Are you sure you want to continue? [y/N] " yorn +if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then + echo -e "\nDropping autogen retention policy." + if docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -execute "drop retention policy autogen on telegraf"; then + echo -e "\nAutogen retention policy dropped from InfluxDb." + else + echo -e "\nSomething went wrong dropping then autogen retention policy from InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + fi +else + echo -e "\nExiting as requested." +fi From 06d77d9972025b8c393a9cb912314fd615cff2ed Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Jun 2021 09:31:32 -0400 Subject: [PATCH 367/378] Update so-common-template.json --- .../templates/so/so-common-template.json | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 3e47fd780..0db294914 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -540,6 +540,130 @@ "zeek":{ "type":"object", "dynamic": true + }, + "aws":{ + "type":"object", + "dynamic": true + }, + "azure":{ + "type":"object", + "dynamic": true + }, + "barracuda":{ + "type":"object", + "dynamic": true + }, + "bluecoat":{ + "type":"object", + "dynamic": true + }, + "cef":{ + "type":"object", + "dynamic": true + }, + "checkpoint":{ + "type":"object", + "dynamic": true + }, + "cisco":{ + "type":"object", + "dynamic": true + }, + "cyberark":{ + "type":"object", + "dynamic": true + }, + "cylance":{ + "type":"object", + "dynamic": true + }, + "f5":{ + "type":"object", + "dynamic": true + }, + "fortinet":{ + "type":"object", + "dynamic": true + }, + "gcp":{ + "type":"object", + "dynamic": true + }, + "google_workspace":{ + "type":"object", + "dynamic": true + }, + "imperva":{ + "type":"object", + "dynamic": true + }, + "infoblox":{ + "type":"object", + "dynamic": true + }, + "juniper":{ + "type":"object", + "dynamic": true + }, + "microsoft":{ + "type":"object", + "dynamic": true + }, + "misp":{ + "type":"object", + "dynamic": true + }, + "netflow":{ + "type":"object", + "dynamic": true + }, + "netscout":{ + "type":"object", + "dynamic": true + }, + "o365":{ + "type":"object", + "dynamic": true + }, + "okta":{ + "type":"object", + "dynamic": true + }, + "proofpoint":{ + "type":"object", + "dynamic": true + }, + "radware":{ + "type":"object", + "dynamic": true + }, + "snort":{ + "type":"object", + "dynamic": true + }, + "snyk":{ + "type":"object", + "dynamic": true + }, + "sonicwall":{ + "type":"object", + "dynamic": true + }, + "sophos":{ + "type":"object", + "dynamic": true + }, + "squid":{ + "type":"object", + "dynamic": true + }, + "tomcat":{ + "type":"object", + "dynamic": true + }, + "zcaler":{ + "type":"object", + "dynamic": true } } } From fcbacd473dce2af14fd516ee070a33834cd6c172 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Jun 2021 09:34:56 -0400 Subject: [PATCH 368/378] Add ELK, redis --- .../templates/so/so-common-template.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 0db294914..26a5f2ec7 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -664,6 +664,22 @@ "zcaler":{ "type":"object", "dynamic": true + }, + "elasticsearch":{ + "type":"object", + "dynamic": true + }, + "kibana":{ + "type":"object", + "dynamic": true + }, + "logstash":{ + "type":"object", + "dynamic": true + }, + "redis":{ + "type":"object", + "dynamic": true } } } From 77ca922f6247629722bd4a182a2b2a0f3fb76e86 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 30 Jun 2021 12:37:30 -0400 Subject: [PATCH 369/378] 2.3.60 --- README.md | 4 ++-- VERIFY_ISO.md | 22 +++++++++++----------- sigs/securityonion-2.3.60.iso.sig | Bin 0 -> 543 bytes 3 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 sigs/securityonion-2.3.60.iso.sig diff --git a/README.md b/README.md index bc784dd22..4bff52b20 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## Security Onion 2.3.52 +## Security Onion 2.3.60 -Security Onion 2.3.52 is here! +Security Onion 2.3.60 is here! ## Screenshots diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index 760df9329..d5664d3a3 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.3.52 ISO image built on 2021/04/27 +### 2.3.60 ISO image built on 2021/04/27 ### Download and Verify -2.3.52 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.3.52.iso +2.3.60 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.3.60.iso -MD5: DF0CCCB0331780F472CC167AEAB55652 -SHA1: 71FAE87E6C0AD99FCC27C50A5E5767D3F2332260 -SHA256: 30E7C4206CC86E94D1657CBE420D2F41C28BC4CC63C51F27C448109EBAF09121 +MD5: 77AF432E46B6DE97C17827EB0E6B1ECD +SHA1: 0098C0383D9CA5FCC35F0320F22C3A7F2A171A3E +SHA256: 10869C21A47A162F347069F52DC203C843672A561C2AC303776525FE8A1F28C7 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.52.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.60.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.52.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.60.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.3.52.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.3.60.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.3.52.iso.sig securityonion-2.3.52.iso +gpg --verify securityonion-2.3.60.iso.sig securityonion-2.3.60.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Sat 05 Jun 2021 06:56:04 PM EDT using RSA key ID FE507013 +gpg: Signature made Wed 30 Jun 2021 10:00:50 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.60.iso.sig b/sigs/securityonion-2.3.60.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..e78b79c9ee202d5094116e6dccc1de90bf33ca3a GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;8)c@h8$2@re`V7LBIa1$`c5C3yjE$f@y;U`0PeVXs& z?V=GHs?wVAw^Dw-J~FqmwS>ds4PiZj_|^#wNy}l}H<-Zz5yKLMis&4WEAekO3G%aF zKxv!!En>8Bxj-!E-hNf??Wx+-A$gCx=9ff!mI(jidXQ8GJt!R)rjA&&rl5ZI4nWm? z98#6?dB6WeyUBnHRRU_nGVN(#4C#!o)Uf)33|*4Z$aUN^KQmhq<#!o{if`awzFMDj z+Kv>cec+u`x!$aK?GW>4k-zyvrzCPS^f$xHuRW9P393gzHd{bmSij77HMtpcnG}q5 z;ySXNoUedf=|8xo|ms@FqH}uC90)T0J!?!^FhE5 z$wlqwzEH$@!0rDO3>!6Nu`d20-4$DI#h07d(Ahv7+u7e$N`KzXeuahiA$$!U=}2Wl zzkIt1U?8dK3nLJMV8AJ>mvgDY+Th#b5`Ir?XHmZ52-y^ix`36ntdDqf7Ju$1W)iEW ztr~5dTbt`HYEKk6+s;|&MVrI953swLC literal 0 HcmV?d00001 From 4109cdec5303fea082db2b3435cfd37b391bb7e1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 30 Jun 2021 15:35:01 -0400 Subject: [PATCH 370/378] Refactor so-docker-prune to prevent exceptions when removing images * Prune containers at beginning of script so stopped containers using old images are removed * Add force=True arg to remove() call to ensure an image is still deleted on the off chance a container is still using that image * Add exception handling to continue removing containers instead of exiting if the script fails to remove a container --- salt/common/tools/sbin/so-docker-prune | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index b1c359b58..3ec2a31a9 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -47,6 +47,10 @@ def get_image_version(string) -> str: def main(quiet): client = docker.from_env() + # Prune old/stopped containers + if not quiet: print('Pruning old containers') + client.container.prune() + image_list = client.images.list(filters={ 'dangling': False }) # Map list of image objects to flattened list of tags (format: "name:version") @@ -74,7 +78,10 @@ def main(quiet): for group in grouped_t_list[2:]: for tag in group: if not quiet: print(f'Removing image {tag}') - client.images.remove(tag) + try: + client.images.remove(tag, force=True) + except docker.errors.ClientError as e: + print(f'Could not remove image {tag}, continuing...') except (docker.errors.APIError, InvalidVersion) as e: print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr) exit(1) From f3041a8d7edfec067fccd4e825c5da907f05fe37 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 30 Jun 2021 16:09:08 -0400 Subject: [PATCH 371/378] Ensure all curl's to Kibana are properly sessioned and/or authenticated depending on elastic auth toggle --- salt/common/tools/sbin/so-kibana-config-export | 4 +++- salt/common/tools/sbin/so-kibana-space-defaults | 4 ++-- salt/kibana/bin/so-kibana-config-load | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-kibana-config-export b/salt/common/tools/sbin/so-kibana-config-export index 636c52229..05454cd76 100755 --- a/salt/common/tools/sbin/so-kibana-config-export +++ b/salt/common/tools/sbin/so-kibana-config-export @@ -23,7 +23,9 @@ KIBANA_HOST={{ MANAGER }} KSO_PORT=5601 OUTFILE="saved_objects.ndjson" -curl -s -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -L $KIBANA_HOST:$KSO_PORT/api/saved_objects/_export -d '{ "type": [ "index-pattern", "config", "visualization", "dashboard", "search" ], "excludeExportDetails": false }' > $OUTFILE + +SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://$KIBANA_HOST:$KSO_PORT/ | grep sid | awk '{print $7}') +{{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -s -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -L $KIBANA_HOST:$KSO_PORT/api/saved_objects/_export -d '{ "type": [ "index-pattern", "config", "visualization", "dashboard", "search" ], "excludeExportDetails": false }' > $OUTFILE # Clean up using PLACEHOLDER sed -i "s/$KIBANA_HOST/PLACEHOLDER/g" $OUTFILE diff --git a/salt/common/tools/sbin/so-kibana-space-defaults b/salt/common/tools/sbin/so-kibana-space-defaults index 48225e2f4..d90cf0c11 100755 --- a/salt/common/tools/sbin/so-kibana-space-defaults +++ b/salt/common/tools/sbin/so-kibana-space-defaults @@ -4,10 +4,10 @@ wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELAST ## This hackery will be removed if using Elastic Auth ## # Let's snag a cookie from Kibana -THECOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') +SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') # Disable certain Features from showing up in the Kibana UI echo echo "Setting up default Space:" -{{ ELASTICCURL }} -b "sid=$THECOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log +{{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log echo diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index 0bbcba375..d42596287 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -11,5 +11,7 @@ sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" +SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + # Load saved objects -{{ ELASTICCURL }} -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log +{{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson >> /opt/so/log/kibana/misc.log From 4dbb869952bd2ae2a69147f340f44b04c34fd52d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 30 Jun 2021 16:21:09 -0400 Subject: [PATCH 372/378] Fix typo --- salt/common/tools/sbin/so-docker-prune | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 3ec2a31a9..a845c4549 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -49,7 +49,7 @@ def main(quiet): # Prune old/stopped containers if not quiet: print('Pruning old containers') - client.container.prune() + client.containers.prune() image_list = client.images.list(filters={ 'dangling': False }) From 7b281abf0c733768d5cf20af84bd3603e7456864 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 17:21:18 -0400 Subject: [PATCH 373/378] migrate script now goes through each day and measurement --- salt/common/tools/sbin/so-influxdb-downsample | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index 7ab5378f7..74a48e6a1 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -14,6 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +{-% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} . /usr/sbin/so-common @@ -25,11 +26,15 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." day=0 startdate=`date` - - while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do + +{% for dest_rp in influxdb.downsample.keys() %} + {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) %} + + while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".\"{{measurement}}\" FROM \"autogen\".\"{{measurement}}\" WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` - echo "Downsampling of $migrationdate started at $startdate and completed at `date`." + + echo "Downsampling of measurement: {{measurement}} from $migrationdate started at $startdate and completed at `date`." newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s") today=$(date +"%s") @@ -38,10 +43,13 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then else ((day=day+1)) startdate=`date` - echo -e "\nDownsampling the next day's worth of data." + echo -e "\nDownsampling the next day's worth of data for measurement: {{measurement}}." fi done + {% endfor %} +{% endfor %} + echo -e "\nInfluxDb data downsampling complete." else From 9235bb35a1dc10b5fe13b6e712f6901022f6020d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 17:30:33 -0400 Subject: [PATCH 374/378] fix jinja whatspace and add defaults --- salt/common/tools/sbin/so-influxdb-downsample | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index 74a48e6a1..b561d2b6a 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -14,7 +14,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{-% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} +{%- import_yaml 'influxdb/defaults.yaml' as default_settings %} +{%- set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} . /usr/sbin/so-common From eb8a030966f322e019a0a1f429279f6834022dfc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 17:41:38 -0400 Subject: [PATCH 375/378] reset vars in jinja loop --- salt/common/tools/sbin/so-influxdb-downsample | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index b561d2b6a..ab8fa1dc0 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -25,12 +25,13 @@ echo -e "\nInfluxDB will now be downsampled. This could take a few hours dependi read -r -p "Are you sure you want to continue? [y/N] " yorn if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." - day=0 - startdate=`date` + {% for dest_rp in influxdb.downsample.keys() %} {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) %} + day=0 + startdate=`date` while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".\"{{measurement}}\" FROM \"autogen\".\"{{measurement}}\" WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` From 108fb1261233c84186b4f2ea1fc23dd7222216f4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 17:53:09 -0400 Subject: [PATCH 376/378] s/Migrating/Downsampling --- salt/common/tools/sbin/so-influxdb-downsample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index ab8fa1dc0..1ecc92611 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -24,7 +24,7 @@ echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling ol echo -e "\nInfluxDB will now be downsampled. This could take a few hours depending on how large the database is and hardware resources available." read -r -p "Are you sure you want to continue? [y/N] " yorn if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then - echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." + echo -e "\nDownsampling InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." {% for dest_rp in influxdb.downsample.keys() %} From 21c9c7b8f45f80956f6fc7f86cebada71f2aed86 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Jul 2021 07:56:45 -0400 Subject: [PATCH 377/378] only render main script if a manager type node --- salt/common/tools/sbin/so-influxdb-downsample | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index 1ecc92611..d1e32ef9f 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -14,8 +14,10 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{%- import_yaml 'influxdb/defaults.yaml' as default_settings %} -{%- set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} +{%- set role = grains.id.split('_') | last %} +{%- if role in ['manager', 'managersearch', 'eval', 'standalone'] %} + {%- import_yaml 'influxdb/defaults.yaml' as default_settings %} + {%- set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} . /usr/sbin/so-common @@ -26,9 +28,8 @@ read -r -p "Are you sure you want to continue? [y/N] " yorn if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then echo -e "\nDownsampling InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." - -{% for dest_rp in influxdb.downsample.keys() %} - {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) %} + {% for dest_rp in influxdb.downsample.keys() -%} + {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) -%} day=0 startdate=`date` @@ -49,11 +50,14 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then fi done - {% endfor %} -{% endfor %} + {% endfor -%} + {% endfor -%} echo -e "\nInfluxDb data downsampling complete." else echo -e "\nExiting as requested." fi +{%- else %} +echo -e "\nThis script can only be run on a node running InfluxDB." +{%- endif %} From d583c79936746cbec51860d029f75d67192feb63 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 1 Jul 2021 13:09:09 -0400 Subject: [PATCH 378/378] 2.3.60 --- VERIFY_ISO.md | 8 ++++---- sigs/securityonion-2.3.60.iso.sig | Bin 543 -> 543 bytes 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/VERIFY_ISO.md b/VERIFY_ISO.md index d5664d3a3..b98cdfb9b 100644 --- a/VERIFY_ISO.md +++ b/VERIFY_ISO.md @@ -6,9 +6,9 @@ 2.3.60 ISO image: https://download.securityonion.net/file/securityonion/securityonion-2.3.60.iso -MD5: 77AF432E46B6DE97C17827EB0E6B1ECD -SHA1: 0098C0383D9CA5FCC35F0320F22C3A7F2A171A3E -SHA256: 10869C21A47A162F347069F52DC203C843672A561C2AC303776525FE8A1F28C7 +MD5: 0470325615C42C206B028EE37A1AD897 +SHA1: 496E70BD529D3B8A02D0B32F68B8F7527C953612 +SHA256: 417E34DFCD63D84A16FF2041DC712F02D9E0515C8B78BDF0EE1037DD13C32030 Signature for ISO image: https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.60.iso.sig @@ -40,7 +40,7 @@ gpg --verify securityonion-2.3.60.iso.sig securityonion-2.3.60.iso The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Wed 30 Jun 2021 10:00:50 AM EDT using RSA key ID FE507013 +gpg: Signature made Thu 01 Jul 2021 10:59:24 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.3.60.iso.sig b/sigs/securityonion-2.3.60.iso.sig index e78b79c9ee202d5094116e6dccc1de90bf33ca3a..c00a5c66491090d7ef2992b32ae12892e88df79e 100644 GIT binary patch literal 543 zcmV+)0^t3L0vrSY0RjL91p;8**h~Nl2@re`V7LBIa1-!S5C3}hMvA}Ri6L7xJi4j> zFszCJy=S#cK3oK=riZv%VRVMH9mLmWxvcOZnW_ChGHo9i13c4am`S;GI>Z{e_e4+X ze@*D61qSOJe|_p9uWar^taTKH}w5Y6(u?< zH7NXs+<-H4bjIIRrg?Z_TDSZ&62w=|@}qjrYRnoVjbNJoP2Txi zIagwHmU>^`4oY0;RpqC++)md3z8P^OaaVaGIW4C1UBEOj z8+_3VGFVyagb~Gf)V~BebUEAzKvJ+sQxxUP!Q&Q#QGT3h#+E)g$lXk(A6|uLQp!Ji zPxzuZTRPR%Z@W0IBjmnwTosYXzgyblq%3ez&x)$cNgI{GP?k3P^hSsO7*7=8ORH>& z;onR{0=9BYCiYO7BN=rE8&Mquh2|_|LG1XT=}-0pklw0EYDHl#>@hxSooF71|7ms5 z#18bDP(kZJ6_onX7L(a6gHz(i!2WqZW^i!NyIZD;>pmHp;L9wI{O)-5ILOFe)5Cus hN0B0jR+y#-m@{3SZOa~s={58!rmdxnt{h?oOdE8}38w%6 literal 543 zcmV+)0^t3L0vrSY0RjL91p;8)c@h8$2@re`V7LBIa1$`c5C3yjE$f@y;U`0PeVXs& z?V=GHs?wVAw^Dw-J~FqmwS>ds4PiZj_|^#wNy}l}H<-Zz5yKLMis&4WEAekO3G%aF zKxv!!En>8Bxj-!E-hNf??Wx+-A$gCx=9ff!mI(jidXQ8GJt!R)rjA&&rl5ZI4nWm? z98#6?dB6WeyUBnHRRU_nGVN(#4C#!o)Uf)33|*4Z$aUN^KQmhq<#!o{if`awzFMDj z+Kv>cec+u`x!$aK?GW>4k-zyvrzCPS^f$xHuRW9P393gzHd{bmSij77HMtpcnG}q5 z;ySXNoUedf=|8xo|ms@FqH}uC90)T0J!?!^FhE5 z$wlqwzEH$@!0rDO3>!6Nu`d20-4$DI#h07d(Ahv7+u7e$N`KzXeuahiA$$!U=}2Wl zzkIt1U?8dK3nLJMV8AJ>mvgDY+Th#b5`Ir?XHmZ52-y^ix`36ntdDqf7Ju$1W)iEW ztr~5dTbt`HYEKk6+s;|&MVrI953swLC