From 7409f1575235719bb40de14b6a695fe481a96a4b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 5 Mar 2021 13:59:29 -0500 Subject: [PATCH 001/195] update all grafana queries that were using autogen to use default --- salt/grafana/dashboards/eval/eval.json | 4 ++-- salt/grafana/dashboards/manager/manager.json | 2 +- salt/grafana/dashboards/managersearch/managersearch.json | 2 +- salt/grafana/dashboards/sensor_nodes/sensor.json | 4 ++-- salt/grafana/dashboards/standalone/standalone.json | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/grafana/dashboards/eval/eval.json b/salt/grafana/dashboards/eval/eval.json index cc4298bb2..1b67267a5 100644 --- a/salt/grafana/dashboards/eval/eval.json +++ b/salt/grafana/dashboards/eval/eval.json @@ -352,7 +352,7 @@ ], "measurement": "zeekcaptureloss", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ @@ -2176,7 +2176,7 @@ ], "measurement": "docker_container_mem", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/manager/manager.json b/salt/grafana/dashboards/manager/manager.json index 2ce913155..a53438dd6 100644 --- a/salt/grafana/dashboards/manager/manager.json +++ b/salt/grafana/dashboards/manager/manager.json @@ -1647,7 +1647,7 @@ ], "measurement": "influxsize", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/managersearch/managersearch.json b/salt/grafana/dashboards/managersearch/managersearch.json index 15bf3cc73..1b71a9e84 100644 --- a/salt/grafana/dashboards/managersearch/managersearch.json +++ b/salt/grafana/dashboards/managersearch/managersearch.json @@ -1631,7 +1631,7 @@ ], "measurement": "influxsize", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/sensor_nodes/sensor.json b/salt/grafana/dashboards/sensor_nodes/sensor.json index ea0a6a63b..f9dbc9f9c 100644 --- a/salt/grafana/dashboards/sensor_nodes/sensor.json +++ b/salt/grafana/dashboards/sensor_nodes/sensor.json @@ -351,7 +351,7 @@ ], "measurement": "zeekcaptureloss", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ @@ -2866,7 +2866,7 @@ ], "measurement": "healthcheck", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 60a5c6c6c..d745f13c8 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -4486,7 +4486,7 @@ ], "measurement": "zeekcaptureloss", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ @@ -5107,7 +5107,7 @@ ], "measurement": "influxsize", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ From 1f9e5ca3ccf1818dee7c6203e408f4c00ddf39d8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 9 Mar 2021 10:31:59 -0500 Subject: [PATCH 002/195] install influxdb python module add test retention policies --- salt/influxdb/init.sls | 32 ++++++++++++++++++++++++++++++++ setup/so-functions | 3 ++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index aace4e827..871b0a16a 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -57,6 +57,31 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +set_autogen_retention_policy: + influxdb_retention_policy.present: + - name: autogen + - database: telegraf + - duration: 1h + - replication: 1 + - default: True + - ssl: True + - unsafeSsl: True + - require: + - docker_container: so-influxdb + +set_so_long_term_retention_policy: + influxdb_retention_policy.present: + - name: so_long_term + - database: telegraf + - duration: 2h + - replication: 1 + - default: False + - ssl: True + - unsafeSsl: True + - require: + - docker_container: so-influxdb + + {% endif %} {% else %} @@ -66,3 +91,10 @@ append_so-influxdb_so-status.conf: - name: {{sls}}_state_not_allowed {% endif %} + +#influxdb: +# retention_policies: +# autogen: +# duration: 1h +# so_long_term: +# duration: 2h \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index 21602f320..5890f4726 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2022,6 +2022,7 @@ saltify() { set_progress_str 7 'Installing salt-master' retry 50 10 "apt-get -y install salt-master=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1 ;; *) # Copy down the gpg keys and install them from the manager @@ -2041,7 +2042,7 @@ saltify() { retry 50 10 "apt-get -y install salt-minion=3002.5+ds-1 salt-common=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 if [[ $OSVER != 'xenial' ]]; then - retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-influxdb" >> "$setup_log" 2>&1 || exit 1 else retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1 fi From c1e4c4cb3049382ef8c695a78d019a623f363612 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 9 Mar 2021 11:50:27 -0500 Subject: [PATCH 003/195] fix pip and python-influxdb install --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 391ac3558..9ea165548 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1917,7 +1917,8 @@ saltify() { reserve_group_ids >> "$setup_log" 2>&1 if [[ ! $is_iso ]]; then logCmd "yum -y install epel-release" - logCmd "yum -y install sqlite argon2 curl mariadb-devel" + logCmd "yum -y install sqlite argon2 curl mariadb-devel python3-pip" + retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1 fi # Download Ubuntu Keys in case manager updates = 1 mkdir -p /opt/so/gpg >> "$setup_log" 2>&1 @@ -2032,7 +2033,6 @@ saltify() { set_progress_str 7 'Installing salt-master' retry 50 10 "apt-get -y install salt-master=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 - retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1 ;; *) # Copy down the gpg keys and install them from the manager @@ -2054,7 +2054,7 @@ saltify() { if [[ $OSVER != 'xenial' ]]; then retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-influxdb" >> "$setup_log" 2>&1 || exit 1 else - retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-influxdb" >> "$setup_log" 2>&1 || exit 1 fi fi } From 75012cdcbabb66980a551ab2bca4ca2cde709fb5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 10 Mar 2021 15:20:11 -0500 Subject: [PATCH 004/195] create rps and cqs --- salt/common/init.sls | 3 ++ salt/influxdb/defaults.yaml | 33 ++++++++++++++++++ salt/influxdb/init.sls | 68 ++++++++++++++++++++++++++----------- 3 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 salt/influxdb/defaults.yaml diff --git a/salt/common/init.sls b/salt/common/init.sls index 16cba3c1a..eb7a413e2 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -105,6 +105,8 @@ commonpkgs: - python3-m2crypto - python3-mysqldb - git + - patch + heldpackages: pkg.installed: - pkgs: @@ -140,6 +142,7 @@ commonpkgs: - lvm2 - openssl - git + - patch heldpackages: pkg.installed: diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml new file mode 100644 index 000000000..57193e6fa --- /dev/null +++ b/salt/influxdb/defaults.yaml @@ -0,0 +1,33 @@ +influxdb: + retention_policies: + autogen: + default: True + duration: 1h + so_long_term: + default: False + duration: 30d + downsample: + so_long_term: + resolution: 30m + measurements: + - cpu + - disk + - diskio + - docker_container_cpu + - docker_container_mem + - docker_container_net + - elasticsearch_indices + - elasticsearch_jvm + - esteps + - healthcheck + - influxsize + - mem + - net + - pcapage + - processes + - redisqueue + - stenodrop + - suridrop + - system + - zeekcaptureloss + - zeekdrop \ No newline at end of file diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 871b0a16a..b20e6f64b 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -5,6 +5,8 @@ {% set MANAGER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{% import_yaml 'influxdb/defaults.yaml' as default_settings %} +{% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} @@ -57,30 +59,31 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb -set_autogen_retention_policy: +{% for rp in influxdb.retention_policies.keys() %} +{{rp}}_retention_policy: influxdb_retention_policy.present: - - name: autogen + - name: {{rp}} - database: telegraf - - duration: 1h + - duration: {{influxdb.retention_policies[rp].duration}} - replication: 1 - - default: True + - default: {{influxdb.retention_policies[rp].get('default', 'False')}} - ssl: True - - unsafeSsl: True - require: - docker_container: so-influxdb +{% endfor %} -set_so_long_term_retention_policy: - influxdb_retention_policy.present: - - name: so_long_term +{% for dest_rp in influxdb.downsample.keys() %} + {% for measurement in influxdb.downsample[dest_rp].measurements %} +so_downsample_{{measurement}}_cq: + influxdb_continuous_query.present: + - name: so_downsample_{{measurement}}_cq - database: telegraf - - duration: 2h - - replication: 1 - - default: False + - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) - ssl: True - - unsafeSsl: True - require: - docker_container: so-influxdb - + {% endfor %} +{% endfor %} {% endif %} @@ -92,9 +95,36 @@ set_so_long_term_retention_policy: {% endif %} -#influxdb: -# retention_policies: -# autogen: -# duration: 1h -# so_long_term: -# duration: 2h \ No newline at end of file +influxdb: + retention_policies: + autogen: + default: True + duration: 1h + so_long_term: + default: False + duration: 2h + downsample: + so_long_term: + resolution: 30m + measurements: + - cpu + - disk + - diskio + - docker_container_cpu + - docker_container_mem + - docker_container_net + - elasticsearch_indices + - elasticsearch_jvm + - esteps + - healthcheck + - influxsize + - mem + - net + - pcapage + - processes + - redisqueue + - stenodrop + - suridrop + - system + - zeekcaptureloss + - zeekdrop \ No newline at end of file From 8fc1656939468b69d6fb2dd5599d1762ed920ae6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 09:24:57 -0500 Subject: [PATCH 005/195] fix timeouts / retries in ssl state --- salt/ssl/init.sls | 110 +++++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 44 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 8d6c65bea..4646b1d07 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -68,8 +68,9 @@ removeesp12dir: - x509: /etc/pki/influxdb.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the talking to influxdb /etc/pki/influxdb.crt: @@ -86,8 +87,9 @@ removeesp12dir: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/influxdb.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 influxkeyperms: file.managed: @@ -111,8 +113,9 @@ influxkeyperms: - x509: /etc/pki/redis.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/redis.crt: x509.certificate_managed: @@ -153,8 +156,9 @@ rediskeyperms: - x509: /etc/pki/filebeat.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Request a cert and drop it where it needs to go to be distributed /etc/pki/filebeat.crt: @@ -175,8 +179,9 @@ rediskeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/filebeat.key -topk8 -out /etc/pki/filebeat.p8 -nocrypt" - onchanges: @@ -232,8 +237,9 @@ fbcrtlink: - x509: /etc/pki/registry.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the docker registry /etc/pki/registry.crt: @@ -250,8 +256,9 @@ fbcrtlink: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/registry.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 regkeyperms: file.managed: @@ -273,8 +280,9 @@ regkeyperms: - x509: /etc/pki/minio.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for minio /etc/pki/minio.crt: @@ -291,8 +299,9 @@ regkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/minio.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 miniokeyperms: file.managed: @@ -332,8 +341,9 @@ miniokeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: @@ -366,8 +376,9 @@ elasticp12perms: - x509: /etc/pki/managerssl.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the reverse proxy /etc/pki/managerssl.crt: @@ -385,8 +396,9 @@ elasticp12perms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 msslkeyperms: file.managed: @@ -409,8 +421,9 @@ msslkeyperms: - x509: /etc/pki/fleet.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/fleet.crt: x509.certificate_managed: @@ -425,8 +438,9 @@ msslkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 fleetkeyperms: file.managed: @@ -456,8 +470,9 @@ fbcertdir: - x509: /opt/so/conf/filebeat/etc/pki/filebeat.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Request a cert and drop it where it needs to go to be distributed /opt/so/conf/filebeat/etc/pki/filebeat.crt: @@ -478,8 +493,9 @@ fbcertdir: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /opt/so/conf/filebeat/etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Convert the key to pkcs#8 so logstash will work correctly. filebeatpkcs: @@ -520,8 +536,9 @@ chownfilebeatp8: - x509: /etc/pki/managerssl.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the reverse proxy /etc/pki/managerssl.crt: @@ -539,8 +556,9 @@ chownfilebeatp8: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 msslkeyperms: file.managed: @@ -563,8 +581,9 @@ msslkeyperms: - x509: /etc/pki/fleet.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/fleet.crt: x509.certificate_managed: @@ -579,8 +598,9 @@ msslkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 fleetkeyperms: file.managed: @@ -606,8 +626,9 @@ fleetkeyperms: - x509: /etc/pki/elasticsearch.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/elasticsearch.crt: x509.certificate_managed: @@ -623,8 +644,9 @@ fleetkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: From 6dba2879c52e65828598f609e0f409aa35c49300 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 09:25:44 -0500 Subject: [PATCH 006/195] change so_long_term rp to 6h for tetsing --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 57193e6fa..9647b70ea 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -5,7 +5,7 @@ influxdb: duration: 1h so_long_term: default: False - duration: 30d + duration: 6h downsample: so_long_term: resolution: 30m From 3385ba2ca2da7590eae56f513e241adc3fe7360c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 09:35:54 -0500 Subject: [PATCH 007/195] verify ssl --- salt/influxdb/init.sls | 42 +++++++----------------------------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index b20e6f64b..f55e255b7 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -68,6 +68,9 @@ append_so-influxdb_so-status.conf: - replication: 1 - default: {{influxdb.retention_policies[rp].get('default', 'False')}} - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb {% endfor %} @@ -80,6 +83,9 @@ so_downsample_{{measurement}}_cq: - database: telegraf - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb {% endfor %} @@ -93,38 +99,4 @@ so_downsample_{{measurement}}_cq: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} - -influxdb: - retention_policies: - autogen: - default: True - duration: 1h - so_long_term: - default: False - duration: 2h - downsample: - so_long_term: - resolution: 30m - measurements: - - cpu - - disk - - diskio - - docker_container_cpu - - docker_container_mem - - docker_container_net - - elasticsearch_indices - - elasticsearch_jvm - - esteps - - healthcheck - - influxsize - - mem - - net - - pcapage - - processes - - redisqueue - - stenodrop - - suridrop - - system - - zeekcaptureloss - - zeekdrop \ No newline at end of file +{% endif %} \ No newline at end of file From 3b74d987c1a1486dc5e720b93814d592ebc35c62 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 10:49:15 -0500 Subject: [PATCH 008/195] fix retry in ca state. add subjectAltName to influxdb.crt --- salt/ca/init.sls | 5 +++-- salt/ssl/init.sls | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/ca/init.sls b/salt/ca/init.sls index 0d35c10c1..485b01d3e 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -43,8 +43,9 @@ pki_private_key: - require: - file: /etc/pki - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 x509_pem_entries: module.run: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 4646b1d07..85854bab4 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -79,6 +79,7 @@ removeesp12dir: - signing_policy: influxdb - public_key: /etc/pki/influxdb.key - CN: {{ manager }} + - subjectAltName: DNS:{{ HOSTNAME }} - days_remaining: 0 - days_valid: 820 - backup: True From c4da57603087b0c6ab9f7bbad5147c439aabc471 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 12:20:32 -0500 Subject: [PATCH 009/195] ensure the presence of the telegraf database --- salt/influxdb/init.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index f55e255b7..608561857 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -59,6 +59,17 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +telegraf_database: + influxdb_database.present: + - name: telegraf + - database: telegraf + - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} + - require: + - docker_container: so-influxdb + {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: influxdb_retention_policy.present: @@ -73,6 +84,7 @@ append_so-influxdb_so-status.conf: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb + - influxdb_database: telegraf_database {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} @@ -88,6 +100,7 @@ so_downsample_{{measurement}}_cq: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb + - influxdb_database: telegraf_database {% endfor %} {% endfor %} From 3a8aea0de69ceb26ec4e115c00dd465001a16aae Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 09:11:48 -0400 Subject: [PATCH 010/195] removing domainstats and freqserver from so-image-common --- salt/common/tools/sbin/so-image-common | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 402ae97f3..a3b9f62e9 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -47,13 +47,11 @@ container_list() { TRUSTED_CONTAINERS=( "so-acng" "so-curator" - "so-domainstats" "so-elastalert" "so-elasticsearch" "so-filebeat" "so-fleet" "so-fleet-launcher" - "so-freqserver" "so-grafana" "so-idstools" "so-influxdb" From 53528d486c0a2793cf45ccda7ee2950bb2f29e4e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 09:44:56 -0400 Subject: [PATCH 011/195] remove minio --- salt/common/tools/sbin/so-image-common | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index a3b9f62e9..be5a327f0 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -58,7 +58,6 @@ container_list() { "so-kibana" "so-kratos" "so-logstash" - "so-minio" "so-mysql" "so-nginx" "so-pcaptools" From 11c3f14b42d64b5581304e383f81b96b6f9b6836 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 10:35:20 -0400 Subject: [PATCH 012/195] end patch files with newline --- salt/salt/files/influxdb_continuous_query.py.patch | 2 +- salt/salt/files/influxdb_retention_policy.py.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/files/influxdb_continuous_query.py.patch b/salt/salt/files/influxdb_continuous_query.py.patch index 049b04782..a596983ff 100644 --- a/salt/salt/files/influxdb_continuous_query.py.patch +++ b/salt/salt/files/influxdb_continuous_query.py.patch @@ -1,4 +1,4 @@ 60c60 < database, name, query, resample_time, coverage_period --- -> database, name, query, resample_time, coverage_period, **client_args \ No newline at end of file +> database, name, query, resample_time, coverage_period, **client_args diff --git a/salt/salt/files/influxdb_retention_policy.py.patch b/salt/salt/files/influxdb_retention_policy.py.patch index b16de431e..79ca3e536 100644 --- a/salt/salt/files/influxdb_retention_policy.py.patch +++ b/salt/salt/files/influxdb_retention_policy.py.patch @@ -1,4 +1,4 @@ 38c38 < hours = int(duration.split("h")) --- -> hours = int(duration.split("h")[0]) \ No newline at end of file +> hours = int(duration.split("h")[0]) From d75fafb19cddfadeeff95be574b6d492b42d145b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 17:30:27 -0400 Subject: [PATCH 013/195] add support for shard_duration to influxdb module and influxdb_retention_policy state - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- .../files/influxdb_retention_policy.py.patch | 12 ++++++++++++ salt/salt/files/influxmod.py.patch | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 salt/salt/files/influxmod.py.patch diff --git a/salt/salt/files/influxdb_retention_policy.py.patch b/salt/salt/files/influxdb_retention_policy.py.patch index 79ca3e536..7180f8934 100644 --- a/salt/salt/files/influxdb_retention_policy.py.patch +++ b/salt/salt/files/influxdb_retention_policy.py.patch @@ -2,3 +2,15 @@ < hours = int(duration.split("h")) --- > hours = int(duration.split("h")[0]) +52c52 +< def present(name, database, duration="7d", replication=1, default=False, **client_args): +--- +> def present(name, database, duration="7d", replication=1, default=False, shard_duration="1d", **client_args): +77c77 +< database, name, duration, replication, default, **client_args +--- +> database, name, duration, replication, shard_duration, default, **client_args +119c119 +< database, name, duration, replication, default, **client_args +--- +> database, name, duration, replication, shard_duration, default, **client_args diff --git a/salt/salt/files/influxmod.py.patch b/salt/salt/files/influxmod.py.patch new file mode 100644 index 000000000..b653265cb --- /dev/null +++ b/salt/salt/files/influxmod.py.patch @@ -0,0 +1,16 @@ +427c427 +< database, name, duration, replication, default=False, **client_args +--- +> database, name, duration, replication, shard_duration, default=False, **client_args +462c462 +< client.create_retention_policy(name, duration, replication, database, default) +--- +> client.create_retention_policy(name, duration, replication, database, default, shard_duration) +468c468 +< database, name, duration, replication, default=False, **client_args +--- +> database, name, duration, replication, shard_duration, default=False, **client_args +504c504 +< client.alter_retention_policy(name, database, duration, replication, default) +--- +> client.alter_retention_policy(name, database, duration, replication, default, shard_duration) From cd0ab5c70909b3234d59a04a5c347aacbf51ec06 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 25 Mar 2021 08:50:32 -0400 Subject: [PATCH 014/195] add support for shard_duration to influxdb module and influxdb_retention_policy state - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- .../files/{influxmod.py.patch => influxdbmod.py.patch} | 0 salt/salt/minion.sls | 7 ++++++- 2 files changed, 6 insertions(+), 1 deletion(-) rename salt/salt/files/{influxmod.py.patch => influxdbmod.py.patch} (100%) diff --git a/salt/salt/files/influxmod.py.patch b/salt/salt/files/influxdbmod.py.patch similarity index 100% rename from salt/salt/files/influxmod.py.patch rename to salt/salt/files/influxdbmod.py.patch diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index fa5c0659e..47cd179ac 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -53,4 +53,9 @@ influxdb_continuous_query.present_patch: influxdb_retention_policy.present_patch: file.patch: - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py - - source: salt://salt/files/influxdb_retention_policy.py.patch \ No newline at end of file + - source: salt://salt/files/influxdb_retention_policy.py.patch + +influxdbmod.py_shard_duration_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py + - source: salt://salt/files/influxdbmod.py.patch \ No newline at end of file From 889e624a8cbf3736d2f45b57b2a92cb7934dac73 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 25 Mar 2021 09:33:10 -0400 Subject: [PATCH 015/195] add shard_duration to state and defaults - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- salt/influxdb/defaults.yaml | 6 ++++-- salt/influxdb/init.sls | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 9647b70ea..88faaedba 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,10 +2,12 @@ influxdb: retention_policies: autogen: default: True - duration: 1h + duration: 1d + shard_duration: 1h so_long_term: default: False - duration: 6h + duration: 7d + shard_duration: 1d downsample: so_long_term: resolution: 30m diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 88f03042e..59673c93b 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -79,6 +79,7 @@ telegraf_database: - name: {{rp}} - database: telegraf - duration: {{influxdb.retention_policies[rp].duration}} + - shard_duration: {{influxdb.retention_policies[rp].shard_duration}} - replication: 1 - default: {{influxdb.retention_policies[rp].get('default', 'False')}} - ssl: True From e0f4abaa093bb9617ed77b64eeb6eab219bd6b98 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Mar 2021 10:36:56 -0400 Subject: [PATCH 016/195] try to do it with just 1 cq, modify defaults for testing --- salt/influxdb/defaults.yaml | 4 ++-- salt/influxdb/init.sls | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 88faaedba..3de7edd56 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,11 +2,11 @@ influxdb: retention_policies: autogen: default: True - duration: 1d + duration: 1h shard_duration: 1h so_long_term: default: False - duration: 7d + duration: 2d shard_duration: 1d downsample: so_long_term: diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 59673c93b..b3f3e5c9c 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -92,13 +92,14 @@ telegraf_database: - file: influxdb_retention_policy.present_patch {% endfor %} +{# {% for dest_rp in influxdb.downsample.keys() %} {% for measurement in influxdb.downsample[dest_rp].measurements %} so_downsample_{{measurement}}_cq: influxdb_continuous_query.present: - name: so_downsample_{{measurement}}_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}), * - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] @@ -109,6 +110,22 @@ so_downsample_{{measurement}}_cq: - file: influxdb_continuous_query.present_patch {% endfor %} {% endfor %} +#} + +so_downsample_cq: + influxdb_continuous_query.present: + - name: so_downsample_cq + - database: telegraf + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} + - require: + - docker_container: so-influxdb + - influxdb_database: telegraf_database + - file: influxdb_continuous_query.present_patch + {% endif %} From d1150f150fb174f6aaec9e13c5916ada0ae76e81 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Mar 2021 10:59:18 -0400 Subject: [PATCH 017/195] loop through the rps --- salt/influxdb/init.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index b3f3e5c9c..1b21612b3 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -112,6 +112,7 @@ so_downsample_{{measurement}}_cq: {% endfor %} #} +{% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq @@ -125,7 +126,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - +{% endfor %} {% endif %} From 30c6d4756a76687b9cbf95d9c9ff5085c538bb66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Mar 2021 09:38:37 -0400 Subject: [PATCH 018/195] change default long term resolution to 5m --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 3de7edd56..927cd7b52 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -10,7 +10,7 @@ influxdb: shard_duration: 1d downsample: so_long_term: - resolution: 30m + resolution: 5m measurements: - cpu - disk From 8e55e0b9944e6d46032b8fa2e00998a3efd9ded3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Mar 2021 13:36:52 -0400 Subject: [PATCH 019/195] start graphing data from so_long_term --- .../dashboards/standalone/standalone.new.json | 7841 +++++++++++++++++ 1 file changed, 7841 insertions(+) create mode 100644 salt/grafana/dashboards/standalone/standalone.new.json diff --git a/salt/grafana/dashboards/standalone/standalone.new.json b/salt/grafana/dashboards/standalone/standalone.new.json new file mode 100644 index 000000000..41c4d3541 --- /dev/null +++ b/salt/grafana/dashboards/standalone/standalone.new.json @@ -0,0 +1,7841 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "This Dashboard provides a general overview of Standalone Mode", + "editable": true, + "gnetId": 2381, + "graphTooltip": 0, + "id": 6, + "iteration": 1617106562807, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 60 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Usage", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* -1 + 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "groupBy": [], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_idle" + ], + "type": "field" + }, + { + "params": [ + "*-1+100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 39, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "uptime" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "{{ SERVERNAME }} - System Uptime", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=", + "value": "so-kibana" + }, + { + "condition": "AND", + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=", + "value": "so-kibana" + }, + { + "condition": "AND", + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Kibana CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "suridrop", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Suricata Packet Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 0 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stenodrop", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - PCAP Packet Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Used", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] + }, + { + "groupBy": [], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Disk Used(/)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 5 + }, + "hiddenSeries": false, + "id": 74, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Used", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] + }, + { + "groupBy": [], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Disk Used(/nsm)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 5 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - InfluxDB CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Logstash CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 5 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Suri CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 5 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Steno CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Interrupt": "#70DBED", + "Nice": "#629E51", + "SoftIRQ": "#EA6460", + "System": "#BF1B00", + "User": "#1F78C1", + "Wait": "#F2C96D", + "cpu.mean": "#629E51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "System", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_system" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "User", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_user" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Nice", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_nice" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Interrupt", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_irq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Wait", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "SoftIRQ", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_softirq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "Percent(%)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "InBound": "#629E51", + "OutBound": "#5195CE", + "net.non_negative_derivative": "super-light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 10 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "net", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "alias": "Outbound", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "net", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_sent" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Management Traffic", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 10 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Suri Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 10 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Steno Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "#cpu": "green", + "1 Minute Average": "#EAB839", + "15 Minute Average": "#BF1B00", + "5 Minute Average": "#E0752D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "#cpu", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "#cpu", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "n_cpus" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "1 Minute Average", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "5 Minute Average", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load5" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "15 Minute Average", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load15" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Load Average", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "InBound": "#629E51", + "OutBound": "#5195CE", + "net.non_negative_derivative": "light-orange" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 15 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "net", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MONINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MONINT }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Monitor Traffic", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 15 + }, + "id": 37, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "healthcheck", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "zeek_restart" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Zeek Restarts via Healthcheck", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 2, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": null, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 259200 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 432000 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 15 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Oldest Pcap", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "pcapage", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "seconds" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "pcapage", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_seconds" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - PCAP Retention", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Blocked": "#BF1B00", + "Running": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 7, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Blocked", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "blocked" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Running", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "processes", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "running" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Sleep", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "processes", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "sleeping" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Processes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 20 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "alias": "Outbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Logstash Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 20 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 20 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Threads", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "processes", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_threads" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Total Threads", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 25 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "alias": "Outbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - InfluxDB Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 25 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "zeekcaptureloss", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "loss" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek Capture Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 25 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "zeekdrop", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek Packet Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - IO Wait", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 30 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] + }, + { + "alias": "Outbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Proxy Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 30 + }, + "hiddenSeries": false, + "id": 69, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "influxsize", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "kbytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "influxsize", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_kbytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - InfluxDB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "deckbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 30 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "redisqueue", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "unparsed" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "redisqueue", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_unparsed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Redis Queue", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Read", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "read_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Write", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "write_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Disk I/O", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 35 + }, + "hiddenSeries": false, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "EPS", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "esteps", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "eps" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "esteps", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_eps" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Estimated EPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "EPS", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 35 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Redis Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 35 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "/ 16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Redis CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Buffered": "#6ED0E0", + "Cached": "#F9934E", + "Free": "#629E51", + "Used": "#58140C" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "decimals": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Used", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Buffered", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffered" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Cached", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "cached" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Free", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "free" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 40 + }, + "id": 33, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "docs_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": "", + "title": "{{ SERVERNAME }} - ES Documents", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "store_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_store_size_in_bytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES Store Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 40 + }, + "hiddenSeries": false, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "threads_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_threads_count" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES Thread Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 40 + }, + "hiddenSeries": false, + "id": 63, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "fielddata_memory_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_fielddata_memory_size_in_bytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES Fielddata Cache Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 67, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Total", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Used", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Memory(Used)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10s", + "value": "10s" + }, + "error": null, + "hide": 0, + "label": null, + "name": "Interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_Interval" + }, + { + "selected": true, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "10s, 1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Standalone Mode - {{ SERVERNAME }} Overview", + "uid": "so_overview", + "version": 1 + } \ No newline at end of file From 4f3b3a787c5cf9348518280193263b69aacf314b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Apr 2021 10:18:01 -0400 Subject: [PATCH 020/195] change defaults for testing, remove measurements list since cq uses wildcard now - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- salt/influxdb/defaults.yaml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 927cd7b52..ce5624be2 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,34 +2,12 @@ influxdb: retention_policies: autogen: default: True - duration: 1h + duration: 2d shard_duration: 1h so_long_term: default: False - duration: 2d + duration: 7d shard_duration: 1d downsample: so_long_term: - resolution: 5m - measurements: - - cpu - - disk - - diskio - - docker_container_cpu - - docker_container_mem - - docker_container_net - - elasticsearch_indices - - elasticsearch_jvm - - esteps - - healthcheck - - influxsize - - mem - - net - - pcapage - - processes - - redisqueue - - stenodrop - - suridrop - - system - - zeekcaptureloss - - zeekdrop \ No newline at end of file + resolution: 5m \ No newline at end of file From 5cda35db0aa0d8d9b6f2ded8a9a4ef959550ab58 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Apr 2021 10:45:54 -0400 Subject: [PATCH 021/195] change defaults for testing - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index ce5624be2..03ac4c3c6 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,7 +2,7 @@ influxdb: retention_policies: autogen: default: True - duration: 2d + duration: 1d shard_duration: 1h so_long_term: default: False From 8eef574342f27c46d581331114cd2d4dbfd5d027 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 08:28:01 -0400 Subject: [PATCH 022/195] install python3-influxdb and create requires --- salt/influxdb/init.sls | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 1b21612b3..67342f826 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -13,6 +13,10 @@ include: - salt.minion +python3-influxdb: + pkg.installed: + - name: python3-influxdb + # Influx DB influxconfdir: file.directory: @@ -62,6 +66,7 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb + telegraf_database: influxdb_database.present: - name: telegraf @@ -72,6 +77,7 @@ telegraf_database: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb + - pkg: python3-influxdb {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: @@ -90,28 +96,9 @@ telegraf_database: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_retention_policy.present_patch + - pkg: python3-influxdb {% endfor %} -{# -{% for dest_rp in influxdb.downsample.keys() %} - {% for measurement in influxdb.downsample[dest_rp].measurements %} -so_downsample_{{measurement}}_cq: - influxdb_continuous_query.present: - - name: so_downsample_{{measurement}}_cq - - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}), * - - ssl: True - - verify_ssl: /etc/pki/ca.crt - - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] - - influxdb_host: {{ MANAGER }} - - require: - - docker_container: so-influxdb - - influxdb_database: telegraf_database - - file: influxdb_continuous_query.present_patch - {% endfor %} -{% endfor %} -#} - {% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: @@ -126,6 +113,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch + - pkg: python3-influxdb {% endfor %} {% endif %} From d792c65ce326e246ae603fcf3b82dd162da4ac60 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 09:25:25 -0400 Subject: [PATCH 023/195] change how influx is patch and python3-influxdb is installed --- salt/influxdb/init.sls | 11 ++++------- salt/salt/minion.sls | 20 +------------------- salt/salt/python3-influxdb.sls | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 26 deletions(-) create mode 100644 salt/salt/python3-influxdb.sls diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 67342f826..55f350099 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -12,10 +12,7 @@ include: - salt.minion - -python3-influxdb: - pkg.installed: - - name: python3-influxdb + - salt.python3-influxdb # Influx DB influxconfdir: @@ -77,7 +74,7 @@ telegraf_database: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb - - pkg: python3-influxdb + - sls: salt.python3-influxdb {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: @@ -96,7 +93,7 @@ telegraf_database: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_retention_policy.present_patch - - pkg: python3-influxdb + - sls: salt.python3-influxdb {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} @@ -113,7 +110,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - - pkg: python3-influxdb + - sls: python3-influxdb {% endfor %} {% endif %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 2ac66c81a..41f9bfaf2 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -67,22 +67,4 @@ salt_minion_service: patch_pkg: pkg.installed: - - name: patch - - -#https://github.com/saltstack/salt/issues/59766 -influxdb_continuous_query.present_patch: - file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_continuous_query.py - - source: salt://salt/files/influxdb_continuous_query.py.patch - -#https://github.com/saltstack/salt/issues/59761 -influxdb_retention_policy.present_patch: - file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py - - source: salt://salt/files/influxdb_retention_policy.py.patch - -influxdbmod.py_shard_duration_patch: - file.patch: - - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py - - source: salt://salt/files/influxdbmod.py.patch \ No newline at end of file + - name: patch \ No newline at end of file diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls new file mode 100644 index 000000000..0f8cad42e --- /dev/null +++ b/salt/salt/python3-influxdb.sls @@ -0,0 +1,23 @@ +python3_influxdb: + pkg.installed: + - name: python3-influxdb + +#https://github.com/saltstack/salt/issues/59766 +influxdb_continuous_query.present_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_continuous_query.py + - source: salt://salt/files/influxdb_continuous_query.py.patch + - pkg: python3_influxdb + +#https://github.com/saltstack/salt/issues/59761 +influxdb_retention_policy.present_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py + - source: salt://salt/files/influxdb_retention_policy.py.patch + - pkg: python3_influxdb + +influxdbmod.py_shard_duration_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py + - source: salt://salt/files/influxdbmod.py.patch + - pkg: python3_influxdb \ No newline at end of file From af86a9dac01fe511677d72718f1ac30675a79fe1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 09:52:22 -0400 Subject: [PATCH 024/195] handle different paths for salt states/modules based on os --- salt/salt/map.jinja | 4 ++++ salt/salt/python3-influxdb.sls | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5d6d980be..489a41dd2 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -4,9 +4,13 @@ {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?', python_shell=True) %} + {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} + {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 0f8cad42e..f0536e36f 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,3 +1,6 @@ +{% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} +{% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} + python3_influxdb: pkg.installed: - name: python3-influxdb @@ -5,19 +8,19 @@ python3_influxdb: #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_continuous_query.py + - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - pkg: python3_influxdb #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py + - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - pkg: python3_influxdb influxdbmod.py_shard_duration_patch: file.patch: - - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py + - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - pkg: python3_influxdb \ No newline at end of file From b53017ee87603f1d15c441bb4913a65b2989fd1e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 10:44:46 -0400 Subject: [PATCH 025/195] ensure salt python modules are installed and patched during soup --- salt/common/tools/sbin/soup | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2a1ddab1c..2d9e0a146 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -729,6 +729,13 @@ echo "" echo "Starting Salt Master service." systemctl start salt-master +sleep 10 + +echo "" +echo "Ensuring python modules for Salt are installed and patched." +salt-call state.apply python3-influxdb -l info queue=True +echo "" + # Only regenerate osquery packages if Fleet is enabled FLEET_MANAGER=$(lookup_pillar fleet_manager) FLEET_NODE=$(lookup_pillar fleet_node) From 0a01d7b0412e14b9d16bb9e317bc5437504cc4a4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 11:14:13 -0400 Subject: [PATCH 026/195] fix var --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 97e61e6e2..febdd4a6c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -167,7 +167,7 @@ gpg_rpm_import() { if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then local RPMKEYSLOC="../salt/repo/client/files/centos/keys" else - local RPMKEYSLOC="$UPDATEDIR/salt/repo/client/files/centos/keys" + local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/centos/keys" fi RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') From 1f654d44440fd1431d911bda96c67ea1db1ba663 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 12:04:58 -0400 Subject: [PATCH 027/195] fix the state apply --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2d9e0a146..05c56008a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -733,7 +733,7 @@ sleep 10 echo "" echo "Ensuring python modules for Salt are installed and patched." -salt-call state.apply python3-influxdb -l info queue=True +salt-call state.apply salt.python3-influxdb -l info queue=True echo "" # Only regenerate osquery packages if Fleet is enabled From 17c95723ec6670b94fb1f03518ffa2b11a35a03e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 16:44:40 -0400 Subject: [PATCH 028/195] update package name --- salt/salt/python3-influxdb.sls | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index f0536e36f..41989816f 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,26 +1,26 @@ {% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} -python3_influxdb: +securityonion_python3_influxdb: pkg.installed: - - name: python3-influxdb + - name: securityonion-python3-influxdb #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - - pkg: python3_influxdb + - pkg: securityonion_python3_influxdb #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - - pkg: python3_influxdb + - pkg: securityonion_python3_influxdb influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - - pkg: python3_influxdb \ No newline at end of file + - pkg: securityonion_python3_influxdb \ No newline at end of file From c9298137b52c1466f087246de153d553d36d6d04 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 17:08:35 -0400 Subject: [PATCH 029/195] adding docker-ce.repo to delete list --- salt/repo/client/map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/repo/client/map.jinja b/salt/repo/client/map.jinja index ccfa1eae2..dedb96783 100644 --- a/salt/repo/client/map.jinja +++ b/salt/repo/client/map.jinja @@ -10,6 +10,7 @@ 'CentOS-Sources.repo', 'CentOS-Vault.repo', 'CentOS-x86_64-kernel.repo', + 'docker-ce.repo', 'epel.repo', 'epel-testing.repo', 'saltstack.repo', From a7247e98121f8dc1ee13760efc4403be3ae8a83b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 17:17:49 -0400 Subject: [PATCH 030/195] update package name --- salt/salt/map.jinja | 2 ++ salt/salt/python3-influxdb.sls | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 489a41dd2..fb6cadb1b 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -6,11 +6,13 @@ {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?', python_shell=True) %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} + {% set PYTHON3INFLUX= 'python3-influxdb' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} + {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 41989816f..36347cb95 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,26 +1,27 @@ {% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} +{% from "salt/map.jinja" import PYTHON3INFLUX with context %} -securityonion_python3_influxdb: +python3_influxdb: pkg.installed: - - name: securityonion-python3-influxdb + - name: {{ PYTHON3INFLUX }} #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - - pkg: securityonion_python3_influxdb + - pkg: python3_influxdb #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - - pkg: securityonion_python3_influxdb + - pkg: python3_influxdb influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - - pkg: securityonion_python3_influxdb \ No newline at end of file + - pkg: python3_influxdb \ No newline at end of file From 5cedf98f555d79c1b80276ebb31ce125f5073a66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:05:52 -0400 Subject: [PATCH 031/195] change path --- salt/salt/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index fb6cadb1b..67ab4a594 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -10,8 +10,8 @@ {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} - {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} - {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} + {% set SALT_STATE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% endif %} From 21077ef26e09c2f1a0578d24e13ee23d01db05d1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:09:11 -0400 Subject: [PATCH 032/195] undo path change --- salt/salt/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 67ab4a594..fb6cadb1b 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -10,8 +10,8 @@ {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} - {% set SALT_STATE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/states' %} - {% set SALT_MODULE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/modules' %} + {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% endif %} From f5ddb084b6f850f23652432a85fe60254f3169c3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:22:44 -0400 Subject: [PATCH 033/195] create salt.helper-packages state, use it to install the patch package --- salt/common/init.sls | 2 -- salt/salt/helper-packages.sls | 3 +++ salt/salt/minion.sls | 1 + salt/salt/python3-influxdb.sls | 15 ++++++++++++--- 4 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 salt/salt/helper-packages.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index 6d2a9aea7..cb6039756 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -95,7 +95,6 @@ commonpkgs: - python3-mysqldb - python3-packaging - git - - patch heldpackages: pkg.installed: @@ -135,7 +134,6 @@ commonpkgs: - lvm2 - openssl - git - - patch heldpackages: pkg.installed: diff --git a/salt/salt/helper-packages.sls b/salt/salt/helper-packages.sls new file mode 100644 index 000000000..93ad76a22 --- /dev/null +++ b/salt/salt/helper-packages.sls @@ -0,0 +1,3 @@ +patch_package: + pkg.installed: + - name: patch \ No newline at end of file diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 41f9bfaf2..0dc8760cb 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -7,6 +7,7 @@ include: - salt + - salt.helper_packages - systemd.reload {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 36347cb95..b939c3380 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -2,6 +2,9 @@ {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} {% from "salt/map.jinja" import PYTHON3INFLUX with context %} +include: + - salt.helper-packages + python3_influxdb: pkg.installed: - name: {{ PYTHON3INFLUX }} @@ -11,17 +14,23 @@ influxdb_continuous_query.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - - pkg: python3_influxdb + - require: + - pkg: python3_influxdb + - pkg: patch_package #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - - pkg: python3_influxdb + - require: + - pkg: python3_influxdb + - pkg: patch_package influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - - pkg: python3_influxdb \ No newline at end of file + - require: + - pkg: python3_influxdb + - pkg: patch_package \ No newline at end of file From 1fbf77d09067ca71c5b913d948cf714bf60904e9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:53:00 -0400 Subject: [PATCH 034/195] fix state name --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 0dc8760cb..2f4c3d2ac 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -7,7 +7,7 @@ include: - salt - - salt.helper_packages + - salt.helper-packages - systemd.reload {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} From 28982e0e0b9fa2cec0f8bd4f1ea734f99551da5a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 19:22:07 -0400 Subject: [PATCH 035/195] fix requirement --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 55f350099..8248c6228 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -110,7 +110,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - - sls: python3-influxdb + - sls: salt.python3-influxdb {% endfor %} {% endif %} From 8dd9564171b145126cfa4d02539c5ce395380def Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 28 Apr 2021 10:17:37 -0400 Subject: [PATCH 036/195] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 58e1f4fa9..678d59d4f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.50 +2.3.60 From 14557983e1714cb14d685b8168a9e7d2be47a54a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 28 Apr 2021 10:24:39 -0400 Subject: [PATCH 037/195] Update raid.sh --- salt/telegraf/scripts/raid.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/telegraf/scripts/raid.sh b/salt/telegraf/scripts/raid.sh index 03e309c38..f237e8021 100644 --- a/salt/telegraf/scripts/raid.sh +++ b/salt/telegraf/scripts/raid.sh @@ -16,6 +16,8 @@ # along with this program. If not, see . THEGREP=$(ps -ef | grep $0 | grep -v $$ | grep -v grep) +RAIDLOG=/var/log/raid/status.log +RAIDSTATUS=$(cat /var/log/raid/status.log) if [ ! "$THEGREP" ]; then From 7a21c44727f2b5b28cbf393040f5ebc1da183349 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 28 Apr 2021 13:27:11 -0400 Subject: [PATCH 038/195] Remove unused and incorrectly formatted osraid metric --- salt/common/tools/sbin/so-raid-status | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin/so-raid-status index 11909e012..e7da94878 100755 --- a/salt/common/tools/sbin/so-raid-status +++ b/salt/common/tools/sbin/so-raid-status @@ -17,18 +17,6 @@ . /usr/sbin/so-common -#check_boss_raid() { -# BOSSBIN=/opt/boss/mvcli -# BOSSRC=$($BOSSBIN info -o vd | grep functional) -# -# if [[ $BOSSRC ]]; then -# # Raid is good -# BOSSRAID=0 -# else -# BOSSRAID=1 -# fi -#} - check_lsi_raid() { # For use for LSI on Ubuntu #MEGA=/opt/MegaRAID/MegeCli/MegaCli64 @@ -66,13 +54,11 @@ mkdir -p /opt/so/log/raid {%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %} #check_boss_raid check_software_raid -#echo "osraid=$BOSSRAID nsmraid=$SWRAID" > /opt/so/log/raid/status.log -echo "osraid=1 nsmraid=$SWRAID" > /opt/so/log/raid/status.log +echo "nsmraid=$SWRAID" > /opt/so/log/raid/status.log {%- elif grains['sosmodel'] in ['SOS1000F', 'SOS1000', 'SOSSN7200', 'SOS10K', 'SOS4000'] %} #check_boss_raid check_lsi_raid -#echo "osraid=$BOSSRAID nsmraid=$LSIRAID" > /opt/so/log/raid/status.log -echo "osraid=1 nsmraid=$LSIRAID" > /opt/so/log/raid/status.log +echo "nsmraid=$LSIRAID" > /opt/so/log/raid/status.log {%- else %} exit 0 {%- endif %} From 5584c4f1aee056a0d8946bb74776157a1f71a859 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 28 Apr 2021 18:01:33 -0400 Subject: [PATCH 039/195] define and install the dependencies for python36-influxdb --- salt/salt/map.jinja | 1 + salt/salt/python3-influxdb.sls | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 21fcb9a58..a4289a6b8 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -15,6 +15,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} + {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index b939c3380..424f06493 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,10 +1,15 @@ {% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} {% from "salt/map.jinja" import PYTHON3INFLUX with context %} +{% from "salt/map.jinja" import PYTHON3INFLUXDEPS with context %} include: - salt.helper-packages +python3_influxdb_dependencies: + pkg.installed: + - pkgs: {{ PYTHON3INFLUXDEPS }} + python3_influxdb: pkg.installed: - name: {{ PYTHON3INFLUX }} From 454b541a2e02e8eddc8182726059ea60eb1eea79 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 28 Apr 2021 18:04:14 -0400 Subject: [PATCH 040/195] merge with dev, change version so test box doesnt try to upgrade to 2.3.60 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 678d59d4f..58e1f4fa9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.60 +2.3.50 From f8e01d5d53d83b3cf3221037abd2f18532c8985a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 06:43:05 -0400 Subject: [PATCH 041/195] let the state retry incase influxdb isnt fully up yet --- salt/influxdb/init.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 8248c6228..22e2c9856 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -75,6 +75,10 @@ telegraf_database: - require: - docker_container: so-influxdb - sls: salt.python3-influxdb + - timeout: 30 + - retry: + attempts: 5 + interval: 30 {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: From 907dbe638886e37947714e74d935270a6f9dde84 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 08:47:33 -0400 Subject: [PATCH 042/195] for for influx to be up so the reliant states dont fail --- salt/influxdb/init.sls | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 22e2c9856..609263bb5 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -63,6 +63,16 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +wait_for_influxdb: + http.query: + - name: 'https://{{MANAGER}}:8086/query?q=SHOW+DATABASES' + - ssl: True + - verify_ssl: False + - status: 200 + - timeout: 30 + - retry: + attempts: 5 + interval: 60 telegraf_database: influxdb_database.present: @@ -75,10 +85,7 @@ telegraf_database: - require: - docker_container: so-influxdb - sls: salt.python3-influxdb - - timeout: 30 - - retry: - attempts: 5 - interval: 30 + - http: wait_for_influxdb {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: From 93ee96b1cd224176f1e5e107cd3839e186b9a4ac Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 10:19:42 -0400 Subject: [PATCH 043/195] Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry --- setup/so-setup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index d7dd3b660..2160929cf 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -918,10 +918,11 @@ success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}') if [[ $success != 0 ]]; then SO_ERROR=1; fi # Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox -if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then +# Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry +if grep -q -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then SO_ERROR=1 - grep --color=never "ERROR" "$setup_log" > "$error_log" + grep --color=never "ERROR" "$setup_log" | grep -qvE "Status .* was not found" > "$error_log" fi if [[ -n $SO_ERROR ]]; then From 02b621bd2cec52d65c4924974ccbebd4e80b4f87 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 29 Apr 2021 11:29:07 -0400 Subject: [PATCH 044/195] Add Kratos to Backups --- salt/common/tools/sbin/so-config-backup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-config-backup b/salt/common/tools/sbin/so-config-backup index c8e504b4a..32be845ae 100755 --- a/salt/common/tools/sbin/so-config-backup +++ b/salt/common/tools/sbin/so-config-backup @@ -35,6 +35,7 @@ if [ ! -f $BACKUPFILE ]; then {%- endfor %} tar -rf $BACKUPFILE /etc/pki tar -rf $BACKUPFILE /etc/salt + tar -rf $BACKUPFILE /opt/so/conf/kratos fi From 76d735ff43eb7f55b59d4bffe6fa6541a5cb556c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 29 Apr 2021 11:49:20 -0400 Subject: [PATCH 045/195] Add ID Fix to nodeID --- salt/sensoroni/files/sensoroni.json | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sensoroni/files/sensoroni.json b/salt/sensoroni/files/sensoroni.json index 378d42373..1a6e6fc8c 100644 --- a/salt/sensoroni/files/sensoroni.json +++ b/salt/sensoroni/files/sensoroni.json @@ -15,6 +15,7 @@ "logFilename": "/opt/sensoroni/logs/sensoroni.log", "logLevel":"info", "agent": { + "nodeId": "{{ grains.host | lower }}", "role": "{{ grains.role }}", "description": "{{ DESCRIPTION }}", "address": "{{ ADDRESS }}", From 44ad8ce8881ff66c5b2a5f0f8ace6b231cf9d2e7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 29 Apr 2021 12:08:07 -0400 Subject: [PATCH 046/195] Switch to the ES-included community_id plugin --- salt/elasticsearch/files/ingest/filterlog | 2 +- salt/elasticsearch/files/ingest/sysmon | 4 ++-- salt/elasticsearch/files/ingest/zeek.common | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/files/ingest/filterlog b/salt/elasticsearch/files/ingest/filterlog index cb914f4a3..3e885fe54 100644 --- a/salt/elasticsearch/files/ingest/filterlog +++ b/salt/elasticsearch/files/ingest/filterlog @@ -51,7 +51,7 @@ }, { "set": { "field": "_index", "value": "so-firewall", "override": true } }, { "set": { "if": "ctx.network?.transport_id == '0'", "field": "network.transport", "value": "icmp", "override": true } }, - {"community_id": { "if": "ctx.network?.transport != null", "field":["source.ip","source.port","destination.ip","destination.port","network.transport"],"target_field":"network.community_id"}}, + {"community_id": {} }, { "set": { "field": "module", "value": "pfsense", "override": true } }, { "set": { "field": "dataset", "value": "firewall", "override": true } }, { "remove": { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"], "ignore_failure": true } } diff --git a/salt/elasticsearch/files/ingest/sysmon b/salt/elasticsearch/files/ingest/sysmon index 599899488..e4db4bcb4 100644 --- a/salt/elasticsearch/files/ingest/sysmon +++ b/salt/elasticsearch/files/ingest/sysmon @@ -1,7 +1,6 @@ { "description" : "sysmon", "processors" : [ - {"community_id": {"if": "ctx.winlog.event_data?.Protocol != null", "field":["winlog.event_data.SourceIp","winlog.event_data.SourcePort","winlog.event_data.DestinationIp","winlog.event_data.DestinationPort","winlog.event_data.Protocol"],"target_field":"network.community_id"}}, { "set": { "field": "event.code", "value": "{{winlog.event_id}}", "override": true } }, { "set": { "field": "event.module", "value": "sysmon", "override": true } }, { "set": { "if": "ctx.winlog?.computer_name != null", "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true } }, @@ -64,6 +63,7 @@ { "rename": { "field": "winlog.event_data.SourceIp", "target_field": "source.ip", "ignore_missing": true } }, { "rename": { "field": "winlog.event_data.SourcePort", "target_field": "source.port", "ignore_missing": true } }, { "rename": { "field": "winlog.event_data.targetFilename", "target_field": "file.target", "ignore_missing": true } }, - { "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } } + { "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } }, + { "community_id": {} } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common index 563f5956b..e7b898c6f 100644 --- a/salt/elasticsearch/files/ingest/zeek.common +++ b/salt/elasticsearch/files/ingest/zeek.common @@ -8,11 +8,11 @@ { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, - {"community_id": {"if": "ctx.network?.transport != null", "field":["message2.id.orig_h","message2.id.orig_p","message2.id.resp_h","message2.id.resp_p","network.transport"],"target_field":"network.community_id"}}, { "rename": { "field": "message2.id.orig_h", "target_field": "source.ip", "ignore_missing": true } }, { "rename": { "field": "message2.id.orig_p", "target_field": "source.port", "ignore_missing": true } }, { "rename": { "field": "message2.id.resp_h", "target_field": "destination.ip", "ignore_missing": true } }, { "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } }, + { "community_id": {} }, { "set": { "if": "ctx.source?.ip != null", "field": "client.ip", "value": "{{source.ip}}" } }, { "set": { "if": "ctx.source?.port != null", "field": "client.port", "value": "{{source.port}}" } }, { "set": { "if": "ctx.destination?.ip != null", "field": "server.ip", "value": "{{destination.ip}}" } }, From 450a01784befcb723409be8e293905f134c4d1c8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:22:31 -0400 Subject: [PATCH 047/195] support installing via pip for ubuntu --- salt/salt/map.jinja | 7 +++++-- salt/salt/python3-influxdb.sls | 10 +++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index a4289a6b8..8a3c26bb8 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -7,7 +7,9 @@ {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} - {% set PYTHON3INFLUX= 'python3-influxdb' %} + {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} + {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] + {% set PYTHONINSTALLER = 'pip' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} @@ -15,7 +17,8 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} - {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} + {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] + {% set PYTHONINSTALLER = 'pkg' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 424f06493..67fde3be2 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -2,16 +2,19 @@ {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} {% from "salt/map.jinja" import PYTHON3INFLUX with context %} {% from "salt/map.jinja" import PYTHON3INFLUXDEPS with context %} +{% from "salt/map.jinja" import PYTHONINSTALLER with context %} include: - salt.helper-packages +{# python3_influxdb_dependencies: - pkg.installed: + {{PYTHONINSTALLER}}.installed: - pkgs: {{ PYTHON3INFLUXDEPS }} +#} python3_influxdb: - pkg.installed: + {{PYTHONINSTALLER}}.installed: - name: {{ PYTHON3INFLUX }} #https://github.com/saltstack/salt/issues/59766 @@ -38,4 +41,5 @@ influxdbmod.py_shard_duration_patch: - source: salt://salt/files/influxdbmod.py.patch - require: - pkg: python3_influxdb - - pkg: patch_package \ No newline at end of file + - pkg: patch_package + - reload_modules: True \ No newline at end of file From e7660d68cb50827b9ed6901b16c4adcb0d7661bc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:25:29 -0400 Subject: [PATCH 048/195] add %} --- salt/salt/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 8a3c26bb8..9cf9331f5 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -8,7 +8,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} - {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] + {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} {% else %} {% set SPLITCHAR = '-' %} From 888d637b67e6444edb012c57d8a3531b36d537ef Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:26:24 -0400 Subject: [PATCH 049/195] add %} --- salt/salt/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 9cf9331f5..9b84b72e2 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -17,7 +17,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} - {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] + {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} {% endif %} From 689a01423fea65468f4b592052c66bdf7eab37c7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:28:31 -0400 Subject: [PATCH 050/195] fix deps --- salt/salt/python3-influxdb.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 67fde3be2..3cad07144 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -23,7 +23,7 @@ influxdb_continuous_query.present_patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - require: - - pkg: python3_influxdb + - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package #https://github.com/saltstack/salt/issues/59761 @@ -32,7 +32,7 @@ influxdb_retention_policy.present_patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - require: - - pkg: python3_influxdb + - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package influxdbmod.py_shard_duration_patch: @@ -40,6 +40,6 @@ influxdbmod.py_shard_duration_patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - require: - - pkg: python3_influxdb + - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package - reload_modules: True \ No newline at end of file From d475e50bef82df40fb59ce15dc087fe088abc314 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:49:15 -0400 Subject: [PATCH 051/195] add deps for ubuntu --- salt/salt/map.jinja | 2 +- salt/salt/python3-influxdb.sls | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 9b84b72e2..5b211e969 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -8,7 +8,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} - {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] %} + {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} {% else %} {% set SPLITCHAR = '-' %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 3cad07144..f65a0c6b4 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -7,11 +7,9 @@ include: - salt.helper-packages -{# python3_influxdb_dependencies: {{PYTHONINSTALLER}}.installed: - pkgs: {{ PYTHON3INFLUXDEPS }} -#} python3_influxdb: {{PYTHONINSTALLER}}.installed: From b6561fd8e2f20a00a9a9c42212c66ceb4ad414ae Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 29 Apr 2021 15:35:22 -0400 Subject: [PATCH 052/195] Update defaultRoute with new path structure --- salt/kibana/etc/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/etc/config.json b/salt/kibana/etc/config.json index 9631e0343..7593a0c27 100644 --- a/salt/kibana/etc/config.json +++ b/salt/kibana/etc/config.json @@ -1,7 +1,7 @@ { "attributes": { "defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29", - "defaultRoute":"/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645", + "defaultRoute":"/kibana/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645", "discover:sampleSize":"100", "dashboard:defaultDarkTheme":true, "theme:darkMode":true, From dda07af4d45a38ae290fc9277e806a0be176525e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 29 Apr 2021 17:44:15 -0400 Subject: [PATCH 053/195] Update Kibana config defaults --- salt/kibana/etc/config.json | 10 ---------- salt/kibana/files/saved_objects.ndjson | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 salt/kibana/etc/config.json diff --git a/salt/kibana/etc/config.json b/salt/kibana/etc/config.json deleted file mode 100644 index 7593a0c27..000000000 --- a/salt/kibana/etc/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ "attributes": - { - "defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29", - "defaultRoute":"/kibana/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645", - "discover:sampleSize":"100", - "dashboard:defaultDarkTheme":true, - "theme:darkMode":true, - "timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}" - } -} diff --git a/salt/kibana/files/saved_objects.ndjson b/salt/kibana/files/saved_objects.ndjson index fee868127..863209d48 100644 --- a/salt/kibana/files/saved_objects.ndjson +++ b/salt/kibana/files/saved_objects.ndjson @@ -460,7 +460,7 @@ {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\",\"time_zone\":\"America/New_York\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":48,\"x\":0,\"y\":0,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":96,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":8,\"y\":20,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":28,\"y\":20,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":0,\"y\":72,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":48,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":24,\"x\":16,\"y\":72,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":28,\"y\":8,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":120,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"event_type\",\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"}]","timeRestore":false,"title":"z16.04 - Sysmon - Logs","version":1},"id":"6d189680-6d62-11e7-8ddb-e71eb260f4a3","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"8cfdeff0-6d6b-11e7-ad64-15aa071374a6","name":"panel_1","type":"visualization"},{"id":"0eb1fd80-6d70-11e7-b09b-f57b22df6524","name":"panel_2","type":"visualization"},{"id":"3072c750-6d71-11e7-b09b-f57b22df6524","name":"panel_3","type":"visualization"},{"id":"7bc74b40-6d71-11e7-b09b-f57b22df6524","name":"panel_4","type":"visualization"},{"id":"13ed0810-6d72-11e7-b09b-f57b22df6524","name":"panel_5","type":"visualization"},{"id":"3b6c92c0-6d72-11e7-b09b-f57b22df6524","name":"panel_6","type":"visualization"},{"id":"e09f6010-6d72-11e7-b09b-f57b22df6524","name":"panel_7","type":"visualization"},{"id":"29611940-6d75-11e7-b09b-f57b22df6524","name":"panel_8","type":"visualization"},{"id":"6b70b840-6d75-11e7-b09b-f57b22df6524","name":"panel_9","type":"visualization"},{"id":"248c1d20-6d6b-11e7-ad64-15aa071374a6","name":"panel_10","type":"search"},{"id":"AWDHHk1sxQT5EBNmq43Y","name":"panel_11","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQyLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} -{"attributes":{"buildNum":33984,"dashboard:defaultDarkTheme":true,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":"100","theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"id":"7.11.2","migrationVersion":{"config":"7.9.0"},"references":[],"type":"config","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ1LDRd"} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.12.1","id":"7.12.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":56,\"x\":0,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":8,\"y\":32,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":20,\"y\":32,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":56,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"columns\":[\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"uid\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":32,\"y\":32,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":40,\"h\":24,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"}]","timeRestore":false,"title":"z16.04 - Bro - Modbus","version":1},"id":"70c005f0-3583-11e7-a588-05992195c551","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"0d168a30-363f-11e7-a6f7-4f44d7bf1c33","name":"panel_1","type":"visualization"},{"id":"20eabd60-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_2","type":"visualization"},{"id":"3c65f500-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_3","type":"visualization"},{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"panel_4","type":"search"},{"id":"178209e0-6e1b-11e7-b553-7f80727663c1","name":"panel_5","type":"visualization"},{"id":"AWDG_9KpxQT5EBNmq4Oo","name":"panel_6","type":"visualization"},{"id":"453f8b90-4a58-11e8-9b0a-f1d33346f773","name":"panel_7","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ4LDRd"} From e32ca284c52c86aee3274bcf0940f6c6d26920c7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 30 Apr 2021 11:10:31 -0400 Subject: [PATCH 054/195] ensure proper version of python3-influxdb is installed prior to running the states that require it --- salt/influxdb/init.sls | 6 ++++++ salt/salt/python3-influxdb.sls | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 609263bb5..cda095a31 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -7,6 +7,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% import_yaml 'influxdb/defaults.yaml' as default_settings %} {% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} +{% from 'salt/map.jinja' import PYTHON3INFLUX with context %} +{% set PYTHONINFLUXVERSION = PYTHON3INFLUX.split("==")[1] | trim %} +{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) | trim %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} @@ -63,6 +66,8 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +# We have to make sure the influxdb module is the right version prior to state run since reload_modules is bugged +{% if PYTHONINFLUXVERSIONINSTALLED == PYTHONINFLUXVERSION %} wait_for_influxdb: http.query: - name: 'https://{{MANAGER}}:8086/query?q=SHOW+DATABASES' @@ -124,6 +129,7 @@ so_downsample_cq: - sls: salt.python3-influxdb {% endfor %} +{% endif %} {% endif %} {% else %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index f65a0c6b4..acadf2161 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -33,11 +33,12 @@ influxdb_retention_policy.present_patch: - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package +# We should be able to set reload_modules: True in this state in order to tell salt to reload its python modules due to us possibly installing +# and possibly modifying modules in this state. This is bugged according to https://github.com/saltstack/salt/issues/24925 influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - require: - {{PYTHONINSTALLER}}: python3_influxdb - - pkg: patch_package - - reload_modules: True \ No newline at end of file + - pkg: patch_package \ No newline at end of file From 038cadeae892c6efdb824cb6274eaee8bf4b4924 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 30 Apr 2021 12:31:57 -0400 Subject: [PATCH 055/195] change version to 2.3.60 to prepare to push to dev --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 58e1f4fa9..678d59d4f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.50 +2.3.60 From d6b9154a88368e710cd5683036b3ec76db4d6dfd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 30 Apr 2021 14:48:51 -0400 Subject: [PATCH 056/195] change how version to be installed is defined to work with centos --- salt/influxdb/init.sls | 4 ++-- salt/salt/map.jinja | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index cda095a31..c6120d292 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -8,8 +8,8 @@ {% import_yaml 'influxdb/defaults.yaml' as default_settings %} {% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} {% from 'salt/map.jinja' import PYTHON3INFLUX with context %} -{% set PYTHONINFLUXVERSION = PYTHON3INFLUX.split("==")[1] | trim %} -{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) | trim %} +{% from 'salt/map.jinja' import PYTHONINFLUXVERSION with context %} +{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5b211e969..b0b9ffb2a 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -7,7 +7,8 @@ {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} - {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} + {% set PYTHONINFLUXVERSION = '5.3.1' %} + {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION %} {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} {% else %} @@ -16,6 +17,7 @@ {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} + {% set PYTHONINFLUXVERSION = '5.3.1' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} From b80dd1ef3eb0f34df9a1f5642f08ba8da36abe02 Mon Sep 17 00:00:00 2001 From: Pete Date: Fri, 30 Apr 2021 20:21:50 +0000 Subject: [PATCH 057/195] fix 5-second sleep using wait here instead of sleep tries to wait until pid 5 exits and generates the error > /usr/sbin/so-playbook-reset: line 25: wait: pid 5 is not a child of this shell --- salt/common/tools/sbin/so-playbook-reset | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-playbook-reset b/salt/common/tools/sbin/so-playbook-reset index 9113fd2b8..ca1d3f57f 100755 --- a/salt/common/tools/sbin/so-playbook-reset +++ b/salt/common/tools/sbin/so-playbook-reset @@ -22,5 +22,5 @@ salt-call state.apply playbook.db_init,playbook,playbook.automation_user_create /usr/sbin/so-soctopus-restart echo "Importing Plays - this will take some time...." -wait 5 -/usr/sbin/so-playbook-ruleupdate \ No newline at end of file +sleep 5 +/usr/sbin/so-playbook-ruleupdate From 19dd9b97d29256247d24ecc9520ace3128116ab2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 3 May 2021 09:40:53 -0400 Subject: [PATCH 058/195] Don't ask for node description on eval and import installs --- setup/so-setup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index d7dd3b660..f85cba628 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -291,7 +291,7 @@ if ! [[ -f $install_opt_file ]]; then collect_hostname fi - whiptail_node_description + [[ ! ( $is_eval || $is_import ) ]] && whiptail_node_description if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then network_init_whiptail @@ -960,4 +960,4 @@ fi install_cleanup >> "$setup_log" 2>&1 -if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit; fi \ No newline at end of file +if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit; fi From 619402cc671bde3c99b4473d11d8effd37ddb752 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 3 May 2021 17:03:30 +0000 Subject: [PATCH 059/195] Add event_data to common template so elastalert/playbook event_data fields can be indexed and searchable --- salt/elasticsearch/templates/so/so-common-template.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index c1f0a6755..8adbdcf13 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -228,7 +228,11 @@ "event":{ "type":"object", "dynamic": true - }, + }, + "event_data":{ + "type":"object", + "dynamic": true + }, "file":{ "type":"object", "dynamic": true From 12306368cfe6125869d32319ef471d76856c2aee Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 08:37:52 -0400 Subject: [PATCH 060/195] add post upgrade function for 2.3.60 soup to migrate influxdb data --- salt/common/tools/sbin/soup | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b3484446f..395ebef36 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -284,6 +284,7 @@ postupgrade_changes() { [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2 [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30 [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40 + [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60 } post_rc1_to_2.3.21() { @@ -304,6 +305,15 @@ post_2.3.30_to_2.3.40() { POSTVERSION=2.3.40 } +post_2.3.5X_to_2.3.60() { + # These are post upgrade changes that need to take place after all the new states have been applied. + echo "" + echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes." + docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$SOUP_LOG" 2>&1 & + + POSTVERSION=2.3.60 +} + rc1_to_rc2() { From 9b065155f46bd609ccf24e9b818a0219ed00d728 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 08:56:13 -0400 Subject: [PATCH 061/195] log iunfluxdb migration to new log --- salt/common/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 395ebef36..b1e326c73 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -24,6 +24,7 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) DEFAULT_SALT_DIR=/opt/so/saltstack/default BATCHSIZE=5 SOUP_LOG=/root/soup.log +INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup add_common() { @@ -309,7 +310,7 @@ post_2.3.5X_to_2.3.60() { # These are post upgrade changes that need to take place after all the new states have been applied. echo "" echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes." - docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$SOUP_LOG" 2>&1 & + docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & POSTVERSION=2.3.60 } From f186a3dde923ce29fe9eccd7a7cd0e451d4c65b4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 09:30:38 -0400 Subject: [PATCH 062/195] make sure user sees influxdb migration message by requiring enter to be pressed to continue --- salt/common/tools/sbin/soup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b1e326c73..b08271e1f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -309,7 +309,9 @@ post_2.3.30_to_2.3.40() { post_2.3.5X_to_2.3.60() { # These are post upgrade changes that need to take place after all the new states have been applied. echo "" - echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes." + echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." + echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" + read -p "Press enter to continue." docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & POSTVERSION=2.3.60 From e2f52765e4ae48e851ddad7675df6239906c43c5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 09:34:42 -0400 Subject: [PATCH 063/195] add newline --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b08271e1f..de968caea 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -312,6 +312,7 @@ post_2.3.5X_to_2.3.60() { echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." + echo "" docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & POSTVERSION=2.3.60 From a0f183916237be25782ea7c1ed8534cb84a052e4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 09:59:16 -0400 Subject: [PATCH 064/195] run in background --- salt/common/tools/sbin/soup | 2 +- salt/influxdb/defaults.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index de968caea..4c82e48c7 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -313,7 +313,7 @@ post_2.3.5X_to_2.3.60() { echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." echo "" - docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & + docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 POSTVERSION=2.3.60 } diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 03ac4c3c6..54e175024 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,12 +2,12 @@ influxdb: retention_policies: autogen: default: True - duration: 1d - shard_duration: 1h + duration: 30d + shard_duration: 1d so_long_term: default: False - duration: 7d - shard_duration: 1d + duration: 0s + shard_duration: 7d downsample: so_long_term: resolution: 5m \ No newline at end of file From 2d91e509faecf7f32bb22b808e2c5817a3248bb7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 10:34:13 -0400 Subject: [PATCH 065/195] update wording --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 4c82e48c7..0e6f84a63 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -309,7 +309,7 @@ post_2.3.30_to_2.3.40() { post_2.3.5X_to_2.3.60() { # These are post upgrade changes that need to take place after all the new states have been applied. echo "" - echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." + echo "Migration of InfluxDB data will begin in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." echo "" From 3dcaa1f6fb8494d08935ceecec14d8e7972b0c32 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 10:37:25 -0400 Subject: [PATCH 066/195] more logging for influxdb migration --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0e6f84a63..07a9804b2 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -313,7 +313,7 @@ post_2.3.5X_to_2.3.60() { echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." echo "" - docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 + docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*" && echo "InfluxDB migration complete." || echo "InfluxDB migration failed."' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 POSTVERSION=2.3.60 } From 0622c77a7f51dda93ddce0bbf2f12b2b8cbb925f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 4 May 2021 10:50:13 -0400 Subject: [PATCH 067/195] Add filebeat modules --- salt/filebeat/modules/activemq.yml.disabled | 19 ++ salt/filebeat/modules/apache.yml.disabled | 19 ++ salt/filebeat/modules/auditd.yml.disabled | 10 + salt/filebeat/modules/aws.yml.disabled | 255 ++++++++++++++++++ salt/filebeat/modules/azure.yml.disabled | 45 ++++ salt/filebeat/modules/barracuda.yml.disabled | 41 +++ salt/filebeat/modules/bluecoat.yml.disabled | 22 ++ salt/filebeat/modules/cef.yml.disabled | 17 ++ salt/filebeat/modules/checkpoint.yml.disabled | 24 ++ salt/filebeat/modules/cisco.yml.disabled | 142 ++++++++++ salt/filebeat/modules/coredns.yml.disabled | 11 + .../filebeat/modules/crowdstrike.yml.disabled | 11 + salt/filebeat/modules/cyberark.yml.disabled | 22 ++ salt/filebeat/modules/cylance.yml.disabled | 22 ++ .../modules/elasticsearch.yml.disabled | 35 +++ salt/filebeat/modules/envoyproxy.yml.disabled | 11 + salt/filebeat/modules/f5.yml.disabled | 41 +++ salt/filebeat/modules/fortinet.yml.disabled | 83 ++++++ salt/filebeat/modules/gcp.yml.disabled | 76 ++++++ .../modules/google_workspace.yml.disabled | 53 ++++ .../filebeat/modules/googlecloud.yml.disabled | 58 ++++ salt/filebeat/modules/gsuite.yml.disabled | 53 ++++ salt/filebeat/modules/haproxy.yml.disabled | 14 + salt/filebeat/modules/ibmmq.yml.disabled | 11 + salt/filebeat/modules/icinga.yml.disabled | 27 ++ salt/filebeat/modules/iis.yml.disabled | 20 ++ salt/filebeat/modules/imperva.yml.disabled | 22 ++ salt/filebeat/modules/infoblox.yml.disabled | 22 ++ salt/filebeat/modules/iptables.yml.disabled | 13 + salt/filebeat/modules/juniper.yml.disabled | 54 ++++ salt/filebeat/modules/kafka.yml.disabled | 15 ++ salt/filebeat/modules/kibana.yml.disabled | 19 ++ salt/filebeat/modules/logstash.yml.disabled | 18 ++ salt/filebeat/modules/microsoft.yml.disabled | 49 ++++ salt/filebeat/modules/misp.yml.disabled | 17 ++ salt/filebeat/modules/mongodb.yml.disabled | 11 + salt/filebeat/modules/mssql.yml.disabled | 11 + salt/filebeat/modules/mysql.yml.disabled | 19 ++ .../modules/mysqlenterprise.yml.disabled | 14 + salt/filebeat/modules/nats.yml.disabled | 11 + salt/filebeat/modules/netflow.yml.disabled | 14 + salt/filebeat/modules/netscout.yml.disabled | 22 ++ salt/filebeat/modules/nginx.yml.disabled | 27 ++ salt/filebeat/modules/o365.yml.disabled | 48 ++++ salt/filebeat/modules/okta.yml.disabled | 10 + salt/filebeat/modules/oracle.yml.disabled | 13 + salt/filebeat/modules/osquery.yml.disabled | 15 ++ salt/filebeat/modules/panw.yml.disabled | 22 ++ salt/filebeat/modules/pensando.yml.disabled | 13 + salt/filebeat/modules/postgresql.yml.disabled | 11 + salt/filebeat/modules/proofpoint.yml.disabled | 22 ++ salt/filebeat/modules/rabbitmq.yml.disabled | 11 + salt/filebeat/modules/radware.yml.disabled | 22 ++ salt/filebeat/modules/redis.yml.disabled | 21 ++ salt/filebeat/modules/santa.yml.disabled | 9 + salt/filebeat/modules/snort.yml.disabled | 22 ++ salt/filebeat/modules/snyk.yml.disabled | 112 ++++++++ salt/filebeat/modules/sonicwall.yml.disabled | 22 ++ salt/filebeat/modules/sophos.yml.disabled | 46 ++++ salt/filebeat/modules/squid.yml.disabled | 22 ++ salt/filebeat/modules/suricata.yml.disabled | 11 + salt/filebeat/modules/system.yml.disabled | 19 ++ .../filebeat/modules/threatintel.yml.disabled | 105 ++++++++ salt/filebeat/modules/tomcat.yml.disabled | 22 ++ salt/filebeat/modules/traefik.yml.disabled | 11 + salt/filebeat/modules/zeek.yml.disabled | 84 ++++++ salt/filebeat/modules/zoom.yml.disabled | 22 ++ salt/filebeat/modules/zscaler.yml.disabled | 22 ++ 68 files changed, 2237 insertions(+) create mode 100644 salt/filebeat/modules/activemq.yml.disabled create mode 100644 salt/filebeat/modules/apache.yml.disabled create mode 100644 salt/filebeat/modules/auditd.yml.disabled create mode 100644 salt/filebeat/modules/aws.yml.disabled create mode 100644 salt/filebeat/modules/azure.yml.disabled create mode 100644 salt/filebeat/modules/barracuda.yml.disabled create mode 100644 salt/filebeat/modules/bluecoat.yml.disabled create mode 100644 salt/filebeat/modules/cef.yml.disabled create mode 100644 salt/filebeat/modules/checkpoint.yml.disabled create mode 100644 salt/filebeat/modules/cisco.yml.disabled create mode 100644 salt/filebeat/modules/coredns.yml.disabled create mode 100644 salt/filebeat/modules/crowdstrike.yml.disabled create mode 100644 salt/filebeat/modules/cyberark.yml.disabled create mode 100644 salt/filebeat/modules/cylance.yml.disabled create mode 100644 salt/filebeat/modules/elasticsearch.yml.disabled create mode 100644 salt/filebeat/modules/envoyproxy.yml.disabled create mode 100644 salt/filebeat/modules/f5.yml.disabled create mode 100644 salt/filebeat/modules/fortinet.yml.disabled create mode 100644 salt/filebeat/modules/gcp.yml.disabled create mode 100644 salt/filebeat/modules/google_workspace.yml.disabled create mode 100644 salt/filebeat/modules/googlecloud.yml.disabled create mode 100644 salt/filebeat/modules/gsuite.yml.disabled create mode 100644 salt/filebeat/modules/haproxy.yml.disabled create mode 100644 salt/filebeat/modules/ibmmq.yml.disabled create mode 100644 salt/filebeat/modules/icinga.yml.disabled create mode 100644 salt/filebeat/modules/iis.yml.disabled create mode 100644 salt/filebeat/modules/imperva.yml.disabled create mode 100644 salt/filebeat/modules/infoblox.yml.disabled create mode 100644 salt/filebeat/modules/iptables.yml.disabled create mode 100644 salt/filebeat/modules/juniper.yml.disabled create mode 100644 salt/filebeat/modules/kafka.yml.disabled create mode 100644 salt/filebeat/modules/kibana.yml.disabled create mode 100644 salt/filebeat/modules/logstash.yml.disabled create mode 100644 salt/filebeat/modules/microsoft.yml.disabled create mode 100644 salt/filebeat/modules/misp.yml.disabled create mode 100644 salt/filebeat/modules/mongodb.yml.disabled create mode 100644 salt/filebeat/modules/mssql.yml.disabled create mode 100644 salt/filebeat/modules/mysql.yml.disabled create mode 100644 salt/filebeat/modules/mysqlenterprise.yml.disabled create mode 100644 salt/filebeat/modules/nats.yml.disabled create mode 100644 salt/filebeat/modules/netflow.yml.disabled create mode 100644 salt/filebeat/modules/netscout.yml.disabled create mode 100644 salt/filebeat/modules/nginx.yml.disabled create mode 100644 salt/filebeat/modules/o365.yml.disabled create mode 100644 salt/filebeat/modules/okta.yml.disabled create mode 100644 salt/filebeat/modules/oracle.yml.disabled create mode 100644 salt/filebeat/modules/osquery.yml.disabled create mode 100644 salt/filebeat/modules/panw.yml.disabled create mode 100644 salt/filebeat/modules/pensando.yml.disabled create mode 100644 salt/filebeat/modules/postgresql.yml.disabled create mode 100644 salt/filebeat/modules/proofpoint.yml.disabled create mode 100644 salt/filebeat/modules/rabbitmq.yml.disabled create mode 100644 salt/filebeat/modules/radware.yml.disabled create mode 100644 salt/filebeat/modules/redis.yml.disabled create mode 100644 salt/filebeat/modules/santa.yml.disabled create mode 100644 salt/filebeat/modules/snort.yml.disabled create mode 100644 salt/filebeat/modules/snyk.yml.disabled create mode 100644 salt/filebeat/modules/sonicwall.yml.disabled create mode 100644 salt/filebeat/modules/sophos.yml.disabled create mode 100644 salt/filebeat/modules/squid.yml.disabled create mode 100644 salt/filebeat/modules/suricata.yml.disabled create mode 100644 salt/filebeat/modules/system.yml.disabled create mode 100644 salt/filebeat/modules/threatintel.yml.disabled create mode 100644 salt/filebeat/modules/tomcat.yml.disabled create mode 100644 salt/filebeat/modules/traefik.yml.disabled create mode 100644 salt/filebeat/modules/zeek.yml.disabled create mode 100644 salt/filebeat/modules/zoom.yml.disabled create mode 100644 salt/filebeat/modules/zscaler.yml.disabled diff --git a/salt/filebeat/modules/activemq.yml.disabled b/salt/filebeat/modules/activemq.yml.disabled new file mode 100644 index 000000000..43536ecbc --- /dev/null +++ b/salt/filebeat/modules/activemq.yml.disabled @@ -0,0 +1,19 @@ +# Module: activemq +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-activemq.html + +- module: activemq + # Audit logs + audit: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Application logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/apache.yml.disabled b/salt/filebeat/modules/apache.yml.disabled new file mode 100644 index 000000000..b923dd581 --- /dev/null +++ b/salt/filebeat/modules/apache.yml.disabled @@ -0,0 +1,19 @@ +# Module: apache +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-apache.html + +- module: apache + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/auditd.yml.disabled b/salt/filebeat/modules/auditd.yml.disabled new file mode 100644 index 000000000..76296ec85 --- /dev/null +++ b/salt/filebeat/modules/auditd.yml.disabled @@ -0,0 +1,10 @@ +# Module: auditd +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-auditd.html + +- module: auditd + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/aws.yml.disabled b/salt/filebeat/modules/aws.yml.disabled new file mode 100644 index 000000000..904bd976c --- /dev/null +++ b/salt/filebeat/modules/aws.yml.disabled @@ -0,0 +1,255 @@ +# Module: aws +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-aws.html + +- module: aws + cloudtrail: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Process CloudTrail logs + # default is true, set to false to skip Cloudtrail logs + # var.process_cloudtrail_logs: false + + # Process CloudTrail Digest logs + # default true, set to false to skip CloudTrail Digest logs + # var.process_digest_logs: false + + # Process CloudTrail Insight logs + # default true, set to false to skip CloudTrail Insight logs + # var.process_insight_logs: false + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + cloudwatch: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + ec2: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + elb: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + s3access: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + + vpcflow: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # Use access_key_id, secret_access_key and/or session_token instead of shared credential file + #var.access_key_id: access_key_id + #var.secret_access_key: secret_access_key + #var.session_token: session_token + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + # AWS IAM Role to assume + #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 diff --git a/salt/filebeat/modules/azure.yml.disabled b/salt/filebeat/modules/azure.yml.disabled new file mode 100644 index 000000000..3b2bc1ecf --- /dev/null +++ b/salt/filebeat/modules/azure.yml.disabled @@ -0,0 +1,45 @@ +# Module: azure +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-azure.html + +- module: azure + # All logs + activitylogs: + enabled: true + var: + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + eventhub: "insights-operational-logs" + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + consumer_group: "$Default" + # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string + connection_string: "" + # the name of the storage account the state/offsets will be stored and updated + storage_account: "" + # the storage account key, this key will be used to authorize access to data in your storage account + storage_account_key: "" + + platformlogs: + enabled: false + # var: + # eventhub: "" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + + auditlogs: + enabled: false + # var: + # eventhub: "insights-logs-auditlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + signinlogs: + enabled: false + # var: + # eventhub: "insights-logs-signinlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" diff --git a/salt/filebeat/modules/barracuda.yml.disabled b/salt/filebeat/modules/barracuda.yml.disabled new file mode 100644 index 000000000..99ff85036 --- /dev/null +++ b/salt/filebeat/modules/barracuda.yml.disabled @@ -0,0 +1,41 @@ +# Module: barracuda +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-barracuda.html + +- module: barracuda + waf: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9503 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + spamfirewall: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9524 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/bluecoat.yml.disabled b/salt/filebeat/modules/bluecoat.yml.disabled new file mode 100644 index 000000000..6550c8eed --- /dev/null +++ b/salt/filebeat/modules/bluecoat.yml.disabled @@ -0,0 +1,22 @@ +# Module: bluecoat +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-bluecoat.html + +- module: bluecoat + director: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9505 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/cef.yml.disabled b/salt/filebeat/modules/cef.yml.disabled new file mode 100644 index 000000000..2de22edcc --- /dev/null +++ b/salt/filebeat/modules/cef.yml.disabled @@ -0,0 +1,17 @@ +# Module: cef +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cef.html + +- module: cef + log: + enabled: true + var: + syslog_host: localhost + syslog_port: 9003 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] diff --git a/salt/filebeat/modules/checkpoint.yml.disabled b/salt/filebeat/modules/checkpoint.yml.disabled new file mode 100644 index 000000000..9d34b8d72 --- /dev/null +++ b/salt/filebeat/modules/checkpoint.yml.disabled @@ -0,0 +1,24 @@ +# Module: checkpoint +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-checkpoint.html + +- module: checkpoint + firewall: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: syslog + + # The interface to listen to UDP based syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The UDP port to listen for syslog traffic. Defaults to 9001. + #var.syslog_port: 9001 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] diff --git a/salt/filebeat/modules/cisco.yml.disabled b/salt/filebeat/modules/cisco.yml.disabled new file mode 100644 index 000000000..9e4658045 --- /dev/null +++ b/salt/filebeat/modules/cisco.yml.disabled @@ -0,0 +1,142 @@ +# Module: cisco +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cisco.html + +- module: cisco + asa: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: syslog + + # The interface to listen to UDP based syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The UDP port to listen for syslog traffic. Defaults to 9001. + #var.syslog_port: 9001 + + # Set the log level from 1 (alerts only) to 7 (include all messages). + # Messages with a log level higher than the specified will be dropped. + # See https://www.cisco.com/c/en/us/td/docs/security/asa/syslog/b_syslog/syslogs-sev-level.html + #var.log_level: 7 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] + + ftd: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: syslog + + # The interface to listen to UDP based syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The UDP port to listen for syslog traffic. Defaults to 9003. + #var.syslog_port: 9003 + + # Set the log level from 1 (alerts only) to 7 (include all messages). + # Messages with a log level higher than the specified will be dropped. + # See https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide/syslogs-sev-level.html + #var.log_level: 7 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] + + ios: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: syslog + + # The interface to listen to UDP based syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The UDP port to listen for syslog traffic. Defaults to 9002. + #var.syslog_port: 9002 + + # Set custom paths for the log files when using file input. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + nexus: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9506 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + meraki: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9525 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + umbrella: + enabled: true + + #var.input: aws-s3 + # AWS SQS queue url + #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue + # Access ID to authenticate with the S3 input + #var.access_key_id: 123456 + # Access key to authenticate with the S3 input + #var.secret_access_key: PASSWORD + # The duration that the received messages are hidden from ReceiveMessage request + #var.visibility_timeout: 300s + # Maximum duration before AWS API request will be interrupted + #var.api_timeout: 120s + + amp: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + + # The API URL + #var.url: https://api.amp.cisco.com/v1/events + # The client ID used as a username for the API requests. + #var.client_id: + # The API key related to the client ID. + #var.api_key: + # How far to look back the first time the module is started. Expects an amount of hours. + #var.first_interval: 24h + # Overriding the default request timeout, optional. + #var.request_timeout: 60s diff --git a/salt/filebeat/modules/coredns.yml.disabled b/salt/filebeat/modules/coredns.yml.disabled new file mode 100644 index 000000000..46e9e55c1 --- /dev/null +++ b/salt/filebeat/modules/coredns.yml.disabled @@ -0,0 +1,11 @@ +# Module: coredns +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-coredns.html + +- module: coredns + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/crowdstrike.yml.disabled b/salt/filebeat/modules/crowdstrike.yml.disabled new file mode 100644 index 000000000..8d2c8531d --- /dev/null +++ b/salt/filebeat/modules/crowdstrike.yml.disabled @@ -0,0 +1,11 @@ +# Module: crowdstrike +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-crowdstrike.html + +- module: crowdstrike + + falcon: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/cyberark.yml.disabled b/salt/filebeat/modules/cyberark.yml.disabled new file mode 100644 index 000000000..e97955adf --- /dev/null +++ b/salt/filebeat/modules/cyberark.yml.disabled @@ -0,0 +1,22 @@ +# Module: cyberark +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cyberark.html + +- module: cyberark + corepas: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9527 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/cylance.yml.disabled b/salt/filebeat/modules/cylance.yml.disabled new file mode 100644 index 000000000..342d654d2 --- /dev/null +++ b/salt/filebeat/modules/cylance.yml.disabled @@ -0,0 +1,22 @@ +# Module: cylance +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cylance.html + +- module: cylance + protect: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9508 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/elasticsearch.yml.disabled b/salt/filebeat/modules/elasticsearch.yml.disabled new file mode 100644 index 000000000..e6074c05e --- /dev/null +++ b/salt/filebeat/modules/elasticsearch.yml.disabled @@ -0,0 +1,35 @@ +# Module: elasticsearch +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-elasticsearch.html + +- module: elasticsearch + # Server log + server: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + gc: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + audit: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + slowlog: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + deprecation: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/envoyproxy.yml.disabled b/salt/filebeat/modules/envoyproxy.yml.disabled new file mode 100644 index 000000000..543b17be5 --- /dev/null +++ b/salt/filebeat/modules/envoyproxy.yml.disabled @@ -0,0 +1,11 @@ +# Module: envoyproxy +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-envoyproxy.html + +- module: envoyproxy + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/f5.yml.disabled b/salt/filebeat/modules/f5.yml.disabled new file mode 100644 index 000000000..959842174 --- /dev/null +++ b/salt/filebeat/modules/f5.yml.disabled @@ -0,0 +1,41 @@ +# Module: f5 +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-f5.html + +- module: f5 + bigipapm: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9504 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + bigipafm: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9528 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/fortinet.yml.disabled b/salt/filebeat/modules/fortinet.yml.disabled new file mode 100644 index 000000000..281b7d788 --- /dev/null +++ b/salt/filebeat/modules/fortinet.yml.disabled @@ -0,0 +1,83 @@ +# Module: fortinet +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-fortinet.html + +- module: fortinet + firewall: + enabled: true + + # Set which input to use between tcp, udp (default) or file. + #var.input: udp + + # The interface to listen to syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The port to listen for syslog traffic. Defaults to 9004. + #var.syslog_port: 9004 + + # Set internal interfaces. used to override parsed network.direction + # based on a tagged interface. Both internal and external interfaces must be + # set to leverage this functionality. + #var.internal_interfaces: [ "LAN" ] + + # Set external interfaces. used to override parsed network.direction + # based on a tagged interface. Both internal and external interfaces must be + # set to leverage this functionality. + #var.external_interfaces: [ "WAN" ] + + clientendpoint: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9510 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + fortimail: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9529 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + fortimanager: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9530 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/gcp.yml.disabled b/salt/filebeat/modules/gcp.yml.disabled new file mode 100644 index 000000000..a09d0fe36 --- /dev/null +++ b/salt/filebeat/modules/gcp.yml.disabled @@ -0,0 +1,76 @@ +# Module: gcp +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-gcp.html + +- module: gcp + vpcflow: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be + # configured to use this topic as a sink for VPC flow logs. + var.topic: gcp-vpc-flowlogs + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + # Set internal networks. This is used to classify network.direction based + # off of what networks are considered "internal" either base off of a CIDR + # block or named network conditions. If this is not specified, then traffic + # direction is determined by whether it is between source and destination + # instance information rather than IP. + # + # For a full list of network conditions see: + # https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network + #var.internal_networks: [ "private" ] + + firewall: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-firewall + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-firewall-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + # Set internal networks. This is used to classify network.direction based + # off of what networks are considered "internal" either base off of a CIDR + # block or named network conditions. If this is not specified, then traffic + # is taken from the direction data in the rule_details event payload. + # + # For a full list of network conditions see: + # https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network + #var.internal_networks: [ "private" ] + + audit: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-audit + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-audit + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/salt/filebeat/modules/google_workspace.yml.disabled b/salt/filebeat/modules/google_workspace.yml.disabled new file mode 100644 index 000000000..6d364af98 --- /dev/null +++ b/salt/filebeat/modules/google_workspace.yml.disabled @@ -0,0 +1,53 @@ +# Module: google_workspace +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-google_workspace.html + +- module: google_workspace + saml: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + user_accounts: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + login: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + admin: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + drive: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + groups: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + diff --git a/salt/filebeat/modules/googlecloud.yml.disabled b/salt/filebeat/modules/googlecloud.yml.disabled new file mode 100644 index 000000000..9a28dc036 --- /dev/null +++ b/salt/filebeat/modules/googlecloud.yml.disabled @@ -0,0 +1,58 @@ +# Module: googlecloud +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-googlecloud.html + +# googlecloud module is deprecated, please use gcp instead +- module: gcp + vpcflow: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be + # configured to use this topic as a sink for VPC flow logs. + var.topic: gcp-vpc-flowlogs + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-vpc-flowlogs-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + firewall: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-firewall + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-firewall-sub + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json + + audit: + enabled: true + + # Google Cloud project ID. + var.project_id: my-gcp-project-id + + # Google Pub/Sub topic containing firewall logs. Stackdriver must be + # configured to use this topic as a sink for firewall logs. + var.topic: gcp-vpc-audit + + # Google Pub/Sub subscription for the topic. Filebeat will create this + # subscription if it does not exist. + var.subscription_name: filebeat-gcp-audit + + # Credentials file for the service account with authorization to read from + # the subscription. + var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/salt/filebeat/modules/gsuite.yml.disabled b/salt/filebeat/modules/gsuite.yml.disabled new file mode 100644 index 000000000..6aec3b65d --- /dev/null +++ b/salt/filebeat/modules/gsuite.yml.disabled @@ -0,0 +1,53 @@ +# Module: gsuite +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-gsuite.html + +# Gsuite module is deprecated and will be removed in future releases. Please use Google Workspace module instead. +- module: gsuite + saml: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + user_accounts: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + login: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + admin: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + drive: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h + groups: + enabled: true + # var.jwt_file: credentials.json + # var.delegated_account: admin@example.com + # var.initial_interval: 24h + # var.http_client_timeout: 60s + # var.user_key: all + # var.interval: 2h diff --git a/salt/filebeat/modules/haproxy.yml.disabled b/salt/filebeat/modules/haproxy.yml.disabled new file mode 100644 index 000000000..b2615dbb8 --- /dev/null +++ b/salt/filebeat/modules/haproxy.yml.disabled @@ -0,0 +1,14 @@ +# Module: haproxy +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-haproxy.html + +- module: haproxy + # All logs + log: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/ibmmq.yml.disabled b/salt/filebeat/modules/ibmmq.yml.disabled new file mode 100644 index 000000000..bfaf3792d --- /dev/null +++ b/salt/filebeat/modules/ibmmq.yml.disabled @@ -0,0 +1,11 @@ +# Module: ibmmq +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-ibmmq.html + +- module: ibmmq + # All logs + errorlog: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/icinga.yml.disabled b/salt/filebeat/modules/icinga.yml.disabled new file mode 100644 index 000000000..a7c3ac6e1 --- /dev/null +++ b/salt/filebeat/modules/icinga.yml.disabled @@ -0,0 +1,27 @@ +# Module: icinga +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-icinga.html + +- module: icinga + # Main logs + main: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Debug logs + debug: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Startup logs + startup: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/iis.yml.disabled b/salt/filebeat/modules/iis.yml.disabled new file mode 100644 index 000000000..44c200ba1 --- /dev/null +++ b/salt/filebeat/modules/iis.yml.disabled @@ -0,0 +1,20 @@ +# Module: iis +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-iis.html + +- module: iis + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + \ No newline at end of file diff --git a/salt/filebeat/modules/imperva.yml.disabled b/salt/filebeat/modules/imperva.yml.disabled new file mode 100644 index 000000000..8e53deaa6 --- /dev/null +++ b/salt/filebeat/modules/imperva.yml.disabled @@ -0,0 +1,22 @@ +# Module: imperva +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-imperva.html + +- module: imperva + securesphere: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9511 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/infoblox.yml.disabled b/salt/filebeat/modules/infoblox.yml.disabled new file mode 100644 index 000000000..9e82f8340 --- /dev/null +++ b/salt/filebeat/modules/infoblox.yml.disabled @@ -0,0 +1,22 @@ +# Module: infoblox +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-infoblox.html + +- module: infoblox + nios: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9512 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/iptables.yml.disabled b/salt/filebeat/modules/iptables.yml.disabled new file mode 100644 index 000000000..1147e14dd --- /dev/null +++ b/salt/filebeat/modules/iptables.yml.disabled @@ -0,0 +1,13 @@ +# Module: iptables +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-iptables.html + +- module: iptables + log: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/juniper.yml.disabled b/salt/filebeat/modules/juniper.yml.disabled new file mode 100644 index 000000000..71112679d --- /dev/null +++ b/salt/filebeat/modules/juniper.yml.disabled @@ -0,0 +1,54 @@ +# Module: juniper +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-juniper.html + +- module: juniper + junos: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9513 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + netscreen: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9523 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local + + srx: + enabled: true + + # Set which input to use between tcp, udp (default) or file. + #var.input: udp + + # The interface to listen to syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The port to listen for syslog traffic. Defaults to 9006. + #var.syslog_port: 9006 diff --git a/salt/filebeat/modules/kafka.yml.disabled b/salt/filebeat/modules/kafka.yml.disabled new file mode 100644 index 000000000..23362c8a1 --- /dev/null +++ b/salt/filebeat/modules/kafka.yml.disabled @@ -0,0 +1,15 @@ +# Module: kafka +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-kafka.html + +- module: kafka + # All logs + log: + enabled: true + + # Set custom paths for Kafka. If left empty, + # Filebeat will look under /opt. + #var.kafka_home: + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/kibana.yml.disabled b/salt/filebeat/modules/kibana.yml.disabled new file mode 100644 index 000000000..a4956c4b6 --- /dev/null +++ b/salt/filebeat/modules/kibana.yml.disabled @@ -0,0 +1,19 @@ +# Module: kibana +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-kibana.html + +- module: kibana + # Server logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Audit logs + audit: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/logstash.yml.disabled b/salt/filebeat/modules/logstash.yml.disabled new file mode 100644 index 000000000..f14229409 --- /dev/null +++ b/salt/filebeat/modules/logstash.yml.disabled @@ -0,0 +1,18 @@ +# Module: logstash +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-logstash.html + +- module: logstash + # logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Slow logs + slowlog: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/microsoft.yml.disabled b/salt/filebeat/modules/microsoft.yml.disabled new file mode 100644 index 000000000..b0a1b10c6 --- /dev/null +++ b/salt/filebeat/modules/microsoft.yml.disabled @@ -0,0 +1,49 @@ +# Module: microsoft +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-microsoft.html + +- module: microsoft + # ATP configuration + defender_atp: + enabled: true + # How often the API should be polled + #var.interval: 5m + + # Oauth Client ID + #var.oauth2.client.id: "" + + # Oauth Client Secret + #var.oauth2.client.secret: "" + + # Oauth Token URL, should include the tenant ID + #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" + m365_defender: + enabled: true + # How often the API should be polled + #var.interval: 5m + + # Oauth Client ID + #var.oauth2.client.id: "" + + # Oauth Client Secret + #var.oauth2.client.secret: "" + + # Oauth Token URL, should include the tenant ID + #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" + dhcp: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9515 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/misp.yml.disabled b/salt/filebeat/modules/misp.yml.disabled new file mode 100644 index 000000000..9a489fa0f --- /dev/null +++ b/salt/filebeat/modules/misp.yml.disabled @@ -0,0 +1,17 @@ +# Module: misp +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-misp.html + +- module: misp + threat: + enabled: true + # API key to access MISP + #var.api_key + + # Array object in MISP response + #var.http_request_body.limit: 1000 + + # URL of the MISP REST API + #var.url + + # You can also pass SSL options. For example: + #var.ssl.verification_mode: none diff --git a/salt/filebeat/modules/mongodb.yml.disabled b/salt/filebeat/modules/mongodb.yml.disabled new file mode 100644 index 000000000..266d2e4e8 --- /dev/null +++ b/salt/filebeat/modules/mongodb.yml.disabled @@ -0,0 +1,11 @@ +# Module: mongodb +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mongodb.html + +- module: mongodb + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/mssql.yml.disabled b/salt/filebeat/modules/mssql.yml.disabled new file mode 100644 index 000000000..bfe4c6e64 --- /dev/null +++ b/salt/filebeat/modules/mssql.yml.disabled @@ -0,0 +1,11 @@ +# Module: mssql +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mssql.html + +- module: mssql + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] diff --git a/salt/filebeat/modules/mysql.yml.disabled b/salt/filebeat/modules/mysql.yml.disabled new file mode 100644 index 000000000..e6be4045b --- /dev/null +++ b/salt/filebeat/modules/mysql.yml.disabled @@ -0,0 +1,19 @@ +# Module: mysql +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mysql.html + +- module: mysql + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Slow logs + slowlog: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/mysqlenterprise.yml.disabled b/salt/filebeat/modules/mysqlenterprise.yml.disabled new file mode 100644 index 000000000..37e10d0eb --- /dev/null +++ b/salt/filebeat/modules/mysqlenterprise.yml.disabled @@ -0,0 +1,14 @@ +# Module: mysqlenterprise +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mysqlenterprise.html + +- module: mysqlenterprise + audit: + enabled: true + + # Sets the input type. Currently only supports file + #var.input: file + + # Set paths for the log files when file input is used. + # Should only be used together with file input + # var.paths: + # - /home/user/mysqlauditlogs/audit.*.log diff --git a/salt/filebeat/modules/nats.yml.disabled b/salt/filebeat/modules/nats.yml.disabled new file mode 100644 index 000000000..65e44962d --- /dev/null +++ b/salt/filebeat/modules/nats.yml.disabled @@ -0,0 +1,11 @@ +# Module: nats +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-nats.html + +- module: nats + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/netflow.yml.disabled b/salt/filebeat/modules/netflow.yml.disabled new file mode 100644 index 000000000..781748b00 --- /dev/null +++ b/salt/filebeat/modules/netflow.yml.disabled @@ -0,0 +1,14 @@ +# Module: netflow +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-netflow.html + +- module: netflow + log: + enabled: true + var: + netflow_host: localhost + netflow_port: 2055 + # internal_networks specifies which networks are considered internal or private + # you can specify either a CIDR block or any of the special named ranges listed + # at: https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network + internal_networks: + - private diff --git a/salt/filebeat/modules/netscout.yml.disabled b/salt/filebeat/modules/netscout.yml.disabled new file mode 100644 index 000000000..215349046 --- /dev/null +++ b/salt/filebeat/modules/netscout.yml.disabled @@ -0,0 +1,22 @@ +# Module: netscout +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-netscout.html + +- module: netscout + sightline: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9502 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/nginx.yml.disabled b/salt/filebeat/modules/nginx.yml.disabled new file mode 100644 index 000000000..e2fa44a78 --- /dev/null +++ b/salt/filebeat/modules/nginx.yml.disabled @@ -0,0 +1,27 @@ +# Module: nginx +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-nginx.html + +- module: nginx + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs + ingress_controller: + enabled: false + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/o365.yml.disabled b/salt/filebeat/modules/o365.yml.disabled new file mode 100644 index 000000000..578ff365d --- /dev/null +++ b/salt/filebeat/modules/o365.yml.disabled @@ -0,0 +1,48 @@ +# Module: o365 +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-o365.html + +- module: o365 + audit: + enabled: true + + # Set the application_id (also known as client ID): + var.application_id: "" + + # Configure the tenants to monitor: + # Use the tenant ID (also known as directory ID) and the domain name. + # var.tenants: + # - id: "tenant_id_1" + # name: "mydomain.onmicrosoft.com" + # - id: "tenant_id_2" + # name: "mycompany.com" + var.tenants: + - id: "" + name: "mytenant.onmicrosoft.com" + + # List of content-types to fetch. By default all known content-types + # are retrieved: + # var.content_type: + # - "Audit.AzureActiveDirectory" + # - "Audit.Exchange" + # - "Audit.SharePoint" + # - "Audit.General" + # - "DLP.All" + + # Use the following settings to enable certificate-based authentication: + # var.certificate: "/path/to/certificate.pem" + # var.key: "/path/to/private_key.pem" + # var.key_passphrase: "myPrivateKeyPassword" + + # Client-secret based authentication: + # Comment the following line if using certificate authentication. + var.client_secret: "" + + # Advanced settings, use with care: + # var.api: + # # Settings for custom endpoints: + # authentication_endpoint: "https://login.microsoftonline.us/" + # resource: "https://manage.office365.us" + # + # max_retention: 168h + # max_requests_per_minute: 2000 + # poll_interval: 3m diff --git a/salt/filebeat/modules/okta.yml.disabled b/salt/filebeat/modules/okta.yml.disabled new file mode 100644 index 000000000..4fc943592 --- /dev/null +++ b/salt/filebeat/modules/okta.yml.disabled @@ -0,0 +1,10 @@ +# Module: okta +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-okta.html + +- module: okta + system: + enabled: true + # You must configure the URL with your Okta domain and provide an + # API token to access the logs API. + #var.url: https://yourOktaDomain/api/v1/logs + #var.api_key: 'yourApiTokenHere' diff --git a/salt/filebeat/modules/oracle.yml.disabled b/salt/filebeat/modules/oracle.yml.disabled new file mode 100644 index 000000000..3bd576ee1 --- /dev/null +++ b/salt/filebeat/modules/oracle.yml.disabled @@ -0,0 +1,13 @@ +# Module: oracle +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-oracle.html + +- module: oracle + database_audit: + enabled: true + + # Set which input to use between syslog or file (default). + #var.input: file + + # Set paths for the log files when file input is used. + # Should only be used together with file input + # var.paths: /home/user/oracleauditlogs/*.aud diff --git a/salt/filebeat/modules/osquery.yml.disabled b/salt/filebeat/modules/osquery.yml.disabled new file mode 100644 index 000000000..7a9a09dd8 --- /dev/null +++ b/salt/filebeat/modules/osquery.yml.disabled @@ -0,0 +1,15 @@ +# Module: osquery +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-osquery.html + +- module: osquery + result: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # If true, all fields created by this module are prefixed with + # `osquery.result`. Set to false to copy the fields in the root + # of the document. The default is true. + #var.use_namespace: true diff --git a/salt/filebeat/modules/panw.yml.disabled b/salt/filebeat/modules/panw.yml.disabled new file mode 100644 index 000000000..eb094a25a --- /dev/null +++ b/salt/filebeat/modules/panw.yml.disabled @@ -0,0 +1,22 @@ +# Module: panw +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-panw.html + +- module: panw + panos: + enabled: true + + # Set which input to use between syslog (default) or file. + #var.input: + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Set internal security zones. used to determine network.direction + # default "trust" + #var.internal_zones: + + # Set external security zones. used to determine network.direction + # default "untrust" + #var.external_zones: + diff --git a/salt/filebeat/modules/pensando.yml.disabled b/salt/filebeat/modules/pensando.yml.disabled new file mode 100644 index 000000000..66bd60d76 --- /dev/null +++ b/salt/filebeat/modules/pensando.yml.disabled @@ -0,0 +1,13 @@ +# Module: pensando +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-pensando.html + +- module: pensando +# Firewall logs + dfw: + enabled: true + var.syslog_host: 0.0.0.0 + var.syslog_port: 9001 + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + # var.paths: diff --git a/salt/filebeat/modules/postgresql.yml.disabled b/salt/filebeat/modules/postgresql.yml.disabled new file mode 100644 index 000000000..804b7f34f --- /dev/null +++ b/salt/filebeat/modules/postgresql.yml.disabled @@ -0,0 +1,11 @@ +# Module: postgresql +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-postgresql.html + +- module: postgresql + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/proofpoint.yml.disabled b/salt/filebeat/modules/proofpoint.yml.disabled new file mode 100644 index 000000000..9aeebd5fe --- /dev/null +++ b/salt/filebeat/modules/proofpoint.yml.disabled @@ -0,0 +1,22 @@ +# Module: proofpoint +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-proofpoint.html + +- module: proofpoint + emailsecurity: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9531 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/rabbitmq.yml.disabled b/salt/filebeat/modules/rabbitmq.yml.disabled new file mode 100644 index 000000000..e61a0a0c9 --- /dev/null +++ b/salt/filebeat/modules/rabbitmq.yml.disabled @@ -0,0 +1,11 @@ +# Module: rabbitmq +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-rabbitmq.html + +- module: rabbitmq + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: ["/var/log/rabbitmq/rabbit@localhost.log*"] diff --git a/salt/filebeat/modules/radware.yml.disabled b/salt/filebeat/modules/radware.yml.disabled new file mode 100644 index 000000000..f9ab3e519 --- /dev/null +++ b/salt/filebeat/modules/radware.yml.disabled @@ -0,0 +1,22 @@ +# Module: radware +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-radware.html + +- module: radware + defensepro: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9518 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/redis.yml.disabled b/salt/filebeat/modules/redis.yml.disabled new file mode 100644 index 000000000..9b621dc2d --- /dev/null +++ b/salt/filebeat/modules/redis.yml.disabled @@ -0,0 +1,21 @@ +# Module: redis +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-redis.html + +- module: redis + # Main logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: ["/var/log/redis/redis-server.log*"] + + # Slow logs, retrieved via the Redis API (SLOWLOG) + slowlog: + enabled: true + + # The Redis hosts to connect to. + #var.hosts: ["localhost:6379"] + + # Optional, the password to use when connecting to Redis. + #var.password: diff --git a/salt/filebeat/modules/santa.yml.disabled b/salt/filebeat/modules/santa.yml.disabled new file mode 100644 index 000000000..1a7363547 --- /dev/null +++ b/salt/filebeat/modules/santa.yml.disabled @@ -0,0 +1,9 @@ +# Module: santa +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-santa.html + +- module: santa + log: + enabled: true + # Set custom paths for the log files. If left empty, + # Filebeat will choose the the default path. + #var.paths: diff --git a/salt/filebeat/modules/snort.yml.disabled b/salt/filebeat/modules/snort.yml.disabled new file mode 100644 index 000000000..8c9bcc471 --- /dev/null +++ b/salt/filebeat/modules/snort.yml.disabled @@ -0,0 +1,22 @@ +# Module: snort +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-snort.html + +- module: snort + log: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9532 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/snyk.yml.disabled b/salt/filebeat/modules/snyk.yml.disabled new file mode 100644 index 000000000..0b13f8155 --- /dev/null +++ b/salt/filebeat/modules/snyk.yml.disabled @@ -0,0 +1,112 @@ +# Module: snyk +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-snyk.html + +- module: snyk + audit: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + # + # What audit type to collect, can be either "group" or "organization". + #var.audit_type: organization + # + # The ID related to the audit_type. If audit type is group, then this value should be + # the group ID and if it is organization it should be the organization ID to collect from. + #var.audit_id: 1235432-asdfdf-2341234-asdgjhg + + # How often the API should be polled, defaults to 1 hour. + #var.interval: 1h + # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). + #var.first_interval: 24h + + # The API token that is created for a specific user, found in the Snyk management dashboard. + #var.api_token: + + # Event filtering. + # All configuration items below is OPTIONAL and the default options will be overwritten + # for each entry that is not commented out. + + # Will return only logs for this specific project. + #var.project_id: "" + # User public ID. Will fetch only audit logs originated from this user's actions. + #var.user_id: "" + # Will return only logs for this specific event. + #var.event: "" + # User email address. Will fetch only audit logs originated from this user's actions. + #var.email_address: "" + + vulnerabilities: + enabled: true + + # Set which input to use between httpjson (default) or file. + #var.input: httpjson + + # How often the API should be polled. Data from the Snyk API is automatically updated + # once per day, so the default interval is 24 hours. + #var.interval: 24h + + # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). + #var.first_interval: 24h + + # The API token that is created for a specific user, found in the Snyk management dashboard. + #var.api_token: + + # The list of org IDs to filter the results by. + # One organization ID per line, starting with a - sign + #var.orgs: + # - 12354-asdfdf-123543-asdsdfg + # - 76554-jhggfd-654342-hgrfasd + + + # Event filtering. + # All configuration items below is OPTIONAL and the default options will be overwritten + # for each entry that is not commented out. + + # The severity levels of issues to filter the results by. + #var.included_severity: + # - high + # - medium + # - low + # + # The exploit maturity levels of issues to filter the results by. + #var.exploit_maturity: + # - mature + # - proof-of-concept + # - no-known-exploit + # - no-data + # + # The type of issues to filter the results by. + #var.types: + # - vuln + # - license + # + # The type of languages to filter the results by. + #var.languages: + # - javascript + # - ruby + # - java + # - scala + # - python + # - golang + # - php + # - dotnet + # - swift + # - docker + # + # Search term to filter issue name by, or an exact CVE or CWE. + #var.identifier: + # - "" + # + # If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. + #var.ignored: false + #var.patched: false + #var.fixable: false + #var.is_fixed: false + #var.is_patchable: false + #var.is_pinnable: false + # + # The priority score ranging between 0-1000 + #var.min_priority_score: 0 + #var.max_priority_score: 1000 + diff --git a/salt/filebeat/modules/sonicwall.yml.disabled b/salt/filebeat/modules/sonicwall.yml.disabled new file mode 100644 index 000000000..de457109d --- /dev/null +++ b/salt/filebeat/modules/sonicwall.yml.disabled @@ -0,0 +1,22 @@ +# Module: sonicwall +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-sonicwall.html + +- module: sonicwall + firewall: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9519 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/sophos.yml.disabled b/salt/filebeat/modules/sophos.yml.disabled new file mode 100644 index 000000000..8fc346540 --- /dev/null +++ b/salt/filebeat/modules/sophos.yml.disabled @@ -0,0 +1,46 @@ +# Module: sophos +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-sophos.html + +- module: sophos + xg: + enabled: true + + # Set which input to use between tcp, udp (default) or file. + #var.input: udp + + # The interface to listen to syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The port to listen for syslog traffic. Defaults to 9004. + #var.syslog_port: 9005 + + # firewall default hostname + #var.default_host_name: firewall.localgroup.local + + # known firewalls + #var.known_devices: + #- serial_number: "1234567890123457" + # hostname: "a.host.local" + #- serial_number: "1234234590678557" + # hostname: "b.host.local" + + + utm: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9533 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/squid.yml.disabled b/salt/filebeat/modules/squid.yml.disabled new file mode 100644 index 000000000..a47807253 --- /dev/null +++ b/salt/filebeat/modules/squid.yml.disabled @@ -0,0 +1,22 @@ +# Module: squid +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-squid.html + +- module: squid + log: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9520 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/suricata.yml.disabled b/salt/filebeat/modules/suricata.yml.disabled new file mode 100644 index 000000000..1edd3f832 --- /dev/null +++ b/salt/filebeat/modules/suricata.yml.disabled @@ -0,0 +1,11 @@ +# Module: suricata +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html + +- module: suricata + # All logs + eve: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/system.yml.disabled b/salt/filebeat/modules/system.yml.disabled new file mode 100644 index 000000000..d633bac04 --- /dev/null +++ b/salt/filebeat/modules/system.yml.disabled @@ -0,0 +1,19 @@ +# Module: system +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-system.html + +- module: system + # Syslog + syslog: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Authorization logs + auth: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/threatintel.yml.disabled b/salt/filebeat/modules/threatintel.yml.disabled new file mode 100644 index 000000000..b461d91e2 --- /dev/null +++ b/salt/filebeat/modules/threatintel.yml.disabled @@ -0,0 +1,105 @@ +# Module: threatintel +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-threatintel.html + +- module: threatintel + abuseurl: + enabled: true + + # Input used for ingesting threat intel data. + var.input: httpjson + + # The URL used for Threat Intel API calls. + var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/ + + # The interval to poll the API for updates. + var.interval: 10m + + abusemalware: + enabled: true + + # Input used for ingesting threat intel data. + var.input: httpjson + + # The URL used for Threat Intel API calls. + var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/ + + # The interval to poll the API for updates. + var.interval: 10m + + misp: + enabled: true + + # Input used for ingesting threat intel data, defaults to JSON. + var.input: httpjson + + # The URL of the MISP instance, should end with "/events/restSearch". + var.url: https://SERVER/events/restSearch + + # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI. + var.api_token: API_KEY + + # Configures the type of SSL verification done, if MISP is running on self signed certificates + # then the certificate would either need to be trusted, or verification_mode set to none. + #var.ssl.verification_mode: none + + # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context. + # For examples please reference the filebeat module documentation. + #var.filters: + # - threat_level: [4, 5] + # - to_ids: true + + # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer + # than the last event that was already ingested. + var.first_interval: 300h + + # The interval to poll the API for updates. + var.interval: 5m + + otx: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The URL used for OTX Threat Intel API calls. + var.url: https://otx.alienvault.com/api/v1/indicators/export + + # The authentication token used to contact the OTX API, can be found on the OTX UI. + var.api_token: API_KEY + + # Optional filters that can be applied to retrieve only specific indicators. + #var.types: "domain,IPv4,hostname,url,FileHash-SHA256" + + # The timeout of the HTTP client connecting to the OTX API + #var.http_client_timeout: 120s + + # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module. + var.lookback_range: 1h + + # How far back to look once the beat starts up for the first time, the value has to be in hours. + var.first_interval: 400h + + # The interval to poll the API for updates + var.interval: 5m + + anomali: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending + # on the type of threat intel source that is needed. + var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects + + # The Username used by anomali Limo, defaults to guest. + #var.username: guest + + # The password used by anomali Limo, defaults to guest. + #var.password: guest + + # How far back to look once the beat starts up for the first time, the value has to be in hours. + var.first_interval: 400h + + # The interval to poll the API for updates + var.interval: 5m diff --git a/salt/filebeat/modules/tomcat.yml.disabled b/salt/filebeat/modules/tomcat.yml.disabled new file mode 100644 index 000000000..84f4619d5 --- /dev/null +++ b/salt/filebeat/modules/tomcat.yml.disabled @@ -0,0 +1,22 @@ +# Module: tomcat +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-tomcat.html + +- module: tomcat + log: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9501 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local diff --git a/salt/filebeat/modules/traefik.yml.disabled b/salt/filebeat/modules/traefik.yml.disabled new file mode 100644 index 000000000..657d5ccd9 --- /dev/null +++ b/salt/filebeat/modules/traefik.yml.disabled @@ -0,0 +1,11 @@ +# Module: traefik +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-traefik.html + +- module: traefik + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/zeek.yml.disabled b/salt/filebeat/modules/zeek.yml.disabled new file mode 100644 index 000000000..0667c6e35 --- /dev/null +++ b/salt/filebeat/modules/zeek.yml.disabled @@ -0,0 +1,84 @@ +# Module: zeek +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html + +- module: zeek + capture_loss: + enabled: true + connection: + enabled: true + dce_rpc: + enabled: true + dhcp: + enabled: true + dnp3: + enabled: true + dns: + enabled: true + dpd: + enabled: true + files: + enabled: true + ftp: + enabled: true + http: + enabled: true + intel: + enabled: true + irc: + enabled: true + kerberos: + enabled: true + modbus: + enabled: true + mysql: + enabled: true + notice: + enabled: true + ntlm: + enabled: true + ocsp: + enabled: true + pe: + enabled: true + radius: + enabled: true + rdp: + enabled: true + rfb: + enabled: true + signature: + enabled: true + sip: + enabled: true + smb_cmd: + enabled: true + smb_files: + enabled: true + smb_mapping: + enabled: true + smtp: + enabled: true + snmp: + enabled: true + socks: + enabled: true + ssh: + enabled: true + ssl: + enabled: true + stats: + enabled: true + syslog: + enabled: true + traceroute: + enabled: true + tunnel: + enabled: true + weird: + enabled: true + x509: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/zoom.yml.disabled b/salt/filebeat/modules/zoom.yml.disabled new file mode 100644 index 000000000..15fa9d4b2 --- /dev/null +++ b/salt/filebeat/modules/zoom.yml.disabled @@ -0,0 +1,22 @@ +# Module: zoom +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zoom.html + +- module: zoom + webhook: + enabled: true + + # The type of input to use + #var.input: http_endpoint + + # The interface to listen for incoming HTTP requests. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.listen_address: localhost + + # The port to bind to + #var.listen_port: 80 + + # The header Zoom uses to send its secret token, defaults to "Authorization" + #secret.header: Authorization + + # The secret token value created by Zoom + #secret.value: ZOOMTOKEN diff --git a/salt/filebeat/modules/zscaler.yml.disabled b/salt/filebeat/modules/zscaler.yml.disabled new file mode 100644 index 000000000..accdec9ea --- /dev/null +++ b/salt/filebeat/modules/zscaler.yml.disabled @@ -0,0 +1,22 @@ +# Module: zscaler +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zscaler.html + +- module: zscaler + zia: + enabled: true + + # Set which input to use between udp (default), tcp or file. + # var.input: udp + # var.syslog_host: localhost + # var.syslog_port: 9521 + + # Set paths for the log files when file input is used. + # var.paths: + + # Toggle output of non-ECS fields (default true). + # var.rsa_fields: true + + # Set custom timezone offset. + # "local" (default) for system timezone. + # "+02:00" for GMT+02:00 + # var.tz_offset: local From fcd1bea4a3b25b1b74c28363769d8256bca3a892 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 12:06:03 -0400 Subject: [PATCH 068/195] remove auto data migration, change duration from 0s to 0d --- salt/common/tools/sbin/soup | 8 -------- salt/influxdb/defaults.yaml | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 07a9804b2..1f6a6e5d7 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -307,14 +307,6 @@ post_2.3.30_to_2.3.40() { } post_2.3.5X_to_2.3.60() { - # These are post upgrade changes that need to take place after all the new states have been applied. - echo "" - echo "Migration of InfluxDB data will begin in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." - echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" - read -p "Press enter to continue." - echo "" - docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*" && echo "InfluxDB migration complete." || echo "InfluxDB migration failed."' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 - POSTVERSION=2.3.60 } diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 54e175024..0c9a09080 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -6,7 +6,7 @@ influxdb: shard_duration: 1d so_long_term: default: False - duration: 0s + duration: 0d shard_duration: 7d downsample: so_long_term: From c5f99b012e6e1995376d5c4deb603911c623d4bb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 13:58:53 -0400 Subject: [PATCH 069/195] comment out creation of cq to test data migration --- salt/influxdb/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index c6120d292..9a94b1c95 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -112,7 +112,7 @@ telegraf_database: - sls: salt.python3-influxdb {% endfor %} -{% for dest_rp in influxdb.downsample.keys() %} +{#% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq @@ -127,7 +127,7 @@ so_downsample_cq: - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - sls: salt.python3-influxdb -{% endfor %} +{% endfor %#} {% endif %} {% endif %} From a1a79719fc759577fe42ea0d368aa17f5b61bd39 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 May 2021 12:07:30 +0000 Subject: [PATCH 070/195] Add ignore above for message keyword field --- salt/elasticsearch/templates/so/so-common-template.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 8adbdcf13..214e5c691 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -320,7 +320,8 @@ "type":"text", "fields":{ "keyword":{ - "type":"keyword" + "type":"keyword", + "ignore_above": 32766 } } }, From 6a639edb052e328307a6bc90ec55041c486a8d4a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 08:33:31 -0400 Subject: [PATCH 071/195] have cq created once again --- salt/influxdb/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 9a94b1c95..c6120d292 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -112,7 +112,7 @@ telegraf_database: - sls: salt.python3-influxdb {% endfor %} -{#% for dest_rp in influxdb.downsample.keys() %} +{% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq @@ -127,7 +127,7 @@ so_downsample_cq: - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - sls: salt.python3-influxdb -{% endfor %#} +{% endfor %} {% endif %} {% endif %} From ce24781446c1ddbf736702dae33fb0d154fe7aeb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 13:29:24 -0400 Subject: [PATCH 072/195] first take at so-infludb-clean --- salt/common/tools/so-influxdb-clear | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 salt/common/tools/so-influxdb-clear diff --git a/salt/common/tools/so-influxdb-clear b/salt/common/tools/so-influxdb-clear new file mode 100644 index 000000000..38ef6491f --- /dev/null +++ b/salt/common/tools/so-influxdb-clear @@ -0,0 +1,42 @@ + +echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." +echo "The duration will need to be specified as an integer followed by the duration unit, without a space." +echo "For example, to purge all data, but retain the past 3 months, specify 3mo for the duration." +echo "The duration units are as follows:" +echo " mo - month(s)" +echo " w - week(s)" +echo " d - day(s)" + +while true; do +echo "" +read -p 'Enter the duration of past data that you would like to retain: ' duration +duration=$(echo $duration | tr '[:upper:]' '[:lower:]') + +originalIFS=$IFS + +case "${duration}" in + + [0-9]+mo$) + break + ;; + + [0-9]+w$) + break + ;; + + [0-9]+d$) + break + ;; + + *) + echo "" + echo "Invalid duration." + ;; + +esac +done + +IFS=$originalIFS + +echo "Cleaning InfluxDb and saving only the past ${duration}" +docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - ${duration}\"" \ No newline at end of file From e53280447415a81ab0063c1277c467798dece9ef Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 13:42:21 -0400 Subject: [PATCH 073/195] move to proper dir --- salt/common/tools/{ => sbin}/so-influxdb-clear | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) rename salt/common/tools/{ => sbin}/so-influxdb-clear (59%) diff --git a/salt/common/tools/so-influxdb-clear b/salt/common/tools/sbin/so-influxdb-clear similarity index 59% rename from salt/common/tools/so-influxdb-clear rename to salt/common/tools/sbin/so-influxdb-clear index 38ef6491f..c6d5446e9 100644 --- a/salt/common/tools/so-influxdb-clear +++ b/salt/common/tools/sbin/so-influxdb-clear @@ -1,3 +1,19 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." From a5ee8fb59d8a37ba71e1a596d20e78b103c97dfe Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 14:56:53 -0400 Subject: [PATCH 074/195] fix the issues with so-influxdb-clear --- salt/common/tools/sbin/so-influxdb-clear | 44 ++++++++++-------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clear b/salt/common/tools/sbin/so-influxdb-clear index c6d5446e9..809b17381 100644 --- a/salt/common/tools/sbin/so-influxdb-clear +++ b/salt/common/tools/sbin/so-influxdb-clear @@ -15,8 +15,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +modurregex="^[0-9]+mo$" +wdurregex="^[0-9]+w$" +ddurregex="^[0-9]+d$" + +echo "" echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." +echo "" echo "For example, to purge all data, but retain the past 3 months, specify 3mo for the duration." echo "The duration units are as follows:" echo " mo - month(s)" @@ -24,35 +30,21 @@ echo " w - week(s)" echo " d - day(s)" while true; do -echo "" -read -p 'Enter the duration of past data that you would like to retain: ' duration -duration=$(echo $duration | tr '[:upper:]' '[:lower:]') + echo "" + read -p 'Enter the duration of past data that you would like to retain: ' duration + duration=$(echo $duration | tr '[:upper:]' '[:lower:]') -originalIFS=$IFS - -case "${duration}" in - - [0-9]+mo$) + if [[ "$duration" =~ $modurregex ]] || [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then break - ;; + fi - [0-9]+w$) - break - ;; - - [0-9]+d$) - break - ;; - - *) - echo "" - echo "Invalid duration." - ;; - -esac + echo "" + echo "Invalid duration." done -IFS=$originalIFS - echo "Cleaning InfluxDb and saving only the past ${duration}" -docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - ${duration}\"" \ No newline at end of file +if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then + echo "InfluxDb clean complete." +else + echo "Something went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." +fi From 44de6110972dd9d857e11b7b066d04ba206c43a1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 14:57:39 -0400 Subject: [PATCH 075/195] rename to so-influxdb-clean --- salt/common/tools/sbin/{so-influxdb-clear => so-influxdb-clean} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/common/tools/sbin/{so-influxdb-clear => so-influxdb-clean} (100%) diff --git a/salt/common/tools/sbin/so-influxdb-clear b/salt/common/tools/sbin/so-influxdb-clean similarity index 100% rename from salt/common/tools/sbin/so-influxdb-clear rename to salt/common/tools/sbin/so-influxdb-clean From db98b7ed276cb0df0661798180771a7379562268 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 15:08:11 -0400 Subject: [PATCH 076/195] verify with user before proceedig to clean --- salt/common/tools/sbin/so-influxdb-clean | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 809b17381..453d5b2e4 100644 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -19,11 +19,9 @@ modurregex="^[0-9]+mo$" wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" -echo "" -echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." +echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." -echo "" -echo "For example, to purge all data, but retain the past 3 months, specify 3mo for the duration." +echo -e "\nFor example, to purge all data, but retain the past 3 months, specify 3mo for the duration." echo "The duration units are as follows:" echo " mo - month(s)" echo " w - week(s)" @@ -38,13 +36,18 @@ while true; do break fi - echo "" - echo "Invalid duration." + echo -e "\nInvalid duration." done -echo "Cleaning InfluxDb and saving only the past ${duration}" -if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then - echo "InfluxDb clean complete." +echo -e "\nInfluxDB will now be cleaned and leave only the past $duration worth of data." +read -r -p "Are you sure you want to continue? [y/N] " yorn +if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then + echo -e "\nCleaning InfluxDb and saving only the past $duration. This may could take several minutes depending on how much data needs to be cleaned." + if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then + echo -e "\nInfluxDb clean complete." + else + echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + fi else - echo "Something went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + echo -e "\nExiting as requested." fi From 0ea4c99102bbd3aecc11c164395ece5a8edd5dc8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 15:32:53 -0400 Subject: [PATCH 077/195] remove support for months as it isnt supported in InfluxQL --- salt/common/tools/sbin/so-influxdb-clean | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 453d5b2e4..954abfa8f 100644 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -15,15 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -modurregex="^[0-9]+mo$" wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." -echo -e "\nFor example, to purge all data, but retain the past 3 months, specify 3mo for the duration." +echo -e "\nFor example, to purge all data, but retain the past 12 weeks, specify 12w for the duration." echo "The duration units are as follows:" -echo " mo - month(s)" echo " w - week(s)" echo " d - day(s)" @@ -32,7 +30,7 @@ while true; do read -p 'Enter the duration of past data that you would like to retain: ' duration duration=$(echo $duration | tr '[:upper:]' '[:lower:]') - if [[ "$duration" =~ $modurregex ]] || [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then + if [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then break fi From 925be17d51be12a6a49caec3f3c5f3d194fa04ad Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 15:59:18 -0400 Subject: [PATCH 078/195] clean some commas in so-influxdb-clean --- salt/common/tools/sbin/so-influxdb-clean | 4 ++-- salt/common/tools/sbin/so-influxdb-migrate | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 salt/common/tools/sbin/so-influxdb-migrate diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 954abfa8f..635ac6a02 100644 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -19,8 +19,8 @@ wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." -echo "The duration will need to be specified as an integer followed by the duration unit, without a space." -echo -e "\nFor example, to purge all data, but retain the past 12 weeks, specify 12w for the duration." +echo "The duration will need to be specified as an integer followed by the duration unit without a space." +echo -e "\nFor example, to purge all data but retain the past 12 weeks, specify 12w for the duration." echo "The duration units are as follows:" echo " w - week(s)" echo " d - day(s)" diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate new file mode 100644 index 000000000..c27997bd5 --- /dev/null +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -0,0 +1,16 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . From 20188549f731ca133d4fc69402f3ea31bc4e0757 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 19:28:16 -0400 Subject: [PATCH 079/195] add the logic for so-influxdb-migrate --- salt/common/tools/sbin/so-influxdb-migrate | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index c27997bd5..3b21d099d 100644 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -14,3 +14,33 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + +echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy." + +echo -e "\nInfluxDB will now be migrated. This could take a few hours depending on how large the database is and hardware resources available." +read -r -p "Are you sure you want to continue? [y/N] " yorn +if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then + echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be moved." + day=0 + startdate=`date` + + while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do + # why 2020-07-21? + echo "Migration of 2020-07-21T00:00:00.0000000Z' + ${day}d started at $startdate and completed at `date`." + + echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" + if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then + echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" + else + echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + fi + + ((day=day+1)) + startdate=`date` + done + + echo -e "\nInfluxDb data migration complete." + +else + echo -e "\nExiting as requested." +fi From 9dbb9f519b40765972e86bc1993796d268a2c5d2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 09:14:49 -0400 Subject: [PATCH 080/195] create so_short_term rp as default so that autogen can just be dropped once data is downsampled --- salt/common/tools/sbin/so-influxdb-migrate | 14 +++++++------- salt/influxdb/defaults.yaml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index 3b21d099d..3411dc39f 100644 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -24,16 +24,16 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then day=0 startdate=`date` - while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do + while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? echo "Migration of 2020-07-21T00:00:00.0000000Z' + ${day}d started at $startdate and completed at `date`." - echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" - if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then - echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" - else - echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." - fi +# echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" +# if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then +# echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" +# else +# echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." +# fi ((day=day+1)) startdate=`date` diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 0c9a09080..7ba83dd6d 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -1,6 +1,6 @@ influxdb: retention_policies: - autogen: + so_short_term: default: True duration: 30d shard_duration: 1d From 37929dbd7d09d0da03919541c792eb0cefa25fde Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 May 2021 13:54:28 +0000 Subject: [PATCH 081/195] Add additional config for Filebeat modules --- pillar/zeek/init.sls | 3 +- salt/filebeat/etc/filebeat.yml | 78 ----------------- salt/filebeat/init.sls | 23 +++++ salt/filebeat/modules/suricata.yml.disabled | 11 --- salt/filebeat/modules/zeek.yml.disabled | 84 ------------------- .../config/so/9000_output_zeek.conf.jinja | 27 +++--- .../config/so/9400_output_suricata.conf.jinja | 6 +- 7 files changed, 41 insertions(+), 191 deletions(-) delete mode 100644 salt/filebeat/modules/suricata.yml.disabled delete mode 100644 salt/filebeat/modules/zeek.yml.disabled diff --git a/pillar/zeek/init.sls b/pillar/zeek/init.sls index 30a59284a..5eeb273b9 100644 --- a/pillar/zeek/init.sls +++ b/pillar/zeek/init.sls @@ -52,5 +52,4 @@ zeek: - frameworks/signatures/detect-windows-shells redef: - LogAscii::use_json = T; - - LogAscii::json_timestamps = JSON::TS_ISO8601; - - CaptureLoss::watch_interval = 5 mins; \ No newline at end of file + - CaptureLoss::watch_interval = 5 mins; diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 0f7c9c778..bd72bc583 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -105,84 +105,6 @@ filebeat.inputs: fields_under_root: true {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} - {%- if ZEEKVER != 'SURICATA' %} - {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} -- type: log - paths: - - /nsm/zeek/logs/current/{{ LOGNAME }}.log - fields: - module: zeek - dataset: {{ LOGNAME }} - category: network - processors: - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: true - close_removed: false - -- type: log - paths: - - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log - fields: - module: zeek - dataset: {{ LOGNAME }} - category: network - imported: true - processors: - - add_tags: - tags: ["import"] - - dissect: - tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}" - field: "log.file.path" - target_prefix: "" - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: false - close_removed: false - {%- endfor %} - {%- endif %} - -- type: log - paths: - - /nsm/suricata/eve*.json - fields: - module: suricata - dataset: common - category: network - - processors: - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: false - close_removed: false - -- type: log - paths: - - /nsm/import/*/suricata/eve*.json - fields: - module: suricata - dataset: common - category: network - imported: true - processors: - - add_tags: - tags: ["import"] - - dissect: - tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}" - field: "log.file.path" - target_prefix: "" - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: false - close_removed: false {%- if STRELKAENABLED == 1 %} - type: log diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 64cdc47fc..8ab200276 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -26,6 +26,12 @@ filebeatetcdir: - user: 939 - group: 939 - makedirs: True +filebeatmoduledir: + file.directory: + - name: /opt/so/conf/filebeat/modules + - user: root + - group: root + - makedirs: True filebeatlogdir: file.directory: - name: /opt/so/log/filebeat @@ -55,6 +61,21 @@ filebeatconfsync: - defaults: INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }} OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }} +# Filebeat module config file +filebeatmoduleconfsync: + file.managed: + - name: /opt/so/conf/filebeat/etc/module-setup.yml + - source: salt://filebeat/etc/module-setup.yml + - user: root + - group: root + - template: jinja +# Sync Filebeat modules +filebeatmodules: + file.recurse: + - name: /opt/so/conf/filebeat/modules + - source: salt://filebeat/modules + - user: root + - group: root so-filebeat: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }} @@ -65,8 +86,10 @@ so-filebeat: - /nsm:/nsm:ro - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - /opt/so/conf/filebeat/etc/module-setup.yml:/usr/share/filebeat/module-setup.yml:ro - /nsm/wazuh/logs/alerts:/wazuh/alerts:ro - /nsm/wazuh/logs/archives:/wazuh/archives:ro + - /opt/so/conf/filebeat/modules:/usr/share/filebeat/modules.d - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw diff --git a/salt/filebeat/modules/suricata.yml.disabled b/salt/filebeat/modules/suricata.yml.disabled deleted file mode 100644 index 1edd3f832..000000000 --- a/salt/filebeat/modules/suricata.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: suricata -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html - -- module: suricata - # All logs - eve: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/zeek.yml.disabled b/salt/filebeat/modules/zeek.yml.disabled deleted file mode 100644 index 0667c6e35..000000000 --- a/salt/filebeat/modules/zeek.yml.disabled +++ /dev/null @@ -1,84 +0,0 @@ -# Module: zeek -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html - -- module: zeek - capture_loss: - enabled: true - connection: - enabled: true - dce_rpc: - enabled: true - dhcp: - enabled: true - dnp3: - enabled: true - dns: - enabled: true - dpd: - enabled: true - files: - enabled: true - ftp: - enabled: true - http: - enabled: true - intel: - enabled: true - irc: - enabled: true - kerberos: - enabled: true - modbus: - enabled: true - mysql: - enabled: true - notice: - enabled: true - ntlm: - enabled: true - ocsp: - enabled: true - pe: - enabled: true - radius: - enabled: true - rdp: - enabled: true - rfb: - enabled: true - signature: - enabled: true - sip: - enabled: true - smb_cmd: - enabled: true - smb_files: - enabled: true - smb_mapping: - enabled: true - smtp: - enabled: true - snmp: - enabled: true - socks: - enabled: true - ssh: - enabled: true - ssl: - enabled: true - stats: - enabled: true - syslog: - enabled: true - traceroute: - enabled: true - tunnel: - enabled: true - weird: - enabled: true - x509: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index d17dc2b22..da798a79d 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -1,19 +1,20 @@ -{%- if grains['role'] == 'so-eval' -%} +%- if grains['role'] == 'so-eval' -%} {%- set ES = salt['pillar.get']('manager:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [module] =~ "zeek" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ ES }}" - index => "so-zeek" - template_name => "so-zeek" - template => "/templates/so-zeek-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } + if [event][module] == 'zeek' { + elasticsearch { + id => "zeek_logs" + pipeline => "filebeat-%{[agent][version]}-zeek-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-zeek-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 5013bafc1..258781f29 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -4,12 +4,12 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [module] =~ "suricata" and "import" not in [tags] { + if [event][module] =~ "suricata" and "import" not in [tags] { elasticsearch { - pipeline => "%{module}.%{dataset}" + pipeline => "filebeat-%{[agent][version]}-suricata-%{[fileset][name]}-pipeline" hosts => "{{ ES }}" index => "so-ids" - template_name => "so-ids" + template_name => "so-ids" template => "/templates/so-ids-template.json" ssl => true ssl_certificate_verification => false From 1b749cf004979b6cd8fc25959ea4d937b9b656fd Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 May 2021 13:55:07 +0000 Subject: [PATCH 082/195] Additional config --- .../tools/sbin/so-filebeat-module-setup | 56 +++++++++++++++++++ salt/filebeat/etc/module-setup.yml | 10 ++++ 2 files changed, 66 insertions(+) create mode 100755 salt/common/tools/sbin/so-filebeat-module-setup create mode 100644 salt/filebeat/etc/module-setup.yml diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup new file mode 100755 index 000000000..a42b0ac80 --- /dev/null +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -0,0 +1,56 @@ +{%- set mainint = salt['pillar.get']('host:mainint') %} +{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} + +#!/bin/bash +# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +default_conf_dir=/opt/so/conf +ELASTICSEARCH_HOST="{{ MYIP }}" +ELASTICSEARCH_PORT=9200 +#ELASTICSEARCH_AUTH="" + +# Define a default directory to load pipelines from +FB_MODULE_YML="/usr/share/filebeat/module-setup.yml" + +if [ "$1" == "" ]; then + echo "No module supplied. Exiting..." +else + # Wait for ElasticSearch to initialize + echo -n "Waiting for ElasticSearch..." + COUNT=0 + ELASTICSEARCH_CONNECTED="no" + while [[ "$COUNT" -le 240 ]]; do + curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi + done + if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo + fi + + echo "Setting up ingest pipeline(s) for $1..." + docker exec -it so-filebeat filebeat setup modules -pipelines -modules $1 -c $FB_MODULE_YML +fi + diff --git a/salt/filebeat/etc/module-setup.yml b/salt/filebeat/etc/module-setup.yml new file mode 100644 index 000000000..431e432b3 --- /dev/null +++ b/salt/filebeat/etc/module-setup.yml @@ -0,0 +1,10 @@ +{%- if grains['role'] in ['so-managersearch', 'so-heavynode', 'so-node'] %} +{%- set MANAGER = salt['grains.get']('host' '') %} +{%- else %} +{%- set MANAGER = salt['grains.get']('master') %} +{%- endif %} + +output.elasticsearch: + enabled: true + hosts: ["https://{{ MANAGER }}:9200"] + ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"] From ee92ba20b04a025eb4e75d49000930724bd772c0 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 May 2021 13:56:39 +0000 Subject: [PATCH 083/195] Add modules path reference --- salt/filebeat/etc/filebeat.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index bd72bc583..46a59f772 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -71,7 +71,13 @@ logging.files: # Set to true to log messages in json format. #logging.json: false + + #========================== Modules configuration ============================ +filebeat.config.modules: + enabled: true + path: ${path.config}/modules.d/*.yml + filebeat.modules: #=========================== Filebeat prospectors ============================= From 728d1f75406be60b5acda5a8eeb6a0b038a8950a Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 May 2021 14:06:17 +0000 Subject: [PATCH 084/195] Make Zeek and Suricata great again --- salt/filebeat/modules/suricata.yml | 12 +++ salt/filebeat/modules/zeek.yml | 122 +++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 salt/filebeat/modules/suricata.yml create mode 100644 salt/filebeat/modules/zeek.yml diff --git a/salt/filebeat/modules/suricata.yml b/salt/filebeat/modules/suricata.yml new file mode 100644 index 000000000..b7cc11e85 --- /dev/null +++ b/salt/filebeat/modules/suricata.yml @@ -0,0 +1,12 @@ +# Module: suricata +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html + +- module: suricata + # All logs + eve: + enabled: true + var.paths: ["/nsm/suricata/eve*.json"] + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/salt/filebeat/modules/zeek.yml b/salt/filebeat/modules/zeek.yml new file mode 100644 index 000000000..9fd61c448 --- /dev/null +++ b/salt/filebeat/modules/zeek.yml @@ -0,0 +1,122 @@ +# Module: zeek +# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html + +- module: zeek + capture_loss: + enabled: false + var.paths: ["/nsm/zeek/logs/current/capture_loss.log"] + connection: + enabled: true + var.paths: ["/nsm/zeek/logs/current/conn.log"] + dce_rpc: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dce_rpc.log"] + dhcp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dhcp.log"] + dnp3: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dnp3.log"] + dns: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dns.log"] + dpd: + enabled: true + var.paths: ["/nsm/zeek/logs/current/dpd.log"] + files: + enabled: true + var.paths: ["/nsm/zeek/logs/current/files.log"] + ftp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/ftp.log"] + http: + enabled: true + var.paths: ["/nsm/zeek/logs/current/http.log"] + intel: + enabled: true + var.paths: ["/nsm/zeek/logs/current/intel.log"] + irc: + enabled: true + var.paths: ["/nsm/zeek/logs/current/irc.log"] + kerberos: + enabled: true + var.paths: ["/nsm/zeek/logs/current/kerberos.log"] + modbus: + enabled: true + var.paths: ["/nsm/zeek/logs/current/modbus.log"] + mysql: + enabled: true + var.paths: ["/nsm/zeek/logs/current/mysql.log"] + notice: + enabled: true + var.paths: ["/nsm/zeek/logs/current/notice.log"] + ntlm: + enabled: true + var.paths: ["/nsm/zeek/logs/current/ntlm.log"] + ocsp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/oscp.log"] + pe: + enabled: true + var.paths: ["/nsm/zeek/logs/current/pe.log"] + radius: + enabled: true + var.paths: ["/nsm/zeek/logs/current/radius.log"] + rdp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/rdp.log"] + rfb: + enabled: true + var.paths: ["/nsm/zeek/logs/current/rfb.log"] + signature: + enabled: true + var.paths: ["/nsm/zeek/logs/current/signature.log"] + sip: + enabled: true + var.paths: ["/nsm/zeek/logs/current/sip.log"] + smb_cmd: + enabled: true + var.paths: ["/nsm/zeek/logs/current/smb_cmd.log"] + smb_files: + enabled: true + var.paths: ["/nsm/zeek/logs/current/smb_files.log"] + smb_mapping: + enabled: true + var.paths: ["/nsm/zeek/logs/current/smb_mapping.log"] + smtp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/smtp.log"] + snmp: + enabled: true + var.paths: ["/nsm/zeek/logs/current/snmp.log"] + socks: + enabled: true + var.paths: ["/nsm/zeek/logs/current/socks.log"] + ssh: + enabled: true + var.paths: ["/nsm/zeek/logs/current/ssh.log"] + ssl: + enabled: true + var.paths: ["/nsm/zeek/logs/current/ssl.log"] + stats: + enabled: false + var.paths: ["/nsm/zeek/logs/current/stats.log"] + syslog: + enabled: false + var.paths: ["/nsm/zeek/logs/current/syslog.log"] + traceroute: + enabled: false + var.paths: ["/nsm/zeek/logs/current/traceroute.log.log"] + tunnel: + enabled: true + var.paths: ["/nsm/zeek/logs/current/tunnel.log"] + weird: + enabled: true + var.paths: ["/nsm/zeek/logs/current/weird.log"] + x509: + enabled: true + var.paths: ["/nsm/zeek/logs/current/x509.log"] + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: From da528e802f2797cdcc4a6b920b87f3476bd74399 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 12:52:47 -0400 Subject: [PATCH 085/195] ensure migration script doesnt migrate the current days data and fix downsample cq to move from so_short_term rp --- salt/common/tools/sbin/so-influxdb-migrate | 21 +++++++++++---------- salt/influxdb/init.sls | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index 3411dc39f..4d65cba95 100644 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -26,17 +26,18 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? - echo "Migration of 2020-07-21T00:00:00.0000000Z' + ${day}d started at $startdate and completed at `date`." + migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` + echo "Migration of $migrationdate started at $startdate and completed at `date`." -# echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" -# if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then -# echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" -# else -# echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." -# fi - - ((day=day+1)) - startdate=`date` + newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s") + today=$(date +"%s") + if [ $newdaytomigrate -ge $today ]; then + break + else + ((day=day+1)) + startdate=`date` + echo -e "\nMigrating the next day's worth of data." + fi done echo -e "\nInfluxDb data migration complete." diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index c6120d292..2617578a4 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,7 +117,7 @@ so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM \"so_short_term\"./.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] From 78240b4b5232513e32521bed339157df32940a9c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 17:49:02 -0400 Subject: [PATCH 086/195] change retry command --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 965b9369c..6cb292aba 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that there is at least 1 minion connected - retry 50 10 "(( $(salt-run manage.up -l error | wc -l) >= 1 ))" + retry 50 10 "salt-run manage.up -l error" $MINIONID echo "" echo "Ensuring python modules for Salt are installed and patched." From b8ad80ae35c0cc352ab3b942efcd4ca82c1329f9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 17:49:40 -0400 Subject: [PATCH 087/195] update comment --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6cb292aba..bf09e2b2e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -801,7 +801,7 @@ else echo "Starting Salt Master service." systemctl start salt-master - # Testing that that salt-master is up by checking that there is at least 1 minion connected + # Testing that that salt-master is up by checking that is it connected to itself retry 50 10 "salt-run manage.up -l error" $MINIONID echo "" From 6474c296e18240ee028c234f83f69eeb9e23bfaf Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 20:26:13 -0400 Subject: [PATCH 088/195] dont need to specify dest rp --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 2617578a4..c6120d292 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,7 +117,7 @@ so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM \"so_short_term\"./.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] From 5d3b147b42aa1040e11af75dae0fec34494c2e8d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 20:32:26 -0400 Subject: [PATCH 089/195] change retry command --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index bf09e2b2e..751a2413c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-run manage.up -l error" $MINIONID + retry 50 10 "salt-run manage.up -l error | grep $MINIONID" echo "" echo "Ensuring python modules for Salt are installed and patched." From 46223e0b3055e8c0af6d424ab23df399421d0627 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 08:59:47 -0400 Subject: [PATCH 090/195] add quotes around minionid --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 751a2413c..6fb139909 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-run manage.up -l error | grep $MINIONID" + retry 50 10 "salt-run manage.up -l error | grep \"$MINIONID\"" echo "" echo "Ensuring python modules for Salt are installed and patched." From d0e084b8eab150719b63925714a8dbc5fce19927 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 10:20:04 -0400 Subject: [PATCH 091/195] change command to test if salt-master is accepting connections --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6fb139909..77a2b6018 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-run manage.up -l error | grep \"$MINIONID\"" + retry 50 10 "salt-call state.show_top -l error" echo "" echo "Ensuring python modules for Salt are installed and patched." From 7b29c6427b5f5efc1ef53052c65e588d992b4a90 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 7 May 2021 10:55:17 -0400 Subject: [PATCH 092/195] Add preliminary error handling in soup --- salt/common/tools/sbin/so-common | 15 ++++ salt/common/tools/sbin/so-image-common | 13 ++- salt/common/tools/sbin/soup | 107 +++++++++++++++++++++---- 3 files changed, 117 insertions(+), 18 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 22c7543ea..f78919226 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -324,6 +324,21 @@ retry() { return 1 } +run_check_net_err() { + local cmd=$1 + local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable + + local exit_code + retry 50 10 "$cmd" + exit_code=$? + + if [[ $exit_code -ne 0 ]]; then + echo "Command failed with error $exit_code" + echo "$err_msg" + exit $exit_code + fi +} + set_os() { if [ -f /etc/redhat-release ]; then OS=centos diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index be5a327f0..c855abd15 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -18,6 +18,7 @@ # NOTE: This script depends on so-common IMAGEREPO=security-onion-solutions +# shellcheck disable=SC2120 container_list() { MANAGERCHECK=$1 @@ -128,7 +129,9 @@ update_docker_containers() { mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 # Let's make sure we have the public key - retry 50 10 "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1 + run_check_net_err \ + "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ + "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" >> "$LOG_FILE" 2>&1 result=$? if [[ $result -eq 0 ]]; then cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 @@ -148,10 +151,14 @@ update_docker_containers() { # Pull down the trusted docker image local image=$i:$VERSION$IMAGE_TAG_SUFFIX - retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1 + run_check_net_err \ + "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \ + "Could not pull pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 # Get signature - retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1 + run_check_net_err \ + "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ + "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 exit 1 diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 8e7f70517..f43a31740 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -20,11 +20,83 @@ UPDATE_DIR=/tmp/sogh/securityonion INSTALLEDVERSION=$(cat /etc/soversion) POSTVERSION=$INSTALLEDVERSION -INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) +INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}') BATCHSIZE=5 SOUP_LOG=/root/soup.log WHATWOULDYOUSAYYAHDOHERE=soup +set -e + +trap 'check_err $? $BASH_LINENO $BASH_COMMAND' EXIT + +check_err() { + local exit_code=$1 + local lineno=$2 + local cmd=$3 + local err_msg="Unknown error occured, please check $SOUP_LOG for details." + + if [[ $exit_code -ne 0 ]]; then + printf '%s' "Soup failed on line $lineno with error $exit_code: " + case $exit_code in + 2) + echo 'No such file or directory' + ;; + 5) + echo 'Interrupted system call' + ;; + 12) + echo 'Out of memory' + ;; + 28) + echo 'No space left on device' + echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html' + ;; + 30) + echo 'Read-only file system' + ;; + 35) + echo 'Resource temporarily unavailable' + ;; + 64) + echo 'Machine is not on the network' + ;; + 67) + echo 'Link has been severed' + ;; + 100) + echo 'Netowrk is down' + ;; + 101) + echo 'Network is unreachable' + ;; + 102) + echo 'Network reset' + ;; + 110) + echo 'Connection timed out' + ;; + 111) + echo 'Connection refused' + ;; + 112) + echo 'Host is down' + ;; + 113) + echo 'No route to host' + ;; + *) + echo '' + echo "$err_msg" + ;; + esac + if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then + echo "$err_msg" + fi + exit $exit_code + fi + +} + add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ @@ -87,9 +159,9 @@ airgap_update_dockers() { docker stop so-dockerregistry docker rm so-dockerregistry echo "Copying the new dockers over" - tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker + tar xvf "$AGDOCKER/registry.tar" -C /nsm/docker-registry/docker echo "Add Registry back" - docker load -i $AGDOCKER/registry_image.tar + docker load -i "$AGDOCKER/registry_image.tar" fi fi } @@ -101,9 +173,9 @@ update_registry() { } check_sudoers() { - if grep -q "so-setup" /etc/sudoers; then - echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." - fi + if grep -q "so-setup" /etc/sudoers; then + echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"." + fi } check_log_size_limit() { @@ -177,7 +249,7 @@ check_os_updates() { echo "Continuing without updating packages" elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" - salt \* -b 5 state.apply patch.os queue=True + run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' else echo "Exiting soup" exit 0 @@ -205,7 +277,7 @@ clone_to_tmp() { if [ -n "$BRANCH" ]; then SOUP_BRANCH="-b $BRANCH" fi - git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git + run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com" cd /tmp if [ ! -f $UPDATE_DIR/VERSION ]; then echo "Update was unable to pull from github. Please check your internet." @@ -586,13 +658,14 @@ upgrade_check() { } upgrade_check_salt() { - NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'}) + NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk '{print $2}') if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then echo "You are already running the correct version of Salt for Security Onion." else UPGRADESALT=1 fi } + upgrade_salt() { SALTUPGRADED=True echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION." @@ -604,7 +677,9 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION" + run_check_net_err \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "Could not update soup, please check $SOUP_LOG for details." echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" @@ -617,7 +692,9 @@ upgrade_salt() { apt-mark unhold "salt-minion" echo "Updating Salt packages and restarting services." echo "" - sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION" + run_check_net_err \ + "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ + "Could not update soup, please check $SOUP_LOG for details." echo "Applying apt hold for Salt." echo "" apt-mark hold "salt-common" @@ -650,7 +727,7 @@ verify_latest_update_script() { } main () { -echo "### Preparing soup at `date` ###" +echo "### Preparing soup at $(date) ###" while getopts ":b" opt; do case "$opt" in b ) # process option b @@ -756,7 +833,7 @@ else echo "Checking if Salt was upgraded." echo "" # Check that Salt was upgraded - SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'}) + SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}') if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." echo "Once the issue is resolved, run soup again." @@ -874,7 +951,7 @@ EOF fi fi -echo "### soup has been served at `date` ###" +echo "### soup has been served at $(date) ###" } cat << EOF @@ -889,6 +966,6 @@ Press Enter to continue or Ctrl-C to cancel. EOF -read input +read -r input main "$@" | tee -a $SOUP_LOG From 009f7617c1c77426c8a20c7ce62b3ec97dec6472 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 12:47:22 -0400 Subject: [PATCH 093/195] check salt-master is responding --- salt/common/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 77a2b6018..51b7bc29c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -834,6 +834,10 @@ else echo "" echo "Starting Salt Master service." systemctl start salt-master + + # Testing that that salt-master is up by checking that is it connected to itself + retry 50 10 "salt-call state.show_top -l error" + echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes From 2e228c8355473c7f366ea2f1e35defab75d8591a Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 7 May 2021 13:22:03 -0400 Subject: [PATCH 094/195] FEATURE: Pivot from Alerts/Hunt to CyberChef #4081 --- salt/soc/files/soc/alerts.actions.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/alerts.actions.json b/salt/soc/files/soc/alerts.actions.json index c0543d8fc..558d10a36 100644 --- a/salt/soc/files/soc/alerts.actions.json +++ b/salt/soc/files/soc/alerts.actions.json @@ -18,6 +18,10 @@ "/joblookup?esid={:soc_id}", "/joblookup?ncid={:network.community_id}" ]}, + { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", + "links": [ + "/cyberchef/#input={value|base64}" + ]}, { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", "links": [ "https://www.google.com/search?q={value}" @@ -26,4 +30,4 @@ "links": [ "https://www.virustotal.com/gui/search/{value}" ]} -] \ No newline at end of file +] From 293fb0a76d87feb207c8eb6db6b0a85aca49b928 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 7 May 2021 13:23:46 -0400 Subject: [PATCH 095/195] FEATURE: Pivot from Alerts/Hunt to CyberChef #4081 --- salt/soc/files/soc/hunt.actions.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/soc/files/soc/hunt.actions.json b/salt/soc/files/soc/hunt.actions.json index c0543d8fc..558d10a36 100644 --- a/salt/soc/files/soc/hunt.actions.json +++ b/salt/soc/files/soc/hunt.actions.json @@ -18,6 +18,10 @@ "/joblookup?esid={:soc_id}", "/joblookup?ncid={:network.community_id}" ]}, + { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", + "links": [ + "/cyberchef/#input={value|base64}" + ]}, { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", "links": [ "https://www.google.com/search?q={value}" @@ -26,4 +30,4 @@ "links": [ "https://www.virustotal.com/gui/search/{value}" ]} -] \ No newline at end of file +] From 9eb63b17f9f571282836b48a1343a1f6c265ff6d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 14:48:02 -0400 Subject: [PATCH 096/195] exit if retry fails --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 51b7bc29c..620737c16 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" + retry 50 10 "salt-call state.show_top -l error" || exit 1 echo "" echo "Ensuring python modules for Salt are installed and patched." @@ -836,7 +836,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" + retry 50 10 "salt-call state.show_top -l error" || exit 1 echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True From f8dd6890b27f6109d8f31a7dbbf3db709d312ea9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 7 May 2021 16:50:59 -0400 Subject: [PATCH 097/195] Unset/set exit on command fail for retries --- salt/common/tools/sbin/soup | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f43a31740..b02a63fc6 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -249,7 +249,9 @@ check_os_updates() { echo "Continuing without updating packages" elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" + set +e run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' + set -e else echo "Exiting soup" exit 0 @@ -277,7 +279,9 @@ clone_to_tmp() { if [ -n "$BRANCH" ]; then SOUP_BRANCH="-b $BRANCH" fi + set +e run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com" + set -e cd /tmp if [ ! -f $UPDATE_DIR/VERSION ]; then echo "Update was unable to pull from github. Please check your internet." @@ -677,9 +681,11 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" + set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." + set -e echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" @@ -692,9 +698,11 @@ upgrade_salt() { apt-mark unhold "salt-minion" echo "Updating Salt packages and restarting services." echo "" + set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." + set -e echo "Applying apt hold for Salt." echo "" apt-mark hold "salt-common" @@ -808,7 +816,9 @@ else check_os_updates else update_registry + set +e update_docker_containers "soup" + set -e fi echo "" From 8b3730748b7a26d58489a9b803337b54e3f5827c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 10:58:29 -0400 Subject: [PATCH 098/195] Add debug line and remove exit command on retry failure --- salt/common/tools/sbin/soup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 398aed29c..fdc983d30 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -662,6 +662,7 @@ upgrade_check() { is_hotfix=true fi else + echo "DEBUG: here" is_hotfix=false fi @@ -889,7 +890,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" || exit 1 + retry 50 10 "salt-call state.show_top -l error" echo "" echo "Ensuring python modules for Salt are installed and patched." @@ -923,7 +924,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" || exit 1 + retry 50 10 "salt-call state.show_top -l error" echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True From a75d4841d0a29eb06ec492fbf0518f3c6bb86b45 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 11:05:24 -0400 Subject: [PATCH 099/195] Add debug lines --- salt/common/tools/sbin/soup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index fdc983d30..d1d3f5e6e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -734,7 +734,7 @@ verify_latest_update_script() { cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ - salt-call state.apply common queue=True + salt-call state.apply -l info common queue=True echo "" echo "soup has been updated. Please run soup again." exit 0 @@ -796,7 +796,9 @@ add_common fi echo "Let's see if we need to update Security Onion." +echo "DEBUG: before upgrade_check" upgrade_check +echo "DEBUG: after upgrade_check" upgrade_space echo "Checking for Salt Master and Minion updates." From d3b08beb534a2b3cd336d7d2dcdb9dc8ab8dc7b0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 11:11:54 -0400 Subject: [PATCH 100/195] Only cat file if it exists --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d1d3f5e6e..6c4af361b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -651,7 +651,7 @@ upgrade_check() { # Let's make sure we actually need to update. NEWVERSION=$(cat $UPDATE_DIR/VERSION) HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX) - CURRENTHOTFIX=$(cat /etc/sohotfix 2>/dev/null) + [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix) if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then echo "Checking to see if there are hotfixes needed" if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then From fec868432f3d240a216b42647cc633a6cd9c168c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 11:59:22 -0400 Subject: [PATCH 101/195] Try to fix bash trap --- salt/common/tools/sbin/soup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6c4af361b..364a769c0 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -28,13 +28,12 @@ WHATWOULDYOUSAYYAHDOHERE=soup set -e -trap 'check_err $? $BASH_LINENO $BASH_COMMAND' EXIT +trap 'check_err $? $BASH_LINENO' EXIT check_err() { local exit_code=$1 local lineno=$2 - local cmd=$3 - local err_msg="Unknown error occured, please check $SOUP_LOG for details." + local err_msg="Unhandled error occured, please check $SOUP_LOG for details." if [[ $exit_code -ne 0 ]]; then printf '%s' "Soup failed on line $lineno with error $exit_code: " @@ -86,7 +85,7 @@ check_err() { echo 'No route to host' ;; *) - echo '' + echo 'Unhandled error' echo "$err_msg" ;; esac From f793450d972cb69743fe7cac96644e934ac275b6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 10 May 2021 13:22:13 -0400 Subject: [PATCH 102/195] Return actual exit code from retry --- salt/common/tools/sbin/so-common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index f78919226..2b2bbf944 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -302,6 +302,7 @@ retry() { cmd=$3 expectedOutput=$4 attempt=0 + local exitcode=0 while [[ $attempt -lt $maxAttempts ]]; do attempt=$((attempt+1)) echo "Executing command with retry support: $cmd" @@ -321,7 +322,7 @@ retry() { sleep $sleepDelay done echo "Command continues to fail; giving up." - return 1 + return $exitcode } run_check_net_err() { From 01a121e0298243dc664be36caa2661b13b9bc020 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 May 2021 15:29:50 -0400 Subject: [PATCH 103/195] Add defaults.yml --- salt/filebeat/thirdpartydefaults.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 salt/filebeat/thirdpartydefaults.yml diff --git a/salt/filebeat/thirdpartydefaults.yml b/salt/filebeat/thirdpartydefaults.yml new file mode 100644 index 000000000..9e5fef988 --- /dev/null +++ b/salt/filebeat/thirdpartydefaults.yml @@ -0,0 +1,17 @@ +thirtd_party_filebeat: + modules: + fortinet: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9004 + clientendpoint: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9510 + fortimail: + enabled: false + var.input: udp + var.syslog_port: 9350 \ No newline at end of file From efc028d0a55b39a39ea484cb9c1a5cd425e63dd6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 10 May 2021 18:08:47 -0400 Subject: [PATCH 104/195] handle the docker port bindings for filebeat modules --- salt/filebeat/init.sls | 9 +++++++++ salt/filebeat/map.jinja | 2 ++ salt/filebeat/thirdpartydefaults.yml | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 salt/filebeat/map.jinja diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 8ab200276..eb4dc116a 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -20,6 +20,8 @@ {% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} {% set MANAGER = salt['grains.get']('master') %} {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} +{% from 'filebeat/map.jinja' import THIRDPARTY with context %} + filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc @@ -98,6 +100,13 @@ so-filebeat: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp - 0.0.0.0:5066:5066/tcp +{% for module in THIRDPARTY.modules.keys() %} + {% for submodule in THIRDPARTY.modules[module] %} + {% if THIRDPARTY.modules[module][submodule].enabled %} + - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/{{ THIRDPARTY.modules[module][submodule]["var.input"] }} + {% endif %} + {% endfor %} +{% endfor %} - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja new file mode 100644 index 000000000..668889227 --- /dev/null +++ b/salt/filebeat/map.jinja @@ -0,0 +1,2 @@ +{% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %} +{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} \ No newline at end of file diff --git a/salt/filebeat/thirdpartydefaults.yml b/salt/filebeat/thirdpartydefaults.yml index 9e5fef988..1e2eb8c23 100644 --- a/salt/filebeat/thirdpartydefaults.yml +++ b/salt/filebeat/thirdpartydefaults.yml @@ -1,4 +1,4 @@ -thirtd_party_filebeat: +third_party_filebeat: modules: fortinet: firewall: From 4012a8276c93e3b6dcf9ff3c22e3085910c5d249 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 May 2021 12:22:25 -0400 Subject: [PATCH 105/195] add template for module .yml file --- salt/filebeat/etc/thirdparty.yml.jinja | 16 ++++++++++++++++ salt/filebeat/init.sls | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 salt/filebeat/etc/thirdparty.yml.jinja diff --git a/salt/filebeat/etc/thirdparty.yml.jinja b/salt/filebeat/etc/thirdparty.yml.jinja new file mode 100644 index 000000000..8c081828e --- /dev/null +++ b/salt/filebeat/etc/thirdparty.yml.jinja @@ -0,0 +1,16 @@ +# DO NOT EDIT THIS FILE +{% for module in THIRDPARTY.modules.keys() -%} +- module: {{ module }} + {%- for fileset in THIRDPARTY.modules[module] %} + {{ fileset }}: + enabled: {{ THIRDPARTY.modules[module][fileset].enabled }} + {#- only manage the settings if the fileset is enabled #} + {%- if THIRDPARTY.modules[module][fileset].enabled %} + {%- for var, value in THIRDPARTY.modules[module][fileset].items() %} + {%- if var|lower != 'enabled' %} + {{ var }}: {{ value }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endfor %} +{% endfor %} \ No newline at end of file diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index eb4dc116a..b3bce806c 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -22,24 +22,28 @@ {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} + filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc - user: 939 - group: 939 - makedirs: True + filebeatmoduledir: file.directory: - name: /opt/so/conf/filebeat/modules - user: root - group: root - makedirs: True + filebeatlogdir: file.directory: - name: /opt/so/log/filebeat - user: 939 - group: 939 - makedirs: True + filebeatpkidir: file.directory: - name: /opt/so/conf/filebeat/etc/pki @@ -52,6 +56,7 @@ fileregistrydir: - user: 939 - group: 939 - makedirs: True + # This needs to be owned by root filebeatconfsync: file.managed: @@ -63,6 +68,7 @@ filebeatconfsync: - defaults: INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }} OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }} + # Filebeat module config file filebeatmoduleconfsync: file.managed: @@ -71,6 +77,7 @@ filebeatmoduleconfsync: - user: root - group: root - template: jinja + # Sync Filebeat modules filebeatmodules: file.recurse: @@ -78,6 +85,15 @@ filebeatmodules: - source: salt://filebeat/modules - user: root - group: root + +thirdparty_module_conf: + file.managed: + - name: /opt/so/conf/filebeat/etc/thirdparty.yml + - source: salt://filebeat/etc/thirdparty.yml.jinja + - template: jinja + - defaults: + THIRDPARTY: {{ THIRDPARTY }} + so-filebeat: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }} From ad67167e9778908534b921c38f27b3bdbc88d162 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 May 2021 12:58:21 -0400 Subject: [PATCH 106/195] remove whitespace control --- salt/filebeat/etc/thirdparty.yml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/thirdparty.yml.jinja b/salt/filebeat/etc/thirdparty.yml.jinja index 8c081828e..186115af4 100644 --- a/salt/filebeat/etc/thirdparty.yml.jinja +++ b/salt/filebeat/etc/thirdparty.yml.jinja @@ -1,5 +1,5 @@ # DO NOT EDIT THIS FILE -{% for module in THIRDPARTY.modules.keys() -%} +{% for module in THIRDPARTY.modules.keys() %} - module: {{ module }} {%- for fileset in THIRDPARTY.modules[module] %} {{ fileset }}: From c379822bf0df0fb52083392498feebc8e22ddc1e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 11 May 2021 12:59:49 -0400 Subject: [PATCH 107/195] Set variable to skip trap if error already handled --- salt/common/tools/sbin/so-common | 1 + salt/common/tools/sbin/soup | 1 + 2 files changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 2b2bbf944..2b22ebbeb 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -334,6 +334,7 @@ run_check_net_err() { exit_code=$? if [[ $exit_code -ne 0 ]]; then + ERR_HANDLED=true echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 364a769c0..c8601fe1a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -35,6 +35,7 @@ check_err() { local lineno=$2 local err_msg="Unhandled error occured, please check $SOUP_LOG for details." + [[ $ERR_HANDLED == true ]] && exit $exit_code if [[ $exit_code -ne 0 ]]; then printf '%s' "Soup failed on line $lineno with error $exit_code: " case $exit_code in From 7bd04deae78f0199afb706afefb8ca4b20588b5e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 12 May 2021 10:45:03 -0400 Subject: [PATCH 108/195] Unset exit on failure for pkill command --- salt/common/tools/sbin/soup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index c8601fe1a..7c97a540a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -834,7 +834,9 @@ else echo "Stopping Salt Minion service." systemctl stop salt-minion echo "Killing any remaining Salt Minion processes." + set +e pkill -9 -ef /usr/bin/salt-minion + set -e echo "" echo "Stopping Salt Master service." systemctl stop salt-master From 807b525c797bd280036bf06d0a63a17693894f44 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 12 May 2021 11:19:33 -0400 Subject: [PATCH 109/195] Temp remove exit on failure + bash trap --- salt/common/tools/sbin/soup | 83 ------------------------------------- 1 file changed, 83 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 7c97a540a..53dd7f176 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -26,77 +26,6 @@ SOUP_LOG=/root/soup.log INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup -set -e - -trap 'check_err $? $BASH_LINENO' EXIT - -check_err() { - local exit_code=$1 - local lineno=$2 - local err_msg="Unhandled error occured, please check $SOUP_LOG for details." - - [[ $ERR_HANDLED == true ]] && exit $exit_code - if [[ $exit_code -ne 0 ]]; then - printf '%s' "Soup failed on line $lineno with error $exit_code: " - case $exit_code in - 2) - echo 'No such file or directory' - ;; - 5) - echo 'Interrupted system call' - ;; - 12) - echo 'Out of memory' - ;; - 28) - echo 'No space left on device' - echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html' - ;; - 30) - echo 'Read-only file system' - ;; - 35) - echo 'Resource temporarily unavailable' - ;; - 64) - echo 'Machine is not on the network' - ;; - 67) - echo 'Link has been severed' - ;; - 100) - echo 'Netowrk is down' - ;; - 101) - echo 'Network is unreachable' - ;; - 102) - echo 'Network reset' - ;; - 110) - echo 'Connection timed out' - ;; - 111) - echo 'Connection refused' - ;; - 112) - echo 'Host is down' - ;; - 113) - echo 'No route to host' - ;; - *) - echo 'Unhandled error' - echo "$err_msg" - ;; - esac - if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then - echo "$err_msg" - fi - exit $exit_code - fi - -} add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ @@ -250,9 +179,7 @@ check_os_updates() { echo "Continuing without updating packages" elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" - set +e run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' - set -e else echo "Exiting soup" exit 0 @@ -280,9 +207,7 @@ clone_to_tmp() { if [ -n "$BRANCH" ]; then SOUP_BRANCH="-b $BRANCH" fi - set +e run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com" - set -e cd /tmp if [ ! -f $UPDATE_DIR/VERSION ]; then echo "Update was unable to pull from github. Please check your internet." @@ -688,11 +613,9 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" - set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." - set -e echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" @@ -705,11 +628,9 @@ upgrade_salt() { apt-mark unhold "salt-minion" echo "Updating Salt packages and restarting services." echo "" - set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." - set -e echo "Applying apt hold for Salt." echo "" apt-mark hold "salt-common" @@ -825,18 +746,14 @@ else check_os_updates else update_registry - set +e update_docker_containers "soup" - set -e fi echo "" echo "Stopping Salt Minion service." systemctl stop salt-minion echo "Killing any remaining Salt Minion processes." - set +e pkill -9 -ef /usr/bin/salt-minion - set -e echo "" echo "Stopping Salt Master service." systemctl stop salt-master From 9ced391c11f255a7fe22b4e740b436398de0983d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 12 May 2021 13:20:59 -0400 Subject: [PATCH 110/195] Fix indent in main(), re-add trap, remove ERR_HANDLED variable --- salt/common/tools/sbin/so-common | 1 - salt/common/tools/sbin/soup | 534 ++++++++++++++++++------------- 2 files changed, 309 insertions(+), 226 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 2b22ebbeb..2b2bbf944 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -334,7 +334,6 @@ run_check_net_err() { exit_code=$? if [[ $exit_code -ne 0 ]]; then - ERR_HANDLED=true echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 53dd7f176..814f9fd1b 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -26,6 +26,72 @@ SOUP_LOG=/root/soup.log INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup +check_err() { + local exit_code=$1 + local lineno=$2 + local err_msg="Unhandled error occured, please check $SOUP_LOG for details." + + if [[ $exit_code -ne 0 ]]; then + printf '%s' "Soup failed on line $lineno with error $exit_code: " + case $exit_code in + 2) + echo 'No such file or directory' + ;; + 5) + echo 'Interrupted system call' + ;; + 12) + echo 'Out of memory' + ;; + 28) + echo 'No space left on device' + echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html' + ;; + 30) + echo 'Read-only file system' + ;; + 35) + echo 'Resource temporarily unavailable' + ;; + 64) + echo 'Machine is not on the network' + ;; + 67) + echo 'Link has been severed' + ;; + 100) + echo 'Netowrk is down' + ;; + 101) + echo 'Network is unreachable' + ;; + 102) + echo 'Network reset' + ;; + 110) + echo 'Connection timed out' + ;; + 111) + echo 'Connection refused' + ;; + 112) + echo 'Host is down' + ;; + 113) + echo 'No route to host' + ;; + *) + echo 'Unhandled error' + echo "$err_msg" + ;; + esac + if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then + echo "$err_msg" + fi + exit $exit_code + fi + +} add_common() { cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/ @@ -48,8 +114,8 @@ airgap_mounted() { echo "Example: /home/user/securityonion-2.X.0.iso" echo "Example: /dev/sdx1" echo "" - read -p 'Enter the location of the iso: ' ISOLOC - if [ -f $ISOLOC ]; then + read -rp 'Enter the location of the iso: ' ISOLOC + if [[ -f $ISOLOC ]]; then # Mounting the ISO image mkdir -p /tmp/soagupdate mount -t iso9660 -o loop $ISOLOC /tmp/soagupdate @@ -61,7 +127,7 @@ airgap_mounted() { else echo "ISO has been mounted!" fi - elif [ -f $ISOLOC/SecurityOnion/VERSION ]; then + elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then ln -s $ISOLOC /tmp/soagupdate echo "Found the update content" else @@ -79,9 +145,9 @@ airgap_mounted() { } airgap_update_dockers() { - if [ $is_airgap -eq 0 ]; then + if [[ $is_airgap -eq 0 ]]; then # Let's copy the tarball - if [ ! -f $AGDOCKER/registry.tar ]; then + if [[ ! -f $AGDOCKER/registry.tar ]]; then echo "Unable to locate registry. Exiting" exit 1 else @@ -179,7 +245,9 @@ check_os_updates() { echo "Continuing without updating packages" elif [[ "$confirm" == [uU] ]]; then echo "Applying Grid Updates" + set +e run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.' + set -e else echo "Exiting soup" exit 0 @@ -207,7 +275,9 @@ clone_to_tmp() { if [ -n "$BRANCH" ]; then SOUP_BRANCH="-b $BRANCH" fi + set +e run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com" + set -e cd /tmp if [ ! -f $UPDATE_DIR/VERSION ]; then echo "Update was unable to pull from github. Please check your internet." @@ -253,7 +323,11 @@ preupgrade_changes_2.3.50_repo() { # We made repo changes in 2.3.50 and this prepares for that on upgrade echo "Checking to see if 2.3.50 repo changes are needed." - [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo + if [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]]; then + up_2.3.3X_to_2.3.50_repo + else + echo "No changes needed." + fi } preupgrade_changes() { @@ -433,7 +507,7 @@ up_2.3.2X_to_2.3.30() { sed -i "/ imagerepo: securityonion/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls # Strelka rule repo pillar addition - if [ $is_airgap -eq 0 ]; then + if [[ $is_airgap -eq 0 ]]; then # Add manager as default Strelka YARA rule repo sed -i "/^strelka:/a \\ repos: \n - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls; else @@ -460,7 +534,7 @@ up_2.3.3X_to_2.3.50_repo() { rm -f "/etc/yum.repos.d/$DELREPO.repo" fi done - if [ $is_airgap -eq 1 ]; then + if [[ $is_airgap -eq 1 ]]; then # Copy the new repo file if not airgap cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/ yum clean all @@ -613,9 +687,11 @@ upgrade_salt() { yum versionlock delete "salt-*" echo "Updating Salt packages and restarting services." echo "" + set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." + set -e echo "Applying yum versionlock for Salt." echo "" yum versionlock add "salt-*" @@ -628,9 +704,11 @@ upgrade_salt() { apt-mark unhold "salt-minion" echo "Updating Salt packages and restarting services." echo "" + set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ "Could not update soup, please check $SOUP_LOG for details." + set -e echo "Applying apt hold for Salt." echo "" apt-mark hold "salt-common" @@ -662,229 +740,235 @@ verify_latest_update_script() { fi } -main () { -echo "### Preparing soup at $(date) ###" -while getopts ":b" opt; do - case "$opt" in - b ) # process option b - shift - BATCHSIZE=$1 - if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then - echo "Batch size must be a number greater than 0." - exit 1 - fi - ;; - \? ) - echo "Usage: cmd [-b]" - ;; - esac -done - -echo "Checking to see if this is a manager." -echo "" -require_manager -set_minionid -echo "Checking to see if this is an airgap install" -echo "" -check_airgap -echo "Found that Security Onion $INSTALLEDVERSION is currently installed." -echo "" -set_os -set_palette -check_elastic_license -echo "" -if [ $is_airgap -eq 0 ]; then - # Let's mount the ISO since this is airgap - airgap_mounted -else - echo "Cloning Security Onion github repo into $UPDATE_DIR." - echo "Removing previous upgrade sources." - rm -rf $UPDATE_DIR - clone_to_tmp -fi -check_os_updates -echo "" -echo "Verifying we have the latest soup script." -verify_latest_update_script -echo "" - -echo "Generating new repo archive" -generate_and_clean_tarballs -if [ -f /usr/sbin/so-image-common ]; then - . /usr/sbin/so-image-common -else -add_common -fi - -echo "Let's see if we need to update Security Onion." -echo "DEBUG: before upgrade_check" -upgrade_check -echo "DEBUG: after upgrade_check" -upgrade_space - -echo "Checking for Salt Master and Minion updates." -upgrade_check_salt - - -if [ "$is_hotfix" == "true" ]; then - echo "Applying $HOTFIXVERSION" - copy_new_files - echo "" - update_version - salt-call state.highstate -l info queue=True +main() { + set -e + trap 'check_err $? $BASH_LINENO' EXIT -else - echo "" - echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." - echo "" - - echo "Updating dockers to $NEWVERSION." - if [ $is_airgap -eq 0 ]; then - airgap_update_dockers - update_centos_repo - yum clean all - check_os_updates - else - update_registry - update_docker_containers "soup" - fi - - echo "" - echo "Stopping Salt Minion service." - systemctl stop salt-minion - echo "Killing any remaining Salt Minion processes." - pkill -9 -ef /usr/bin/salt-minion - echo "" - echo "Stopping Salt Master service." - systemctl stop salt-master - echo "" - - preupgrade_changes_2.3.50_repo - - # Does salt need upgraded. If so update it. - if [ "$UPGRADESALT" == "1" ]; then - echo "Upgrading Salt" - # Update the repo files so it can actually upgrade - upgrade_salt - fi - - echo "Checking if Salt was upgraded." - echo "" - # Check that Salt was upgraded - SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}') - if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then - echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." - echo "Once the issue is resolved, run soup again." - echo "Exiting." - echo "" - exit 1 - else - echo "Salt upgrade success." - echo "" - fi - - preupgrade_changes - echo "" - - if [ $is_airgap -eq 0 ]; then - echo "Updating Rule Files to the Latest." - update_airgap_rules - fi - - # Only update the repo if its airgap - if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then - update_centos_repo - fi - - echo "" - echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR." - copy_new_files - echo "" - update_version - - echo "" - echo "Locking down Salt Master for upgrade" - masterlock - - echo "" - echo "Starting Salt Master service." - systemctl start salt-master - - # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" - - echo "" - echo "Ensuring python modules for Salt are installed and patched." - salt-call state.apply salt.python3-influxdb -l info queue=True - echo "" - - # Only regenerate osquery packages if Fleet is enabled - FLEET_MANAGER=$(lookup_pillar fleet_manager) - FLEET_NODE=$(lookup_pillar fleet_node) - if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then - echo "" - echo "Regenerating Osquery Packages.... This will take several minutes." - salt-call state.apply fleet.event_gen-packages -l info queue=True - echo "" - fi - - echo "" - echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." - salt-call state.highstate -l info queue=True - echo "" - echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete." - - echo "" - echo "Stopping Salt Master to remove ACL" - systemctl stop salt-master - - masterunlock - - echo "" - echo "Starting Salt Master service." - systemctl start salt-master - - # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" - - echo "Running a highstate. This could take several minutes." - salt-call state.highstate -l info queue=True - postupgrade_changes - unmount_update - thehive_maint - - if [ "$UPGRADESALT" == "1" ]; then - if [ $is_airgap -eq 0 ]; then - echo "" - echo "Cleaning repos on remote Security Onion nodes." - salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all" - echo "" - fi - fi - - check_sudoers - - if [[ -n $lsl_msg ]]; then - case $lsl_msg in - 'distributed') - echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect." - echo " -> We recommend checking and adjusting the values as necessary." - echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/" + echo "### Preparing soup at $(date) ###" + while getopts ":b" opt; do + case "$opt" in + b ) # process option b + shift + BATCHSIZE=$1 + if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then + echo "Batch size must be a number greater than 0." + exit 1 + fi ;; - 'single-node') - # We can assume the lsl_details array has been set if lsl_msg has this value - echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}." - echo " -> We recommend checking and adjusting the value as necessary." - echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls" + \? ) + echo "Usage: cmd [-b]" ;; esac + done + + echo "Checking to see if this is a manager." + echo "" + require_manager + set_minionid + echo "Checking to see if this is an airgap install" + echo "" + check_airgap + echo "Found that Security Onion $INSTALLEDVERSION is currently installed." + echo "" + set_os + set_palette + check_elastic_license + echo "" + if [[ $is_airgap -eq 0 ]]; then + # Let's mount the ISO since this is airgap + airgap_mounted + else + echo "Cloning Security Onion github repo into $UPDATE_DIR." + echo "Removing previous upgrade sources." + rm -rf $UPDATE_DIR + clone_to_tmp + fi + check_os_updates + echo "" + echo "Verifying we have the latest soup script." + verify_latest_update_script + echo "" + + echo "Generating new repo archive" + generate_and_clean_tarballs + if [ -f /usr/sbin/so-image-common ]; then + . /usr/sbin/so-image-common + else + add_common fi - NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) + echo "Let's see if we need to update Security Onion." + echo "DEBUG: before upgrade_check" + upgrade_check + echo "DEBUG: after upgrade_check" + upgrade_space - if [ $NUM_MINIONS -gt 1 ]; then + echo "Checking for Salt Master and Minion updates." + upgrade_check_salt - cat << EOF + + if [ "$is_hotfix" == "true" ]; then + echo "Applying $HOTFIXVERSION" + copy_new_files + echo "" + update_version + salt-call state.highstate -l info queue=True + else + echo "" + echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION." + echo "" + + echo "Updating dockers to $NEWVERSION." + if [[ $is_airgap -eq 0 ]]; then + airgap_update_dockers + update_centos_repo + yum clean all + check_os_updates + else + update_registry + set +e + update_docker_containers "soup" + set -e + fi + + echo "" + echo "Stopping Salt Minion service." + systemctl stop salt-minion + echo "Killing any remaining Salt Minion processes." + set +e + pkill -9 -ef /usr/bin/salt-minion + set -e + echo "" + echo "Stopping Salt Master service." + systemctl stop salt-master + echo "" + + preupgrade_changes_2.3.50_repo + + # Does salt need upgraded. If so update it. + if [[ $UPGRADESALT -eq 1 ]]; then + echo "Upgrading Salt" + # Update the repo files so it can actually upgrade + upgrade_salt + fi + + echo "Checking if Salt was upgraded." + echo "" + # Check that Salt was upgraded + SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}') + if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then + echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG." + echo "Once the issue is resolved, run soup again." + echo "Exiting." + echo "" + exit 1 + else + echo "Salt upgrade success." + echo "" + fi + + preupgrade_changes + echo "" + + if [[ $is_airgap -eq 0 ]]; then + echo "Updating Rule Files to the Latest." + update_airgap_rules + fi + + # Only update the repo if its airgap + if [[ $is_airgap -eq 0 ]] && [[ $UPGRADESALT -ne 1 ]]; then + update_centos_repo + fi + + echo "" + echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR." + copy_new_files + echo "" + update_version + + echo "" + echo "Locking down Salt Master for upgrade" + masterlock + + echo "" + echo "Starting Salt Master service." + systemctl start salt-master + + # Testing that that salt-master is up by checking that is it connected to itself + retry 50 10 "salt-call state.show_top -l error" + + echo "" + echo "Ensuring python modules for Salt are installed and patched." + salt-call state.apply salt.python3-influxdb -l info queue=True + echo "" + + # Only regenerate osquery packages if Fleet is enabled + FLEET_MANAGER=$(lookup_pillar fleet_manager) + FLEET_NODE=$(lookup_pillar fleet_node) + if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then + echo "" + echo "Regenerating Osquery Packages.... This will take several minutes." + salt-call state.apply fleet.event_gen-packages -l info queue=True + echo "" + fi + + echo "" + echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." + salt-call state.highstate -l info queue=True + echo "" + echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete." + + echo "" + echo "Stopping Salt Master to remove ACL" + systemctl stop salt-master + + masterunlock + + echo "" + echo "Starting Salt Master service." + systemctl start salt-master + + # Testing that that salt-master is up by checking that is it connected to itself + retry 50 10 "salt-call state.show_top -l error" + + echo "Running a highstate. This could take several minutes." + salt-call state.highstate -l info queue=True + postupgrade_changes + unmount_update + thehive_maint + + if [[ $UPGRADESALT -eq 1 ]]; then + if [[ $is_airgap -eq 0 ]]; then + echo "" + echo "Cleaning repos on remote Security Onion nodes." + salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all" + echo "" + fi + fi + + check_sudoers + + if [[ -n $lsl_msg ]]; then + case $lsl_msg in + 'distributed') + echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect." + echo " -> We recommend checking and adjusting the values as necessary." + echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/" + ;; + 'single-node') + # We can assume the lsl_details array has been set if lsl_msg has this value + echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}." + echo " -> We recommend checking and adjusting the value as necessary." + echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls" + ;; + esac + fi + + NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) + + if [[ $NUM_MINIONS -gt 1 ]]; then + + cat << EOF @@ -898,10 +982,10 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html EOF + fi fi -fi -echo "### soup has been served at $(date) ###" + echo "### soup has been served at $(date) ###" } cat << EOF From 192b5db25a5c3d2746da34f8cf8d79c8a388ba28 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 12 May 2021 15:26:39 -0400 Subject: [PATCH 111/195] Add true to end of functions ending with shorthand comparison Functions ending with test using [[ ]] && will trip set -e, so adding true to the last line of the function will prevent the function from returning a nonzero code --- salt/common/tools/sbin/soup | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 814f9fd1b..e5c09a487 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -28,11 +28,10 @@ WHATWOULDYOUSAYYAHDOHERE=soup check_err() { local exit_code=$1 - local lineno=$2 local err_msg="Unhandled error occured, please check $SOUP_LOG for details." if [[ $exit_code -ne 0 ]]; then - printf '%s' "Soup failed on line $lineno with error $exit_code: " + printf '%s' "Soup failed with error $exit_code: " case $exit_code in 2) echo 'No such file or directory' @@ -323,11 +322,8 @@ preupgrade_changes_2.3.50_repo() { # We made repo changes in 2.3.50 and this prepares for that on upgrade echo "Checking to see if 2.3.50 repo changes are needed." - if [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]]; then - up_2.3.3X_to_2.3.50_repo - else - echo "No changes needed." - fi + [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo + true } preupgrade_changes() { @@ -340,6 +336,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20 [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30 [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50 + true } postupgrade_changes() { @@ -350,6 +347,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30 [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40 [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60 + true } post_rc1_to_2.3.21() { @@ -661,7 +659,6 @@ upgrade_check() { is_hotfix=true fi else - echo "DEBUG: here" is_hotfix=false fi @@ -742,7 +739,7 @@ verify_latest_update_script() { main() { set -e - trap 'check_err $? $BASH_LINENO' EXIT + trap 'check_err $?' EXIT echo "### Preparing soup at $(date) ###" while getopts ":b" opt; do @@ -876,7 +873,7 @@ main() { fi # Only update the repo if its airgap - if [[ $is_airgap -eq 0 ]] && [[ $UPGRADESALT -ne 1 ]]; then + if [[ $is_airgap -eq 0 && $UPGRADESALT -ne 1 ]]; then update_centos_repo fi From 68a667ee7cb6de7c8e829939f9e4042c5cc63890 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 May 2021 15:31:19 -0400 Subject: [PATCH 112/195] Add thirfpartydefaults.yml --- ...efaults.yml => securityoniondefaults.yaml} | 0 salt/filebeat/thirdpartydefaults.yaml | 275 ++++++++++++++++++ 2 files changed, 275 insertions(+) rename salt/filebeat/{thirdpartydefaults.yml => securityoniondefaults.yaml} (100%) create mode 100644 salt/filebeat/thirdpartydefaults.yaml diff --git a/salt/filebeat/thirdpartydefaults.yml b/salt/filebeat/securityoniondefaults.yaml similarity index 100% rename from salt/filebeat/thirdpartydefaults.yml rename to salt/filebeat/securityoniondefaults.yaml diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml new file mode 100644 index 000000000..027ec4595 --- /dev/null +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -0,0 +1,275 @@ +third_party_filebeat: + modules: + aws: + cloudtrail: + enabled: false + cloudwatch: + enabled: false + ec2: + enabled: false + elb: + enabled: false + s3access: + enabled: false + vpcflow: + enabled: false + azure: + activitylogs: + enabled: false + platformlogs: + enabled: false + auditlogs: + enabled: false + signinlogs: + enabled: false + barracuda: + waf: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9503 + spamfirewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9524 + bluecoat: + director: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9505 + cef: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9003 + checkpoint: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9505 + cisco: + asa: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9001 + ftd: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9003 + ios: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9002 + nexus: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9506 + meraki: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9525 + umbrella: + enabled: false + amp: + enabled: false + cyberark: + corepas: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9527 + cylance: + protect: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9508 + f5: + bigipapm: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9504 + bigipafm: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9528 + fortinet: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9004 + clientendpoint: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9510 + fortimail: + enabled: false + var.input: udp + var.syslog_port: 9350 + gcp: + vpcflow: + enabled: false + firewall: + enabled: false + audit: + enabled: false + google_workspace: + saml: + enabled: false + user_accounts: + enabled: false + login: + enabled: false + admin: + enabled: false + drive: + enabled: false + groups: + enabled: false + imperva: + securesphere: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9511 + infoblox: + nios: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9512 + juniper: + junos: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9513 + netscreen: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9523 + srx: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9006 + microsoft: + defender_atp: + enabled: false + m365_defender: + enabled: false + dhcp: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9515 + misp: + threat: + enabled: false + netflow: + log: + enabled: false + var.netflow_host: 0.0.0.0 + var.netflow_port: 2055 + var.internal_networks: + - private + netscout: + sightline: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9502 + o365: + audit: + enabled: false + okta: + enabled: false + pesando: + dfw: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9001 + proofpoint: + emailsecurity: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9531 + radware: + defensepro: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9518 + snort: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9532 + snyk: + audit: + enabled: false + vulnerabilities: + enabled: false + sonicwall: + firewall: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9519 + sophos: + xg: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9005 + utm: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9533 + squid: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9520 + threatintel: + abuseurl: + enabled: false + abusemalware: + enabled: false + misp: + enabled: false + otx: + enabled: false + anomali: + enabled: false + tomcat: + log: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9501 + zscaler: + zia: + enabled: false + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9521 From 74f2a61b25c3697908283660c73ab7b463bfb0e5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 May 2021 09:06:47 -0400 Subject: [PATCH 113/195] install influxdb and grafana during setup prior to final highstate --- salt/influxdb/init.sls | 4 ++-- setup/so-setup | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index c6120d292..9ef9aaae2 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -1,6 +1,8 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} + {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} {% set MANAGER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} @@ -11,8 +13,6 @@ {% from 'salt/map.jinja' import PYTHONINFLUXVERSION with context %} {% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) %} -{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} - include: - salt.minion - salt.python3-influxdb diff --git a/setup/so-setup b/setup/so-setup index 1d8eba9ed..4573da330 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -837,6 +837,13 @@ echo "1" > /root/accept_changes set_progress_str 73 "Update playbook rules" so-playbook-ruleupdate >> /root/setup_playbook_rule_update.log 2>&1 & fi + + if [[ "$GRAFANA" = 1 ]]; then + set_progress_str 74 "Installing InfluxDB and Grafana" + salt-call state.apply -l info influxdb >> $setup_log 2>&1 + salt-call state.apply -l info grafana >> $setup_log 2>&1 + fi + fi if [[ "$OSQUERY" = 1 ]]; then From d3af06e7a4a47d3ff902690834a1f2ec6da7cccc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 13 May 2021 11:00:42 -0400 Subject: [PATCH 114/195] handle exception if influxdb module doesnt exist --- salt/influxdb/init.sls | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 9ef9aaae2..3a309db47 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -1,9 +1,10 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} +{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} + {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} -{% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} {% set MANAGER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} @@ -11,7 +12,7 @@ {% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} {% from 'salt/map.jinja' import PYTHON3INFLUX with context %} {% from 'salt/map.jinja' import PYTHONINFLUXVERSION with context %} -{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) %} +{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c \"exec('try:import influxdb; print (influxdb.__version__)\\nexcept:print(\\'Module Not Found\\')')\"", python_shell=True) %} include: - salt.minion From 71a74a665653f209c31354eed53ee345007b213d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 13 May 2021 13:07:16 -0400 Subject: [PATCH 115/195] Added updated script and core modules --- .../tools/sbin/so-filebeat-module-setup | 50 +++++++++---------- salt/filebeat/securityoniondefaults.yaml | 45 +++++++++++------ 2 files changed, 54 insertions(+), 41 deletions(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index a42b0ac80..6616854eb 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -25,32 +25,30 @@ ELASTICSEARCH_PORT=9200 # Define a default directory to load pipelines from FB_MODULE_YML="/usr/share/filebeat/module-setup.yml" -if [ "$1" == "" ]; then - echo "No module supplied. Exiting..." -else - # Wait for ElasticSearch to initialize - echo -n "Waiting for ElasticSearch..." - COUNT=0 - ELASTICSEARCH_CONNECTED="no" - while [[ "$COUNT" -le 240 ]]; do - curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" - if [ $? -eq 0 ]; then - ELASTICSEARCH_CONNECTED="yes" - echo "connected!" - break - else - ((COUNT+=1)) - sleep 1 - echo -n "." - fi - done - if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then - echo - echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" - echo - fi - echo "Setting up ingest pipeline(s) for $1..." - docker exec -it so-filebeat filebeat setup modules -pipelines -modules $1 -c $FB_MODULE_YML +# Wait for ElasticSearch to initialize +echo -n "Waiting for ElasticSearch..." +COUNT=0 +ELASTICSEARCH_CONNECTED="no" +while [[ "$COUNT" -le 240 ]]; do + curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + if [ $? -eq 0 ]; then + ELASTICSEARCH_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi +done +if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then + echo + echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" + echo fi +echo "Setting up ingest pipeline(s)" +docker exec -it so-filebeat filebeat setup modules -pipelines -modules activemq,apache,auditd,aws,azure,barracuda,bluecoat,cef,checkpoint,cisco,coredns,crowdstrike,cyberark,cylance,elasticsearch,envoyproxy,f5,fortinet,gcp,google_workspace,googlecloud,gsuite,haproxy,ibmmq,icinga,iis,imperva,infoblox,iptables,juniper,kafka,kibana,logstash,microsoft,misp,mondogb,mssql,mysql,mysqlenterprise,nats,netflow,netscout,nginx,o365,okta,osquery,panw,pensando,postgresql,rabbitmq,radware,redis,santa,snort,snyk,sonicwall,sophos,squid,suricata,system,threatintel,tomcat,traefik,zeek,zoom,zscaler -c $FB_MODULE_YML + + diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 1e2eb8c23..e6406af8c 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -1,17 +1,32 @@ -third_party_filebeat: +securityonion_filebeat: modules: - fortinet: - firewall: - enabled: false - var.input: udp - var.syslog_host: 0.0.0.0 - var.syslog_port: 9004 - clientendpoint: + elasticsearch: + server: + enabled: true + var.paths: ["/logs/elasticsearch/*.log"] + kibana: + log: + enabled: true + var.paths: ["/logs/kibana/kibana.log"] + logstash: + log: + enabled: true + var.paths: ["/logs/logstash.log"] + redis: + log: + enabled: true + var.paths: ["/logs/redis.log"] + suricata: + eve: + enabled: true + var.paths: ["/nsm/suricata/eve*.json"] + {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} + {%- if ZEEKVER != 'SURICATA' %} + zeek: + {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} + {{ LOGNAME }}: enabled: false - var.input: udp - var.syslog_host: 0.0.0.0 - var.syslog_port: 9510 - fortimail: - enabled: false - var.input: udp - var.syslog_port: 9350 \ No newline at end of file + var.paths: ["/nsm/zeek/logs/current/{{ LOGNAME }}.log"] + {%- endfor %} + {%- endif %} + {%- endif %} \ No newline at end of file From 3859f6464ab0ab6b418c2d39561196a848e28df5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 May 2021 08:56:42 -0400 Subject: [PATCH 116/195] dont be quiet on first grep --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 4573da330..4c364ded6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -926,7 +926,7 @@ if [[ $success != 0 ]]; then SO_ERROR=1; fi # Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox # Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry -if grep -q -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then +if grep -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then SO_ERROR=1 grep --color=never "ERROR" "$setup_log" | grep -qvE "Status .* was not found" > "$error_log" From 231e07dbbda9a33091be927c70965ddcab26cec7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 May 2021 15:19:45 -0400 Subject: [PATCH 117/195] circumvent file.patch putting ERROR in log if patch doesnt need applied --- salt/salt/python3-influxdb.sls | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index acadf2161..055390a27 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -15,6 +15,9 @@ python3_influxdb: {{PYTHONINSTALLER}}.installed: - name: {{ PYTHON3INFLUX }} +# We circumvent the file.patch state putting ERROR in the log by using the unless and file.touch below +# https://github.com/saltstack/salt/pull/47010 and https://github.com/saltstack/salt/issues/52329 + #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: @@ -23,8 +26,15 @@ influxdb_continuous_query.present_patch: - require: - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package + - unless: ls /opt/so/state/influxdb_continuous_query.py.patched -#https://github.com/saltstack/salt/issues/59761 +influxdb_continuous_query.py.patched: + file.touch: + - name: /opt/so/state/influxdb_continuous_query.py.patched + - onchanges: + - file: influxdb_continuous_query.present_patch + + #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py @@ -32,6 +42,13 @@ influxdb_retention_policy.present_patch: - require: - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package + - unless: ls /opt/so/state/influxdb_retention_policy.py.patched + +influxdb_retention_policy.py.patched: + file.touch: + - name: /opt/so/state/influxdb_retention_policy.py.patched + - onchanges: + - file: influxdb_retention_policy.present_patch # We should be able to set reload_modules: True in this state in order to tell salt to reload its python modules due to us possibly installing # and possibly modifying modules in this state. This is bugged according to https://github.com/saltstack/salt/issues/24925 @@ -41,4 +58,13 @@ influxdbmod.py_shard_duration_patch: - source: salt://salt/files/influxdbmod.py.patch - require: - {{PYTHONINSTALLER}}: python3_influxdb - - pkg: patch_package \ No newline at end of file + - pkg: patch_package + - unless: ls /opt/so/state/influxdbmod.py.patched + +influxdbmod.py.patched: + file.touch: + - name: /opt/so/state/influxdbmod.py.patched + - onchanges: + - file: influxdbmod.py_shard_duration_patch + + From 54f9e3ff9df127d1ddd2706db5109f414dc4bd4c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 14 May 2021 16:24:16 -0400 Subject: [PATCH 118/195] remove leading space on comment line --- salt/salt/python3-influxdb.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 055390a27..7a59c4303 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -34,7 +34,7 @@ influxdb_continuous_query.py.patched: - onchanges: - file: influxdb_continuous_query.present_patch - #https://github.com/saltstack/salt/issues/59761 +#https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py From 3625453668953d6d6a7c18cea26e4c16909a6344 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 11:00:28 -0400 Subject: [PATCH 119/195] Don't unmount airgap directory if not airgap --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index e5c09a487..676c5b0bc 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -931,7 +931,7 @@ main() { echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes - unmount_update + [[ $is_airgap -eq 0 ]] && unmount_update thehive_maint if [[ $UPGRADESALT -eq 1 ]]; then From 2bcd51b21c3771e3ee488a7a63b6342e8481bebe Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 15:10:57 -0400 Subject: [PATCH 120/195] Fix error message --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 676c5b0bc..323e755b2 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -687,7 +687,7 @@ upgrade_salt() { set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ - "Could not update soup, please check $SOUP_LOG for details." + "Could not update salt, please check $SOUP_LOG or scrollback for details." set -e echo "Applying yum versionlock for Salt." echo "" @@ -704,7 +704,7 @@ upgrade_salt() { set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ - "Could not update soup, please check $SOUP_LOG for details." + "Could not update salt, please check $SOUP_LOG or scrollback for details." set -e echo "Applying apt hold for Salt." echo "" From 992b76a0f09c40b6e7631767397e94039c5a79f0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 15:38:10 -0400 Subject: [PATCH 121/195] Remove debug lines --- salt/common/tools/sbin/soup | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 323e755b2..2f27939e6 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -795,9 +795,7 @@ main() { fi echo "Let's see if we need to update Security Onion." - echo "DEBUG: before upgrade_check" upgrade_check - echo "DEBUG: after upgrade_check" upgrade_space echo "Checking for Salt Master and Minion updates." From b89091cc7d441361e571f31d1dd723606088ebb7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 15:58:25 -0400 Subject: [PATCH 122/195] Try retrying in curl instead of shell function --- salt/common/tools/sbin/so-common | 10 ++++++++-- salt/common/tools/sbin/so-image-common | 10 ++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 2b2bbf944..d5450f096 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -328,10 +328,16 @@ retry() { run_check_net_err() { local cmd=$1 local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable + local no_retry=$3 local exit_code - retry 50 10 "$cmd" - exit_code=$? + if [[ -z $no_retry ]]; then + retry 50 10 "$cmd" + exit_code=$? + else + eval "$cmd" + exit_code=$? + fi if [[ $exit_code -ne 0 ]]; then echo "Command failed with error $exit_code" diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index c855abd15..fa1d6c6dd 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -130,8 +130,9 @@ update_docker_containers() { # Let's make sure we have the public key run_check_net_err \ - "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ - "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" >> "$LOG_FILE" 2>&1 + "curl --retry 20 --retry-delay 30 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ + "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \ + noretry >> "$LOG_FILE" 2>&1 result=$? if [[ $result -eq 0 ]]; then cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 @@ -157,8 +158,9 @@ update_docker_containers() { # Get signature run_check_net_err \ - "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ - "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " >> "$LOG_FILE" 2>&1 + "curl --retry 20 --retry-delay 30 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ + "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " \ + noretry >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 exit 1 From 3f238f7a4ad911e605c5c859d9edaf564a012aa4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 16:02:52 -0400 Subject: [PATCH 123/195] Set flag so trap doesn't repeat info --- salt/common/tools/sbin/so-common | 1 + salt/common/tools/sbin/soup | 1 + 2 files changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index d5450f096..0390a2bf2 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -340,6 +340,7 @@ run_check_net_err() { fi if [[ $exit_code -ne 0 ]]; then + ERR_HANDLED=true echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2f27939e6..9a1afb2bb 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -30,6 +30,7 @@ check_err() { local exit_code=$1 local err_msg="Unhandled error occured, please check $SOUP_LOG for details." + [[ $ERR_HANDLED == true ]] && exit $exit_code if [[ $exit_code -ne 0 ]]; then printf '%s' "Soup failed with error $exit_code: " case $exit_code in From 4e1fba5b38183cad4c479fab7149946c06c8be83 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 16:04:13 -0400 Subject: [PATCH 124/195] Only echo error code if not using retry --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 0390a2bf2..0b8cd97aa 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -341,7 +341,7 @@ run_check_net_err() { if [[ $exit_code -ne 0 ]]; then ERR_HANDLED=true - echo "Command failed with error $exit_code" + [[ -z $no_retry ]] && echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code fi From ce9554281ee2d83567a9cd1f2b5a9a7a3f1eab13 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 16:08:34 -0400 Subject: [PATCH 125/195] Fix backwards logic --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 0b8cd97aa..c6b0c0bd8 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -341,7 +341,7 @@ run_check_net_err() { if [[ $exit_code -ne 0 ]]; then ERR_HANDLED=true - [[ -z $no_retry ]] && echo "Command failed with error $exit_code" + [[ -z $no_retry ]] || echo "Command failed with error $exit_code" echo "$err_msg" exit $exit_code fi From dbd5ef70c995eb6a8065487d9e3908307e41bf26 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 17 May 2021 16:19:31 -0400 Subject: [PATCH 126/195] Change retry delay + count --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index c6b0c0bd8..3ab260eb8 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -332,7 +332,7 @@ run_check_net_err() { local exit_code if [[ -z $no_retry ]]; then - retry 50 10 "$cmd" + retry 20 30 "$cmd" exit_code=$? else eval "$cmd" From bdd53ed5e3a806ed26786a43df20b85ce9002294 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 18 May 2021 12:23:40 -0400 Subject: [PATCH 127/195] Change retry delay + count --- salt/common/tools/sbin/so-common | 2 +- salt/common/tools/sbin/so-image-common | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 3ab260eb8..55350d702 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -332,7 +332,7 @@ run_check_net_err() { local exit_code if [[ -z $no_retry ]]; then - retry 20 30 "$cmd" + retry 5 60 "$cmd" exit_code=$? else eval "$cmd" diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index fa1d6c6dd..2c61838b5 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -130,7 +130,7 @@ update_docker_containers() { # Let's make sure we have the public key run_check_net_err \ - "curl --retry 20 --retry-delay 30 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ + "curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \ "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \ noretry >> "$LOG_FILE" 2>&1 result=$? @@ -158,7 +158,7 @@ update_docker_containers() { # Get signature run_check_net_err \ - "curl --retry 20 --retry-delay 30 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ + "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " \ noretry >> "$LOG_FILE" 2>&1 if [[ $? -ne 0 ]]; then From f445186f1ebb2b28fd3f6592e8b07c98fce06fd5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 18 May 2021 13:38:55 -0400 Subject: [PATCH 128/195] Remove redundant error messages --- salt/common/tools/sbin/so-image-common | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 2c61838b5..f840e07ab 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -136,9 +136,6 @@ update_docker_containers() { result=$? if [[ $result -eq 0 ]]; then cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1 - else - echo "Failed to pull signature key file: $result" - exit 1 fi # Download the containers from the interwebs @@ -159,12 +156,8 @@ update_docker_containers() { # Get signature run_check_net_err \ "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \ - "Could not pull signature file, please ensure connectivity to https://sigs.securityonion.net " \ + "Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \ noretry >> "$LOG_FILE" 2>&1 - if [[ $? -ne 0 ]]; then - echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1 - exit 1 - fi # Dump our hash values DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image) From ac42cba50b6e4679d5dd976bfb1f11fa8428eec6 Mon Sep 17 00:00:00 2001 From: Masaya-A <68965261+Masaya-A@users.noreply.github.com> Date: Wed, 19 May 2021 09:06:02 +0900 Subject: [PATCH 129/195] Adding MAC Address for NIC List --- setup/so-functions | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index dfe79f674..bc5d9d1ca 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1348,15 +1348,16 @@ filter_unused_nics() { nic_list=() for nic in "${filtered_nics[@]}"; do + local nic_mac=$(cat "/sys/class/net/${nic}/address") case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in 1) - nic_list+=("$nic" "Link UP " "OFF") + nic_list+=("$nic" "$nic_mac" "Link UP " "OFF") ;; 0) - nic_list+=("$nic" "Link DOWN " "OFF") + nic_list+=("$nic" "$nic_mac" "Link DOWN " "OFF") ;; *) - nic_list+=("$nic" "Link UNKNOWN " "OFF") + nic_list+=("$nic" "$nic_mac" "Link UNKNOWN " "OFF") ;; esac done From 6af126b87200e616864e5788798790919928b84a Mon Sep 17 00:00:00 2001 From: Masaya-A <68965261+Masaya-A@users.noreply.github.com> Date: Wed, 19 May 2021 10:02:19 +0900 Subject: [PATCH 130/195] Fix array --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index bc5d9d1ca..8cdbc2596 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1351,13 +1351,13 @@ filter_unused_nics() { local nic_mac=$(cat "/sys/class/net/${nic}/address") case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in 1) - nic_list+=("$nic" "$nic_mac" "Link UP " "OFF") + nic_list+=("$nic" "$nic_mac Link UP " "OFF") ;; 0) - nic_list+=("$nic" "$nic_mac" "Link DOWN " "OFF") + nic_list+=("$nic" "$nic_mac Link DOWN " "OFF") ;; *) - nic_list+=("$nic" "$nic_mac" "Link UNKNOWN " "OFF") + nic_list+=("$nic" "$nic_mac Link UNKNOWN " "OFF") ;; esac done From 3450219bc7073fb692259ed185fc88eca2dd516e Mon Sep 17 00:00:00 2001 From: Masaya-A <68965261+Masaya-A@users.noreply.github.com> Date: Thu, 20 May 2021 09:33:14 +0900 Subject: [PATCH 131/195] Drop error to /dev/null --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 8cdbc2596..5ce3d6dee 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1348,7 +1348,7 @@ filter_unused_nics() { nic_list=() for nic in "${filtered_nics[@]}"; do - local nic_mac=$(cat "/sys/class/net/${nic}/address") + local nic_mac=$(cat "/sys/class/net/${nic}/address" 2>/dev/null) case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in 1) nic_list+=("$nic" "$nic_mac Link UP " "OFF") From c1dd4dafe47001101e949d5a8fbefaca0d97f987 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 21 May 2021 12:41:10 -0400 Subject: [PATCH 132/195] Fix influx state --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index bafcbd53b..346d971fd 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -3,7 +3,7 @@ {% set GRAFANA = salt['pillar.get']('manager:grafana', '0') %} -{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] or (grains.role == 'so-eval' and GRAFANA == 1) %} {% set MANAGER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} From 99d41d1606245cc0d75520545e55bf9b27dd1e9b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 11:29:40 -0400 Subject: [PATCH 133/195] Add ending newline to soup --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 9a1afb2bb..eb15305aa 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -999,3 +999,4 @@ EOF read -r input main "$@" | tee -a $SOUP_LOG + \ No newline at end of file From 686c7c5a6c5cf5010851381900f1cd8f9ea3a2f5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 13:26:43 -0400 Subject: [PATCH 134/195] Add exception handling for docker API error to so-docker-prune --- salt/common/tools/sbin/so-docker-prune | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index f6c043ef3..754a0f171 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -38,7 +38,7 @@ def get_image_version(string) -> str: Version(ver) except InvalidVersion: # Strip the last substring following a hyphen for automated branches - ver = '-'.join(ver.split('-')[:-1]) + ver = '-'.join(ver.split('-')[:-1]) return ver @@ -73,8 +73,12 @@ def main(quiet): for tag in group: if not quiet: print(f'Removing image {tag}') client.images.remove(tag) - except InvalidVersion as e: - print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr) + except (docker.errors.APIError, InvalidVersion) as e: + print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr) + exit(1) + except Exception as e: + print('Unhandled exception occurred:') + print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr) exit(1) if no_prunable and not quiet: @@ -86,4 +90,4 @@ if __name__ == "__main__": main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False) args = main_parser.parse_args(sys.argv[1:]) - main(args.quiet) \ No newline at end of file + main(args.quiet) From dcb89b704a373cf804b318b2c261e97e39dcb4e2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 13:45:12 -0400 Subject: [PATCH 135/195] Move sensoroni state out of the * block of top.sls Resolves #3559 --- salt/top.sls | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/top.sls b/salt/top.sls index 8a12aaa26..f3ccc9dda 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -38,7 +38,6 @@ base: - patch.os.schedule - motd - salt.minion-check - - sensoroni - salt.lasthighstate '*_helixsensor and G@saltversion:{{saltversion}}': @@ -47,6 +46,7 @@ base: - ca - ssl - registry + - sensoroni - telegraf - firewall - idstools @@ -66,6 +66,7 @@ base: - match: compound - ca - ssl + - sensoroni - telegraf - firewall - nginx @@ -92,6 +93,7 @@ base: - ca - ssl - registry + - sensoroni - manager - nginx - telegraf @@ -160,6 +162,7 @@ base: - ca - ssl - registry + - sensoroni - nginx - telegraf - influxdb @@ -220,6 +223,7 @@ base: - ca - ssl - registry + - sensoroni - manager - nginx - telegraf @@ -290,6 +294,7 @@ base: - match: compound - ca - ssl + - sensoroni - nginx - telegraf - firewall @@ -320,6 +325,7 @@ base: - ca - ssl - registry + - sensoroni - nginx - telegraf - influxdb @@ -382,6 +388,7 @@ base: - match: compound - ca - ssl + - sensoroni - nginx - telegraf - firewall @@ -424,6 +431,7 @@ base: - match: compound - ca - ssl + - sensoroni - nginx - telegraf - firewall @@ -441,6 +449,7 @@ base: - ca - ssl - registry + - sensoroni - manager - nginx - soc From ffd5bfc48063aeb501def0030748b93091dd6678 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 15:25:03 -0400 Subject: [PATCH 136/195] Force images from automated branches to a very high semver --- salt/common/tools/sbin/so-docker-prune | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 754a0f171..2dd183c10 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -37,8 +37,10 @@ def get_image_version(string) -> str: try: Version(ver) except InvalidVersion: - # Strip the last substring following a hyphen for automated branches - ver = '-'.join(ver.split('-')[:-1]) + # Also return a very high (but less than 'latest') semver for automated branches + # since the image will most likely be the latest version + if any(substr in ver for substr in ['bravo', 'delta', 'foxtrot', 'kilo']): + return '99999.9.9' return ver From 3bc0def02af00ab5741f903722f8316e68d3bfe1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 24 May 2021 16:45:05 -0400 Subject: [PATCH 137/195] Add failure message to salt-master check --- salt/common/tools/sbin/soup | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index eb15305aa..d2b96b194 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -890,8 +890,10 @@ main() { echo "Starting Salt Master service." systemctl start salt-master - # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" + # Testing that salt-master is up by checking that is it connected to itself + set +e + retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details." + set -e echo "" echo "Ensuring python modules for Salt are installed and patched." @@ -924,9 +926,11 @@ main() { echo "Starting Salt Master service." systemctl start salt-master - # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" - + # Testing that salt-master is up by checking that is it connected to itself + set +e + retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details." + set -e + echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes From 5e5d30a3774a49b2e2128e2c554b5eb6a7a4f53b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 08:26:25 -0400 Subject: [PATCH 138/195] Fix 3rd party modules --- salt/filebeat/thirdpartydefaults.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml index 027ec4595..cfb8d10d9 100644 --- a/salt/filebeat/thirdpartydefaults.yaml +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -195,9 +195,10 @@ third_party_filebeat: var.syslog_port: 9502 o365: audit: - enabled: false + enabled: false okta: - enabled: false + system: + enabled: false pesando: dfw: enabled: false From dbc4ffd69ad757d59d9e3d8351a7f1c1510fa0e6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 25 May 2021 09:20:45 -0400 Subject: [PATCH 139/195] Fix typo --- salt/common/tools/sbin/so-image-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index f840e07ab..9b6e2174a 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -151,7 +151,7 @@ update_docker_containers() { local image=$i:$VERSION$IMAGE_TAG_SUFFIX run_check_net_err \ "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \ - "Could not pull pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 + "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 # Get signature run_check_net_err \ From 7e008378bae365a16ae4d9741e3346eebcae04c0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 25 May 2021 09:23:44 -0400 Subject: [PATCH 140/195] Replace string with variable, remove unnecessary text --- salt/common/tools/sbin/soup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d2b96b194..7fca8ceb9 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -688,7 +688,7 @@ upgrade_salt() { set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \ - "Could not update salt, please check $SOUP_LOG or scrollback for details." + "Could not update salt, please check $SOUP_LOG for details." set -e echo "Applying yum versionlock for Salt." echo "" @@ -705,7 +705,7 @@ upgrade_salt() { set +e run_check_net_err \ "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \ - "Could not update salt, please check $SOUP_LOG or scrollback for details." + "Could not update salt, please check $SOUP_LOG for details." set -e echo "Applying apt hold for Salt." echo "" @@ -892,7 +892,7 @@ main() { # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details." + retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "" @@ -928,7 +928,7 @@ main() { # Testing that salt-master is up by checking that is it connected to itself set +e - retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check /root/soup.log for details." + retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details." set -e echo "Running a highstate. This could take several minutes." From 1e564c2140d0f53e3b5c90ba89bd3ec8b7e3471b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 10:22:36 -0400 Subject: [PATCH 141/195] Fix zeek jinja --- salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index da798a79d..486d22bfe 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -1,4 +1,4 @@ -%- if grains['role'] == 'so-eval' -%} +{%- if grains['role'] == 'so-eval' -%} {%- set ES = salt['pillar.get']('manager:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} From 192cec182550135ad711b4c58b61cbe86c5ae7bb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 25 May 2021 11:25:12 -0400 Subject: [PATCH 142/195] Change how version with dashes are handled by so-docker-prune --- salt/common/tools/sbin/so-docker-prune | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-docker-prune b/salt/common/tools/sbin/so-docker-prune index 2dd183c10..b1c359b58 100755 --- a/salt/common/tools/sbin/so-docker-prune +++ b/salt/common/tools/sbin/so-docker-prune @@ -32,15 +32,15 @@ def get_image_version(string) -> str: ver = string.split(':')[-1] if ver == 'latest': # Version doesn't like "latest", so use a high semver - return '999999.9.9' + return '99999.9.9' else: try: Version(ver) except InvalidVersion: - # Also return a very high (but less than 'latest') semver for automated branches - # since the image will most likely be the latest version - if any(substr in ver for substr in ['bravo', 'delta', 'foxtrot', 'kilo']): - return '99999.9.9' + # Also return a very high semver for any version + # with a dash in it since it will likely be a dev version of some kind + if '-' in ver: + return '999999.9.9' return ver From 94af55a95104cd4229a315343973c8167b6cadd6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 25 May 2021 11:25:37 -0400 Subject: [PATCH 143/195] Fix typo --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 7fca8ceb9..d1261fe30 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -60,7 +60,7 @@ check_err() { echo 'Link has been severed' ;; 100) - echo 'Netowrk is down' + echo 'Network is down' ;; 101) echo 'Network is unreachable' From 2aacd5b9b6cf339052d674595531f515c3842fff Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 16:40:50 -0400 Subject: [PATCH 144/195] so defaults filebeat modules --- salt/filebeat/etc/module_config.yml.jinja | 16 ++++++++++++++++ salt/filebeat/etc/thirdparty.yml.jinja | 16 ---------------- salt/filebeat/init.sls | 19 ++++++++++--------- salt/filebeat/map.jinja | 5 ++++- 4 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 salt/filebeat/etc/module_config.yml.jinja delete mode 100644 salt/filebeat/etc/thirdparty.yml.jinja diff --git a/salt/filebeat/etc/module_config.yml.jinja b/salt/filebeat/etc/module_config.yml.jinja new file mode 100644 index 000000000..7cd624895 --- /dev/null +++ b/salt/filebeat/etc/module_config.yml.jinja @@ -0,0 +1,16 @@ +# DO NOT EDIT THIS FILE +{% for module in MODULES.modules.keys() %} +- module: {{ module }} + {%- for fileset in MODULES.modules[module] %} + {{ fileset }}: + enabled: {{ MODULES.modules[module][fileset].enabled }} + {#- only manage the settings if the fileset is enabled #} + {%- if MODULES.modules[module][fileset].enabled %} + {%- for var, value in MODULES.modules[module][fileset].items() %} + {%- if var|lower != 'enabled' %} + {{ var }}: {{ value }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- endfor %} +{% endfor %} diff --git a/salt/filebeat/etc/thirdparty.yml.jinja b/salt/filebeat/etc/thirdparty.yml.jinja deleted file mode 100644 index 186115af4..000000000 --- a/salt/filebeat/etc/thirdparty.yml.jinja +++ /dev/null @@ -1,16 +0,0 @@ -# DO NOT EDIT THIS FILE -{% for module in THIRDPARTY.modules.keys() %} -- module: {{ module }} - {%- for fileset in THIRDPARTY.modules[module] %} - {{ fileset }}: - enabled: {{ THIRDPARTY.modules[module][fileset].enabled }} - {#- only manage the settings if the fileset is enabled #} - {%- if THIRDPARTY.modules[module][fileset].enabled %} - {%- for var, value in THIRDPARTY.modules[module][fileset].items() %} - {%- if var|lower != 'enabled' %} - {{ var }}: {{ value }} - {%- endif %} - {%- endfor %} - {%- endif %} - {%- endfor %} -{% endfor %} \ No newline at end of file diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index b3bce806c..b1a91b133 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -21,6 +21,7 @@ {% set MANAGER = salt['grains.get']('master') %} {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} +{% from 'filebeat/map.jinja' import SO with context %} filebeatetcdir: @@ -78,21 +79,21 @@ filebeatmoduleconfsync: - group: root - template: jinja -# Sync Filebeat modules -filebeatmodules: - file.recurse: - - name: /opt/so/conf/filebeat/modules - - source: salt://filebeat/modules - - user: root - - group: root +sodefaults_module_conf: + file.managed: + - name: /opt/so/conf/filebeat/etc/securityonion.yml + - source: salt://filebeat/etc/module_config.yml.jinja + - template: jinja + - defaults: + MODULES: {{ SO }} thirdparty_module_conf: file.managed: - name: /opt/so/conf/filebeat/etc/thirdparty.yml - - source: salt://filebeat/etc/thirdparty.yml.jinja + - source: salt://filebeat/etc/module_config.yml.jinja - template: jinja - defaults: - THIRDPARTY: {{ THIRDPARTY }} + MODULES: {{ THIRDPARTY }} so-filebeat: docker_container.running: diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index 668889227..aaae60f31 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -1,2 +1,5 @@ {% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %} -{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} \ No newline at end of file +{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} + +{% import_yaml 'filebeat/securityoniondefaults.yaml' as SO %} +{#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} From 9363fc153c3c7b45bdf1c82530530e7c8b92398f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 16:44:13 -0400 Subject: [PATCH 145/195] Fix pillar for module --- salt/filebeat/securityoniondefaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index e6406af8c..8bcecd618 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -1,3 +1,4 @@ +{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %} securityonion_filebeat: modules: elasticsearch: From 0de5c6f204186342d90b928fca5b1019b22b26cb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 16:52:02 -0400 Subject: [PATCH 146/195] fix sodefault modules --- salt/filebeat/map.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index aaae60f31..6ae6e7cff 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -1,5 +1,6 @@ {% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %} {% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %} -{% import_yaml 'filebeat/securityoniondefaults.yaml' as SO %} +{% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} +{% set SO = SODEFAULTS.securityonion_filebeat %} {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} From cd3e355f848536e2c7d7241c498f9c51315d8ebe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 16:54:20 -0400 Subject: [PATCH 147/195] Fix zeek depth --- salt/zeek/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 02c1cc1ba..4e597f597 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -76,9 +76,9 @@ zeekpolicysync: # Ensure the zeek spool tree (and state.db) ownership is correct zeekspoolownership: file.directory: - - name: /nsm/zeek/spool + - name: /nsm/zeek - user: 937 - - max_depth: 0 + - max_depth: 1 - recurse: - user From 543154f037453377c0780d4375dc4b52613b14db Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 May 2021 16:58:18 -0400 Subject: [PATCH 148/195] Remove old modules --- salt/filebeat/init.sls | 4 +- salt/filebeat/modules/activemq.yml.disabled | 19 -- salt/filebeat/modules/apache.yml.disabled | 19 -- salt/filebeat/modules/auditd.yml.disabled | 10 - salt/filebeat/modules/aws.yml.disabled | 255 ------------------ salt/filebeat/modules/azure.yml.disabled | 45 ---- salt/filebeat/modules/barracuda.yml.disabled | 41 --- salt/filebeat/modules/bluecoat.yml.disabled | 22 -- salt/filebeat/modules/cef.yml.disabled | 17 -- salt/filebeat/modules/checkpoint.yml.disabled | 24 -- salt/filebeat/modules/cisco.yml.disabled | 142 ---------- salt/filebeat/modules/coredns.yml.disabled | 11 - .../filebeat/modules/crowdstrike.yml.disabled | 11 - salt/filebeat/modules/cyberark.yml.disabled | 22 -- salt/filebeat/modules/cylance.yml.disabled | 22 -- .../modules/elasticsearch.yml.disabled | 35 --- salt/filebeat/modules/envoyproxy.yml.disabled | 11 - salt/filebeat/modules/f5.yml.disabled | 41 --- salt/filebeat/modules/fortinet.yml.disabled | 83 ------ salt/filebeat/modules/gcp.yml.disabled | 76 ------ .../modules/google_workspace.yml.disabled | 53 ---- .../filebeat/modules/googlecloud.yml.disabled | 58 ---- salt/filebeat/modules/gsuite.yml.disabled | 53 ---- salt/filebeat/modules/haproxy.yml.disabled | 14 - salt/filebeat/modules/ibmmq.yml.disabled | 11 - salt/filebeat/modules/icinga.yml.disabled | 27 -- salt/filebeat/modules/iis.yml.disabled | 20 -- salt/filebeat/modules/imperva.yml.disabled | 22 -- salt/filebeat/modules/infoblox.yml.disabled | 22 -- salt/filebeat/modules/iptables.yml.disabled | 13 - salt/filebeat/modules/juniper.yml.disabled | 54 ---- salt/filebeat/modules/kafka.yml.disabled | 15 -- salt/filebeat/modules/kibana.yml.disabled | 19 -- salt/filebeat/modules/logstash.yml.disabled | 18 -- salt/filebeat/modules/microsoft.yml.disabled | 49 ---- salt/filebeat/modules/misp.yml.disabled | 17 -- salt/filebeat/modules/mongodb.yml.disabled | 11 - salt/filebeat/modules/mssql.yml.disabled | 11 - salt/filebeat/modules/mysql.yml.disabled | 19 -- .../modules/mysqlenterprise.yml.disabled | 14 - salt/filebeat/modules/nats.yml.disabled | 11 - salt/filebeat/modules/netflow.yml.disabled | 14 - salt/filebeat/modules/netscout.yml.disabled | 22 -- salt/filebeat/modules/nginx.yml.disabled | 27 -- salt/filebeat/modules/o365.yml.disabled | 48 ---- salt/filebeat/modules/okta.yml.disabled | 10 - salt/filebeat/modules/oracle.yml.disabled | 13 - salt/filebeat/modules/osquery.yml.disabled | 15 -- salt/filebeat/modules/panw.yml.disabled | 22 -- salt/filebeat/modules/pensando.yml.disabled | 13 - salt/filebeat/modules/postgresql.yml.disabled | 11 - salt/filebeat/modules/proofpoint.yml.disabled | 22 -- salt/filebeat/modules/rabbitmq.yml.disabled | 11 - salt/filebeat/modules/radware.yml.disabled | 22 -- salt/filebeat/modules/redis.yml.disabled | 21 -- salt/filebeat/modules/santa.yml.disabled | 9 - salt/filebeat/modules/snort.yml.disabled | 22 -- salt/filebeat/modules/snyk.yml.disabled | 112 -------- salt/filebeat/modules/sonicwall.yml.disabled | 22 -- salt/filebeat/modules/sophos.yml.disabled | 46 ---- salt/filebeat/modules/squid.yml.disabled | 22 -- salt/filebeat/modules/suricata.yml | 12 - salt/filebeat/modules/system.yml.disabled | 19 -- .../filebeat/modules/threatintel.yml.disabled | 105 -------- salt/filebeat/modules/tomcat.yml.disabled | 22 -- salt/filebeat/modules/traefik.yml.disabled | 11 - salt/filebeat/modules/zeek.yml | 122 --------- salt/filebeat/modules/zoom.yml.disabled | 22 -- salt/filebeat/modules/zscaler.yml.disabled | 22 -- salt/filebeat/securityoniondefaults.yaml | 2 +- 70 files changed, 3 insertions(+), 2279 deletions(-) delete mode 100644 salt/filebeat/modules/activemq.yml.disabled delete mode 100644 salt/filebeat/modules/apache.yml.disabled delete mode 100644 salt/filebeat/modules/auditd.yml.disabled delete mode 100644 salt/filebeat/modules/aws.yml.disabled delete mode 100644 salt/filebeat/modules/azure.yml.disabled delete mode 100644 salt/filebeat/modules/barracuda.yml.disabled delete mode 100644 salt/filebeat/modules/bluecoat.yml.disabled delete mode 100644 salt/filebeat/modules/cef.yml.disabled delete mode 100644 salt/filebeat/modules/checkpoint.yml.disabled delete mode 100644 salt/filebeat/modules/cisco.yml.disabled delete mode 100644 salt/filebeat/modules/coredns.yml.disabled delete mode 100644 salt/filebeat/modules/crowdstrike.yml.disabled delete mode 100644 salt/filebeat/modules/cyberark.yml.disabled delete mode 100644 salt/filebeat/modules/cylance.yml.disabled delete mode 100644 salt/filebeat/modules/elasticsearch.yml.disabled delete mode 100644 salt/filebeat/modules/envoyproxy.yml.disabled delete mode 100644 salt/filebeat/modules/f5.yml.disabled delete mode 100644 salt/filebeat/modules/fortinet.yml.disabled delete mode 100644 salt/filebeat/modules/gcp.yml.disabled delete mode 100644 salt/filebeat/modules/google_workspace.yml.disabled delete mode 100644 salt/filebeat/modules/googlecloud.yml.disabled delete mode 100644 salt/filebeat/modules/gsuite.yml.disabled delete mode 100644 salt/filebeat/modules/haproxy.yml.disabled delete mode 100644 salt/filebeat/modules/ibmmq.yml.disabled delete mode 100644 salt/filebeat/modules/icinga.yml.disabled delete mode 100644 salt/filebeat/modules/iis.yml.disabled delete mode 100644 salt/filebeat/modules/imperva.yml.disabled delete mode 100644 salt/filebeat/modules/infoblox.yml.disabled delete mode 100644 salt/filebeat/modules/iptables.yml.disabled delete mode 100644 salt/filebeat/modules/juniper.yml.disabled delete mode 100644 salt/filebeat/modules/kafka.yml.disabled delete mode 100644 salt/filebeat/modules/kibana.yml.disabled delete mode 100644 salt/filebeat/modules/logstash.yml.disabled delete mode 100644 salt/filebeat/modules/microsoft.yml.disabled delete mode 100644 salt/filebeat/modules/misp.yml.disabled delete mode 100644 salt/filebeat/modules/mongodb.yml.disabled delete mode 100644 salt/filebeat/modules/mssql.yml.disabled delete mode 100644 salt/filebeat/modules/mysql.yml.disabled delete mode 100644 salt/filebeat/modules/mysqlenterprise.yml.disabled delete mode 100644 salt/filebeat/modules/nats.yml.disabled delete mode 100644 salt/filebeat/modules/netflow.yml.disabled delete mode 100644 salt/filebeat/modules/netscout.yml.disabled delete mode 100644 salt/filebeat/modules/nginx.yml.disabled delete mode 100644 salt/filebeat/modules/o365.yml.disabled delete mode 100644 salt/filebeat/modules/okta.yml.disabled delete mode 100644 salt/filebeat/modules/oracle.yml.disabled delete mode 100644 salt/filebeat/modules/osquery.yml.disabled delete mode 100644 salt/filebeat/modules/panw.yml.disabled delete mode 100644 salt/filebeat/modules/pensando.yml.disabled delete mode 100644 salt/filebeat/modules/postgresql.yml.disabled delete mode 100644 salt/filebeat/modules/proofpoint.yml.disabled delete mode 100644 salt/filebeat/modules/rabbitmq.yml.disabled delete mode 100644 salt/filebeat/modules/radware.yml.disabled delete mode 100644 salt/filebeat/modules/redis.yml.disabled delete mode 100644 salt/filebeat/modules/santa.yml.disabled delete mode 100644 salt/filebeat/modules/snort.yml.disabled delete mode 100644 salt/filebeat/modules/snyk.yml.disabled delete mode 100644 salt/filebeat/modules/sonicwall.yml.disabled delete mode 100644 salt/filebeat/modules/sophos.yml.disabled delete mode 100644 salt/filebeat/modules/squid.yml.disabled delete mode 100644 salt/filebeat/modules/suricata.yml delete mode 100644 salt/filebeat/modules/system.yml.disabled delete mode 100644 salt/filebeat/modules/threatintel.yml.disabled delete mode 100644 salt/filebeat/modules/tomcat.yml.disabled delete mode 100644 salt/filebeat/modules/traefik.yml.disabled delete mode 100644 salt/filebeat/modules/zeek.yml delete mode 100644 salt/filebeat/modules/zoom.yml.disabled delete mode 100644 salt/filebeat/modules/zscaler.yml.disabled diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index b1a91b133..18ca9b8c1 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -81,7 +81,7 @@ filebeatmoduleconfsync: sodefaults_module_conf: file.managed: - - name: /opt/so/conf/filebeat/etc/securityonion.yml + - name: /opt/so/conf/filebeat/modules/securityonion.yml - source: salt://filebeat/etc/module_config.yml.jinja - template: jinja - defaults: @@ -89,7 +89,7 @@ sodefaults_module_conf: thirdparty_module_conf: file.managed: - - name: /opt/so/conf/filebeat/etc/thirdparty.yml + - name: /opt/so/conf/filebeat/modules/thirdparty.yml - source: salt://filebeat/etc/module_config.yml.jinja - template: jinja - defaults: diff --git a/salt/filebeat/modules/activemq.yml.disabled b/salt/filebeat/modules/activemq.yml.disabled deleted file mode 100644 index 43536ecbc..000000000 --- a/salt/filebeat/modules/activemq.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: activemq -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-activemq.html - -- module: activemq - # Audit logs - audit: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Application logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/apache.yml.disabled b/salt/filebeat/modules/apache.yml.disabled deleted file mode 100644 index b923dd581..000000000 --- a/salt/filebeat/modules/apache.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: apache -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-apache.html - -- module: apache - # Access logs - access: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Error logs - error: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/auditd.yml.disabled b/salt/filebeat/modules/auditd.yml.disabled deleted file mode 100644 index 76296ec85..000000000 --- a/salt/filebeat/modules/auditd.yml.disabled +++ /dev/null @@ -1,10 +0,0 @@ -# Module: auditd -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-auditd.html - -- module: auditd - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/aws.yml.disabled b/salt/filebeat/modules/aws.yml.disabled deleted file mode 100644 index 904bd976c..000000000 --- a/salt/filebeat/modules/aws.yml.disabled +++ /dev/null @@ -1,255 +0,0 @@ -# Module: aws -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-aws.html - -- module: aws - cloudtrail: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Process CloudTrail logs - # default is true, set to false to skip Cloudtrail logs - # var.process_cloudtrail_logs: false - - # Process CloudTrail Digest logs - # default true, set to false to skip CloudTrail Digest logs - # var.process_digest_logs: false - - # Process CloudTrail Insight logs - # default true, set to false to skip CloudTrail Insight logs - # var.process_insight_logs: false - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - cloudwatch: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - ec2: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - elb: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - s3access: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 - - vpcflow: - enabled: false - - # AWS SQS queue url - #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - #var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - #var.credential_profile_name: fb-aws - - # Use access_key_id, secret_access_key and/or session_token instead of shared credential file - #var.access_key_id: access_key_id - #var.secret_access_key: secret_access_key - #var.session_token: session_token - - # The duration that the received messages are hidden from ReceiveMessage request - # Default to be 300s - #var.visibility_timeout: 300s - - # Maximum duration before AWS API request will be interrupted - # Default to be 120s - #var.api_timeout: 120s - - # Custom endpoint used to access AWS APIs - #var.endpoint: amazonaws.com - - # AWS IAM Role to assume - #var.role_arn: arn:aws:iam::123456789012:role/test-mb - - # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. - #var.fips_enabled: false - - # The maximum number of messages to return from SQS. Valid values: 1 to 10. - #var.max_number_of_messages: 5 diff --git a/salt/filebeat/modules/azure.yml.disabled b/salt/filebeat/modules/azure.yml.disabled deleted file mode 100644 index 3b2bc1ecf..000000000 --- a/salt/filebeat/modules/azure.yml.disabled +++ /dev/null @@ -1,45 +0,0 @@ -# Module: azure -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-azure.html - -- module: azure - # All logs - activitylogs: - enabled: true - var: - # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub - eventhub: "insights-operational-logs" - # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module - consumer_group: "$Default" - # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string - connection_string: "" - # the name of the storage account the state/offsets will be stored and updated - storage_account: "" - # the storage account key, this key will be used to authorize access to data in your storage account - storage_account_key: "" - - platformlogs: - enabled: false - # var: - # eventhub: "" - # consumer_group: "$Default" - # connection_string: "" - # storage_account: "" - # storage_account_key: "" - - - auditlogs: - enabled: false - # var: - # eventhub: "insights-logs-auditlogs" - # consumer_group: "$Default" - # connection_string: "" - # storage_account: "" - # storage_account_key: "" - signinlogs: - enabled: false - # var: - # eventhub: "insights-logs-signinlogs" - # consumer_group: "$Default" - # connection_string: "" - # storage_account: "" - # storage_account_key: "" diff --git a/salt/filebeat/modules/barracuda.yml.disabled b/salt/filebeat/modules/barracuda.yml.disabled deleted file mode 100644 index 99ff85036..000000000 --- a/salt/filebeat/modules/barracuda.yml.disabled +++ /dev/null @@ -1,41 +0,0 @@ -# Module: barracuda -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-barracuda.html - -- module: barracuda - waf: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9503 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - spamfirewall: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9524 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/bluecoat.yml.disabled b/salt/filebeat/modules/bluecoat.yml.disabled deleted file mode 100644 index 6550c8eed..000000000 --- a/salt/filebeat/modules/bluecoat.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: bluecoat -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-bluecoat.html - -- module: bluecoat - director: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9505 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/cef.yml.disabled b/salt/filebeat/modules/cef.yml.disabled deleted file mode 100644 index 2de22edcc..000000000 --- a/salt/filebeat/modules/cef.yml.disabled +++ /dev/null @@ -1,17 +0,0 @@ -# Module: cef -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cef.html - -- module: cef - log: - enabled: true - var: - syslog_host: localhost - syslog_port: 9003 - - # Set internal security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.internal_zones: [ "Internal" ] - - # Set external security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.external_zones: [ "External" ] diff --git a/salt/filebeat/modules/checkpoint.yml.disabled b/salt/filebeat/modules/checkpoint.yml.disabled deleted file mode 100644 index 9d34b8d72..000000000 --- a/salt/filebeat/modules/checkpoint.yml.disabled +++ /dev/null @@ -1,24 +0,0 @@ -# Module: checkpoint -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-checkpoint.html - -- module: checkpoint - firewall: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: syslog - - # The interface to listen to UDP based syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The UDP port to listen for syslog traffic. Defaults to 9001. - #var.syslog_port: 9001 - - # Set internal security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.internal_zones: [ "Internal" ] - - # Set external security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.external_zones: [ "External" ] diff --git a/salt/filebeat/modules/cisco.yml.disabled b/salt/filebeat/modules/cisco.yml.disabled deleted file mode 100644 index 9e4658045..000000000 --- a/salt/filebeat/modules/cisco.yml.disabled +++ /dev/null @@ -1,142 +0,0 @@ -# Module: cisco -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cisco.html - -- module: cisco - asa: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: syslog - - # The interface to listen to UDP based syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The UDP port to listen for syslog traffic. Defaults to 9001. - #var.syslog_port: 9001 - - # Set the log level from 1 (alerts only) to 7 (include all messages). - # Messages with a log level higher than the specified will be dropped. - # See https://www.cisco.com/c/en/us/td/docs/security/asa/syslog/b_syslog/syslogs-sev-level.html - #var.log_level: 7 - - # Set internal security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.internal_zones: [ "Internal" ] - - # Set external security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.external_zones: [ "External" ] - - ftd: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: syslog - - # The interface to listen to UDP based syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The UDP port to listen for syslog traffic. Defaults to 9003. - #var.syslog_port: 9003 - - # Set the log level from 1 (alerts only) to 7 (include all messages). - # Messages with a log level higher than the specified will be dropped. - # See https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide/syslogs-sev-level.html - #var.log_level: 7 - - # Set internal security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.internal_zones: [ "Internal" ] - - # Set external security zones. used to override parsed network.direction - # based on zone egress and ingress - #var.external_zones: [ "External" ] - - ios: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: syslog - - # The interface to listen to UDP based syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The UDP port to listen for syslog traffic. Defaults to 9002. - #var.syslog_port: 9002 - - # Set custom paths for the log files when using file input. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - nexus: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9506 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - meraki: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9525 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - umbrella: - enabled: true - - #var.input: aws-s3 - # AWS SQS queue url - #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue - # Access ID to authenticate with the S3 input - #var.access_key_id: 123456 - # Access key to authenticate with the S3 input - #var.secret_access_key: PASSWORD - # The duration that the received messages are hidden from ReceiveMessage request - #var.visibility_timeout: 300s - # Maximum duration before AWS API request will be interrupted - #var.api_timeout: 120s - - amp: - enabled: true - - # Set which input to use between httpjson (default) or file. - #var.input: httpjson - - # The API URL - #var.url: https://api.amp.cisco.com/v1/events - # The client ID used as a username for the API requests. - #var.client_id: - # The API key related to the client ID. - #var.api_key: - # How far to look back the first time the module is started. Expects an amount of hours. - #var.first_interval: 24h - # Overriding the default request timeout, optional. - #var.request_timeout: 60s diff --git a/salt/filebeat/modules/coredns.yml.disabled b/salt/filebeat/modules/coredns.yml.disabled deleted file mode 100644 index 46e9e55c1..000000000 --- a/salt/filebeat/modules/coredns.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: coredns -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-coredns.html - -- module: coredns - # Fileset for native deployment - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/crowdstrike.yml.disabled b/salt/filebeat/modules/crowdstrike.yml.disabled deleted file mode 100644 index 8d2c8531d..000000000 --- a/salt/filebeat/modules/crowdstrike.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: crowdstrike -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-crowdstrike.html - -- module: crowdstrike - - falcon: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/cyberark.yml.disabled b/salt/filebeat/modules/cyberark.yml.disabled deleted file mode 100644 index e97955adf..000000000 --- a/salt/filebeat/modules/cyberark.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: cyberark -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cyberark.html - -- module: cyberark - corepas: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9527 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/cylance.yml.disabled b/salt/filebeat/modules/cylance.yml.disabled deleted file mode 100644 index 342d654d2..000000000 --- a/salt/filebeat/modules/cylance.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: cylance -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-cylance.html - -- module: cylance - protect: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9508 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/elasticsearch.yml.disabled b/salt/filebeat/modules/elasticsearch.yml.disabled deleted file mode 100644 index e6074c05e..000000000 --- a/salt/filebeat/modules/elasticsearch.yml.disabled +++ /dev/null @@ -1,35 +0,0 @@ -# Module: elasticsearch -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-elasticsearch.html - -- module: elasticsearch - # Server log - server: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - gc: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - audit: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - slowlog: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - deprecation: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/envoyproxy.yml.disabled b/salt/filebeat/modules/envoyproxy.yml.disabled deleted file mode 100644 index 543b17be5..000000000 --- a/salt/filebeat/modules/envoyproxy.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: envoyproxy -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-envoyproxy.html - -- module: envoyproxy - # Fileset for native deployment - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/f5.yml.disabled b/salt/filebeat/modules/f5.yml.disabled deleted file mode 100644 index 959842174..000000000 --- a/salt/filebeat/modules/f5.yml.disabled +++ /dev/null @@ -1,41 +0,0 @@ -# Module: f5 -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-f5.html - -- module: f5 - bigipapm: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9504 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - bigipafm: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9528 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/fortinet.yml.disabled b/salt/filebeat/modules/fortinet.yml.disabled deleted file mode 100644 index 281b7d788..000000000 --- a/salt/filebeat/modules/fortinet.yml.disabled +++ /dev/null @@ -1,83 +0,0 @@ -# Module: fortinet -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-fortinet.html - -- module: fortinet - firewall: - enabled: true - - # Set which input to use between tcp, udp (default) or file. - #var.input: udp - - # The interface to listen to syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The port to listen for syslog traffic. Defaults to 9004. - #var.syslog_port: 9004 - - # Set internal interfaces. used to override parsed network.direction - # based on a tagged interface. Both internal and external interfaces must be - # set to leverage this functionality. - #var.internal_interfaces: [ "LAN" ] - - # Set external interfaces. used to override parsed network.direction - # based on a tagged interface. Both internal and external interfaces must be - # set to leverage this functionality. - #var.external_interfaces: [ "WAN" ] - - clientendpoint: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9510 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - fortimail: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9529 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - fortimanager: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9530 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/gcp.yml.disabled b/salt/filebeat/modules/gcp.yml.disabled deleted file mode 100644 index a09d0fe36..000000000 --- a/salt/filebeat/modules/gcp.yml.disabled +++ /dev/null @@ -1,76 +0,0 @@ -# Module: gcp -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-gcp.html - -- module: gcp - vpcflow: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be - # configured to use this topic as a sink for VPC flow logs. - var.topic: gcp-vpc-flowlogs - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-vpc-flowlogs-sub - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json - - # Set internal networks. This is used to classify network.direction based - # off of what networks are considered "internal" either base off of a CIDR - # block or named network conditions. If this is not specified, then traffic - # direction is determined by whether it is between source and destination - # instance information rather than IP. - # - # For a full list of network conditions see: - # https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network - #var.internal_networks: [ "private" ] - - firewall: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing firewall logs. Stackdriver must be - # configured to use this topic as a sink for firewall logs. - var.topic: gcp-vpc-firewall - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-firewall-sub - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json - - # Set internal networks. This is used to classify network.direction based - # off of what networks are considered "internal" either base off of a CIDR - # block or named network conditions. If this is not specified, then traffic - # is taken from the direction data in the rule_details event payload. - # - # For a full list of network conditions see: - # https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network - #var.internal_networks: [ "private" ] - - audit: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing firewall logs. Stackdriver must be - # configured to use this topic as a sink for firewall logs. - var.topic: gcp-vpc-audit - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-audit - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/salt/filebeat/modules/google_workspace.yml.disabled b/salt/filebeat/modules/google_workspace.yml.disabled deleted file mode 100644 index 6d364af98..000000000 --- a/salt/filebeat/modules/google_workspace.yml.disabled +++ /dev/null @@ -1,53 +0,0 @@ -# Module: google_workspace -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-google_workspace.html - -- module: google_workspace - saml: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - user_accounts: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - login: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - admin: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - drive: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - groups: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - diff --git a/salt/filebeat/modules/googlecloud.yml.disabled b/salt/filebeat/modules/googlecloud.yml.disabled deleted file mode 100644 index 9a28dc036..000000000 --- a/salt/filebeat/modules/googlecloud.yml.disabled +++ /dev/null @@ -1,58 +0,0 @@ -# Module: googlecloud -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-googlecloud.html - -# googlecloud module is deprecated, please use gcp instead -- module: gcp - vpcflow: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing VPC flow logs. Stackdriver must be - # configured to use this topic as a sink for VPC flow logs. - var.topic: gcp-vpc-flowlogs - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-vpc-flowlogs-sub - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json - - firewall: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing firewall logs. Stackdriver must be - # configured to use this topic as a sink for firewall logs. - var.topic: gcp-vpc-firewall - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-firewall-sub - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json - - audit: - enabled: true - - # Google Cloud project ID. - var.project_id: my-gcp-project-id - - # Google Pub/Sub topic containing firewall logs. Stackdriver must be - # configured to use this topic as a sink for firewall logs. - var.topic: gcp-vpc-audit - - # Google Pub/Sub subscription for the topic. Filebeat will create this - # subscription if it does not exist. - var.subscription_name: filebeat-gcp-audit - - # Credentials file for the service account with authorization to read from - # the subscription. - var.credentials_file: ${path.config}/gcp-service-account-xyz.json diff --git a/salt/filebeat/modules/gsuite.yml.disabled b/salt/filebeat/modules/gsuite.yml.disabled deleted file mode 100644 index 6aec3b65d..000000000 --- a/salt/filebeat/modules/gsuite.yml.disabled +++ /dev/null @@ -1,53 +0,0 @@ -# Module: gsuite -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-gsuite.html - -# Gsuite module is deprecated and will be removed in future releases. Please use Google Workspace module instead. -- module: gsuite - saml: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - user_accounts: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - login: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - admin: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - drive: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h - groups: - enabled: true - # var.jwt_file: credentials.json - # var.delegated_account: admin@example.com - # var.initial_interval: 24h - # var.http_client_timeout: 60s - # var.user_key: all - # var.interval: 2h diff --git a/salt/filebeat/modules/haproxy.yml.disabled b/salt/filebeat/modules/haproxy.yml.disabled deleted file mode 100644 index b2615dbb8..000000000 --- a/salt/filebeat/modules/haproxy.yml.disabled +++ /dev/null @@ -1,14 +0,0 @@ -# Module: haproxy -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-haproxy.html - -- module: haproxy - # All logs - log: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/ibmmq.yml.disabled b/salt/filebeat/modules/ibmmq.yml.disabled deleted file mode 100644 index bfaf3792d..000000000 --- a/salt/filebeat/modules/ibmmq.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: ibmmq -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-ibmmq.html - -- module: ibmmq - # All logs - errorlog: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/icinga.yml.disabled b/salt/filebeat/modules/icinga.yml.disabled deleted file mode 100644 index a7c3ac6e1..000000000 --- a/salt/filebeat/modules/icinga.yml.disabled +++ /dev/null @@ -1,27 +0,0 @@ -# Module: icinga -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-icinga.html - -- module: icinga - # Main logs - main: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Debug logs - debug: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Startup logs - startup: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/iis.yml.disabled b/salt/filebeat/modules/iis.yml.disabled deleted file mode 100644 index 44c200ba1..000000000 --- a/salt/filebeat/modules/iis.yml.disabled +++ /dev/null @@ -1,20 +0,0 @@ -# Module: iis -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-iis.html - -- module: iis - # Access logs - access: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Error logs - error: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - \ No newline at end of file diff --git a/salt/filebeat/modules/imperva.yml.disabled b/salt/filebeat/modules/imperva.yml.disabled deleted file mode 100644 index 8e53deaa6..000000000 --- a/salt/filebeat/modules/imperva.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: imperva -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-imperva.html - -- module: imperva - securesphere: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9511 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/infoblox.yml.disabled b/salt/filebeat/modules/infoblox.yml.disabled deleted file mode 100644 index 9e82f8340..000000000 --- a/salt/filebeat/modules/infoblox.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: infoblox -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-infoblox.html - -- module: infoblox - nios: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9512 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/iptables.yml.disabled b/salt/filebeat/modules/iptables.yml.disabled deleted file mode 100644 index 1147e14dd..000000000 --- a/salt/filebeat/modules/iptables.yml.disabled +++ /dev/null @@ -1,13 +0,0 @@ -# Module: iptables -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-iptables.html - -- module: iptables - log: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/juniper.yml.disabled b/salt/filebeat/modules/juniper.yml.disabled deleted file mode 100644 index 71112679d..000000000 --- a/salt/filebeat/modules/juniper.yml.disabled +++ /dev/null @@ -1,54 +0,0 @@ -# Module: juniper -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-juniper.html - -- module: juniper - junos: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9513 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - netscreen: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9523 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local - - srx: - enabled: true - - # Set which input to use between tcp, udp (default) or file. - #var.input: udp - - # The interface to listen to syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The port to listen for syslog traffic. Defaults to 9006. - #var.syslog_port: 9006 diff --git a/salt/filebeat/modules/kafka.yml.disabled b/salt/filebeat/modules/kafka.yml.disabled deleted file mode 100644 index 23362c8a1..000000000 --- a/salt/filebeat/modules/kafka.yml.disabled +++ /dev/null @@ -1,15 +0,0 @@ -# Module: kafka -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-kafka.html - -- module: kafka - # All logs - log: - enabled: true - - # Set custom paths for Kafka. If left empty, - # Filebeat will look under /opt. - #var.kafka_home: - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/kibana.yml.disabled b/salt/filebeat/modules/kibana.yml.disabled deleted file mode 100644 index a4956c4b6..000000000 --- a/salt/filebeat/modules/kibana.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: kibana -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-kibana.html - -- module: kibana - # Server logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Audit logs - audit: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/logstash.yml.disabled b/salt/filebeat/modules/logstash.yml.disabled deleted file mode 100644 index f14229409..000000000 --- a/salt/filebeat/modules/logstash.yml.disabled +++ /dev/null @@ -1,18 +0,0 @@ -# Module: logstash -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-logstash.html - -- module: logstash - # logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Slow logs - slowlog: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/microsoft.yml.disabled b/salt/filebeat/modules/microsoft.yml.disabled deleted file mode 100644 index b0a1b10c6..000000000 --- a/salt/filebeat/modules/microsoft.yml.disabled +++ /dev/null @@ -1,49 +0,0 @@ -# Module: microsoft -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-microsoft.html - -- module: microsoft - # ATP configuration - defender_atp: - enabled: true - # How often the API should be polled - #var.interval: 5m - - # Oauth Client ID - #var.oauth2.client.id: "" - - # Oauth Client Secret - #var.oauth2.client.secret: "" - - # Oauth Token URL, should include the tenant ID - #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" - m365_defender: - enabled: true - # How often the API should be polled - #var.interval: 5m - - # Oauth Client ID - #var.oauth2.client.id: "" - - # Oauth Client Secret - #var.oauth2.client.secret: "" - - # Oauth Token URL, should include the tenant ID - #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" - dhcp: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9515 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/misp.yml.disabled b/salt/filebeat/modules/misp.yml.disabled deleted file mode 100644 index 9a489fa0f..000000000 --- a/salt/filebeat/modules/misp.yml.disabled +++ /dev/null @@ -1,17 +0,0 @@ -# Module: misp -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-misp.html - -- module: misp - threat: - enabled: true - # API key to access MISP - #var.api_key - - # Array object in MISP response - #var.http_request_body.limit: 1000 - - # URL of the MISP REST API - #var.url - - # You can also pass SSL options. For example: - #var.ssl.verification_mode: none diff --git a/salt/filebeat/modules/mongodb.yml.disabled b/salt/filebeat/modules/mongodb.yml.disabled deleted file mode 100644 index 266d2e4e8..000000000 --- a/salt/filebeat/modules/mongodb.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: mongodb -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mongodb.html - -- module: mongodb - # All logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/mssql.yml.disabled b/salt/filebeat/modules/mssql.yml.disabled deleted file mode 100644 index bfe4c6e64..000000000 --- a/salt/filebeat/modules/mssql.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: mssql -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mssql.html - -- module: mssql - # Fileset for native deployment - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*'] diff --git a/salt/filebeat/modules/mysql.yml.disabled b/salt/filebeat/modules/mysql.yml.disabled deleted file mode 100644 index e6be4045b..000000000 --- a/salt/filebeat/modules/mysql.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: mysql -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mysql.html - -- module: mysql - # Error logs - error: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Slow logs - slowlog: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/mysqlenterprise.yml.disabled b/salt/filebeat/modules/mysqlenterprise.yml.disabled deleted file mode 100644 index 37e10d0eb..000000000 --- a/salt/filebeat/modules/mysqlenterprise.yml.disabled +++ /dev/null @@ -1,14 +0,0 @@ -# Module: mysqlenterprise -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-mysqlenterprise.html - -- module: mysqlenterprise - audit: - enabled: true - - # Sets the input type. Currently only supports file - #var.input: file - - # Set paths for the log files when file input is used. - # Should only be used together with file input - # var.paths: - # - /home/user/mysqlauditlogs/audit.*.log diff --git a/salt/filebeat/modules/nats.yml.disabled b/salt/filebeat/modules/nats.yml.disabled deleted file mode 100644 index 65e44962d..000000000 --- a/salt/filebeat/modules/nats.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: nats -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-nats.html - -- module: nats - # All logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/netflow.yml.disabled b/salt/filebeat/modules/netflow.yml.disabled deleted file mode 100644 index 781748b00..000000000 --- a/salt/filebeat/modules/netflow.yml.disabled +++ /dev/null @@ -1,14 +0,0 @@ -# Module: netflow -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-netflow.html - -- module: netflow - log: - enabled: true - var: - netflow_host: localhost - netflow_port: 2055 - # internal_networks specifies which networks are considered internal or private - # you can specify either a CIDR block or any of the special named ranges listed - # at: https://www.elastic.co/guide/en/beats/filebeat/current/defining-processors.html#condition-network - internal_networks: - - private diff --git a/salt/filebeat/modules/netscout.yml.disabled b/salt/filebeat/modules/netscout.yml.disabled deleted file mode 100644 index 215349046..000000000 --- a/salt/filebeat/modules/netscout.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: netscout -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-netscout.html - -- module: netscout - sightline: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9502 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/nginx.yml.disabled b/salt/filebeat/modules/nginx.yml.disabled deleted file mode 100644 index e2fa44a78..000000000 --- a/salt/filebeat/modules/nginx.yml.disabled +++ /dev/null @@ -1,27 +0,0 @@ -# Module: nginx -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-nginx.html - -- module: nginx - # Access logs - access: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Error logs - error: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs - ingress_controller: - enabled: false - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/o365.yml.disabled b/salt/filebeat/modules/o365.yml.disabled deleted file mode 100644 index 578ff365d..000000000 --- a/salt/filebeat/modules/o365.yml.disabled +++ /dev/null @@ -1,48 +0,0 @@ -# Module: o365 -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-o365.html - -- module: o365 - audit: - enabled: true - - # Set the application_id (also known as client ID): - var.application_id: "" - - # Configure the tenants to monitor: - # Use the tenant ID (also known as directory ID) and the domain name. - # var.tenants: - # - id: "tenant_id_1" - # name: "mydomain.onmicrosoft.com" - # - id: "tenant_id_2" - # name: "mycompany.com" - var.tenants: - - id: "" - name: "mytenant.onmicrosoft.com" - - # List of content-types to fetch. By default all known content-types - # are retrieved: - # var.content_type: - # - "Audit.AzureActiveDirectory" - # - "Audit.Exchange" - # - "Audit.SharePoint" - # - "Audit.General" - # - "DLP.All" - - # Use the following settings to enable certificate-based authentication: - # var.certificate: "/path/to/certificate.pem" - # var.key: "/path/to/private_key.pem" - # var.key_passphrase: "myPrivateKeyPassword" - - # Client-secret based authentication: - # Comment the following line if using certificate authentication. - var.client_secret: "" - - # Advanced settings, use with care: - # var.api: - # # Settings for custom endpoints: - # authentication_endpoint: "https://login.microsoftonline.us/" - # resource: "https://manage.office365.us" - # - # max_retention: 168h - # max_requests_per_minute: 2000 - # poll_interval: 3m diff --git a/salt/filebeat/modules/okta.yml.disabled b/salt/filebeat/modules/okta.yml.disabled deleted file mode 100644 index 4fc943592..000000000 --- a/salt/filebeat/modules/okta.yml.disabled +++ /dev/null @@ -1,10 +0,0 @@ -# Module: okta -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-okta.html - -- module: okta - system: - enabled: true - # You must configure the URL with your Okta domain and provide an - # API token to access the logs API. - #var.url: https://yourOktaDomain/api/v1/logs - #var.api_key: 'yourApiTokenHere' diff --git a/salt/filebeat/modules/oracle.yml.disabled b/salt/filebeat/modules/oracle.yml.disabled deleted file mode 100644 index 3bd576ee1..000000000 --- a/salt/filebeat/modules/oracle.yml.disabled +++ /dev/null @@ -1,13 +0,0 @@ -# Module: oracle -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-oracle.html - -- module: oracle - database_audit: - enabled: true - - # Set which input to use between syslog or file (default). - #var.input: file - - # Set paths for the log files when file input is used. - # Should only be used together with file input - # var.paths: /home/user/oracleauditlogs/*.aud diff --git a/salt/filebeat/modules/osquery.yml.disabled b/salt/filebeat/modules/osquery.yml.disabled deleted file mode 100644 index 7a9a09dd8..000000000 --- a/salt/filebeat/modules/osquery.yml.disabled +++ /dev/null @@ -1,15 +0,0 @@ -# Module: osquery -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-osquery.html - -- module: osquery - result: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # If true, all fields created by this module are prefixed with - # `osquery.result`. Set to false to copy the fields in the root - # of the document. The default is true. - #var.use_namespace: true diff --git a/salt/filebeat/modules/panw.yml.disabled b/salt/filebeat/modules/panw.yml.disabled deleted file mode 100644 index eb094a25a..000000000 --- a/salt/filebeat/modules/panw.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: panw -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-panw.html - -- module: panw - panos: - enabled: true - - # Set which input to use between syslog (default) or file. - #var.input: - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Set internal security zones. used to determine network.direction - # default "trust" - #var.internal_zones: - - # Set external security zones. used to determine network.direction - # default "untrust" - #var.external_zones: - diff --git a/salt/filebeat/modules/pensando.yml.disabled b/salt/filebeat/modules/pensando.yml.disabled deleted file mode 100644 index 66bd60d76..000000000 --- a/salt/filebeat/modules/pensando.yml.disabled +++ /dev/null @@ -1,13 +0,0 @@ -# Module: pensando -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-pensando.html - -- module: pensando -# Firewall logs - dfw: - enabled: true - var.syslog_host: 0.0.0.0 - var.syslog_port: 9001 - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - # var.paths: diff --git a/salt/filebeat/modules/postgresql.yml.disabled b/salt/filebeat/modules/postgresql.yml.disabled deleted file mode 100644 index 804b7f34f..000000000 --- a/salt/filebeat/modules/postgresql.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: postgresql -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-postgresql.html - -- module: postgresql - # All logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/proofpoint.yml.disabled b/salt/filebeat/modules/proofpoint.yml.disabled deleted file mode 100644 index 9aeebd5fe..000000000 --- a/salt/filebeat/modules/proofpoint.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: proofpoint -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-proofpoint.html - -- module: proofpoint - emailsecurity: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9531 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/rabbitmq.yml.disabled b/salt/filebeat/modules/rabbitmq.yml.disabled deleted file mode 100644 index e61a0a0c9..000000000 --- a/salt/filebeat/modules/rabbitmq.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: rabbitmq -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-rabbitmq.html - -- module: rabbitmq - # All logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: ["/var/log/rabbitmq/rabbit@localhost.log*"] diff --git a/salt/filebeat/modules/radware.yml.disabled b/salt/filebeat/modules/radware.yml.disabled deleted file mode 100644 index f9ab3e519..000000000 --- a/salt/filebeat/modules/radware.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: radware -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-radware.html - -- module: radware - defensepro: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9518 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/redis.yml.disabled b/salt/filebeat/modules/redis.yml.disabled deleted file mode 100644 index 9b621dc2d..000000000 --- a/salt/filebeat/modules/redis.yml.disabled +++ /dev/null @@ -1,21 +0,0 @@ -# Module: redis -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-redis.html - -- module: redis - # Main logs - log: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: ["/var/log/redis/redis-server.log*"] - - # Slow logs, retrieved via the Redis API (SLOWLOG) - slowlog: - enabled: true - - # The Redis hosts to connect to. - #var.hosts: ["localhost:6379"] - - # Optional, the password to use when connecting to Redis. - #var.password: diff --git a/salt/filebeat/modules/santa.yml.disabled b/salt/filebeat/modules/santa.yml.disabled deleted file mode 100644 index 1a7363547..000000000 --- a/salt/filebeat/modules/santa.yml.disabled +++ /dev/null @@ -1,9 +0,0 @@ -# Module: santa -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-santa.html - -- module: santa - log: - enabled: true - # Set custom paths for the log files. If left empty, - # Filebeat will choose the the default path. - #var.paths: diff --git a/salt/filebeat/modules/snort.yml.disabled b/salt/filebeat/modules/snort.yml.disabled deleted file mode 100644 index 8c9bcc471..000000000 --- a/salt/filebeat/modules/snort.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: snort -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-snort.html - -- module: snort - log: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9532 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/snyk.yml.disabled b/salt/filebeat/modules/snyk.yml.disabled deleted file mode 100644 index 0b13f8155..000000000 --- a/salt/filebeat/modules/snyk.yml.disabled +++ /dev/null @@ -1,112 +0,0 @@ -# Module: snyk -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-snyk.html - -- module: snyk - audit: - enabled: true - - # Set which input to use between httpjson (default) or file. - #var.input: httpjson - # - # What audit type to collect, can be either "group" or "organization". - #var.audit_type: organization - # - # The ID related to the audit_type. If audit type is group, then this value should be - # the group ID and if it is organization it should be the organization ID to collect from. - #var.audit_id: 1235432-asdfdf-2341234-asdgjhg - - # How often the API should be polled, defaults to 1 hour. - #var.interval: 1h - # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). - #var.first_interval: 24h - - # The API token that is created for a specific user, found in the Snyk management dashboard. - #var.api_token: - - # Event filtering. - # All configuration items below is OPTIONAL and the default options will be overwritten - # for each entry that is not commented out. - - # Will return only logs for this specific project. - #var.project_id: "" - # User public ID. Will fetch only audit logs originated from this user's actions. - #var.user_id: "" - # Will return only logs for this specific event. - #var.event: "" - # User email address. Will fetch only audit logs originated from this user's actions. - #var.email_address: "" - - vulnerabilities: - enabled: true - - # Set which input to use between httpjson (default) or file. - #var.input: httpjson - - # How often the API should be polled. Data from the Snyk API is automatically updated - # once per day, so the default interval is 24 hours. - #var.interval: 24h - - # How far to look back the first time the module starts up. (Only works with full days, 24 hours, 48 hours etc). - #var.first_interval: 24h - - # The API token that is created for a specific user, found in the Snyk management dashboard. - #var.api_token: - - # The list of org IDs to filter the results by. - # One organization ID per line, starting with a - sign - #var.orgs: - # - 12354-asdfdf-123543-asdsdfg - # - 76554-jhggfd-654342-hgrfasd - - - # Event filtering. - # All configuration items below is OPTIONAL and the default options will be overwritten - # for each entry that is not commented out. - - # The severity levels of issues to filter the results by. - #var.included_severity: - # - high - # - medium - # - low - # - # The exploit maturity levels of issues to filter the results by. - #var.exploit_maturity: - # - mature - # - proof-of-concept - # - no-known-exploit - # - no-data - # - # The type of issues to filter the results by. - #var.types: - # - vuln - # - license - # - # The type of languages to filter the results by. - #var.languages: - # - javascript - # - ruby - # - java - # - scala - # - python - # - golang - # - php - # - dotnet - # - swift - # - docker - # - # Search term to filter issue name by, or an exact CVE or CWE. - #var.identifier: - # - "" - # - # If set to true, only include issues which are ignored, if set to false, only include issues which are not ignored. - #var.ignored: false - #var.patched: false - #var.fixable: false - #var.is_fixed: false - #var.is_patchable: false - #var.is_pinnable: false - # - # The priority score ranging between 0-1000 - #var.min_priority_score: 0 - #var.max_priority_score: 1000 - diff --git a/salt/filebeat/modules/sonicwall.yml.disabled b/salt/filebeat/modules/sonicwall.yml.disabled deleted file mode 100644 index de457109d..000000000 --- a/salt/filebeat/modules/sonicwall.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: sonicwall -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-sonicwall.html - -- module: sonicwall - firewall: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9519 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/sophos.yml.disabled b/salt/filebeat/modules/sophos.yml.disabled deleted file mode 100644 index 8fc346540..000000000 --- a/salt/filebeat/modules/sophos.yml.disabled +++ /dev/null @@ -1,46 +0,0 @@ -# Module: sophos -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-sophos.html - -- module: sophos - xg: - enabled: true - - # Set which input to use between tcp, udp (default) or file. - #var.input: udp - - # The interface to listen to syslog traffic. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.syslog_host: localhost - - # The port to listen for syslog traffic. Defaults to 9004. - #var.syslog_port: 9005 - - # firewall default hostname - #var.default_host_name: firewall.localgroup.local - - # known firewalls - #var.known_devices: - #- serial_number: "1234567890123457" - # hostname: "a.host.local" - #- serial_number: "1234234590678557" - # hostname: "b.host.local" - - - utm: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9533 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/squid.yml.disabled b/salt/filebeat/modules/squid.yml.disabled deleted file mode 100644 index a47807253..000000000 --- a/salt/filebeat/modules/squid.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: squid -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-squid.html - -- module: squid - log: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9520 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/suricata.yml b/salt/filebeat/modules/suricata.yml deleted file mode 100644 index b7cc11e85..000000000 --- a/salt/filebeat/modules/suricata.yml +++ /dev/null @@ -1,12 +0,0 @@ -# Module: suricata -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html - -- module: suricata - # All logs - eve: - enabled: true - var.paths: ["/nsm/suricata/eve*.json"] - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/system.yml.disabled b/salt/filebeat/modules/system.yml.disabled deleted file mode 100644 index d633bac04..000000000 --- a/salt/filebeat/modules/system.yml.disabled +++ /dev/null @@ -1,19 +0,0 @@ -# Module: system -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-system.html - -- module: system - # Syslog - syslog: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Authorization logs - auth: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/threatintel.yml.disabled b/salt/filebeat/modules/threatintel.yml.disabled deleted file mode 100644 index b461d91e2..000000000 --- a/salt/filebeat/modules/threatintel.yml.disabled +++ /dev/null @@ -1,105 +0,0 @@ -# Module: threatintel -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-threatintel.html - -- module: threatintel - abuseurl: - enabled: true - - # Input used for ingesting threat intel data. - var.input: httpjson - - # The URL used for Threat Intel API calls. - var.url: https://urlhaus-api.abuse.ch/v1/urls/recent/ - - # The interval to poll the API for updates. - var.interval: 10m - - abusemalware: - enabled: true - - # Input used for ingesting threat intel data. - var.input: httpjson - - # The URL used for Threat Intel API calls. - var.url: https://urlhaus-api.abuse.ch/v1/payloads/recent/ - - # The interval to poll the API for updates. - var.interval: 10m - - misp: - enabled: true - - # Input used for ingesting threat intel data, defaults to JSON. - var.input: httpjson - - # The URL of the MISP instance, should end with "/events/restSearch". - var.url: https://SERVER/events/restSearch - - # The authentication token used to contact the MISP API. Found when looking at user account in the MISP UI. - var.api_token: API_KEY - - # Configures the type of SSL verification done, if MISP is running on self signed certificates - # then the certificate would either need to be trusted, or verification_mode set to none. - #var.ssl.verification_mode: none - - # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context. - # For examples please reference the filebeat module documentation. - #var.filters: - # - threat_level: [4, 5] - # - to_ids: true - - # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer - # than the last event that was already ingested. - var.first_interval: 300h - - # The interval to poll the API for updates. - var.interval: 5m - - otx: - enabled: true - - # Input used for ingesting threat intel data - var.input: httpjson - - # The URL used for OTX Threat Intel API calls. - var.url: https://otx.alienvault.com/api/v1/indicators/export - - # The authentication token used to contact the OTX API, can be found on the OTX UI. - var.api_token: API_KEY - - # Optional filters that can be applied to retrieve only specific indicators. - #var.types: "domain,IPv4,hostname,url,FileHash-SHA256" - - # The timeout of the HTTP client connecting to the OTX API - #var.http_client_timeout: 120s - - # How many hours to look back for each request, should be close to the configured interval. Deduplication of events is handled by the module. - var.lookback_range: 1h - - # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 400h - - # The interval to poll the API for updates - var.interval: 5m - - anomali: - enabled: true - - # Input used for ingesting threat intel data - var.input: httpjson - - # The URL used for Threat Intel API calls. Limo has multiple different possibilities for URL's depending - # on the type of threat intel source that is needed. - var.url: https://limo.anomali.com/api/v1/taxii2/feeds/collections/41/objects - - # The Username used by anomali Limo, defaults to guest. - #var.username: guest - - # The password used by anomali Limo, defaults to guest. - #var.password: guest - - # How far back to look once the beat starts up for the first time, the value has to be in hours. - var.first_interval: 400h - - # The interval to poll the API for updates - var.interval: 5m diff --git a/salt/filebeat/modules/tomcat.yml.disabled b/salt/filebeat/modules/tomcat.yml.disabled deleted file mode 100644 index 84f4619d5..000000000 --- a/salt/filebeat/modules/tomcat.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: tomcat -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-tomcat.html - -- module: tomcat - log: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9501 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/modules/traefik.yml.disabled b/salt/filebeat/modules/traefik.yml.disabled deleted file mode 100644 index 657d5ccd9..000000000 --- a/salt/filebeat/modules/traefik.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: traefik -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-traefik.html - -- module: traefik - # Access logs - access: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/zeek.yml b/salt/filebeat/modules/zeek.yml deleted file mode 100644 index 9fd61c448..000000000 --- a/salt/filebeat/modules/zeek.yml +++ /dev/null @@ -1,122 +0,0 @@ -# Module: zeek -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html - -- module: zeek - capture_loss: - enabled: false - var.paths: ["/nsm/zeek/logs/current/capture_loss.log"] - connection: - enabled: true - var.paths: ["/nsm/zeek/logs/current/conn.log"] - dce_rpc: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dce_rpc.log"] - dhcp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dhcp.log"] - dnp3: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dnp3.log"] - dns: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dns.log"] - dpd: - enabled: true - var.paths: ["/nsm/zeek/logs/current/dpd.log"] - files: - enabled: true - var.paths: ["/nsm/zeek/logs/current/files.log"] - ftp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/ftp.log"] - http: - enabled: true - var.paths: ["/nsm/zeek/logs/current/http.log"] - intel: - enabled: true - var.paths: ["/nsm/zeek/logs/current/intel.log"] - irc: - enabled: true - var.paths: ["/nsm/zeek/logs/current/irc.log"] - kerberos: - enabled: true - var.paths: ["/nsm/zeek/logs/current/kerberos.log"] - modbus: - enabled: true - var.paths: ["/nsm/zeek/logs/current/modbus.log"] - mysql: - enabled: true - var.paths: ["/nsm/zeek/logs/current/mysql.log"] - notice: - enabled: true - var.paths: ["/nsm/zeek/logs/current/notice.log"] - ntlm: - enabled: true - var.paths: ["/nsm/zeek/logs/current/ntlm.log"] - ocsp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/oscp.log"] - pe: - enabled: true - var.paths: ["/nsm/zeek/logs/current/pe.log"] - radius: - enabled: true - var.paths: ["/nsm/zeek/logs/current/radius.log"] - rdp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/rdp.log"] - rfb: - enabled: true - var.paths: ["/nsm/zeek/logs/current/rfb.log"] - signature: - enabled: true - var.paths: ["/nsm/zeek/logs/current/signature.log"] - sip: - enabled: true - var.paths: ["/nsm/zeek/logs/current/sip.log"] - smb_cmd: - enabled: true - var.paths: ["/nsm/zeek/logs/current/smb_cmd.log"] - smb_files: - enabled: true - var.paths: ["/nsm/zeek/logs/current/smb_files.log"] - smb_mapping: - enabled: true - var.paths: ["/nsm/zeek/logs/current/smb_mapping.log"] - smtp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/smtp.log"] - snmp: - enabled: true - var.paths: ["/nsm/zeek/logs/current/snmp.log"] - socks: - enabled: true - var.paths: ["/nsm/zeek/logs/current/socks.log"] - ssh: - enabled: true - var.paths: ["/nsm/zeek/logs/current/ssh.log"] - ssl: - enabled: true - var.paths: ["/nsm/zeek/logs/current/ssl.log"] - stats: - enabled: false - var.paths: ["/nsm/zeek/logs/current/stats.log"] - syslog: - enabled: false - var.paths: ["/nsm/zeek/logs/current/syslog.log"] - traceroute: - enabled: false - var.paths: ["/nsm/zeek/logs/current/traceroute.log.log"] - tunnel: - enabled: true - var.paths: ["/nsm/zeek/logs/current/tunnel.log"] - weird: - enabled: true - var.paths: ["/nsm/zeek/logs/current/weird.log"] - x509: - enabled: true - var.paths: ["/nsm/zeek/logs/current/x509.log"] - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/zoom.yml.disabled b/salt/filebeat/modules/zoom.yml.disabled deleted file mode 100644 index 15fa9d4b2..000000000 --- a/salt/filebeat/modules/zoom.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: zoom -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zoom.html - -- module: zoom - webhook: - enabled: true - - # The type of input to use - #var.input: http_endpoint - - # The interface to listen for incoming HTTP requests. Defaults to - # localhost. Set to 0.0.0.0 to bind to all available interfaces. - #var.listen_address: localhost - - # The port to bind to - #var.listen_port: 80 - - # The header Zoom uses to send its secret token, defaults to "Authorization" - #secret.header: Authorization - - # The secret token value created by Zoom - #secret.value: ZOOMTOKEN diff --git a/salt/filebeat/modules/zscaler.yml.disabled b/salt/filebeat/modules/zscaler.yml.disabled deleted file mode 100644 index accdec9ea..000000000 --- a/salt/filebeat/modules/zscaler.yml.disabled +++ /dev/null @@ -1,22 +0,0 @@ -# Module: zscaler -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zscaler.html - -- module: zscaler - zia: - enabled: true - - # Set which input to use between udp (default), tcp or file. - # var.input: udp - # var.syslog_host: localhost - # var.syslog_port: 9521 - - # Set paths for the log files when file input is used. - # var.paths: - - # Toggle output of non-ECS fields (default true). - # var.rsa_fields: true - - # Set custom timezone offset. - # "local" (default) for system timezone. - # "+02:00" for GMT+02:00 - # var.tz_offset: local diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 8bcecd618..19826a708 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -26,7 +26,7 @@ securityonion_filebeat: zeek: {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} {{ LOGNAME }}: - enabled: false + enabled: true var.paths: ["/nsm/zeek/logs/current/{{ LOGNAME }}.log"] {%- endfor %} {%- endif %} From dfaf40f58359ab0896c84a47dc85c4b5d5d27280 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 17:14:26 -0400 Subject: [PATCH 149/195] add zeekloglookup to translate zeeklogs to filebeat filesets --- salt/filebeat/init.sls | 1 - salt/filebeat/map.jinja | 4 ++++ salt/filebeat/securityoniondefaults.yaml | 11 +++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index b1a91b133..7850f4eb6 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -23,7 +23,6 @@ {% from 'filebeat/map.jinja' import THIRDPARTY with context %} {% from 'filebeat/map.jinja' import SO with context %} - filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index 6ae6e7cff..b5df8fea5 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -4,3 +4,7 @@ {% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} {% set SO = SODEFAULTS.securityonion_filebeat %} {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} + +{% set ZEEKLOGLOOKUP = { + 'conn': 'connection', +} %} diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 8bcecd618..300e7f42f 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -1,4 +1,6 @@ {%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %} +{% from 'filebeat/map.jinja' import ZEEKLOGLOOKUP with context %} + securityonion_filebeat: modules: elasticsearch: @@ -25,9 +27,14 @@ securityonion_filebeat: {%- if ZEEKVER != 'SURICATA' %} zeek: {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} - {{ LOGNAME }}: + {% if LOGNAME in ZEEKLOGLOOKUP.keys() %} + {% set FILESET = ZEEKLOGLOOKUP.get(LOGNAME) %} + {% else %} + {% set FILESET = LOGNAME %} + {% endif %} + {{ FILESET }}: enabled: false var.paths: ["/nsm/zeek/logs/current/{{ LOGNAME }}.log"] {%- endfor %} {%- endif %} - {%- endif %} \ No newline at end of file + {%- endif %} From 525d4325c7530e4e137b956a954c523c868651d9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 May 2021 17:18:58 -0400 Subject: [PATCH 150/195] define ZEEKLOGLOOKUP in the yaml --- salt/filebeat/map.jinja | 4 ---- salt/filebeat/securityoniondefaults.yaml | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/filebeat/map.jinja b/salt/filebeat/map.jinja index b5df8fea5..6ae6e7cff 100644 --- a/salt/filebeat/map.jinja +++ b/salt/filebeat/map.jinja @@ -4,7 +4,3 @@ {% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %} {% set SO = SODEFAULTS.securityonion_filebeat %} {#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#} - -{% set ZEEKLOGLOOKUP = { - 'conn': 'connection', -} %} diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 58eef8361..0a1459d6b 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -1,5 +1,7 @@ {%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %} -{% from 'filebeat/map.jinja' import ZEEKLOGLOOKUP with context %} +{% set ZEEKLOGLOOKUP = { + 'conn': 'connection', +} %} securityonion_filebeat: modules: From 6da37966d928ef1af1d0a98dfbed7ee712e70b56 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 26 May 2021 09:32:25 -0400 Subject: [PATCH 151/195] Update wording for iso location prompt in soup --- salt/common/tools/sbin/soup | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d1261fe30..f7fa4e211 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -107,14 +107,15 @@ airgap_mounted() { echo "The ISO is already mounted" else echo "" - echo "Looks like we need access to the upgrade content" - echo "" - echo "If you just copied the .iso file over you can specify the path." - echo "If you burned the ISO to a disk the standard way you can specify the device." - echo "Example: /home/user/securityonion-2.X.0.iso" - echo "Example: /dev/sdx1" - echo "" - read -rp 'Enter the location of the iso: ' ISOLOC + cat << EOF +In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. + +For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. + +Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom. + +EOF + read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC if [[ -f $ISOLOC ]]; then # Mounting the ISO image mkdir -p /tmp/soagupdate From 0187c9d6df5aeb4a62c7c963231f4d0f7b2cefe6 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 26 May 2021 09:51:37 -0400 Subject: [PATCH 152/195] Adds so-pcap-export --- salt/common/tools/sbin/so-pcap-export | 26 +++++++++++++++++++++ salt/common/tools/sbin/so-suricata-testrule | 5 ++++ 2 files changed, 31 insertions(+) create mode 100644 salt/common/tools/sbin/so-pcap-export diff --git a/salt/common/tools/sbin/so-pcap-export b/salt/common/tools/sbin/so-pcap-export new file mode 100644 index 000000000..076b4eae6 --- /dev/null +++ b/salt/common/tools/sbin/so-pcap-export @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +if [ $# -lt 2 ]; then + echo "Usage: $0 Output-Filename" + exit 1 +fi + +docker exec -it so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap + +echo "" +echo "If successful, the output was written to: /nsm/pcapout/$2.pcap" \ No newline at end of file diff --git a/salt/common/tools/sbin/so-suricata-testrule b/salt/common/tools/sbin/so-suricata-testrule index ac4b81d3c..f9f23e70f 100755 --- a/salt/common/tools/sbin/so-suricata-testrule +++ b/salt/common/tools/sbin/so-suricata-testrule @@ -23,6 +23,11 @@ TESTPCAP=$2 . /usr/sbin/so-common +if [ $# -lt 2 ]; then + echo "Usage: $0 " + exit 1 +fi + echo "" echo "===============" echo "Running all.rules and $TESTRULE against the following pcap: $TESTPCAP" From cb40a76247141f45ccb2886e3637e2386a3a1b76 Mon Sep 17 00:00:00 2001 From: John Bernal <59663876+j-bernal@users.noreply.github.com> Date: Wed, 26 May 2021 09:55:14 -0400 Subject: [PATCH 153/195] Update so-whiptail Updated Zeek capitalization when prompting for the number of processes. --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 5eca2d39a..b7fc50495 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -64,7 +64,7 @@ whiptail_basic_zeek() { [ -n "$TESTING" ] && return BASICZEEK=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter the number of zeek processes:" 10 75 "$1" 3>&1 1>&2 2>&3) + "Enter the number of Zeek processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 34d4eedf672cb523942c14f16c8a57ef1036fbe9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 May 2021 10:11:47 -0400 Subject: [PATCH 154/195] Remove old modules --- salt/common/tools/sbin/so-filebeat-module-setup | 8 +++++++- salt/common/tools/sbin/so-zeek-logs | 6 ------ salt/filebeat/etc/module_config.yml.jinja | 2 +- salt/filebeat/securityoniondefaults.yaml | 2 ++ salt/filebeat/thirdpartydefaults.yaml | 17 ----------------- salt/zeek/init.sls | 2 ++ setup/so-functions | 6 ------ setup/so-whiptail | 6 ------ 8 files changed, 12 insertions(+), 37 deletions(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 6616854eb..5aefe3ac2 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -49,6 +49,12 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Setting up ingest pipeline(s)" -docker exec -it so-filebeat filebeat setup modules -pipelines -modules activemq,apache,auditd,aws,azure,barracuda,bluecoat,cef,checkpoint,cisco,coredns,crowdstrike,cyberark,cylance,elasticsearch,envoyproxy,f5,fortinet,gcp,google_workspace,googlecloud,gsuite,haproxy,ibmmq,icinga,iis,imperva,infoblox,iptables,juniper,kafka,kibana,logstash,microsoft,misp,mondogb,mssql,mysql,mysqlenterprise,nats,netflow,netscout,nginx,o365,okta,osquery,panw,pensando,postgresql,rabbitmq,radware,redis,santa,snort,snyk,sonicwall,sophos,squid,suricata,system,threatintel,tomcat,traefik,zeek,zoom,zscaler -c $FB_MODULE_YML + +for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler +do + echo "Loading $MODULE" + docker exec -it so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML + sleep 2 +done diff --git a/salt/common/tools/sbin/so-zeek-logs b/salt/common/tools/sbin/so-zeek-logs index 551213580..521f7c9dc 100755 --- a/salt/common/tools/sbin/so-zeek-logs +++ b/salt/common/tools/sbin/so-zeek-logs @@ -14,7 +14,6 @@ whiptail_manager_adv_service_zeeklogs() { "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ - "dhcpv6" "DHCP IPv6 Logs" ON \ "dnp3" "DNP3 Logs" ON \ "dns" "DNS Logs" ON \ "dpd" "DPD Logs" ON \ @@ -25,25 +24,20 @@ whiptail_manager_adv_service_zeeklogs() { "irc" "IRC Chat Logs" ON \ "kerberos" "Kerberos Logs" ON \ "modbus" "MODBUS Logs" ON \ - "mqtt" "MQTT Logs" ON \ "notice" "Zeek Notice Logs" ON \ "ntlm" "NTLM Logs" ON \ - "openvpn" "OPENVPN Logs" ON \ "pe" "PE Logs" ON \ "radius" "Radius Logs" ON \ "rfb" "RFB Logs" ON \ "rdp" "RDP Logs" ON \ - "signatures" "Signatures Logs" ON \ "sip" "SIP Logs" ON \ "smb_files" "SMB Files Logs" ON \ "smb_mapping" "SMB Mapping Logs" ON \ "smtp" "SMTP Logs" ON \ "snmp" "SNMP Logs" ON \ - "software" "Software Logs" ON \ "ssh" "SSH Logs" ON \ "ssl" "SSL Logs" ON \ "syslog" "Syslog Logs" ON \ - "telnet" "Telnet Logs" ON \ "tunnel" "Tunnel Logs" ON \ "weird" "Zeek Weird Logs" ON \ "mysql" "MySQL Logs" ON \ diff --git a/salt/filebeat/etc/module_config.yml.jinja b/salt/filebeat/etc/module_config.yml.jinja index 7cd624895..8f4fbd7bf 100644 --- a/salt/filebeat/etc/module_config.yml.jinja +++ b/salt/filebeat/etc/module_config.yml.jinja @@ -3,7 +3,7 @@ - module: {{ module }} {%- for fileset in MODULES.modules[module] %} {{ fileset }}: - enabled: {{ MODULES.modules[module][fileset].enabled }} + enabled: {{ MODULES.modules[module][fileset].enabled|string|lower }} {#- only manage the settings if the fileset is enabled #} {%- if MODULES.modules[module][fileset].enabled %} {%- for var, value in MODULES.modules[module][fileset].items() %} diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index 0a1459d6b..f503e5de1 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -21,6 +21,8 @@ securityonion_filebeat: log: enabled: true var.paths: ["/logs/redis.log"] + slowlog: + enabled: false suricata: eve: enabled: true diff --git a/salt/filebeat/thirdpartydefaults.yaml b/salt/filebeat/thirdpartydefaults.yaml index cfb8d10d9..1b378f84b 100644 --- a/salt/filebeat/thirdpartydefaults.yaml +++ b/salt/filebeat/thirdpartydefaults.yaml @@ -199,12 +199,6 @@ third_party_filebeat: okta: system: enabled: false - pesando: - dfw: - enabled: false - var.input: udp - var.syslog_host: 0.0.0.0 - var.syslog_port: 9001 proofpoint: emailsecurity: enabled: false @@ -251,17 +245,6 @@ third_party_filebeat: var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9520 - threatintel: - abuseurl: - enabled: false - abusemalware: - enabled: false - misp: - enabled: false - otx: - enabled: false - anomali: - enabled: false tomcat: log: enabled: false diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 4e597f597..ce0d6dccd 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -183,6 +183,8 @@ so-zeek: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }} - start: {{ START }} - privileged: True + - ulimits: + - core=0 - binds: - /nsm/zeek/logs:/nsm/zeek/logs:rw - /nsm/zeek/spool:/nsm/zeek/spool:rw diff --git a/setup/so-functions b/setup/so-functions index 5ce3d6dee..aefaa2cd5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2827,7 +2827,6 @@ zeek_logs_enabled() { " - conn"\ " - dce_rpc"\ " - dhcp"\ - " - dhcpv6"\ " - dnp3"\ " - dns"\ " - dpd"\ @@ -2838,25 +2837,20 @@ zeek_logs_enabled() { " - irc"\ " - kerberos"\ " - modbus"\ - " - mqtt"\ " - notice"\ " - ntlm"\ - " - openvpn"\ " - pe"\ " - radius"\ " - rfb"\ " - rdp"\ - " - signatures"\ " - sip"\ " - smb_files"\ " - smb_mapping"\ " - smtp"\ " - snmp"\ - " - software"\ " - ssh"\ " - ssl"\ " - syslog"\ - " - telnet"\ " - tunnel"\ " - weird"\ " - mysql"\ diff --git a/setup/so-whiptail b/setup/so-whiptail index 5eca2d39a..fdcc4acc9 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1154,7 +1154,6 @@ whiptail_manager_adv_service_zeeklogs() { "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ - "dhcpv6" "DHCP IPv6 Logs" ON \ "dnp3" "DNP3 Logs" ON \ "dns" "DNS Logs" ON \ "dpd" "DPD Logs" ON \ @@ -1165,25 +1164,20 @@ whiptail_manager_adv_service_zeeklogs() { "irc" "IRC Chat Logs" ON \ "kerberos" "Kerberos Logs" ON \ "modbus" "MODBUS Logs" ON \ - "mqtt" "MQTT Logs" ON \ "notice" "Zeek Notice Logs" ON \ "ntlm" "NTLM Logs" ON \ - "openvpn" "OPENVPN Logs" ON \ "pe" "PE Logs" ON \ "radius" "Radius Logs" ON \ "rfb" "RFB Logs" ON \ "rdp" "RDP Logs" ON \ - "signatures" "Signatures Logs" ON \ "sip" "SIP Logs" ON \ "smb_files" "SMB Files Logs" ON \ "smb_mapping" "SMB Mapping Logs" ON \ "smtp" "SMTP Logs" ON \ "snmp" "SNMP Logs" ON \ - "software" "Software Logs" ON \ "ssh" "SSH Logs" ON \ "ssl" "SSL Logs" ON \ "syslog" "Syslog Logs" ON \ - "telnet" "Telnet Logs" ON \ "tunnel" "Tunnel Logs" ON \ "weird" "Zeek Weird Logs" ON \ "mysql" "MySQL Logs" ON \ From 842aa97f7e9dd32ea9974e2d0db8bd8e04a08397 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 May 2021 11:00:18 -0400 Subject: [PATCH 155/195] load filebeat modules when es container starts and if fb container is running --- salt/filebeat/init.sls | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 372656038..312e815c9 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -22,6 +22,13 @@ {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% from 'filebeat/map.jinja' import THIRDPARTY with context %} {% from 'filebeat/map.jinja' import SO with context %} +{% set ES_INCLUDED_NODES = ['so-standalone'] %} + +#only include elastic state for certain nodes +{% if grains.role in ES_INCLUDED_NODES %} +include: + - elasticsearch +{% endif %} filebeatetcdir: file.directory: @@ -126,6 +133,16 @@ so-filebeat: - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml +{% if grains.role in ES_INCLUDED_NODES %} +run_module_setup: + cmd.run: + - name: /usr/sbin/so-filebeat-module-setup + - require: + - docker_container: so-filebeat + - onchanges_in: + - docker_container: so-elasticsearch +{% endif %} + append_so-filebeat_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf From b525cfc787b6c662f861720e394ec2ca4d72c4b5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 May 2021 11:07:53 -0400 Subject: [PATCH 156/195] Remove old modules --- setup/so-functions | 6 ------ 1 file changed, 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index aefaa2cd5..bef4f384d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2862,7 +2862,6 @@ zeek_logs_enabled() { " - conn"\ " - dce_rpc"\ " - dhcp"\ - " - dhcpv6"\ " - dnp3"\ " - dns"\ " - dpd"\ @@ -2873,24 +2872,19 @@ zeek_logs_enabled() { " - irc"\ " - kerberos"\ " - modbus"\ - " - mqtt"\ " - notice"\ " - ntlm"\ - " - openvpn"\ " - pe"\ " - radius"\ " - rfb"\ " - rdp"\ - " - signatures"\ " - sip"\ " - smb_files"\ " - smb_mapping"\ " - smtp"\ " - snmp"\ - " - software"\ " - ssh"\ " - ssl"\ - " - telnet"\ " - tunnel"\ " - weird"\ " - mysql"\ From ee675546ac751615ac9fa0fac00b5540bba4aaa5 Mon Sep 17 00:00:00 2001 From: doug Date: Wed, 26 May 2021 14:09:00 -0400 Subject: [PATCH 157/195] add menu.actions.json and update soc.json --- salt/soc/files/soc/menu.actions.json | 33 ++++++++++++++++++++++++++++ salt/soc/files/soc/soc.json | 8 +++++-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 salt/soc/files/soc/menu.actions.json diff --git a/salt/soc/files/soc/menu.actions.json b/salt/soc/files/soc/menu.actions.json new file mode 100644 index 000000000..558d10a36 --- /dev/null +++ b/salt/soc/files/soc/menu.actions.json @@ -0,0 +1,33 @@ +[ + { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", + "links": [ + "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" + ]}, + { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", + "links": [ + "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", + "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\") | groupby event.module event.dataset", + "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", + "/#/hunt?q=(\"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", + "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset", + "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset", + "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset" + ]}, + { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", + "links": [ + "/joblookup?esid={:soc_id}", + "/joblookup?ncid={:network.community_id}" + ]}, + { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", + "links": [ + "/cyberchef/#input={value|base64}" + ]}, + { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", + "links": [ + "https://www.google.com/search?q={value}" + ]}, + { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", + "links": [ + "https://www.virustotal.com/gui/search/{value}" + ]} +] diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 5dfd364d2..a3326ab4a 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -17,6 +17,7 @@ {%- import_json "soc/files/soc/hunt.queries.json" as hunt_queries %} {%- import_json "soc/files/soc/hunt.actions.json" as hunt_actions %} {%- import_json "soc/files/soc/hunt.eventfields.json" as hunt_eventfields %} +{%- import_json "soc/files/soc/menu.actions.json" as menu_actions %} {%- import_json "soc/files/soc/tools.json" as tools %} {%- set DNET = salt['pillar.get']('global:dockernet', '172.17.0.0') %} @@ -123,8 +124,11 @@ "queryBaseFilter": "", "queryToggleFilters": [], "queries": {{ hunt_queries | json }}, - "actions": {{ hunt_actions | json }} + "actions": {{ menu_actions | json }} }, + "job": { + "actions": {{ menu_actions | json }} + }, "alerts": { "advanced": false, "groupItemsPerPage": 50, @@ -143,7 +147,7 @@ { "name": "escalated", "filter": "event.escalated:true", "enabled": false, "exclusive": true, "enablesToggles":["acknowledged"] } ], "queries": {{ alerts_queries | json }}, - "actions": {{ alerts_actions | json }} + "actions": {{ menu_actions | json }} } } } From bfcde15a24ac0c39d649c2150a3546c1476228f8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 May 2021 14:22:14 -0400 Subject: [PATCH 158/195] elastic pipeline test --- .../config/so/9050_output_elasticsearch.jinja | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja new file mode 100644 index 000000000..eb0d8ef0c --- /dev/null +++ b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja @@ -0,0 +1,20 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +output { + if [event][module] == 'elasticsearch' { + elasticsearch { + id => "elastic_logs" + pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} From aeea5701e490e69254b9fd6d1470a9e8db67da65 Mon Sep 17 00:00:00 2001 From: doug Date: Wed, 26 May 2021 16:34:05 -0400 Subject: [PATCH 159/195] completely disable both alerts.actions.json and hunt.actions.json --- salt/soc/files/soc/alerts.actions.json | 34 +------------------------- salt/soc/files/soc/hunt.actions.json | 34 +------------------------- salt/soc/files/soc/soc.json | 2 -- 3 files changed, 2 insertions(+), 68 deletions(-) diff --git a/salt/soc/files/soc/alerts.actions.json b/salt/soc/files/soc/alerts.actions.json index 558d10a36..1addf23c6 100644 --- a/salt/soc/files/soc/alerts.actions.json +++ b/salt/soc/files/soc/alerts.actions.json @@ -1,33 +1 @@ -[ - { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", - "links": [ - "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" - ]}, - { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", - "links": [ - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset", - "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset", - "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset" - ]}, - { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", - "links": [ - "/joblookup?esid={:soc_id}", - "/joblookup?ncid={:network.community_id}" - ]}, - { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", - "links": [ - "/cyberchef/#input={value|base64}" - ]}, - { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", - "links": [ - "https://www.google.com/search?q={value}" - ]}, - { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", - "links": [ - "https://www.virustotal.com/gui/search/{value}" - ]} -] +This file is no longer used. Please use menu.actions.json instead. diff --git a/salt/soc/files/soc/hunt.actions.json b/salt/soc/files/soc/hunt.actions.json index 558d10a36..1addf23c6 100644 --- a/salt/soc/files/soc/hunt.actions.json +++ b/salt/soc/files/soc/hunt.actions.json @@ -1,33 +1 @@ -[ - { "name": "actionHunt", "description": "actionHuntHelp", "icon": "fa-crosshairs", "target": "", - "links": [ - "/#/hunt?q=\"{value|escape}\" | groupby event.module event.dataset" - ]}, - { "name": "actionCorrelate", "description": "actionCorrelateHelp", "icon": "fab fa-searchengin", "target": "", - "links": [ - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:log.id.uid}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.fuid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=(\"{:log.id.uid}\" OR \"{:network.community_id}\") | groupby event.module event.dataset", - "/#/hunt?q=\"{:log.id.fuid}\" | groupby event.module event.dataset", - "/#/hunt?q=\"{:log.id.uid}\" | groupby event.module event.dataset", - "/#/hunt?q=\"{:network.community_id}\" | groupby event.module event.dataset" - ]}, - { "name": "actionPcap", "description": "actionPcapHelp", "icon": "fa-stream", "target": "", - "links": [ - "/joblookup?esid={:soc_id}", - "/joblookup?ncid={:network.community_id}" - ]}, - { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", - "links": [ - "/cyberchef/#input={value|base64}" - ]}, - { "name": "actionGoogle", "description": "actionGoogleHelp", "icon": "fab fa-google", "target": "_blank", - "links": [ - "https://www.google.com/search?q={value}" - ]}, - { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "fa-external-link-alt", "target": "_blank", - "links": [ - "https://www.virustotal.com/gui/search/{value}" - ]} -] +This file is no longer used. Please use menu.actions.json instead. diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index a3326ab4a..8048970f3 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -12,10 +12,8 @@ {%- set CACHE_EXPIRATION = salt['pillar.get']('sensoroni:cache_expiration_ms', 0) %} {%- set ES_FIELDCAPS_CACHE = salt['pillar.get']('sensoroni:es_fieldcaps_cache_ms', '300000') %} {%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %} -{%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %} {%- import_json "soc/files/soc/alerts.eventfields.json" as alerts_eventfields %} {%- import_json "soc/files/soc/hunt.queries.json" as hunt_queries %} -{%- import_json "soc/files/soc/hunt.actions.json" as hunt_actions %} {%- import_json "soc/files/soc/hunt.eventfields.json" as hunt_eventfields %} {%- import_json "soc/files/soc/menu.actions.json" as menu_actions %} {%- import_json "soc/files/soc/tools.json" as tools %} From b23ce7462ea8b256eb1ccc9d23e0c055733a23d8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 27 May 2021 11:26:25 -0400 Subject: [PATCH 160/195] add depenency --- salt/filebeat/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 312e815c9..5cabaf828 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -138,6 +138,7 @@ run_module_setup: cmd.run: - name: /usr/sbin/so-filebeat-module-setup - require: + - file: filebeatmoduleconfsync - docker_container: so-filebeat - onchanges_in: - docker_container: so-elasticsearch From ada8255af04aaf952c2c9a92f52507ce9ba5e757 Mon Sep 17 00:00:00 2001 From: doug Date: Fri, 28 May 2021 08:59:40 -0400 Subject: [PATCH 161/195] bump version to 7.13.0 --- salt/kibana/files/saved_objects.ndjson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/kibana/files/saved_objects.ndjson b/salt/kibana/files/saved_objects.ndjson index 863209d48..21ef5a94a 100644 --- a/salt/kibana/files/saved_objects.ndjson +++ b/salt/kibana/files/saved_objects.ndjson @@ -460,7 +460,7 @@ {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\",\"time_zone\":\"America/New_York\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":48,\"x\":0,\"y\":0,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":96,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":8,\"y\":20,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":28,\"y\":20,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":0,\"y\":72,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":48,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":24,\"x\":16,\"y\":72,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":28,\"y\":8,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":120,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"event_type\",\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"}]","timeRestore":false,"title":"z16.04 - Sysmon - Logs","version":1},"id":"6d189680-6d62-11e7-8ddb-e71eb260f4a3","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"8cfdeff0-6d6b-11e7-ad64-15aa071374a6","name":"panel_1","type":"visualization"},{"id":"0eb1fd80-6d70-11e7-b09b-f57b22df6524","name":"panel_2","type":"visualization"},{"id":"3072c750-6d71-11e7-b09b-f57b22df6524","name":"panel_3","type":"visualization"},{"id":"7bc74b40-6d71-11e7-b09b-f57b22df6524","name":"panel_4","type":"visualization"},{"id":"13ed0810-6d72-11e7-b09b-f57b22df6524","name":"panel_5","type":"visualization"},{"id":"3b6c92c0-6d72-11e7-b09b-f57b22df6524","name":"panel_6","type":"visualization"},{"id":"e09f6010-6d72-11e7-b09b-f57b22df6524","name":"panel_7","type":"visualization"},{"id":"29611940-6d75-11e7-b09b-f57b22df6524","name":"panel_8","type":"visualization"},{"id":"6b70b840-6d75-11e7-b09b-f57b22df6524","name":"panel_9","type":"visualization"},{"id":"248c1d20-6d6b-11e7-ad64-15aa071374a6","name":"panel_10","type":"search"},{"id":"AWDHHk1sxQT5EBNmq43Y","name":"panel_11","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQyLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} -{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.12.1","id":"7.12.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.13.0","id":"7.13.0","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":56,\"x\":0,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":8,\"y\":32,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":20,\"y\":32,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":56,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"columns\":[\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"uid\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":32,\"y\":32,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":40,\"h\":24,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"}]","timeRestore":false,"title":"z16.04 - Bro - Modbus","version":1},"id":"70c005f0-3583-11e7-a588-05992195c551","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"0d168a30-363f-11e7-a6f7-4f44d7bf1c33","name":"panel_1","type":"visualization"},{"id":"20eabd60-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_2","type":"visualization"},{"id":"3c65f500-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_3","type":"visualization"},{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"panel_4","type":"search"},{"id":"178209e0-6e1b-11e7-b553-7f80727663c1","name":"panel_5","type":"visualization"},{"id":"AWDG_9KpxQT5EBNmq4Oo","name":"panel_6","type":"visualization"},{"id":"453f8b90-4a58-11e8-9b0a-f1d33346f773","name":"panel_7","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ4LDRd"} @@ -730,4 +730,4 @@ {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - Syslog - Severity (Donut)","uiStateJSON":"{}","version":1,"visState":"{\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"syslog.severity.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":25,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Severity\"}}],\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"syslog.severity.keyword: Descending\",\"aggType\":\"terms\"}]}},\"title\":\"Security Onion - Syslog - Severity (Donut)\"}"},"id":"fc8d41a0-777b-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwNDExLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{}"},"savedSearchRefName":"search_0","title":"Security Onion - Connections - Top Source IPs","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - Connections - Top Source IPs\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"source.ip: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Source IP\"}}]}"},"id":"fd8b4640-6e9f-11ea-9266-1fd14ca6af34","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"9b333020-6e9f-11ea-9266-1fd14ca6af34","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwNDEyLDRd"} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"event.module:strelka\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":7,\"i\":\"a2e0a619-a5c5-40d9-8593-e60f13ae22bf\"},\"panelIndex\":\"a2e0a619-a5c5-40d9-8593-e60f13ae22bf\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":8,\"y\":0,\"w\":21,\"h\":7,\"i\":\"566a9d04-f2dc-4868-9625-97a19d985703\"},\"panelIndex\":\"566a9d04-f2dc-4868-9625-97a19d985703\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":29,\"y\":0,\"w\":19,\"h\":7,\"i\":\"f247ec64-c278-4e05-ac4d-983bea9dfb7d\"},\"panelIndex\":\"f247ec64-c278-4e05-ac4d-983bea9dfb7d\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":7,\"w\":12,\"h\":20,\"i\":\"6e80a142-ab0e-4fd3-891c-e495b78a1625\"},\"panelIndex\":\"6e80a142-ab0e-4fd3-891c-e495b78a1625\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":12,\"y\":7,\"w\":11,\"h\":20,\"i\":\"292cc879-6bc0-4541-ba92-3b3c5f4e3368\"},\"panelIndex\":\"292cc879-6bc0-4541-ba92-3b3c5f4e3368\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":23,\"y\":7,\"w\":14,\"h\":20,\"i\":\"66979b2c-e7c1-4291-91ac-16537b7f9ec3\"},\"panelIndex\":\"66979b2c-e7c1-4291-91ac-16537b7f9ec3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":37,\"y\":7,\"w\":11,\"h\":20,\"i\":\"8bb1cf98-0401-4a2d-9dd8-deca08205a22\"},\"panelIndex\":\"8bb1cf98-0401-4a2d-9dd8-deca08205a22\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":27,\"w\":8,\"h\":20,\"i\":\"393f3cec-3ee0-4275-b319-f307e7a260c6\"},\"panelIndex\":\"393f3cec-3ee0-4275-b319-f307e7a260c6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":8,\"y\":27,\"w\":15,\"h\":20,\"i\":\"0e8800a9-a6f5-4a79-8370-61713f584886\"},\"panelIndex\":\"0e8800a9-a6f5-4a79-8370-61713f584886\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":23,\"y\":27,\"w\":25,\"h\":20,\"i\":\"be9a0a2a-d8c6-4d15-b5d7-d5599d0482a3\"},\"panelIndex\":\"be9a0a2a-d8c6-4d15-b5d7-d5599d0482a3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.6.1\",\"gridData\":{\"x\":0,\"y\":47,\"w\":48,\"h\":27,\"i\":\"40296d2b-cb6f-423f-989c-3fdaa82d2aad\"},\"panelIndex\":\"40296d2b-cb6f-423f-989c-3fdaa82d2aad\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_10\"}]","timeRestore":false,"title":"Security Onion - Strelka","version":1},"id":"ff689c50-75f3-11ea-9565-7315f4ee5cac","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"8cfec8c0-6ec2-11ea-9266-1fd14ca6af34","name":"panel_0","type":"visualization"},{"id":"d04b5130-6e99-11ea-9266-1fd14ca6af34","name":"panel_1","type":"visualization"},{"id":"23ed13a0-6e9a-11ea-9266-1fd14ca6af34","name":"panel_2","type":"visualization"},{"id":"7a88adc0-75f0-11ea-9565-7315f4ee5cac","name":"panel_3","type":"visualization"},{"id":"49cfe850-772c-11ea-bee5-af7f7c7b8e05","name":"panel_4","type":"visualization"},{"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","name":"panel_5","type":"visualization"},{"id":"ce9e03f0-772c-11ea-bee5-af7f7c7b8e05","name":"panel_6","type":"visualization"},{"id":"a7ebb450-772c-11ea-bee5-af7f7c7b8e05","name":"panel_7","type":"visualization"},{"id":"08c0b770-772e-11ea-bee5-af7f7c7b8e05","name":"panel_8","type":"visualization"},{"id":"e087c7d0-772d-11ea-bee5-af7f7c7b8e05","name":"panel_9","type":"visualization"},{"id":"8b6f3150-72a2-11ea-8dd2-9d8795a1200b","name":"panel_10","type":"search"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwNDEzLDRd"} -{"exportedCount":732,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file +{"exportedCount":732,"missingRefCount":0,"missingReferences":[]} From ba011581ef0c1c2f35bd9925e4c1ddcfde1462c0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 28 May 2021 10:14:58 -0400 Subject: [PATCH 162/195] Add version to ending summary --- setup/so-whiptail | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-whiptail b/setup/so-whiptail index b7fc50495..d492d10cc 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -447,6 +447,7 @@ whiptail_end_settings() { # BASIC INFO (NETWORK, HOSTNAME, DESCRIPTION, ETC) read -r -d '' end_msg <<- EOM + Security Onion Version: $SOVERSION Node Type: $install_type Hostname: $HOSTNAME EOM From d55a9e6274ad381944504295af93a5905f0fbcfb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 28 May 2021 10:20:19 -0400 Subject: [PATCH 163/195] Add version to all whiptail titles --- setup/so-whiptail | 200 +++++++++++++++++++++++----------------------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index d492d10cc..09792459d 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -22,7 +22,7 @@ whiptail_airgap() { local node_str='node' [[ $is_manager || $is_import ]] && node_str='manager' - INTERWEBS=$(whiptail --title "Security Onion Setup" --menu \ + INTERWEBS=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu \ "How should this $node_str be installed?" 10 60 2 \ "Standard " "This $node_str has internet accesss" \ "Airgap " "This $node_str does not have internet access" 3>&1 1>&2 2>&3 ) @@ -42,7 +42,7 @@ whiptail_avoid_default_hostname() { You can choose to use this default hostname anyway, or change it to a new hostname. EOM - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --yesno "$message" 11 75 \ --yes-button "Use Anyway" --no-button "Change" --defaultno } @@ -51,7 +51,7 @@ whiptail_basic_suri() { [ -n "$TESTING" ] && return - BASICSURI=$(whiptail --title "Security Onion Setup" --inputbox \ + BASICSURI=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the number of Suricata processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -63,7 +63,7 @@ whiptail_basic_zeek() { [ -n "$TESTING" ] && return - BASICZEEK=$(whiptail --title "Security Onion Setup" --inputbox \ + BASICZEEK=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the number of Zeek processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -75,7 +75,7 @@ whiptail_bond_nics_mtu() { [ -n "$TESTING" ] && return # Set the MTU on the monitor interface - MTU=$(whiptail --title "Security Onion Setup" --inputbox \ + MTU=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the MTU for the monitor NICs:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -84,7 +84,7 @@ whiptail_bond_nics_mtu() { whiptail_cancel() { - whiptail --title "Security Onion Setup" --msgbox "Cancelling Setup." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Cancelling Setup." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -104,7 +104,7 @@ whiptail_check_exitstatus() { whiptail_cancel ;; 255) - whiptail --title "Security Onion Setup" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 exit ;; esac @@ -114,7 +114,7 @@ whiptail_components_adv_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 } whiptail_create_admin_user() { @@ -187,7 +187,7 @@ whiptail_create_soremote_user() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Set a password for the soremote user. This account is used for adding sensors remotely." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Set a password for the soremote user. This account is used for adding sensors remotely." 8 75 } @@ -258,11 +258,11 @@ whiptail_requirements_error() { [ -n "$TESTING" ] && return if [[ $(echo "$requirement_needed" | tr '[:upper:]' '[:lower:]') == 'nics' ]]; then - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --msgbox "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select OK to exit setup and reconfigure the machine." 10 75 # Same as whiptail_cancel, but changed the wording to exit instead of cancel. - whiptail --title "Security Onion Setup" --msgbox "Exiting Setup. No changes have been made." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Exiting Setup. No changes have been made." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -272,7 +272,7 @@ whiptail_requirements_error() { fi exit else - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select YES to continue anyway, or select NO to cancel." 10 75 local exitstatus=$? @@ -298,7 +298,7 @@ whiptail_storage_requirements() { EOM whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --yesno "$message" \ 14 75 @@ -310,7 +310,7 @@ whiptail_cur_close_days() { [ -n "$TESTING" ] && return - CURCLOSEDAYS=$(whiptail --title "Security Onion Setup" --inputbox \ + CURCLOSEDAYS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Please specify the threshold (in days) at which Elasticsearch indices will be closed:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -322,7 +322,7 @@ whiptail_dhcp_or_static() { [ -n "$TESTING" ] && return - address_type=$(whiptail --title "Security Onion Setup" --radiolist \ + address_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose how to set up your management interface:" 20 78 4 \ "STATIC" "Set a static IPv4 address" ON \ "DHCP" "Use DHCP to configure the Management Interface" OFF 3>&1 1>&2 2>&3 ) @@ -355,7 +355,7 @@ whiptail_dhcp_warn() { EOM whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --"$window_type" "$dhcp_message" \ 14 75 @@ -366,7 +366,7 @@ whiptail_dhcp_warn() { whiptail_dhcp_or_static ;; 255) - whiptail --title "Security Onion Setup" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 exit ;; esac @@ -380,7 +380,7 @@ whiptail_dockernet_check(){ [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno \ + whiptail --title "Security Onion Setup - $SOVERSION" --yesno \ "Do you want to keep the default Docker IP range?\n\nIf you are unsure, please accept the default option of Yes." 10 75 } @@ -389,7 +389,7 @@ whiptail_dockernet_net() { [ -n "$TESTING" ] && return - DOCKERNET=$(whiptail --title "Security Onion Setup" --inputbox \ + DOCKERNET=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "\nEnter a /24 size network range for docker to use WITHOUT the /24 suffix. This range will be used on ALL nodes." 11 65 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -409,7 +409,7 @@ whiptail_enable_components() { STRELKA=0 if [[ $is_eval ]]; then - COMPONENTS=$(whiptail --title "Security Onion Setup" --checklist \ + COMPONENTS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ "Select Components to install:" 20 75 8 \ GRAFANA "Enable Grafana for system monitoring" ON \ OSQUERY "Enable Fleet with osquery" ON \ @@ -418,7 +418,7 @@ if [[ $is_eval ]]; then PLAYBOOK "Enable Playbook" ON \ STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3) else - COMPONENTS=$(whiptail --title "Security Onion Setup" --checklist \ + COMPONENTS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ "Select Components to install:" 20 75 7 \ OSQUERY "Enable Fleet with osquery" ON \ WAZUH "Enable Wazuh" ON \ @@ -658,7 +658,7 @@ whiptail_eval_adv() { [ -n "$TESTING" ] && return - EVALADVANCED=$(whiptail --title "Security Onion Setup" --radiolist \ + EVALADVANCED=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose your eval install:" 20 75 4 \ "BASIC" "Install basic components for evaluation" ON \ "ADVANCED" "Choose additional components to be installed" OFF 3>&1 1>&2 2>&3 ) @@ -685,7 +685,7 @@ whiptail_gauge_post_setup() { else local msg=$1 - whiptail --title "Security Onion Setup" --gauge "$msg" 6 60 96 + whiptail --title "Security Onion Setup - $SOVERSION" --gauge "$msg" 6 60 96 fi } @@ -693,7 +693,7 @@ whiptail_helix_apikey() { [ -n "$TESTING" ] && return - HELIXAPIKEY=$(whiptail --title "Security Onion Setup" --inputbox \ + HELIXAPIKEY=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your Helix API Key: \n \nThis can be set later using so-helix-apikey" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -707,7 +707,7 @@ whiptail_homenet_manager() { [ -n "$TESTING" ] && return - HNMANAGER=$(whiptail --title "Security Onion Setup" --inputbox \ + HNMANAGER=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -719,13 +719,13 @@ whiptail_homenet_sensor_inherit() { [ -n "$TESTING" ] && return # Ask to inherit from manager - whiptail --title "Security Onion Setup" --yesno "Do you want to inherit the HOME_NET from the Manager?" 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to inherit the HOME_NET from the Manager?" 8 75 } whiptail_homenet_sensor() { [ -n "$TESTING" ] && return - HNSENSOR=$(whiptail --title "Security Onion Setup" --inputbox \ + HNSENSOR=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -738,7 +738,7 @@ whiptail_install_type() { [ -n "$TESTING" ] && return # What kind of install are we doing? - install_type=$(whiptail --title "Security Onion Setup" --radiolist \ + install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose install type:" 12 65 5 \ "EVAL" "Evaluation mode (not for production) " ON \ "STANDALONE" "Standalone production install " OFF \ @@ -764,7 +764,7 @@ whiptail_install_type_dist() { [ -n "$TESTING" ] && return - install_type=$(whiptail --title "Security Onion Setup" --radiolist \ + install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose distributed node type:" 13 60 6 \ "MANAGER" "Start a new grid " ON \ "SENSOR" "Create a forward only sensor " OFF \ @@ -792,14 +792,14 @@ whiptail_install_type_other() { # so-analyst-install will only work with a working network connection # so only show it on network installs for now if [[ $setup_type == 'network' ]]; then - install_type=$(whiptail --title "Security Onion Setup" --radiolist \ + install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose distributed node type:" 9 65 2 \ "ANALYST" "Quit setup and run so-analyst-install " ON \ "HELIXSENSOR" "Create a Helix sensor " OFF \ 3>&1 1>&2 2>&3 ) else - install_type=$(whiptail --title "Security Onion Setup" --radiolist \ + install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose distributed node type:" 8 65 1 \ "HELIXSENSOR" "Create a Helix sensor " ON \ 3>&1 1>&2 2>&3 @@ -815,7 +815,7 @@ whiptail_install_type_other() { whiptail_invalid_input() { # TODO: This should accept a list of arguments to specify what general pattern the input should follow [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox " Invalid input, please try again." 7 40 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox " Invalid input, please try again." 7 40 } @@ -829,13 +829,13 @@ whiptail_invalid_proxy() { Error was: ${proxy_test_err} EOM - whiptail --title "Security Onion Setup" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60 } whiptail_invalid_string() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 } @@ -843,21 +843,21 @@ whiptail_invalid_pass_characters_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 } whiptail_invalid_pass_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Please choose a more secure password." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please choose a more secure password." 8 75 } whiptail_invalid_user_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Please enter a valid email address." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please enter a valid email address." 8 75 } whiptail_invalid_hostname() { @@ -868,7 +868,7 @@ whiptail_invalid_hostname() { the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', \ and hyphen ('-')" | tr -d '\t') - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --msgbox "$error_message" 10 75 } @@ -891,7 +891,7 @@ whiptail_log_size_limit() { By default, this is set to ${percentage}% of the disk space allotted for /nsm. EOM - log_size_limit=$(whiptail --title "Security Onion Setup" --inputbox "$message" 11 75 "$1" 3>&1 1>&2 2>&3) + log_size_limit=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "$message" 11 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -901,7 +901,7 @@ whiptail_log_size_limit() { whiptail_first_menu_iso() { [ -n "$TESTING" ] && return - option=$(whiptail --title "Security Onion Setup" --menu "Select an option" 10 75 2 \ + option=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "Select an option" 10 75 2 \ "Install " "Run the standard Security Onion installation " \ "Configure Network " "Configure networking only " \ 3>&1 1>&2 2>&3 @@ -915,7 +915,7 @@ whiptail_make_changes() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -926,7 +926,7 @@ whiptail_management_interface_dns() { [ -n "$TESTING" ] && return - MDNS=$(whiptail --title "Security Onion Setup" --inputbox \ + MDNS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your DNS servers separated by commas:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -938,7 +938,7 @@ whiptail_management_interface_dns_search() { [ -n "$TESTING" ] && return - MSEARCH=$(whiptail --title "Security Onion Setup" --inputbox \ + MSEARCH=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your DNS search domain:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -949,7 +949,7 @@ whiptail_management_interface_gateway() { [ -n "$TESTING" ] && return - MGATEWAY=$(whiptail --title "Security Onion Setup" --inputbox \ + MGATEWAY=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your gateway's IPv4 address:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -959,7 +959,7 @@ whiptail_management_interface_gateway() { whiptail_management_interface_ip_mask() { [ -n "$TESTING" ] && return - manager_ip_mask=$(whiptail --title "Security Onion Setup" --inputbox \ + manager_ip_mask=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your IPv4 address with CIDR mask (e.g. 192.168.1.2/24):" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -970,7 +970,7 @@ whiptail_management_interface_ip() { [ -n "$TESTING" ] && return - MIP=$(whiptail --title "Security Onion Setup" --inputbox \ + MIP=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your IP address:" 10 60 X.X.X.X 3>&1 1>&2 2>&3) local exitstatus=$? @@ -981,7 +981,7 @@ whiptail_management_interface_mask() { [ -n "$TESTING" ] && return - MMASK=$(whiptail --title "Security Onion Setup" --inputbox \ + MMASK=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the bit mask for your subnet:" 10 60 24 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1060,7 +1060,7 @@ whiptail_net_method() { local option_count=$(( ${#options[@]} / 2 )) - network_traffic=$(whiptail --title "Security Onion Setup" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) + network_traffic=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1071,7 +1071,7 @@ whiptail_net_method() { whiptail_net_setup_complete() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --msgbox "Successfully set up networking, setup will now exit." 7 75 exit 0 } @@ -1085,7 +1085,7 @@ whiptail_network_init_notice() { Select OK to continue. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" 9 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 9 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } @@ -1094,7 +1094,7 @@ whiptail_management_server() { [ -n "$TESTING" ] && return - MSRV=$(whiptail --title "Security Onion Setup" --inputbox \ + MSRV=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your Manager Server hostname: \nIt is CASE SENSITIVE!" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1105,7 +1105,7 @@ whiptail_management_server() { whiptail_manager_ip() { [ -n "$TESTING" ] && return - MSRVIP=$(whiptail --title "Security Onion Setup" --inputbox \ + MSRVIP=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your Manager Server IP Address:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1117,7 +1117,7 @@ whiptail_manager_adv() { [ -n "$TESTING" ] && return - MANAGERADV=$(whiptail --title "Security Onion Setup" --radiolist \ + MANAGERADV=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose which type of manager to install:" 20 75 4 \ "BASIC" "Install manager with recommended settings" ON \ "ADVANCED" "Do additional configuration to the manager" OFF 3>&1 1>&2 2>&3 ) @@ -1132,7 +1132,7 @@ whiptail_manager_adv_escluster(){ [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno \ + whiptail --title "Security Onion Setup - $SOVERSION" --yesno \ "Do you want to set up a traditional ES cluster for using replicas and/or Hot-Warm indices? Recommended only for those who have experience with ES clustering! " 12 75 } @@ -1142,7 +1142,7 @@ whiptail_manager_adv_escluster_name(){ [ -n "$TESTING" ] && return - ESCLUSTERNAME=$(whiptail --title "Security Onion Setup" --inputbox \ + ESCLUSTERNAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter a name for your ES cluster!" 10 75 "$1" 3>&1 1>&2 2>&3) } @@ -1151,7 +1151,7 @@ whiptail_manager_adv_service_zeeklogs() { [ -n "$TESTING" ] && return - BLOGS=$(whiptail --title "Security Onion Setup" --checklist "Please select logs to send:" 24 75 12 \ + BLOGS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist "Please select logs to send:" 24 75 12 \ "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ @@ -1213,13 +1213,13 @@ whiptail_manager_error() { Would you like to continue anyway? EOM - whiptail --title "Security Onion Setup" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 } whiptail_manager_updates_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup"\ + whiptail --title "Security Onion Setup - $SOVERSION"\ --msgbox "Updating through the manager node requires the manager to have internet access, press ENTER to continue."\ 8 75 @@ -1230,7 +1230,7 @@ whiptail_manager_updates_warning() { whiptail_manager_unreachable() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Setup cannot determine if $1 is listening on port 22. Please check the address entered and try again." 7 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Setup cannot determine if $1 is listening on port 22. Please check the address entered and try again." 7 75 } whiptail_metadata_tool() { @@ -1248,7 +1248,7 @@ whiptail_metadata_tool() { EOM # Legacy variable naming - ZEEKVERSION=$(whiptail --title "Security Onion Setup" --menu "$message" 20 75 2 \ + ZEEKVERSION=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "$message" 20 75 2 \ "Zeek " "Use Zeek (Bro) for metadata and Suricata for NIDS alerts" \ "Suricata " "Use Suricata for both metadata and NIDS alerts" 3>&1 1>&2 2>&3) @@ -1262,7 +1262,7 @@ whiptail_nids() { [ -n "$TESTING" ] && return - NIDS=$(whiptail --title "Security Onion Setup" --radiolist \ + NIDS=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose which IDS to run: \n\n(Snort 3.0 support will be added once it is out of beta.)" 25 75 4 \ "Suricata" "Suricata" ON \ "Snort" "Placeholder for Snort 3.0 " OFF 3>&1 1>&2 2>&3 ) @@ -1276,7 +1276,7 @@ whiptail_network_notice() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1286,14 +1286,14 @@ whiptail_network_notice() { whiptail_net_reinit() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "The management interface has already been configured. Do you want to reconfigure it?" 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "The management interface has already been configured. Do you want to reconfigure it?" 8 75 } whiptail_node_advanced() { [ -n "$TESTING" ] && return - NODESETUP=$(whiptail --title "Security Onion Setup" --radiolist \ + NODESETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "What type of config would you like to use?:" 20 75 4 \ "NODEBASIC" "Install Search Node with recommended settings" ON \ "NODEADVANCED" "Advanced Node Setup" OFF 3>&1 1>&2 2>&3 ) @@ -1306,7 +1306,7 @@ whiptail_node_advanced() { whiptail_node_description() { [ -n "$TESTING" ] && return - NODE_DESCRIPTION=$(whiptail --title "Security Onion Setup" \ + NODE_DESCRIPTION=$(whiptail --title "Security Onion Setup - $SOVERSION" \ --inputbox "Enter a short description for the node or press ENTER to leave blank:" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1317,7 +1317,7 @@ whiptail_node_es_heap() { [ -n "$TESTING" ] && return - NODE_ES_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ + NODE_ES_HEAP_SIZE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter ES heap size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1329,7 +1329,7 @@ whiptail_node_ls_heap() { [ -n "$TESTING" ] && return - NODE_LS_HEAP_SIZE=$(whiptail --title "Security Onion Setup" --inputbox \ + NODE_LS_HEAP_SIZE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter Logstash heap size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1341,7 +1341,7 @@ whiptail_node_ls_input_threads() { [ -n "$TESTING" ] && return - LSINPUTTHREADS=$(whiptail --title "Security Onion Setup" --inputbox \ + LSINPUTTHREADS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter number of Logstash input threads:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1354,7 +1354,7 @@ whiptail_node_ls_pipline_batchsize() { [ -n "$TESTING" ] && return - LSPIPELINEBATCH=$(whiptail --title "Security Onion Setup" --inputbox \ + LSPIPELINEBATCH=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter Logstash pipeline batch size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1366,7 +1366,7 @@ whiptail_node_ls_pipeline_worker() { [ -n "$TESTING" ] && return - LSPIPELINEWORKERS=$(whiptail --title "Security Onion Setup" --inputbox \ + LSPIPELINEWORKERS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter number of Logstash pipeline workers:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1377,13 +1377,13 @@ whiptail_node_ls_pipeline_worker() { whiptail_ntp_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Would you like to configure ntp servers?" 7 44 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Would you like to configure ntp servers?" 7 44 } whiptail_ntp_servers() { [ -n "$TESTING" ] && return - ntp_string=$(whiptail --title "Security Onion Setup" \ + ntp_string=$(whiptail --title "Security Onion Setup - $SOVERSION" \ --inputbox "Input the NTP server(s) you would like to use, separated by commas:" 8 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1394,7 +1394,7 @@ whiptail_oinkcode() { [ -n "$TESTING" ] && return - OINKCODE=$(whiptail --title "Security Onion Setup" --inputbox \ + OINKCODE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter your ET Pro or oinkcode:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1407,7 +1407,7 @@ whiptail_oinkcode() { whiptail_passwords_dont_match() { - whiptail --title "Security Onion Setup" --msgbox "Passwords don't match. Please re-enter." 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Passwords don't match. Please re-enter." 8 75 } @@ -1415,7 +1415,7 @@ whiptail_patch_name_new_schedule() { [ -n "$TESTING" ] && return - PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup" --inputbox \ + PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "What name do you want to give this OS patch schedule? This schedule needs to be named uniquely. Available schedules can be found on the manager under /opt/so/salt/patch/os/schedules/.yml" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1426,7 +1426,7 @@ whiptail_patch_schedule() { [ -n "$TESTING" ] && return - patch_schedule=$(whiptail --title "Security Onion Setup" --radiolist \ + patch_schedule=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Choose OS patch schedule.\n\nThis schedule will update the operating system packages but will NOT update Security Onion related tools such as Zeek, Elasticsearch, Kibana, SaltStack, etc." 20 75 5 \ "Automatic" "Updates installed every 8 hours if available" ON \ "Manual" "Updates will be installed manually" OFF \ @@ -1442,7 +1442,7 @@ whiptail_patch_schedule_import() { [ -n "$TESTING" ] && return unset PATCHSCHEDULENAME - PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup" --inputbox \ + PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the name of the OS patch schedule you want to inherit. \nAvailable schedules can be found on the manager under /opt/so/salt/patch/os/schedules/.yml" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1454,7 +1454,7 @@ whiptail_patch_schedule_select_days() { [ -n "$TESTING" ] && return # Select the days to patch - PATCHSCHEDULEDAYS=$(whiptail --title "Security Onion Setup" --checklist \ + PATCHSCHEDULEDAYS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ "Which days do you want to apply OS patches?" 15 75 8 \ Monday "" OFF \ Tuesday "" ON \ @@ -1478,7 +1478,7 @@ whiptail_patch_schedule_select_hours() { [ -n "$TESTING" ] && return # Select the hours to patch - PATCHSCHEDULEHOURS=$(whiptail --title "Security Onion Setup" --checklist \ + PATCHSCHEDULEHOURS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ "At which time, UTC, do you want to apply OS patches on the selected days?" 22 75 13 \ 00:00 "" OFF \ 01:00 "" OFF \ @@ -1519,7 +1519,7 @@ whiptail_proxy_ask() { local pkg_mngr if [[ $OS = 'centos' ]]; then pkg_mngr="yum"; else pkg_mngr='apt'; fi - whiptail --title "Security Onion Setup" --yesno "Do you want to proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment?" 9 65 --defaultno + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment?" 9 65 --defaultno } whiptail_proxy_addr() { @@ -1532,7 +1532,7 @@ whiptail_proxy_addr() { If your proxy requires a username and password do not include them in your input. Setup will ask for those values next. EOM - proxy_addr=$(whiptail --title "Security Onion Setup" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) + proxy_addr=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1541,13 +1541,13 @@ whiptail_proxy_addr() { whiptail_proxy_auth_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Does your proxy require authentication?" 7 60 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Does your proxy require authentication?" 7 60 } whiptail_proxy_auth_user() { [ -n "$TESTING" ] && return - proxy_user=$(whiptail --title "Security Onion Setup" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) + proxy_user=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1558,7 +1558,7 @@ whiptail_proxy_auth_pass() { [ -n "$TESTING" ] && return - proxy_pass=$(whiptail --title "Security Onion Setup" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) + proxy_pass=$(whiptail --title "Security Onion Setup - $SOVERSION" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1572,7 +1572,7 @@ whiptail_requirements_error() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 10 75 local exitstatus=$? @@ -1584,7 +1584,7 @@ whiptail_rule_setup() { [ -n "$TESTING" ] && return # Get pulled pork info - RULESETUP=$(whiptail --title "Security Onion Setup" --radiolist \ + RULESETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "Which IDS ruleset would you like to use?\n\nThis manager server is responsible for downloading the IDS ruleset from the Internet.\n\nSensors then pull a copy of this ruleset from the manager server.\n\nIf you select a commercial ruleset, it is your responsibility to purchase enough licenses for all of your sensors in compliance with your vendor's policies." 20 75 4 \ "ETOPEN" "Emerging Threats Open" ON \ "ETPRO" "Emerging Threats PRO" OFF \ @@ -1600,7 +1600,7 @@ whiptail_sensor_config() { [ -n "$TESTING" ] && return - NSMSETUP=$(whiptail --title "Security Onion Setup" --radiolist \ + NSMSETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "What type of configuration would you like to use?" 20 75 4 \ "BASIC" "Install NSM components with recommended settings" ON \ "ADVANCED" "Configure each component individually" OFF 3>&1 1>&2 2>&3 ) @@ -1642,7 +1642,7 @@ whiptail_sensor_nics() { for bond_nic in "${BNICS[@]}"; do if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ 8 75 exit @@ -1654,7 +1654,7 @@ whiptail_set_hostname() { [ -n "$TESTING" ] && return - HOSTNAME=$(whiptail --title "Security Onion Setup" --inputbox \ + HOSTNAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the hostname (not FQDN) you would like to set:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1665,7 +1665,7 @@ whiptail_set_redirect() { [ -n "$TESTING" ] && return - REDIRECTINFO=$(whiptail --title "Security Onion Setup" --radiolist \ + REDIRECTINFO=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "How would you like to access the web interface?\n\nSecurity Onion uses strict cookie enforcement, so whatever you choose here will be the only way that you can access the web interface.\n\nIf you choose something other than IP address, then you'll need to ensure that you can resolve the name via DNS or hosts entry. If you are unsure, please select IP." 20 75 4 \ "IP" "Use IP address to access the web interface" ON \ "HOSTNAME" "Use hostname to access the web interface" OFF \ @@ -1678,7 +1678,7 @@ whiptail_set_redirect_host() { [ -n "$TESTING" ] && return - REDIRECTHOST=$(whiptail --title "Security Onion Setup" --inputbox \ + REDIRECTHOST=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ "Enter the Hostname, IP, or FQDN you would like to use for the web interface:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1706,7 +1706,7 @@ whiptail_setup_complete() { Press ENTER to reboot. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" 12 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 12 75 } whiptail_setup_failed() { @@ -1726,13 +1726,13 @@ whiptail_setup_failed() { Press Ok to exit. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" $height 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" $height 75 } whiptail_so_allow_yesno() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" \ + whiptail --title "Security Onion Setup - $SOVERSION" \ --yesno "Do you want to run so-allow to allow access to the web tools?" \ 8 75 } @@ -1741,7 +1741,7 @@ whiptail_so_allow() { [ -n "$TESTING" ] && return - ALLOW_CIDR=$(whiptail --title "Security Onion Setup" \ + ALLOW_CIDR=$(whiptail --title "Security Onion Setup - $SOVERSION" \ --inputbox "Enter a single IP address or an IP range, in CIDR notation, to allow:" \ 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1761,7 +1761,7 @@ whiptail_ssh_key_copy_notice() { Select OK to continue. EOM - whiptail --title "Security Onion Setup" --msgbox "$message" 11 75 + whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 11 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } @@ -1800,7 +1800,7 @@ whiptail_storage_requirements() { EOM whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --yesno "$message" \ 14 75 @@ -1812,7 +1812,7 @@ whiptail_strelka_rules() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Do you want to enable the default YARA rules for Strelka?" 8 75 + whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to enable the default YARA rules for Strelka?" 8 75 local exitstatus=$? @@ -1852,7 +1852,7 @@ whiptail_node_updates() { [ -n "$TESTING" ] && return - NODEUPDATES=$(whiptail --title "Security Onion Setup" --radiolist \ + NODEUPDATES=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ "How would you like to download OS package updates for your grid?" 20 75 4 \ "MANAGER" "Manager node is proxy for updates." ON \ "OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) @@ -1877,7 +1877,7 @@ whiptail_you_sure() { EOM whiptail \ - --title "Security Onion Setup" \ + --title "Security Onion Setup - $SOVERSION" \ --yesno "$you_sure_text" \ 20 75 From 73a0b313805fa116e6ad68d9195bb01d58916215 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Jun 2021 12:12:20 -0400 Subject: [PATCH 164/195] elastic pipeline enable --- salt/pipeline/load.sls | 4 ++++ salt/top.sls | 1 + 2 files changed, 5 insertions(+) create mode 100644 salt/pipeline/load.sls diff --git a/salt/pipeline/load.sls b/salt/pipeline/load.sls new file mode 100644 index 000000000..a43450d0a --- /dev/null +++ b/salt/pipeline/load.sls @@ -0,0 +1,4 @@ +load_elastic_pipelines: + cmd.run: + - name: /usr/sbin/so-filebeat-module-setup + \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index 8a12aaa26..340f83825 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -285,6 +285,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_searchnode and G@saltversion:{{saltversion}}': - match: compound From fd1de624c815071afb9d25d078ad0452c8bfbb51 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Jun 2021 14:50:21 -0400 Subject: [PATCH 165/195] Disable TTY for filebeat script --- salt/common/tools/sbin/so-filebeat-module-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 5aefe3ac2..21d94b44f 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -53,7 +53,7 @@ echo "Setting up ingest pipeline(s)" for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler do echo "Loading $MODULE" - docker exec -it so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML + docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML sleep 2 done From 5983eae3a8f7907d2bae84cd44dec295faea443f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Jun 2021 17:47:13 -0400 Subject: [PATCH 166/195] fix filebeat module syntax --- .../tools/sbin/so-filebeat-module-setup | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 21d94b44f..d7706366e 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -47,14 +47,21 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'" echo fi +echo "Testing to see if the pipelines are already applied" +ESVER=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \") +PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) -echo "Setting up ingest pipeline(s)" +if [[ "$PIPELINES" -lt 5 ]]; then + echo "Setting up ingest pipeline(s)" -for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler -do - echo "Loading $MODULE" - docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML - sleep 2 -done + for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler + do + echo "Loading $MODULE" + docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML + sleep 2 + done +else + exit 0 +fi From c757d21360ce261fc77e18b7249ff7d6076fbd24 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 09:38:59 -0400 Subject: [PATCH 167/195] Increase default SOC API and ES timeouts from 2m to 5m --- salt/nginx/etc/nginx.conf | 12 ++++++------ salt/soc/files/soc/soc.json | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index ea820442b..7a238fa54 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -175,8 +175,8 @@ http { auth_request_set $userid $upstream_http_x_kratos_authenticated_identity_id; proxy_set_header x-user-id $userid; proxy_pass http://{{ manager_ip }}:9822/; - proxy_read_timeout 90; - proxy_connect_timeout 90; + proxy_read_timeout 300; + proxy_connect_timeout 300; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -261,8 +261,8 @@ http { auth_request /auth/sessions/whoami; rewrite /kibana/(.*) /$1 break; proxy_pass http://{{ manager_ip }}:5601/; - proxy_read_timeout 90; - proxy_connect_timeout 90; + proxy_read_timeout 300; + proxy_connect_timeout 300; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -341,8 +341,8 @@ http { location /soctopus/ { proxy_pass http://{{ manager_ip }}:7000/; - proxy_read_timeout 90; - proxy_connect_timeout 90; + proxy_read_timeout 300; + proxy_connect_timeout 300; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 8048970f3..e275ec28b 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -50,7 +50,8 @@ "username": "", "password": "", "cacheMs": {{ ES_FIELDCAPS_CACHE }}, - "verifyCert": false + "verifyCert": false, + "timeoutMs": {{ API_TIMEOUT }} }, "influxdb": { {%- if grains['role'] in ['so-import'] or (grains['role'] == 'so-eval' and GRAFANA == 0) %} From e00fe0a732a111d89bf8eae4a243ebb25ae7f251 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 10:02:11 -0400 Subject: [PATCH 168/195] Enable for all modes --- salt/elasticsearch/files/elasticsearch.yml | 2 +- salt/top.sls | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index fed45bf79..a1d4c836b 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -36,7 +36,7 @@ xpack.security.authc: roles: superuser authz_exception: true node.name: {{ grains.host }} -script.max_compilations_rate: 1000/1m +script.max_compilations_rate: 2000/1m {%- if TRUECLUSTER is sameas true %} {%- if grains.role == 'so-manager' %} {%- if salt['pillar.get']('nodestab', {}) %} diff --git a/salt/top.sls b/salt/top.sls index 340f83825..99388fdcd 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -153,6 +153,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_manager and G@saltversion:{{saltversion}}': - match: compound @@ -213,6 +214,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_standalone and G@saltversion:{{saltversion}}': - match: compound @@ -314,6 +316,7 @@ base: {%- endif %} - schedule - docker_clean + - pipeline.load '*_managersearch and G@saltversion:{{saltversion}}': - match: compound @@ -378,6 +381,7 @@ base: - domainstats {%- endif %} - docker_clean + - pipeline.load '*_heavynode and G@saltversion:{{saltversion}}': - match: compound @@ -420,6 +424,7 @@ base: {%- endif %} - schedule - docker_clean + - pipeline.load '*_fleet and G@saltversion:{{saltversion}}': - match: compound @@ -463,3 +468,4 @@ base: - zeek - schedule - docker_clean + - pipeline.load From afbf7de9e38238baf4f275dfb60c9375a633c2de Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 2 Jun 2021 11:05:43 -0400 Subject: [PATCH 169/195] Remove empty lines in iso location prompt --- salt/common/tools/sbin/soup | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index f7fa4e211..db0d26989 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -109,9 +109,7 @@ airgap_mounted() { echo "" cat << EOF In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. - For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. - Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom. EOF From b3f2c60065a6fe80c15cd1af27bc3da05d378d1f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 2 Jun 2021 12:38:32 -0400 Subject: [PATCH 170/195] Whiptail title fixes - Use a variable for the title - Fix cases where the whiptail title wasn't changed previously --- salt/common/tools/sbin/so-common | 18 +-- salt/common/tools/sbin/so-zeek-logs | 6 +- salt/common/tools/sbin/soup | 1 + setup/so-functions | 5 +- setup/so-setup | 3 +- setup/so-variables | 3 + setup/so-whiptail | 200 ++++++++++++++-------------- 7 files changed, 121 insertions(+), 115 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index ec0ad390a..b770a5551 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -153,16 +153,16 @@ Do you agree to the terms of the Elastic License? If so, type AGREE to accept the Elastic License and continue. Otherwise, press Enter to exit this program without making any changes. EOM -AGREED=$(whiptail --title "Security Onion Setup" --inputbox \ -"$message" 20 75 3>&1 1>&2 2>&3) + AGREED=$(whiptail --title "$whiptail_title" --inputbox \ + "$message" 20 75 3>&1 1>&2 2>&3) -if [ "${AGREED^^}" = 'AGREE' ]; then - mkdir -p /opt/so/state - touch /opt/so/state/yeselastic.txt -else - echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion." - exit 1 -fi + if [ "${AGREED^^}" = 'AGREE' ]; then + mkdir -p /opt/so/state + touch /opt/so/state/yeselastic.txt + else + echo "Starting in 2.3.40 you must accept the Elastic license if you want to run Security Onion." + exit 1 + fi } diff --git a/salt/common/tools/sbin/so-zeek-logs b/salt/common/tools/sbin/so-zeek-logs index 551213580..788312df5 100755 --- a/salt/common/tools/sbin/so-zeek-logs +++ b/salt/common/tools/sbin/so-zeek-logs @@ -10,7 +10,7 @@ zeek_logs_enabled() { } whiptail_manager_adv_service_zeeklogs() { - BLOGS=$(whiptail --title "Security Onion Setup" --checklist "Please Select Logs to Send:" 24 78 12 \ + BLOGS=$(whiptail --title "so-zeek-logs" --checklist "Please Select Logs to Send:" 24 78 12 \ "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ @@ -61,10 +61,10 @@ whiptail_manager_adv_service_zeeklogs return_code=$? case $return_code in 1) - whiptail --title "Security Onion Setup" --msgbox "Cancelling. No changes have been made." 8 75 + whiptail --title "so-zeek-logs" --msgbox "Cancelling. No changes have been made." 8 75 ;; 255) - whiptail --title "Security Onion Setup" --msgbox "Whiptail error occured, exiting." 8 75 + whiptail --title "so-zeek-logs" --msgbox "Whiptail error occured, exiting." 8 75 ;; *) zeek_logs_enabled diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index d1261fe30..198761428 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -25,6 +25,7 @@ BATCHSIZE=5 SOUP_LOG=/root/soup.log INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup +whiptail_title='Security Onion UPdater' check_err() { local exit_code=$1 diff --git a/setup/so-functions b/setup/so-functions index 5ce3d6dee..19e0c804f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -896,7 +896,10 @@ compare_main_nic_ip() { This is not a supported configuration, please remediate and rerun setup. EOM - [[ -n $TESTING ]] || whiptail --title "Security Onion Setup" --msgbox "$message" 10 75 + + [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]] || local whiptail_title="so-monitor-add" + + [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 10 75 kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 fi else diff --git a/setup/so-setup b/setup/so-setup index 4c364ded6..8760a39de 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -108,11 +108,10 @@ catch() { automated=no progress() { - local title='Security Onion Setup' local msg=${1:-'Please wait while installing...'} if [ $automated == no ]; then - whiptail --title "$title" --gauge "$msg" 6 70 0 # append to text + whiptail --title "$whiptail_title" --gauge "$msg" 6 70 0 # append to text else cat >> $setup_log 2>&1 fi diff --git a/setup/so-variables b/setup/so-variables index 676cba4f0..a69ef9e1b 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -75,3 +75,6 @@ export net_init_file ntp_string="0.pool.ntp.org,1.pool.ntp.org" export ntp_string + +whiptail_title="Security Onion Setup - $SOVERSION" +export whiptail_title diff --git a/setup/so-whiptail b/setup/so-whiptail index 09792459d..458713591 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -22,7 +22,7 @@ whiptail_airgap() { local node_str='node' [[ $is_manager || $is_import ]] && node_str='manager' - INTERWEBS=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu \ + INTERWEBS=$(whiptail --title "$whiptail_title" --menu \ "How should this $node_str be installed?" 10 60 2 \ "Standard " "This $node_str has internet accesss" \ "Airgap " "This $node_str does not have internet access" 3>&1 1>&2 2>&3 ) @@ -42,7 +42,7 @@ whiptail_avoid_default_hostname() { You can choose to use this default hostname anyway, or change it to a new hostname. EOM - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --yesno "$message" 11 75 \ --yes-button "Use Anyway" --no-button "Change" --defaultno } @@ -51,7 +51,7 @@ whiptail_basic_suri() { [ -n "$TESTING" ] && return - BASICSURI=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + BASICSURI=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the number of Suricata processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -63,7 +63,7 @@ whiptail_basic_zeek() { [ -n "$TESTING" ] && return - BASICZEEK=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + BASICZEEK=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the number of Zeek processes:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -75,7 +75,7 @@ whiptail_bond_nics_mtu() { [ -n "$TESTING" ] && return # Set the MTU on the monitor interface - MTU=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MTU=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the MTU for the monitor NICs:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -84,7 +84,7 @@ whiptail_bond_nics_mtu() { whiptail_cancel() { - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Cancelling Setup." 8 75 + whiptail --title "$whiptail_title" --msgbox "Cancelling Setup." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -104,7 +104,7 @@ whiptail_check_exitstatus() { whiptail_cancel ;; 255) - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 + whiptail --title "$whiptail_title" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 exit ;; esac @@ -114,7 +114,7 @@ whiptail_components_adv_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 + whiptail --title "$whiptail_title" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 75 } whiptail_create_admin_user() { @@ -187,7 +187,7 @@ whiptail_create_soremote_user() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Set a password for the soremote user. This account is used for adding sensors remotely." 8 75 + whiptail --title "$whiptail_title" --msgbox "Set a password for the soremote user. This account is used for adding sensors remotely." 8 75 } @@ -258,11 +258,11 @@ whiptail_requirements_error() { [ -n "$TESTING" ] && return if [[ $(echo "$requirement_needed" | tr '[:upper:]' '[:lower:]') == 'nics' ]]; then - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --msgbox "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select OK to exit setup and reconfigure the machine." 10 75 # Same as whiptail_cancel, but changed the wording to exit instead of cancel. - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Exiting Setup. No changes have been made." 8 75 + whiptail --title "$whiptail_title" --msgbox "Exiting Setup. No changes have been made." 8 75 if [ -d "/root/installtmp" ]; then { echo "/root/installtmp exists"; @@ -272,7 +272,7 @@ whiptail_requirements_error() { fi exit else - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select YES to continue anyway, or select NO to cancel." 10 75 local exitstatus=$? @@ -298,7 +298,7 @@ whiptail_storage_requirements() { EOM whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --yesno "$message" \ 14 75 @@ -310,7 +310,7 @@ whiptail_cur_close_days() { [ -n "$TESTING" ] && return - CURCLOSEDAYS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + CURCLOSEDAYS=$(whiptail --title "$whiptail_title" --inputbox \ "Please specify the threshold (in days) at which Elasticsearch indices will be closed:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -322,7 +322,7 @@ whiptail_dhcp_or_static() { [ -n "$TESTING" ] && return - address_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + address_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose how to set up your management interface:" 20 78 4 \ "STATIC" "Set a static IPv4 address" ON \ "DHCP" "Use DHCP to configure the Management Interface" OFF 3>&1 1>&2 2>&3 ) @@ -355,7 +355,7 @@ whiptail_dhcp_warn() { EOM whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --"$window_type" "$dhcp_message" \ 14 75 @@ -366,7 +366,7 @@ whiptail_dhcp_warn() { whiptail_dhcp_or_static ;; 255) - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 + whiptail --title "$whiptail_title" --msgbox "Whiptail error occured, exiting. Check log for details." 8 75 exit ;; esac @@ -380,7 +380,7 @@ whiptail_dockernet_check(){ [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno \ + whiptail --title "$whiptail_title" --yesno \ "Do you want to keep the default Docker IP range?\n\nIf you are unsure, please accept the default option of Yes." 10 75 } @@ -389,7 +389,7 @@ whiptail_dockernet_net() { [ -n "$TESTING" ] && return - DOCKERNET=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + DOCKERNET=$(whiptail --title "$whiptail_title" --inputbox \ "\nEnter a /24 size network range for docker to use WITHOUT the /24 suffix. This range will be used on ALL nodes." 11 65 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -409,7 +409,7 @@ whiptail_enable_components() { STRELKA=0 if [[ $is_eval ]]; then - COMPONENTS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ + COMPONENTS=$(whiptail --title "$whiptail_title" --checklist \ "Select Components to install:" 20 75 8 \ GRAFANA "Enable Grafana for system monitoring" ON \ OSQUERY "Enable Fleet with osquery" ON \ @@ -418,7 +418,7 @@ if [[ $is_eval ]]; then PLAYBOOK "Enable Playbook" ON \ STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3) else - COMPONENTS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ + COMPONENTS=$(whiptail --title "$whiptail_title" --checklist \ "Select Components to install:" 20 75 7 \ OSQUERY "Enable Fleet with osquery" ON \ WAZUH "Enable Wazuh" ON \ @@ -658,7 +658,7 @@ whiptail_eval_adv() { [ -n "$TESTING" ] && return - EVALADVANCED=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + EVALADVANCED=$(whiptail --title "$whiptail_title" --radiolist \ "Choose your eval install:" 20 75 4 \ "BASIC" "Install basic components for evaluation" ON \ "ADVANCED" "Choose additional components to be installed" OFF 3>&1 1>&2 2>&3 ) @@ -685,7 +685,7 @@ whiptail_gauge_post_setup() { else local msg=$1 - whiptail --title "Security Onion Setup - $SOVERSION" --gauge "$msg" 6 60 96 + whiptail --title "$whiptail_title" --gauge "$msg" 6 60 96 fi } @@ -693,7 +693,7 @@ whiptail_helix_apikey() { [ -n "$TESTING" ] && return - HELIXAPIKEY=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + HELIXAPIKEY=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your Helix API Key: \n \nThis can be set later using so-helix-apikey" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -707,7 +707,7 @@ whiptail_homenet_manager() { [ -n "$TESTING" ] && return - HNMANAGER=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + HNMANAGER=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -719,13 +719,13 @@ whiptail_homenet_sensor_inherit() { [ -n "$TESTING" ] && return # Ask to inherit from manager - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to inherit the HOME_NET from the Manager?" 8 75 + whiptail --title "$whiptail_title" --yesno "Do you want to inherit the HOME_NET from the Manager?" 8 75 } whiptail_homenet_sensor() { [ -n "$TESTING" ] && return - HNSENSOR=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + HNSENSOR=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your home network(s), separating CIDR blocks with a comma (,):" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -738,7 +738,7 @@ whiptail_install_type() { [ -n "$TESTING" ] && return # What kind of install are we doing? - install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + install_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose install type:" 12 65 5 \ "EVAL" "Evaluation mode (not for production) " ON \ "STANDALONE" "Standalone production install " OFF \ @@ -764,7 +764,7 @@ whiptail_install_type_dist() { [ -n "$TESTING" ] && return - install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + install_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose distributed node type:" 13 60 6 \ "MANAGER" "Start a new grid " ON \ "SENSOR" "Create a forward only sensor " OFF \ @@ -792,14 +792,14 @@ whiptail_install_type_other() { # so-analyst-install will only work with a working network connection # so only show it on network installs for now if [[ $setup_type == 'network' ]]; then - install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + install_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose distributed node type:" 9 65 2 \ "ANALYST" "Quit setup and run so-analyst-install " ON \ "HELIXSENSOR" "Create a Helix sensor " OFF \ 3>&1 1>&2 2>&3 ) else - install_type=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + install_type=$(whiptail --title "$whiptail_title" --radiolist \ "Choose distributed node type:" 8 65 1 \ "HELIXSENSOR" "Create a Helix sensor " ON \ 3>&1 1>&2 2>&3 @@ -815,7 +815,7 @@ whiptail_install_type_other() { whiptail_invalid_input() { # TODO: This should accept a list of arguments to specify what general pattern the input should follow [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox " Invalid input, please try again." 7 40 + whiptail --title "$whiptail_title" --msgbox " Invalid input, please try again." 7 40 } @@ -829,13 +829,13 @@ whiptail_invalid_proxy() { Error was: ${proxy_test_err} EOM - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60 + whiptail --title "$whiptail_title" --yesno "$message" --yes-button "Enter Again" --no-button "Skip" 11 60 } whiptail_invalid_string() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 + whiptail --title "$whiptail_title" --msgbox "Invalid input, please try again.\n\nThe $1 cannot contain spaces." 9 45 } @@ -843,21 +843,21 @@ whiptail_invalid_pass_characters_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 + whiptail --title "$whiptail_title" --msgbox "Password is invalid. Please exclude single quotes, double quotes and backslashes from the password." 8 75 } whiptail_invalid_pass_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please choose a more secure password." 8 75 + whiptail --title "$whiptail_title" --msgbox "Please choose a more secure password." 8 75 } whiptail_invalid_user_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Please enter a valid email address." 8 75 + whiptail --title "$whiptail_title" --msgbox "Please enter a valid email address." 8 75 } whiptail_invalid_hostname() { @@ -868,7 +868,7 @@ whiptail_invalid_hostname() { the ASCII letters 'A-Z' and 'a-z' (case-sensitive), the digits '0' through '9', \ and hyphen ('-')" | tr -d '\t') - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --msgbox "$error_message" 10 75 } @@ -891,7 +891,7 @@ whiptail_log_size_limit() { By default, this is set to ${percentage}% of the disk space allotted for /nsm. EOM - log_size_limit=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "$message" 11 75 "$1" 3>&1 1>&2 2>&3) + log_size_limit=$(whiptail --title "$whiptail_title" --inputbox "$message" 11 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -901,7 +901,7 @@ whiptail_log_size_limit() { whiptail_first_menu_iso() { [ -n "$TESTING" ] && return - option=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "Select an option" 10 75 2 \ + option=$(whiptail --title "$whiptail_title" --menu "Select an option" 10 75 2 \ "Install " "Run the standard Security Onion installation " \ "Configure Network " "Configure networking only " \ 3>&1 1>&2 2>&3 @@ -915,7 +915,7 @@ whiptail_make_changes() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 + whiptail --title "$whiptail_title" --yesno "We are going to set this machine up as a $install_type. Please press YES to make changes or NO to cancel." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -926,7 +926,7 @@ whiptail_management_interface_dns() { [ -n "$TESTING" ] && return - MDNS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MDNS=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your DNS servers separated by commas:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -938,7 +938,7 @@ whiptail_management_interface_dns_search() { [ -n "$TESTING" ] && return - MSEARCH=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MSEARCH=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your DNS search domain:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -949,7 +949,7 @@ whiptail_management_interface_gateway() { [ -n "$TESTING" ] && return - MGATEWAY=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MGATEWAY=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your gateway's IPv4 address:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -959,7 +959,7 @@ whiptail_management_interface_gateway() { whiptail_management_interface_ip_mask() { [ -n "$TESTING" ] && return - manager_ip_mask=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + manager_ip_mask=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your IPv4 address with CIDR mask (e.g. 192.168.1.2/24):" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -970,7 +970,7 @@ whiptail_management_interface_ip() { [ -n "$TESTING" ] && return - MIP=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MIP=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your IP address:" 10 60 X.X.X.X 3>&1 1>&2 2>&3) local exitstatus=$? @@ -981,7 +981,7 @@ whiptail_management_interface_mask() { [ -n "$TESTING" ] && return - MMASK=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MMASK=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the bit mask for your subnet:" 10 60 24 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1060,7 +1060,7 @@ whiptail_net_method() { local option_count=$(( ${#options[@]} / 2 )) - network_traffic=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) + network_traffic=$(whiptail --title "$whiptail_title" --menu "$msg" $height 75 $option_count "${options[@]}" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1071,7 +1071,7 @@ whiptail_net_method() { whiptail_net_setup_complete() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --msgbox "Successfully set up networking, setup will now exit." 7 75 exit 0 } @@ -1085,7 +1085,7 @@ whiptail_network_init_notice() { Select OK to continue. EOM - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 9 75 + whiptail --title "$whiptail_title" --msgbox "$message" 9 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } @@ -1094,7 +1094,7 @@ whiptail_management_server() { [ -n "$TESTING" ] && return - MSRV=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MSRV=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your Manager Server hostname: \nIt is CASE SENSITIVE!" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1105,7 +1105,7 @@ whiptail_management_server() { whiptail_manager_ip() { [ -n "$TESTING" ] && return - MSRVIP=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + MSRVIP=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your Manager Server IP Address:" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1117,7 +1117,7 @@ whiptail_manager_adv() { [ -n "$TESTING" ] && return - MANAGERADV=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + MANAGERADV=$(whiptail --title "$whiptail_title" --radiolist \ "Choose which type of manager to install:" 20 75 4 \ "BASIC" "Install manager with recommended settings" ON \ "ADVANCED" "Do additional configuration to the manager" OFF 3>&1 1>&2 2>&3 ) @@ -1132,7 +1132,7 @@ whiptail_manager_adv_escluster(){ [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno \ + whiptail --title "$whiptail_title" --yesno \ "Do you want to set up a traditional ES cluster for using replicas and/or Hot-Warm indices? Recommended only for those who have experience with ES clustering! " 12 75 } @@ -1142,7 +1142,7 @@ whiptail_manager_adv_escluster_name(){ [ -n "$TESTING" ] && return - ESCLUSTERNAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + ESCLUSTERNAME=$(whiptail --title "$whiptail_title" --inputbox \ "Enter a name for your ES cluster!" 10 75 "$1" 3>&1 1>&2 2>&3) } @@ -1151,7 +1151,7 @@ whiptail_manager_adv_service_zeeklogs() { [ -n "$TESTING" ] && return - BLOGS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist "Please select logs to send:" 24 75 12 \ + BLOGS=$(whiptail --title "$whiptail_title" --checklist "Please select logs to send:" 24 75 12 \ "conn" "Connection Logging" ON \ "dce_rpc" "RPC Logs" ON \ "dhcp" "DHCP Logs" ON \ @@ -1213,13 +1213,13 @@ whiptail_manager_error() { Would you like to continue anyway? EOM - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 + whiptail --title "$whiptail_title" --yesno "$msg" 13 75 || whiptail_check_exitstatus 1 } whiptail_manager_updates_warning() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION"\ + whiptail --title "$whiptail_title"\ --msgbox "Updating through the manager node requires the manager to have internet access, press ENTER to continue."\ 8 75 @@ -1230,7 +1230,7 @@ whiptail_manager_updates_warning() { whiptail_manager_unreachable() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Setup cannot determine if $1 is listening on port 22. Please check the address entered and try again." 7 75 + whiptail --title "$whiptail_title" --msgbox "Setup cannot determine if $1 is listening on port 22. Please check the address entered and try again." 7 75 } whiptail_metadata_tool() { @@ -1248,7 +1248,7 @@ whiptail_metadata_tool() { EOM # Legacy variable naming - ZEEKVERSION=$(whiptail --title "Security Onion Setup - $SOVERSION" --menu "$message" 20 75 2 \ + ZEEKVERSION=$(whiptail --title "$whiptail_title" --menu "$message" 20 75 2 \ "Zeek " "Use Zeek (Bro) for metadata and Suricata for NIDS alerts" \ "Suricata " "Use Suricata for both metadata and NIDS alerts" 3>&1 1>&2 2>&3) @@ -1262,7 +1262,7 @@ whiptail_nids() { [ -n "$TESTING" ] && return - NIDS=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + NIDS=$(whiptail --title "$whiptail_title" --radiolist \ "Choose which IDS to run: \n\n(Snort 3.0 support will be added once it is out of beta.)" 25 75 4 \ "Suricata" "Suricata" ON \ "Snort" "Placeholder for Snort 3.0 " OFF 3>&1 1>&2 2>&3 ) @@ -1276,7 +1276,7 @@ whiptail_network_notice() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75 + whiptail --title "$whiptail_title" --yesno "Since this is a network install we assume the management interface, DNS, Hostname, etc are already set up. Select YES to continue." 8 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1286,14 +1286,14 @@ whiptail_network_notice() { whiptail_net_reinit() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "The management interface has already been configured. Do you want to reconfigure it?" 8 75 + whiptail --title "$whiptail_title" --yesno "The management interface has already been configured. Do you want to reconfigure it?" 8 75 } whiptail_node_advanced() { [ -n "$TESTING" ] && return - NODESETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + NODESETUP=$(whiptail --title "$whiptail_title" --radiolist \ "What type of config would you like to use?:" 20 75 4 \ "NODEBASIC" "Install Search Node with recommended settings" ON \ "NODEADVANCED" "Advanced Node Setup" OFF 3>&1 1>&2 2>&3 ) @@ -1306,7 +1306,7 @@ whiptail_node_advanced() { whiptail_node_description() { [ -n "$TESTING" ] && return - NODE_DESCRIPTION=$(whiptail --title "Security Onion Setup - $SOVERSION" \ + NODE_DESCRIPTION=$(whiptail --title "$whiptail_title" \ --inputbox "Enter a short description for the node or press ENTER to leave blank:" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1317,7 +1317,7 @@ whiptail_node_es_heap() { [ -n "$TESTING" ] && return - NODE_ES_HEAP_SIZE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + NODE_ES_HEAP_SIZE=$(whiptail --title "$whiptail_title" --inputbox \ "Enter ES heap size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1329,7 +1329,7 @@ whiptail_node_ls_heap() { [ -n "$TESTING" ] && return - NODE_LS_HEAP_SIZE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + NODE_LS_HEAP_SIZE=$(whiptail --title "$whiptail_title" --inputbox \ "Enter Logstash heap size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1341,7 +1341,7 @@ whiptail_node_ls_input_threads() { [ -n "$TESTING" ] && return - LSINPUTTHREADS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + LSINPUTTHREADS=$(whiptail --title "$whiptail_title" --inputbox \ "Enter number of Logstash input threads:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1354,7 +1354,7 @@ whiptail_node_ls_pipline_batchsize() { [ -n "$TESTING" ] && return - LSPIPELINEBATCH=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + LSPIPELINEBATCH=$(whiptail --title "$whiptail_title" --inputbox \ "Enter Logstash pipeline batch size:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1366,7 +1366,7 @@ whiptail_node_ls_pipeline_worker() { [ -n "$TESTING" ] && return - LSPIPELINEWORKERS=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + LSPIPELINEWORKERS=$(whiptail --title "$whiptail_title" --inputbox \ "Enter number of Logstash pipeline workers:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1377,13 +1377,13 @@ whiptail_node_ls_pipeline_worker() { whiptail_ntp_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Would you like to configure ntp servers?" 7 44 + whiptail --title "$whiptail_title" --yesno "Would you like to configure ntp servers?" 7 44 } whiptail_ntp_servers() { [ -n "$TESTING" ] && return - ntp_string=$(whiptail --title "Security Onion Setup - $SOVERSION" \ + ntp_string=$(whiptail --title "$whiptail_title" \ --inputbox "Input the NTP server(s) you would like to use, separated by commas:" 8 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1394,7 +1394,7 @@ whiptail_oinkcode() { [ -n "$TESTING" ] && return - OINKCODE=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + OINKCODE=$(whiptail --title "$whiptail_title" --inputbox \ "Enter your ET Pro or oinkcode:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1407,7 +1407,7 @@ whiptail_oinkcode() { whiptail_passwords_dont_match() { - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "Passwords don't match. Please re-enter." 8 75 + whiptail --title "$whiptail_title" --msgbox "Passwords don't match. Please re-enter." 8 75 } @@ -1415,7 +1415,7 @@ whiptail_patch_name_new_schedule() { [ -n "$TESTING" ] && return - PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + PATCHSCHEDULENAME=$(whiptail --title "$whiptail_title" --inputbox \ "What name do you want to give this OS patch schedule? This schedule needs to be named uniquely. Available schedules can be found on the manager under /opt/so/salt/patch/os/schedules/.yml" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1426,7 +1426,7 @@ whiptail_patch_schedule() { [ -n "$TESTING" ] && return - patch_schedule=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + patch_schedule=$(whiptail --title "$whiptail_title" --radiolist \ "Choose OS patch schedule.\n\nThis schedule will update the operating system packages but will NOT update Security Onion related tools such as Zeek, Elasticsearch, Kibana, SaltStack, etc." 20 75 5 \ "Automatic" "Updates installed every 8 hours if available" ON \ "Manual" "Updates will be installed manually" OFF \ @@ -1442,7 +1442,7 @@ whiptail_patch_schedule_import() { [ -n "$TESTING" ] && return unset PATCHSCHEDULENAME - PATCHSCHEDULENAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + PATCHSCHEDULENAME=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the name of the OS patch schedule you want to inherit. \nAvailable schedules can be found on the manager under /opt/so/salt/patch/os/schedules/.yml" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1454,7 +1454,7 @@ whiptail_patch_schedule_select_days() { [ -n "$TESTING" ] && return # Select the days to patch - PATCHSCHEDULEDAYS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ + PATCHSCHEDULEDAYS=$(whiptail --title "$whiptail_title" --checklist \ "Which days do you want to apply OS patches?" 15 75 8 \ Monday "" OFF \ Tuesday "" ON \ @@ -1478,7 +1478,7 @@ whiptail_patch_schedule_select_hours() { [ -n "$TESTING" ] && return # Select the hours to patch - PATCHSCHEDULEHOURS=$(whiptail --title "Security Onion Setup - $SOVERSION" --checklist \ + PATCHSCHEDULEHOURS=$(whiptail --title "$whiptail_title" --checklist \ "At which time, UTC, do you want to apply OS patches on the selected days?" 22 75 13 \ 00:00 "" OFF \ 01:00 "" OFF \ @@ -1519,7 +1519,7 @@ whiptail_proxy_ask() { local pkg_mngr if [[ $OS = 'centos' ]]; then pkg_mngr="yum"; else pkg_mngr='apt'; fi - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment?" 9 65 --defaultno + whiptail --title "$whiptail_title" --yesno "Do you want to proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment?" 9 65 --defaultno } whiptail_proxy_addr() { @@ -1532,7 +1532,7 @@ whiptail_proxy_addr() { If your proxy requires a username and password do not include them in your input. Setup will ask for those values next. EOM - proxy_addr=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) + proxy_addr=$(whiptail --title "$whiptail_title" --inputbox "$message" 13 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1541,13 +1541,13 @@ whiptail_proxy_addr() { whiptail_proxy_auth_ask() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Does your proxy require authentication?" 7 60 + whiptail --title "$whiptail_title" --yesno "Does your proxy require authentication?" 7 60 } whiptail_proxy_auth_user() { [ -n "$TESTING" ] && return - proxy_user=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) + proxy_user=$(whiptail --title "$whiptail_title" --inputbox "Please input the proxy user:" 8 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1558,7 +1558,7 @@ whiptail_proxy_auth_pass() { [ -n "$TESTING" ] && return - proxy_pass=$(whiptail --title "Security Onion Setup - $SOVERSION" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) + proxy_pass=$(whiptail --title "$whiptail_title" --passwordbox "Please input the proxy password:" 8 60 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1572,7 +1572,7 @@ whiptail_requirements_error() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Press YES to continue anyway, or press NO to cancel." 10 75 local exitstatus=$? @@ -1584,7 +1584,7 @@ whiptail_rule_setup() { [ -n "$TESTING" ] && return # Get pulled pork info - RULESETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + RULESETUP=$(whiptail --title "$whiptail_title" --radiolist \ "Which IDS ruleset would you like to use?\n\nThis manager server is responsible for downloading the IDS ruleset from the Internet.\n\nSensors then pull a copy of this ruleset from the manager server.\n\nIf you select a commercial ruleset, it is your responsibility to purchase enough licenses for all of your sensors in compliance with your vendor's policies." 20 75 4 \ "ETOPEN" "Emerging Threats Open" ON \ "ETPRO" "Emerging Threats PRO" OFF \ @@ -1600,7 +1600,7 @@ whiptail_sensor_config() { [ -n "$TESTING" ] && return - NSMSETUP=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + NSMSETUP=$(whiptail --title "$whiptail_title" --radiolist \ "What type of configuration would you like to use?" 20 75 4 \ "BASIC" "Install NSM components with recommended settings" ON \ "ADVANCED" "Configure each component individually" OFF 3>&1 1>&2 2>&3 ) @@ -1642,7 +1642,7 @@ whiptail_sensor_nics() { for bond_nic in "${BNICS[@]}"; do if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ 8 75 exit @@ -1654,7 +1654,7 @@ whiptail_set_hostname() { [ -n "$TESTING" ] && return - HOSTNAME=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + HOSTNAME=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the hostname (not FQDN) you would like to set:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1665,7 +1665,7 @@ whiptail_set_redirect() { [ -n "$TESTING" ] && return - REDIRECTINFO=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + REDIRECTINFO=$(whiptail --title "$whiptail_title" --radiolist \ "How would you like to access the web interface?\n\nSecurity Onion uses strict cookie enforcement, so whatever you choose here will be the only way that you can access the web interface.\n\nIf you choose something other than IP address, then you'll need to ensure that you can resolve the name via DNS or hosts entry. If you are unsure, please select IP." 20 75 4 \ "IP" "Use IP address to access the web interface" ON \ "HOSTNAME" "Use hostname to access the web interface" OFF \ @@ -1678,7 +1678,7 @@ whiptail_set_redirect_host() { [ -n "$TESTING" ] && return - REDIRECTHOST=$(whiptail --title "Security Onion Setup - $SOVERSION" --inputbox \ + REDIRECTHOST=$(whiptail --title "$whiptail_title" --inputbox \ "Enter the Hostname, IP, or FQDN you would like to use for the web interface:" 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1706,7 +1706,7 @@ whiptail_setup_complete() { Press ENTER to reboot. EOM - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 12 75 + whiptail --title "$whiptail_title" --msgbox "$message" 12 75 } whiptail_setup_failed() { @@ -1726,13 +1726,13 @@ whiptail_setup_failed() { Press Ok to exit. EOM - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" $height 75 + whiptail --title "$whiptail_title" --msgbox "$message" $height 75 } whiptail_so_allow_yesno() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" \ + whiptail --title "$whiptail_title" \ --yesno "Do you want to run so-allow to allow access to the web tools?" \ 8 75 } @@ -1741,7 +1741,7 @@ whiptail_so_allow() { [ -n "$TESTING" ] && return - ALLOW_CIDR=$(whiptail --title "Security Onion Setup - $SOVERSION" \ + ALLOW_CIDR=$(whiptail --title "$whiptail_title" \ --inputbox "Enter a single IP address or an IP range, in CIDR notation, to allow:" \ 10 75 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -1761,7 +1761,7 @@ whiptail_ssh_key_copy_notice() { Select OK to continue. EOM - whiptail --title "Security Onion Setup - $SOVERSION" --msgbox "$message" 11 75 + whiptail --title "$whiptail_title" --msgbox "$message" 11 75 local exitstatus=$? whiptail_check_exitstatus $exitstatus } @@ -1800,7 +1800,7 @@ whiptail_storage_requirements() { EOM whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --yesno "$message" \ 14 75 @@ -1812,7 +1812,7 @@ whiptail_strelka_rules() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup - $SOVERSION" --yesno "Do you want to enable the default YARA rules for Strelka?" 8 75 + whiptail --title "$whiptail_title" --yesno "Do you want to enable the default YARA rules for Strelka?" 8 75 local exitstatus=$? @@ -1852,7 +1852,7 @@ whiptail_node_updates() { [ -n "$TESTING" ] && return - NODEUPDATES=$(whiptail --title "Security Onion Setup - $SOVERSION" --radiolist \ + NODEUPDATES=$(whiptail --title "$whiptail_title" --radiolist \ "How would you like to download OS package updates for your grid?" 20 75 4 \ "MANAGER" "Manager node is proxy for updates." ON \ "OPEN" "Each node connects to the Internet for updates" OFF 3>&1 1>&2 2>&3 ) @@ -1877,7 +1877,7 @@ whiptail_you_sure() { EOM whiptail \ - --title "Security Onion Setup - $SOVERSION" \ + --title "$whiptail_title" \ --yesno "$you_sure_text" \ 20 75 From 7b7111e12c15d107ad5b5ce68fa4e633d84c4210 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 13:53:39 -0400 Subject: [PATCH 171/195] Fix some hunt queries --- salt/elasticsearch/files/elasticsearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index a1d4c836b..af7cec1fa 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -36,7 +36,7 @@ xpack.security.authc: roles: superuser authz_exception: true node.name: {{ grains.host }} -script.max_compilations_rate: 2000/1m +script.max_compilations_rate: 20000/1m {%- if TRUECLUSTER is sameas true %} {%- if grains.role == 'so-manager' %} {%- if salt['pillar.get']('nodestab', {}) %} From e8cc88174f9918fd6575d20fe2a6feb723853556 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 13:55:05 -0400 Subject: [PATCH 172/195] Fix some hunt queries --- salt/soc/files/soc/hunt.queries.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index 93295364d..9d4cd85bd 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -2,8 +2,8 @@ { "name": "Default Query", "description": "Show all events grouped by the origin host", "query": "* | groupby observer.name"}, { "name": "Log Type", "description": "Show all events grouped by module and dataset", "query": "* | groupby event.module event.dataset"}, { "name": "Elastalerts", "description": "", "query": "_type:elastalert | groupby rule.name"}, - { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.dataset: alert | groupby event.module"}, - { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert", "query": "event.category: network AND event.dataset: alert | groupby rule.category rule.gid rule.uuid rule.name"}, + { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.kind: alert | groupby event.module"}, + { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert", "query": "event.category: network AND event.kind: alert | groupby rule.category rule.gid rule.uuid rule.name"}, { "name": "Wazuh/OSSEC Alerts", "description": "Show all Wazuh alerts at Level 5 or higher grouped by category", "query": "event.module:ossec AND event.dataset:alert AND rule.level:>4 | groupby rule.category rule.name"}, { "name": "Wazuh/OSSEC Alerts", "description": "Show all Wazuh alerts at Level 4 or lower grouped by category", "query": "event.module:ossec AND event.dataset:alert AND rule.level:<5 | groupby rule.category rule.name"}, { "name": "Wazuh/OSSEC Users and Commands", "description": "Show all Wazuh alerts grouped by username and command line", "query": "event.module:ossec AND event.dataset:alert | groupby user.escalated.keyword process.command_line"}, @@ -11,11 +11,11 @@ { "name": "Sysmon Events", "description": "Show all Sysmon logs grouped by event type", "query": "event.module:sysmon | groupby event.dataset"}, { "name": "Sysmon Usernames", "description": "Show all Sysmon logs grouped by username", "query": "event.module:sysmon | groupby event.dataset, user.name.keyword"}, { "name": "Strelka", "description": "Show all Strelka logs grouped by file type", "query": "event.module:strelka | groupby file.mime_type"}, - { "name": "Zeek Notice", "description": "Show notices from Zeek", "query": "event.dataset:notice | groupby notice.note notice.message"}, - { "name": "Connections", "description": "Connections grouped by IP and Port", "query": "event.dataset:conn | groupby source.ip destination.ip network.protocol destination.port"}, - { "name": "Connections", "description": "Connections grouped by Service", "query": "event.dataset:conn | groupby network.protocol destination.port"}, - { "name": "Connections", "description": "Connections grouped by destination country", "query": "event.dataset:conn | groupby destination.geo.country_name"}, - { "name": "Connections", "description": "Connections grouped by source country", "query": "event.dataset:conn | groupby source.geo.country_name"}, + { "name": "Zeek Notice", "description": "Show notices from Zeek", "query": "event.dataset:zeek.notice | groupby notice.note notice.message"}, + { "name": "Connections", "description": "Connections grouped by IP and Port", "query": "event.dataset:zeek.connection | groupby source.ip destination.ip network.protocol destination.port"}, + { "name": "Connections", "description": "Connections grouped by Service", "query": "event.dataset:zeek.connection | groupby network.protocol destination.port"}, + { "name": "Connections", "description": "Connections grouped by destination country", "query": "event.dataset:zeek.connection | groupby destination.geo.country_name"}, + { "name": "Connections", "description": "Connections grouped by source country", "query": "event.dataset:zeek.connection | groupby source.geo.country_name"}, { "name": "DCE_RPC", "description": "DCE_RPC grouped by operation", "query": "event.dataset:dce_rpc | groupby dce_rpc.operation"}, { "name": "DHCP", "description": "DHCP leases", "query": "event.dataset:dhcp | groupby host.hostname client.address"}, { "name": "DHCP", "description": "DHCP grouped by message type", "query": "event.dataset:dhcp | groupby dhcp.message_types"}, From e42db3cd2d1b4719749422dcf9911d3311de80bc Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 14:05:02 -0400 Subject: [PATCH 173/195] Fix some hunt queries --- salt/common/tools/sbin/so-filebeat-module-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index d7706366e..7a6ae7446 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -49,7 +49,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then fi echo "Testing to see if the pipelines are already applied" ESVER=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \") -PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) +PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c) if [[ "$PIPELINES" -lt 5 ]]; then echo "Setting up ingest pipeline(s)" From 9c9bcac61ba6899f1eb0afa3248c2d0269d6f5a4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 2 Jun 2021 15:01:14 -0400 Subject: [PATCH 174/195] Update DNS queries --- salt/soc/files/soc/hunt.queries.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index 9d4cd85bd..c220060dd 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -20,11 +20,11 @@ { "name": "DHCP", "description": "DHCP leases", "query": "event.dataset:dhcp | groupby host.hostname client.address"}, { "name": "DHCP", "description": "DHCP grouped by message type", "query": "event.dataset:dhcp | groupby dhcp.message_types"}, { "name": "DNP3", "description": "DNP3 grouped by reply", "query": "event.dataset:dnp3 | groupby dnp3.fc_reply"}, - { "name": "DNS", "description": "DNS queries grouped by port", "query": "event.dataset:dns | groupby dns.query.name destination.port"}, - { "name": "DNS", "description": "DNS queries grouped by type", "query": "event.dataset:dns | groupby dns.query.type_name destination.port"}, - { "name": "DNS", "description": "DNS queries grouped by response code", "query": "event.dataset:dns | groupby dns.response.code_name destination.port"}, - { "name": "DNS", "description": "DNS highest registered domain", "query": "event.dataset:dns | groupby dns.highest_registered_domain.keyword destination.port"}, - { "name": "DNS", "description": "DNS grouped by parent domain", "query": "event.dataset:dns | groupby dns.parent_domain.keyword destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by port", "query": "_exists_:dns.id | groupby dns.question.name destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by type", "query": "_exists_:dns.id | groupby dns.question.type destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by response code", "query": "_exists_:dns.id | groupby dns.response_code destination.port"}, + { "name": "DNS", "description": "DNS highest registered domain", "query": "_exists_:dns.id | groupby dns.question.top_level_domain destination.port"}, + { "name": "DNS", "description": "DNS grouped by parent domain", "query": "_exists_:dns.id | groupby dns.question.registered_domain destination.port"}, { "name": "DPD", "description": "Dynamic Protocol Detection errors", "query": "event.dataset:dpd | groupby error.reason"}, { "name": "Files", "description": "Files grouped by mimetype", "query": "event.dataset:file | groupby file.mime_type source.ip"}, { "name": "Files", "description": "Files grouped by source", "query": "event.dataset:file | groupby file.source source.ip"}, From a1b34e7a8840b7730817c20e3f812fe5ef9e9c39 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 15:30:14 -0400 Subject: [PATCH 175/195] Fix Suricata index name --- .../pipelines/config/so/9400_output_suricata.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 258781f29..999b270cf 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -9,7 +9,7 @@ output { pipeline => "filebeat-%{[agent][version]}-suricata-%{[fileset][name]}-pipeline" hosts => "{{ ES }}" index => "so-ids" - template_name => "so-ids" + template_name => "so-ids-%{+YYYY.MM.dd}" template => "/templates/so-ids-template.json" ssl => true ssl_certificate_verification => false From db48c15f1d099be45e8178ac94c093159b7eb161 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 15:33:18 -0400 Subject: [PATCH 176/195] Create event.kind field and rename dataset to be module[dot]dataset --- salt/elasticsearch/files/ingest/ossec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/ossec b/salt/elasticsearch/files/ingest/ossec index 868de2798..1c5a0fd42 100644 --- a/salt/elasticsearch/files/ingest/ossec +++ b/salt/elasticsearch/files/ingest/ossec @@ -63,7 +63,8 @@ { "rename": { "field": "fields.module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, { "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } }, { "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } }, - { "set": { "if": "ctx.rule != null && ctx.rule.name != null", "field": "event.dataset", "value": "alert", "override": true } }, + { "set": { "if": "ctx.rule != null && ctx.rule.name != null", "field": "event.dataset", "value": "ossec.alert", "override": true } }, + { "set": { "if": "ctx.rule != null && ctx.rule.name != null", "field": "event.kind", "value": "alert", "override": true } }, { "pipeline": { "name": "common" } } ] } From 4c74e7f3089a57fd68cdd04c6a90c145ee5175b0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 15:35:26 -0400 Subject: [PATCH 177/195] Add event.kind and set name to module[dot]dataset --- salt/elasticsearch/files/ingest/strelka.file | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index e5e8560f8..cf2772305 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -53,7 +53,8 @@ { "set": { "if": "ctx.exiftool?.FileDirectory != null", "field": "file.directory", "value": "{{exiftool.FileDirectory}}", "ignore_failure": true }}, { "set": { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem", "value": "{{exiftool.Subsystem}}", "ignore_failure": true }}, { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "rule.name", "value": "{{scan.yara.matches.0}}" }}, - { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "dataset", "value": "alert", "override": true }}, + { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "dataset", "value": "strelka.alert", "override": true }}, + { "set": { "if": "ctx.scan?.yara?.matches != null", "field": "event.kind", "value": "alert", "override": true }}, { "rename": { "field": "file.flavors.mime", "target_field": "file.mime_type", "ignore_missing": true }}, { "set": { "if": "ctx.rule?.name != null && ctx.rule?.score == null", "field": "event.severity", "value": 3, "override": true } }, { "convert" : { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}}, From 4241bb08b85101673a08ffcdbecfc155d3ca7bc9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 16:37:43 -0400 Subject: [PATCH 178/195] Add suricata/zeek until we migrate templates --- .../templates/so/so-common-template.json | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 214e5c691..919cb521a 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -527,12 +527,20 @@ "version":{ "type":"long" } - } - }, + } + }, "x509":{ "type":"object", "dynamic": true - } + }, + "suricata":{ + "type":"object", + "dynamic": true + }, + "zeek":{ + "type":"object", + "dynamic": true + }, } } } From cba719b3a02b52d75d957612337070446b3ef66d Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Jun 2021 16:42:09 -0400 Subject: [PATCH 179/195] Remove extra comma --- salt/elasticsearch/templates/so/so-common-template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 919cb521a..2d1ef2a21 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -540,7 +540,7 @@ "zeek":{ "type":"object", "dynamic": true - }, + } } } } From 719d841353e927b5fa108ac153d4dee272d6a984 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Jun 2021 20:15:03 -0400 Subject: [PATCH 180/195] Update saved objects --- salt/kibana/files/saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/saved_objects.ndjson b/salt/kibana/files/saved_objects.ndjson index 21ef5a94a..639a38bfc 100644 --- a/salt/kibana/files/saved_objects.ndjson +++ b/salt/kibana/files/saved_objects.ndjson @@ -460,7 +460,7 @@ {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\",\"time_zone\":\"America/New_York\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":48,\"x\":0,\"y\":0,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":96,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":8,\"y\":20,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":28,\"x\":28,\"y\":20,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":0,\"y\":72,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":48,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":24,\"x\":16,\"y\":72,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":20,\"h\":12,\"x\":28,\"y\":8,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_9\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":120,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"event_type\",\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_10\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"15\"},\"panelIndex\":\"15\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_11\"}]","timeRestore":false,"title":"z16.04 - Sysmon - Logs","version":1},"id":"6d189680-6d62-11e7-8ddb-e71eb260f4a3","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"8cfdeff0-6d6b-11e7-ad64-15aa071374a6","name":"panel_1","type":"visualization"},{"id":"0eb1fd80-6d70-11e7-b09b-f57b22df6524","name":"panel_2","type":"visualization"},{"id":"3072c750-6d71-11e7-b09b-f57b22df6524","name":"panel_3","type":"visualization"},{"id":"7bc74b40-6d71-11e7-b09b-f57b22df6524","name":"panel_4","type":"visualization"},{"id":"13ed0810-6d72-11e7-b09b-f57b22df6524","name":"panel_5","type":"visualization"},{"id":"3b6c92c0-6d72-11e7-b09b-f57b22df6524","name":"panel_6","type":"visualization"},{"id":"e09f6010-6d72-11e7-b09b-f57b22df6524","name":"panel_7","type":"visualization"},{"id":"29611940-6d75-11e7-b09b-f57b22df6524","name":"panel_8","type":"visualization"},{"id":"6b70b840-6d75-11e7-b09b-f57b22df6524","name":"panel_9","type":"visualization"},{"id":"248c1d20-6d6b-11e7-ad64-15aa071374a6","name":"panel_10","type":"search"},{"id":"AWDHHk1sxQT5EBNmq43Y","name":"panel_11","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQyLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"},"savedSearchRefName":"search_0","title":"SMB - Action (Pie Chart)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"SMB - Action (Pie Chart)\",\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"action.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"6f883480-3aad-11e7-8b17-0d8709b02c80","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"19849f30-3aab-11e7-8b17-0d8709b02c80","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQzLDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - SSL - Subject","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Security Onion - SSL - Subject\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"ssl.certificate.subject.keyword: Descending\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"ssl.certificate.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Subject\"}}]}"},"id":"6fccb600-75ec-11ea-9565-7315f4ee5cac","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ0LDRd"} -{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.13.0","id":"7.13.0","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} +{"attributes":{"buildNum":39457,"defaultIndex":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute":"/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize":100,"theme:darkMode":true,"timepicker:timeDefaults":"{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion":"7.13.1","id":"7.13.1","migrationVersion":{"config":"7.12.0"},"references":[],"type":"config","updated_at":"2021-04-29T21:42:52.430Z","version":"WzY3NTUsM10="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Strelka - File - MIME Flavors","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Strelka - File - MIME Flavors\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"file.flavors.mime.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"70243970-772c-11ea-bee5-af7f7c7b8e05","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ2LDRd"} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Modbus - Log Count","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}}","version":1,"visState":"{\"title\":\"Modbus - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}"},"id":"AWDG_9KpxQT5EBNmq4Oo","migrationVersion":{"visualization":"7.11.0"},"references":[{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ3LDRd"} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\",\"default_field\":\"*\"}},\"language\":\"lucene\"},\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":true,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":56,\"x\":0,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":32,\"h\":8,\"x\":16,\"y\":0,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":8,\"y\":32,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":12,\"h\":24,\"x\":20,\"y\":32,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":48,\"h\":24,\"x\":0,\"y\":56,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"columns\":[\"source_ip\",\"source_port\",\"destination_ip\",\"destination_port\",\"uid\",\"_id\"],\"sort\":[\"@timestamp\",\"desc\"],\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":16,\"h\":24,\"x\":32,\"y\":32,\"i\":\"9\"},\"panelIndex\":\"9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":8,\"h\":8,\"x\":8,\"y\":0,\"i\":\"10\"},\"panelIndex\":\"10\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"w\":40,\"h\":24,\"x\":8,\"y\":8,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_7\"}]","timeRestore":false,"title":"z16.04 - Bro - Modbus","version":1},"id":"70c005f0-3583-11e7-a588-05992195c551","migrationVersion":{"dashboard":"7.11.0"},"references":[{"id":"b3b449d0-3429-11e7-9d52-4f090484f59e","name":"panel_0","type":"visualization"},{"id":"0d168a30-363f-11e7-a6f7-4f44d7bf1c33","name":"panel_1","type":"visualization"},{"id":"20eabd60-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_2","type":"visualization"},{"id":"3c65f500-380b-11e7-a1cc-ebc6a7e70e84","name":"panel_3","type":"visualization"},{"id":"52dc9fe0-342e-11e7-9e93-53b62e1857b2","name":"panel_4","type":"search"},{"id":"178209e0-6e1b-11e7-b553-7f80727663c1","name":"panel_5","type":"visualization"},{"id":"AWDG_9KpxQT5EBNmq4Oo","name":"panel_6","type":"visualization"},{"id":"453f8b90-4a58-11e8-9b0a-f1d33346f773","name":"panel_7","type":"visualization"}],"type":"dashboard","updated_at":"2021-03-19T14:35:12.119Z","version":"WzcwMTQ4LDRd"} From d1a185aaae34c650c0b9f251b29c87df909e47a6 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Jun 2021 10:59:14 -0400 Subject: [PATCH 181/195] Further standardize whiptail titles --- setup/so-whiptail | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 458713591..258a7b0ac 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -121,7 +121,7 @@ whiptail_create_admin_user() { [ -n "$TESTING" ] && return - ADMINUSER=$(whiptail --title "Security Onion Install" --inputbox \ + ADMINUSER=$(whiptail --title "$whiptail_title" --inputbox \ "Please enter a username for a new system admin user: \nThe local onion account will be disabled during this install" 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -132,7 +132,7 @@ whiptail_create_admin_user_password1() { [ -n "$TESTING" ] && return - ADMINPASS1=$(whiptail --title "Security Onion Install" --passwordbox \ + ADMINPASS1=$(whiptail --title "$whiptail_title" --passwordbox \ "Enter a password for $ADMINUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -143,7 +143,7 @@ whiptail_create_admin_user_password2() { [ -n "$TESTING" ] && return - ADMINPASS2=$(whiptail --title "Security Onion Install" --passwordbox \ + ADMINPASS2=$(whiptail --title "$whiptail_title" --passwordbox \ "Re-enter a password for $ADMINUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -155,7 +155,7 @@ whiptail_create_fleet_node_user() { [ -n "$TESTING" ] && return - FLEETNODEUSER=$(whiptail --title "Security Onion Install" --inputbox \ + FLEETNODEUSER=$(whiptail --title "$whiptail_title" --inputbox \ "Please enter an email for use as the username for the Fleet admin user:" 10 60 "$1" 3>&1 1>&2 2>&3) } @@ -164,7 +164,7 @@ whiptail_create_fleet_node_user_password1() { [ -n "$TESTING" ] && return - FLEETNODEPASSWD1=$(whiptail --title "Security Onion Install" --passwordbox \ + FLEETNODEPASSWD1=$(whiptail --title "$whiptail_title" --passwordbox \ "Enter a password for $FLEETNODEUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -175,7 +175,7 @@ whiptail_create_fleet_node_user_password2() { [ -n "$TESTING" ] && return - FLEETNODEPASSWD2=$(whiptail --title "Security Onion Install" --passwordbox \ + FLEETNODEPASSWD2=$(whiptail --title "$whiptail_title" --passwordbox \ "Re-enter a password for $FLEETNODEUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -195,7 +195,7 @@ whiptail_create_soremote_user_password1() { [ -n "$TESTING" ] && return - SOREMOTEPASS1=$(whiptail --title "Security Onion Install" --passwordbox \ + SOREMOTEPASS1=$(whiptail --title "$whiptail_title" --passwordbox \ "Enter a password for user soremote:" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -207,7 +207,7 @@ whiptail_create_soremote_user_password2() { [ -n "$TESTING" ] && return - SOREMOTEPASS2=$(whiptail --title "Security Onion Install" --passwordbox \ + SOREMOTEPASS2=$(whiptail --title "$whiptail_title" --passwordbox \ "Re-enter a password for user soremote:" 10 75 3>&1 1>&2 2>&3) local exitstatus=$? @@ -219,7 +219,7 @@ whiptail_create_web_user() { [ -n "$TESTING" ] && return - WEBUSER=$(whiptail --title "Security Onion Install" --inputbox \ + WEBUSER=$(whiptail --title "$whiptail_title" --inputbox \ "Please enter an email address to create an administrator account for the web interface.\n\nThis will also be used for TheHive, Cortex, and Fleet." 12 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -230,7 +230,7 @@ whiptail_create_web_user_password1() { [ -n "$TESTING" ] && return - WEBPASSWD1=$(whiptail --title "Security Onion Install" --passwordbox \ + WEBPASSWD1=$(whiptail --title "$whiptail_title" --passwordbox \ "Enter a password for $WEBUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -241,7 +241,7 @@ whiptail_create_web_user_password2() { [ -n "$TESTING" ] && return - WEBPASSWD2=$(whiptail --title "Security Onion Install" --passwordbox \ + WEBPASSWD2=$(whiptail --title "$whiptail_title" --passwordbox \ "Re-enter a password for $WEBUSER:" 10 60 3>&1 1>&2 2>&3) local exitstatus=$? @@ -671,7 +671,7 @@ whiptail_fleet_custom_hostname() { [ -n "$TESTING" ] && return - FLEETCUSTOMHOSTNAME=$(whiptail --title "Security Onion Install" --inputbox \ + FLEETCUSTOMHOSTNAME=$(whiptail --title "$whiptail_title" --inputbox \ "What FQDN should osquery clients use for connections to this Fleet node? Leave blank if the local system hostname will be used." 10 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? @@ -994,13 +994,13 @@ whiptail_management_nic() { filter_unused_nics - MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) + MNIC=$(whiptail --title "$whiptail_title" --radiolist "Please select your management NIC:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus while [ -z "$MNIC" ] do - MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) + MNIC=$(whiptail --title "$whiptail_title" --radiolist "Please select your management NIC:" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus done @@ -1624,13 +1624,13 @@ whiptail_sensor_nics() { local list_type="checklist" fi - BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) + BNICS=$(whiptail --title "$whiptail_title" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus while [ -z "$BNICS" ] do - BNICS=$(whiptail --title "NIC Setup" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) + BNICS=$(whiptail --title "$whiptail_title" --$list_type "$menu_text" 20 75 12 "${nic_list[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus done @@ -1838,7 +1838,7 @@ whiptail_suricata_pins() { local PROCS=$lb_procs fi - SURIPINS=$(whiptail --noitem --title "Pin Suricata CPUS" --checklist "Please select $PROCS cores to pin Suricata to:" 20 75 12 "${filtered_core_str[@]}" 3>&1 1>&2 2>&3 ) + SURIPINS=$(whiptail --noitem --title "$whiptail_title" --checklist "Please select $PROCS cores to pin Suricata to:" 20 75 12 "${filtered_core_str[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus @@ -1902,7 +1902,7 @@ whiptail_zeek_pins() { local PROCS=$lb_procs fi - ZEEKPINS=$(whiptail --noitem --title "Pin Zeek CPUS" --checklist "Please select $PROCS cores to pin Zeek to:" 20 75 12 "${cpu_core_list_whiptail[@]}" 3>&1 1>&2 2>&3 ) + ZEEKPINS=$(whiptail --noitem --title "$whiptail_title" --checklist "Please select $PROCS cores to pin Zeek to:" 20 75 12 "${cpu_core_list_whiptail[@]}" 3>&1 1>&2 2>&3 ) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 3aff3ac7e4757b44276f83d429c6ed8e028ca2d7 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Jun 2021 11:00:20 -0400 Subject: [PATCH 182/195] Change logic to check for unmanaged nics Resolves issue mentioned in #4327 --- setup/so-whiptail | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 258a7b0ac..cbf74680f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1640,13 +1640,15 @@ whiptail_sensor_nics() { IFS=' ' read -ra BNICS <<< "$BNICS" for bond_nic in "${BNICS[@]}"; do - if [[ "${nmcli_dev_status_list}" =~ $bond_nic\:unmanaged ]]; then - whiptail \ - --title "$whiptail_title" \ - --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ - 8 75 - exit - fi + for dev_status in "${nmcli_dev_status_list[@]}"; do + if [[ $dev_status == "${bond_nic}:unmanaged" ]]; then + whiptail \ + --title "$whiptail_title" \ + --msgbox "$bond_nic is unmanaged by Network Manager. Please remove it from other network management tools then re-run setup." \ + 8 75 + exit + fi + done done } From d1d09d4aab198b3781e62a0906d2bcfd3eb19969 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 3 Jun 2021 14:20:52 -0400 Subject: [PATCH 183/195] Remove useless variable assignment --- setup/so-functions | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 19e0c804f..2a9554033 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -897,8 +897,6 @@ compare_main_nic_ip() { This is not a supported configuration, please remediate and rerun setup. EOM - [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]] || local whiptail_title="so-monitor-add" - [[ -n $TESTING ]] || whiptail --title "$whiptail_title" --msgbox "$message" 10 75 kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1 fi From 56eb220ed6eedfd25e912beffeaf5a00c2f26f1f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 09:52:05 -0400 Subject: [PATCH 184/195] Revert to SO taxonomy for zeek and suricata --- salt/filebeat/etc/filebeat.yml | 77 ++++++++++++++++++++++++ salt/filebeat/securityoniondefaults.yaml | 20 +----- 2 files changed, 78 insertions(+), 19 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 46a59f772..f933cee2e 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -111,7 +111,84 @@ filebeat.inputs: fields_under_root: true {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} + {%- if ZEEKVER != 'SURICATA' %} + {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} +- type: log + paths: + - /nsm/zeek/logs/current/{{ LOGNAME }}.log + fields: + module: zeek + dataset: {{ LOGNAME }} + category: network + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + + fields_under_root: true + clean_removed: true + close_removed: false +- type: log + paths: + - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log + fields: + module: zeek + dataset: {{ LOGNAME }} + category: network + imported: true + processors: + - add_tags: + tags: ["import"] + - dissect: + tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}" + field: "log.file.path" + target_prefix: "" + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + + fields_under_root: true + clean_removed: false + close_removed: false + {%- endfor %} + {%- endif %} + +- type: log + paths: + - /nsm/suricata/eve*.json + fields: + module: suricata + dataset: common + category: network + + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + + fields_under_root: true + clean_removed: false + close_removed: false + +- type: log + paths: + - /nsm/import/*/suricata/eve*.json + fields: + module: suricata + dataset: common + category: network + imported: true + processors: + - add_tags: + tags: ["import"] + - dissect: + tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}" + field: "log.file.path" + target_prefix: "" + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + + fields_under_root: true + clean_removed: false + close_removed: false {%- if STRELKAENABLED == 1 %} - type: log paths: diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index f503e5de1..cd215e242 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -23,22 +23,4 @@ securityonion_filebeat: var.paths: ["/logs/redis.log"] slowlog: enabled: false - suricata: - eve: - enabled: true - var.paths: ["/nsm/suricata/eve*.json"] - {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} - {%- if ZEEKVER != 'SURICATA' %} - zeek: - {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} - {% if LOGNAME in ZEEKLOGLOOKUP.keys() %} - {% set FILESET = ZEEKLOGLOOKUP.get(LOGNAME) %} - {% else %} - {% set FILESET = LOGNAME %} - {% endif %} - {{ FILESET }}: - enabled: true - var.paths: ["/nsm/zeek/logs/current/{{ LOGNAME }}.log"] - {%- endfor %} - {%- endif %} - {%- endif %} + \ No newline at end of file From 3e138cbc6dda28b3de009239f61b0e6c63f6a169 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 13:14:46 -0400 Subject: [PATCH 185/195] Revert to SO taxonomy for zeek and suricata --- .../config/so/9000_output_zeek.conf.jinja | 25 +++++++++---------- .../config/so/9050_output_elasticsearch.jinja | 24 ++++++++---------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index 486d22bfe..d17dc2b22 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -4,17 +4,16 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [event][module] == 'zeek' { - elasticsearch { - id => "zeek_logs" - pipeline => "filebeat-%{[agent][version]}-zeek-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-zeek-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } + if [module] =~ "zeek" and "import" not in [tags] { + elasticsearch { + pipeline => "%{module}.%{dataset}" + hosts => "{{ ES }}" + index => "so-zeek" + template_name => "so-zeek" + template => "/templates/so-zeek-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } } diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja index eb0d8ef0c..5013bafc1 100644 --- a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja @@ -4,17 +4,15 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [event][module] == 'elasticsearch' { - elasticsearch { - id => "elastic_logs" - pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } + if [module] =~ "suricata" and "import" not in [tags] { + elasticsearch { + pipeline => "%{module}.%{dataset}" + hosts => "{{ ES }}" + index => "so-ids" + template_name => "so-ids" + template => "/templates/so-ids-template.json" + ssl => true + ssl_certificate_verification => false + } + } } From a959ec1eb1787b736df121c362863e48fb3f11a5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 13:23:31 -0400 Subject: [PATCH 186/195] Revert to SO taxonomy for zeek and suricata --- .../config/so/9050_output_elasticsearch.jinja | 24 ++++++++++--------- .../config/so/9400_output_suricata.conf.jinja | 8 +++---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja index 5013bafc1..eb0d8ef0c 100644 --- a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja @@ -4,15 +4,17 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [module] =~ "suricata" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ ES }}" - index => "so-ids" - template_name => "so-ids" - template => "/templates/so-ids-template.json" - ssl => true - ssl_certificate_verification => false - } - } + if [event][module] == 'elasticsearch' { + elasticsearch { + id => "elastic_logs" + pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 999b270cf..b56f35a29 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -4,15 +4,15 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [event][module] =~ "suricata" and "import" not in [tags] { + if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { - pipeline => "filebeat-%{[agent][version]}-suricata-%{[fileset][name]}-pipeline" + pipeline => "%{module}.%{dataset}" hosts => "{{ ES }}" index => "so-ids" - template_name => "so-ids-%{+YYYY.MM.dd}" + template_name => "so-ids" template => "/templates/so-ids-template.json" ssl => true ssl_certificate_verification => false } } -} +} \ No newline at end of file From 88eea03f97cf8d87c5b18d9bf343c1b8bd15f810 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 13:36:50 -0400 Subject: [PATCH 187/195] Revert to SO taxonomy for zeek and suricata --- salt/soc/files/soc/hunt.queries.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/salt/soc/files/soc/hunt.queries.json b/salt/soc/files/soc/hunt.queries.json index c220060dd..2aaef8e59 100644 --- a/salt/soc/files/soc/hunt.queries.json +++ b/salt/soc/files/soc/hunt.queries.json @@ -2,8 +2,8 @@ { "name": "Default Query", "description": "Show all events grouped by the origin host", "query": "* | groupby observer.name"}, { "name": "Log Type", "description": "Show all events grouped by module and dataset", "query": "* | groupby event.module event.dataset"}, { "name": "Elastalerts", "description": "", "query": "_type:elastalert | groupby rule.name"}, - { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.kind: alert | groupby event.module"}, - { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert", "query": "event.category: network AND event.kind: alert | groupby rule.category rule.gid rule.uuid rule.name"}, + { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.dataset: alert | groupby event.module"}, + { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert", "query": "event.category: network AND event.dataset: alert | groupby rule.category rule.gid rule.uuid rule.name"}, { "name": "Wazuh/OSSEC Alerts", "description": "Show all Wazuh alerts at Level 5 or higher grouped by category", "query": "event.module:ossec AND event.dataset:alert AND rule.level:>4 | groupby rule.category rule.name"}, { "name": "Wazuh/OSSEC Alerts", "description": "Show all Wazuh alerts at Level 4 or lower grouped by category", "query": "event.module:ossec AND event.dataset:alert AND rule.level:<5 | groupby rule.category rule.name"}, { "name": "Wazuh/OSSEC Users and Commands", "description": "Show all Wazuh alerts grouped by username and command line", "query": "event.module:ossec AND event.dataset:alert | groupby user.escalated.keyword process.command_line"}, @@ -11,20 +11,20 @@ { "name": "Sysmon Events", "description": "Show all Sysmon logs grouped by event type", "query": "event.module:sysmon | groupby event.dataset"}, { "name": "Sysmon Usernames", "description": "Show all Sysmon logs grouped by username", "query": "event.module:sysmon | groupby event.dataset, user.name.keyword"}, { "name": "Strelka", "description": "Show all Strelka logs grouped by file type", "query": "event.module:strelka | groupby file.mime_type"}, - { "name": "Zeek Notice", "description": "Show notices from Zeek", "query": "event.dataset:zeek.notice | groupby notice.note notice.message"}, - { "name": "Connections", "description": "Connections grouped by IP and Port", "query": "event.dataset:zeek.connection | groupby source.ip destination.ip network.protocol destination.port"}, - { "name": "Connections", "description": "Connections grouped by Service", "query": "event.dataset:zeek.connection | groupby network.protocol destination.port"}, - { "name": "Connections", "description": "Connections grouped by destination country", "query": "event.dataset:zeek.connection | groupby destination.geo.country_name"}, - { "name": "Connections", "description": "Connections grouped by source country", "query": "event.dataset:zeek.connection | groupby source.geo.country_name"}, + { "name": "Zeek Notice", "description": "Show notices from Zeek", "query": "event.dataset:notice | groupby notice.note notice.message"}, + { "name": "Connections", "description": "Connections grouped by IP and Port", "query": "event.dataset:conn | groupby source.ip destination.ip network.protocol destination.port"}, + { "name": "Connections", "description": "Connections grouped by Service", "query": "event.dataset:conn | groupby network.protocol destination.port"}, + { "name": "Connections", "description": "Connections grouped by destination country", "query": "event.dataset:conn | groupby destination.geo.country_name"}, + { "name": "Connections", "description": "Connections grouped by source country", "query": "event.dataset:conn | groupby source.geo.country_name"}, { "name": "DCE_RPC", "description": "DCE_RPC grouped by operation", "query": "event.dataset:dce_rpc | groupby dce_rpc.operation"}, { "name": "DHCP", "description": "DHCP leases", "query": "event.dataset:dhcp | groupby host.hostname client.address"}, { "name": "DHCP", "description": "DHCP grouped by message type", "query": "event.dataset:dhcp | groupby dhcp.message_types"}, { "name": "DNP3", "description": "DNP3 grouped by reply", "query": "event.dataset:dnp3 | groupby dnp3.fc_reply"}, - { "name": "DNS", "description": "DNS queries grouped by port", "query": "_exists_:dns.id | groupby dns.question.name destination.port"}, - { "name": "DNS", "description": "DNS queries grouped by type", "query": "_exists_:dns.id | groupby dns.question.type destination.port"}, - { "name": "DNS", "description": "DNS queries grouped by response code", "query": "_exists_:dns.id | groupby dns.response_code destination.port"}, - { "name": "DNS", "description": "DNS highest registered domain", "query": "_exists_:dns.id | groupby dns.question.top_level_domain destination.port"}, - { "name": "DNS", "description": "DNS grouped by parent domain", "query": "_exists_:dns.id | groupby dns.question.registered_domain destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by port", "query": "event.dataset:dns | groupby dns.query.name destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by type", "query": "event.dataset:dns | groupby dns.query.type_name destination.port"}, + { "name": "DNS", "description": "DNS queries grouped by response code", "query": "event.dataset:dns | groupby dns.response.code_name destination.port"}, + { "name": "DNS", "description": "DNS highest registered domain", "query": "event.dataset:dns | groupby dns.highest_registered_domain.keyword destination.port"}, + { "name": "DNS", "description": "DNS grouped by parent domain", "query": "event.dataset:dns | groupby dns.parent_domain.keyword destination.port"}, { "name": "DPD", "description": "Dynamic Protocol Detection errors", "query": "event.dataset:dpd | groupby error.reason"}, { "name": "Files", "description": "Files grouped by mimetype", "query": "event.dataset:file | groupby file.mime_type source.ip"}, { "name": "Files", "description": "Files grouped by source", "query": "event.dataset:file | groupby file.source source.ip"}, @@ -63,4 +63,4 @@ { "name": "x509", "description": "x.509 grouped by name and issuer", "query": "event.dataset:x509 | groupby x509.san_dns x509.certificate.issuer"}, { "name": "x509", "description": "x.509 grouped by name and subject", "query": "event.dataset:x509 | groupby x509.san_dns x509.certificate.subject"}, { "name": "Firewall", "description": "Firewall events grouped by action", "query": "event.dataset:firewall | groupby rule.action"} - ] + ] \ No newline at end of file From 33db9023eb2334262a0bafaba9bc5780777fb6b4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 8 Jun 2021 13:50:39 -0400 Subject: [PATCH 188/195] Revert to SO taxonomy for zeek and suricata --- pillar/logstash/search.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 2da8e6c59..8306ec58b 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -7,6 +7,7 @@ logstash: - so/9000_output_zeek.conf.jinja - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja + - so/9050_output_elasticsearch.jinja - so/9100_output_osquery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja From 264080546c97d0826b6c5576678ef5d205775a43 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 11:37:27 -0400 Subject: [PATCH 189/195] Add log path --- salt/filebeat/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 5cabaf828..175213008 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -119,6 +119,7 @@ so-filebeat: - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro + - /opt/so/log:/logs:ro - port_bindings: - 0.0.0.0:514:514/udp - 0.0.0.0:514:514/tcp From 579ff8c0b41291a369b067e2942a057dd1aa2781 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 11:40:17 -0400 Subject: [PATCH 190/195] Add verbosity to checkin --- salt/common/tools/sbin/so-checkin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-checkin b/salt/common/tools/sbin/so-checkin index 90eae6b1e..c70701b71 100755 --- a/salt/common/tools/sbin/so-checkin +++ b/salt/common/tools/sbin/so-checkin @@ -17,4 +17,4 @@ . /usr/sbin/so-common -salt-call state.highstate +salt-call state.highstate -linfo From 4c90a0ed7e02550ec68b48dc2eee479402256c8b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 12:04:32 -0400 Subject: [PATCH 191/195] Add templates for SO logs --- ...ja => 9050_output_elasticsearch_log.jinja} | 0 .../config/so/9050_output_kibana_log.jinja | 20 +++++++++++++++++++ .../config/so/9050_output_logstash_log.jinja | 20 +++++++++++++++++++ .../config/so/9050_output_redis_log.jinja | 20 +++++++++++++++++++ 4 files changed, 60 insertions(+) rename salt/logstash/pipelines/config/so/{9050_output_elasticsearch.jinja => 9050_output_elasticsearch_log.jinja} (100%) create mode 100644 salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja create mode 100644 salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja create mode 100644 salt/logstash/pipelines/config/so/9050_output_redis_log.jinja diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja similarity index 100% rename from salt/logstash/pipelines/config/so/9050_output_elasticsearch.jinja rename to salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja diff --git a/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja b/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja new file mode 100644 index 000000000..bf79e1073 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja @@ -0,0 +1,20 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +output { + if [event][module] == 'kibana' { + elasticsearch { + id => "kibana_logs" + pipeline => "filebeat-%{[agent][version]}-kibana-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} diff --git a/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja b/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja new file mode 100644 index 000000000..df6fba0e0 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja @@ -0,0 +1,20 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +output { + if [event][module] == 'logstash' { + elasticsearch { + id => "logstash_logs" + pipeline => "filebeat-%{[agent][version]}-logstash-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} diff --git a/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja b/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja new file mode 100644 index 000000000..9cc37de35 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja @@ -0,0 +1,20 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +output { + if [event][module] == 'redis' { + elasticsearch { + id => "redis_logs" + pipeline => "filebeat-%{[agent][version]}-redis-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-grid-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} From 1c7741fdbe6055ec123c8bd4c28370bb2db5a2b2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 12:38:19 -0400 Subject: [PATCH 192/195] Add templates for SO logs --- salt/elasticsearch/templates/so/so-common-template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 2d1ef2a21..54e786cdc 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -1,5 +1,5 @@ { - "index_patterns": ["so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"], + "index_patterns": ["so-grid-*","so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"], "version":50001, "order":10, "settings":{ From 7fba904f750273d84ea53c7ef2b0c2eb5de8acae Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 9 Jun 2021 15:32:39 -0400 Subject: [PATCH 193/195] Dynamix Pipelines take 1 --- pillar/logstash/search.sls | 2 +- .../pipelines/config/so/0009_input_beats.conf | 5 +++++ .../so/9050_output_elasticsearch_log.jinja | 20 ------------------- ...=> 9050_output_filebeatmodules.conf.jinja} | 10 +++++----- .../config/so/9050_output_kibana_log.jinja | 20 ------------------- .../config/so/9050_output_logstash_log.jinja | 20 ------------------- 6 files changed, 11 insertions(+), 66 deletions(-) delete mode 100644 salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja rename salt/logstash/pipelines/config/so/{9050_output_redis_log.jinja => 9050_output_filebeatmodules.conf.jinja} (70%) delete mode 100644 salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja delete mode 100644 salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 8306ec58b..10fab2ed1 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -7,7 +7,7 @@ logstash: - so/9000_output_zeek.conf.jinja - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja - - so/9050_output_elasticsearch.jinja + - so/9050_output_filebeatmodules.conf.jinja - so/9100_output_osquery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf index a5c1d491c..31ba798c9 100644 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ b/salt/logstash/pipelines/config/so/0009_input_beats.conf @@ -3,4 +3,9 @@ input { port => "5044" tags => [ "beat-ext" ] } +} +filter { + mutate { + rename => {“@metadata” => “metadata”} + } } \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja deleted file mode 100644 index eb0d8ef0c..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja +++ /dev/null @@ -1,20 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -output { - if [event][module] == 'elasticsearch' { - elasticsearch { - id => "elastic_logs" - pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja similarity index 70% rename from salt/logstash/pipelines/config/so/9050_output_redis_log.jinja rename to salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja index 9cc37de35..20e9f0c0a 100644 --- a/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja @@ -4,12 +4,12 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [event][module] == 'redis' { + if [metadata][pipeline] { elasticsearch { - id => "redis_logs" - pipeline => "filebeat-%{[agent][version]}-redis-%{[fileset][name]}-pipeline" + id => "filebeat_modules_metadata_pipeline" + pipeline => "%{[metadata][pipeline]}" hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" + index => "so-%{[event][module]}-%{+YYYY.MM.dd}" template_name => "so-common" template => "/templates/so-common-template.json" template_overwrite => true @@ -17,4 +17,4 @@ output { ssl_certificate_verification => false } } -} +} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja b/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja deleted file mode 100644 index bf79e1073..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja +++ /dev/null @@ -1,20 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -output { - if [event][module] == 'kibana' { - elasticsearch { - id => "kibana_logs" - pipeline => "filebeat-%{[agent][version]}-kibana-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja b/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja deleted file mode 100644 index df6fba0e0..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja +++ /dev/null @@ -1,20 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -output { - if [event][module] == 'logstash' { - elasticsearch { - id => "logstash_logs" - pipeline => "filebeat-%{[agent][version]}-logstash-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } -} From 1bef1d565235d24a21cd62394576616fcc421406 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 10 Jun 2021 08:16:00 -0400 Subject: [PATCH 194/195] Update to apply to any so-prefixed index --- salt/elasticsearch/templates/so/so-common-template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 54e786cdc..3e47fd780 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -1,5 +1,5 @@ { - "index_patterns": ["so-grid-*","so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"], + "index_patterns": ["so-*"], "version":50001, "order":10, "settings":{ From 12d4d4a4f757bef7b6287958bba4c0847eb25dd2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 10 Jun 2021 09:19:15 -0400 Subject: [PATCH 195/195] Dynamix Pipelines take 2 --- .../templates/so/so-common-template.json | 2 +- salt/filebeat/securityoniondefaults.yaml | 15 ++++++++++----- .../pipelines/config/so/0009_input_beats.conf | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 54e786cdc..3e47fd780 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -1,5 +1,5 @@ { - "index_patterns": ["so-grid-*","so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"], + "index_patterns": ["so-*"], "version":50001, "order":10, "settings":{ diff --git a/salt/filebeat/securityoniondefaults.yaml b/salt/filebeat/securityoniondefaults.yaml index cd215e242..be4f81bd1 100644 --- a/salt/filebeat/securityoniondefaults.yaml +++ b/salt/filebeat/securityoniondefaults.yaml @@ -2,25 +2,30 @@ {% set ZEEKLOGLOOKUP = { 'conn': 'connection', } %} - securityonion_filebeat: modules: + {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone','so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %} elasticsearch: server: enabled: true var.paths: ["/logs/elasticsearch/*.log"] - kibana: - log: - enabled: true - var.paths: ["/logs/kibana/kibana.log"] logstash: log: enabled: true var.paths: ["/logs/logstash.log"] + {%- endif %} + {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %} + kibana: + log: + enabled: true + var.paths: ["/logs/kibana/kibana.log"] + {%- endif %} + {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode'] %} redis: log: enabled: true var.paths: ["/logs/redis.log"] slowlog: enabled: false + {%- endif %} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf index 31ba798c9..9ca55b184 100644 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ b/salt/logstash/pipelines/config/so/0009_input_beats.conf @@ -6,6 +6,6 @@ input { } filter { mutate { - rename => {“@metadata” => “metadata”} + rename => {"@metadata" => "metadata"} } } \ No newline at end of file