diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index cff5fdcdf..bc2a7c86b 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -8,7 +8,6 @@ {% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %} {% set ELASTALERT = salt['pillar.get']('elastalert:enabled', True) %} {% set ELASTICSEARCH = salt['pillar.get']('elasticsearch:enabled', True) %} -{% set FILEBEAT = salt['pillar.get']('filebeat:enabled', True) %} {% set KIBANA = salt['pillar.get']('kibana:enabled', True) %} {% set LOGSTASH = salt['pillar.get']('logstash:enabled', True) %} {% set CURATOR = salt['pillar.get']('curator:enabled', True) %} @@ -188,7 +187,6 @@ 'pcap', 'suricata', 'healthcheck', - 'filebeat', 'schedule', 'tcpreplay', 'docker_clean' @@ -204,10 +202,6 @@ ], }, grain='role') %} - {% if FILEBEAT and grains.role in ['so-helixsensor', 'so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import', 'so-receiver'] %} - {% do allowed_states.append('filebeat') %} - {% endif %} - {% if (PLAYBOOK != 0) and grains.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone'] %} {% do allowed_states.append('mysql') %} {% endif %} diff --git a/salt/common/tools/sbin/so-elastic-auth-password-reset b/salt/common/tools/sbin/so-elastic-auth-password-reset index 17404e953..5bc5230c3 100755 --- a/salt/common/tools/sbin/so-elastic-auth-password-reset +++ b/salt/common/tools/sbin/so-elastic-auth-password-reset @@ -95,8 +95,6 @@ function soUserSync() { $(dirname $0)/so-user sync printf "\nApplying logstash state to the appropriate nodes.\n\n" salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode' state.apply logstash queue=True - printf "\nApplying filebeat state to the appropriate nodes.\n\n" - salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-searchnode or G@role:so-heavynode or G@role:so-sensor or G@role:so-fleet' state.apply filebeat queue=True printf "\nApplying kibana state to the appropriate nodes.\n\n" salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True printf "\nApplying curator state to the appropriate nodes.\n\n" diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear index 08da4021c..fca676172 100755 --- a/salt/common/tools/sbin/so-elastic-clear +++ b/salt/common/tools/sbin/so-elastic-clear @@ -54,17 +54,10 @@ if [ $SKIP -ne 1 ]; then if [ "$INPUT" != "AGREE" ] ; then exit 0; fi fi -# Check to see if Logstash/Filebeat are running +# Check to see if Logstash are running LS_ENABLED=$(so-status | grep logstash) -FB_ENABLED=$(so-status | grep filebeat) EA_ENABLED=$(so-status | grep elastalert) -if [ ! -z "$FB_ENABLED" ]; then - - /usr/sbin/so-filebeat-stop - -fi - if [ ! -z "$LS_ENABLED" ]; then /usr/sbin/so-logstash-stop @@ -86,13 +79,7 @@ do curl -K /opt/so/conf/elasticsearch/curl.config-XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1 done -#Start Logstash/Filebeat -if [ ! -z "$FB_ENABLED" ]; then - - /usr/sbin/so-filebeat-start - -fi - +#Start Logstash if [ ! -z "$LS_ENABLED" ]; then /usr/sbin/so-logstash-start diff --git a/salt/common/tools/sbin/so-elastic-restart b/salt/common/tools/sbin/so-elastic-restart index 4fb8ae10b..67988193f 100755 --- a/salt/common/tools/sbin/so-elastic-restart +++ b/salt/common/tools/sbin/so-elastic-restart @@ -22,10 +22,6 @@ /usr/sbin/so-restart logstash $1 {%- endif %} -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-sensor']%} -/usr/sbin/so-restart filebeat $1 -{%- endif %} - {%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} /usr/sbin/so-restart curator $1 {%- endif %} diff --git a/salt/common/tools/sbin/so-elastic-start b/salt/common/tools/sbin/so-elastic-start index 04c076662..fd78d1859 100755 --- a/salt/common/tools/sbin/so-elastic-start +++ b/salt/common/tools/sbin/so-elastic-start @@ -22,10 +22,6 @@ /usr/sbin/so-start logstash $1 {%- endif %} -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-sensor']%} -/usr/sbin/so-start filebeat $1 -{%- endif %} - {%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} /usr/sbin/so-start curator $1 {%- endif %} diff --git a/salt/common/tools/sbin/so-elastic-stop b/salt/common/tools/sbin/so-elastic-stop index 45e8fd18b..88350a8fe 100755 --- a/salt/common/tools/sbin/so-elastic-stop +++ b/salt/common/tools/sbin/so-elastic-stop @@ -22,10 +22,6 @@ /usr/sbin/so-stop logstash $1 {%- endif %} -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-sensor']%} -/usr/sbin/so-stop filebeat $1 -{%- endif %} - {%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} /usr/sbin/so-stop curator $1 {%- endif %} diff --git a/salt/common/tools/sbin/so-restart b/salt/common/tools/sbin/so-restart index 3790625f7..dfedf290b 100755 --- a/salt/common/tools/sbin/so-restart +++ b/salt/common/tools/sbin/so-restart @@ -7,7 +7,7 @@ -# Usage: so-restart filebeat | kibana | playbook +# Usage: so-restart kibana | playbook . /usr/sbin/so-common @@ -27,5 +27,5 @@ if [ $# -ge 1 ]; then *) docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;; esac else - echo -e "\nPlease provide an argument by running like so-restart $component, or by using the component-specific script.\nEx. so-restart filebeat, or so-filebeat-restart\n" + echo -e "\nPlease provide an argument by running like so-restart $component, or by using the component-specific script.\nEx. so-restart logstash, or so-logstash-restart\n" fi diff --git a/salt/common/tools/sbin/so-start b/salt/common/tools/sbin/so-start index 6e208a6af..fbf3e4300 100755 --- a/salt/common/tools/sbin/so-start +++ b/salt/common/tools/sbin/so-start @@ -7,7 +7,7 @@ -# Usage: so-start all | filebeat | kibana | playbook +# Usage: so-start all | kibana | playbook . /usr/sbin/so-common @@ -27,5 +27,5 @@ if [ $# -ge 1 ]; then *) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; esac else - echo -e "\nPlease provide an argument by running like so-start $component, or by using the component-specific script.\nEx. so-start filebeat, or so-filebeat-start\n" + echo -e "\nPlease provide an argument by running like so-start $component, or by using the component-specific script.\nEx. so-start logstash, or so-logstash-start\n" fi diff --git a/salt/common/tools/sbin/so-stop b/salt/common/tools/sbin/so-stop index 3538b2fd3..32e24f83a 100755 --- a/salt/common/tools/sbin/so-stop +++ b/salt/common/tools/sbin/so-stop @@ -7,7 +7,7 @@ -# Usage: so-stop filebeat | kibana | playbook | thehive +# Usage: so-stop kibana | playbook | thehive . /usr/sbin/so-common @@ -20,6 +20,6 @@ if [ $# -ge 1 ]; then *) docker stop so-$1 ; docker rm so-$1 ;; esac else - echo -e "\nPlease provide an argument by running like so-stop $component, or by using the component-specific script.\nEx. so-stop filebeat, or so-filebeat-stop\n" + echo -e "\nPlease provide an argument by running like so-stop $component, or by using the component-specific script.\nEx. so-stop logstash, or so-logstash-stop\n" fi diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/files/bin/so-curator-close index 4d6fbe602..885cb4502 100644 --- a/salt/curator/files/bin/so-curator-close +++ b/salt/curator/files/bin/so-curator-close @@ -13,8 +13,6 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} - /usr/sbin/so-curator-closed-delete > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; @@ -27,6 +25,3 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; -{% for INDEX in MODULESMERGED.modules.keys() -%} -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% endfor -%} diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/files/bin/so-curator-cluster-close index 95d882619..0929149ed 100644 --- a/salt/curator/files/bin/so-curator-cluster-close +++ b/salt/curator/files/bin/so-curator-cluster-close @@ -13,8 +13,6 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} - docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-close.yml > /dev/null 2>&1; @@ -25,6 +23,3 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-close.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-close.yml > /dev/null 2>&1; -{% for INDEX in MODULESMERGED.modules.keys() -%} -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-close.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% endfor -%} diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/files/bin/so-curator-cluster-delete index 9ec5129af..2d71f725d 100644 --- a/salt/curator/files/bin/so-curator-cluster-delete +++ b/salt/curator/files/bin/so-curator-cluster-delete @@ -13,8 +13,6 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} - docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-delete.yml > /dev/null 2>&1; @@ -25,6 +23,3 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-delete.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-delete.yml > /dev/null 2>&1; -{% for INDEX in MODULESMERGED.modules.keys() -%} -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-delete.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% endfor -%} diff --git a/salt/curator/files/bin/so-curator-cluster-warm b/salt/curator/files/bin/so-curator-cluster-warm index 7de6dd391..f868caf0d 100644 --- a/salt/curator/files/bin/so-curator-cluster-warm +++ b/salt/curator/files/bin/so-curator-cluster-warm @@ -14,8 +14,6 @@ read lastPID < $lf [ ! -z "$lastPID" -a -d /proc/$lastPID ] && exit echo $$ > $lf -{% from 'filebeat/modules.map.jinja' import MODULESMERGED with context %} - docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-zeek-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-beats-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-firewall-warm.yml > /dev/null 2>&1; @@ -26,6 +24,3 @@ docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/cur docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-ossec-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-strelka-warm.yml > /dev/null 2>&1; docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-syslog-warm.yml > /dev/null 2>&1; -{% for INDEX in MODULESMERGED.modules.keys() -%} -docker exec so-curator curator --config /etc/curator/config/curator.yml /etc/curator/action/so-{{ INDEX }}-warm.yml > /dev/null 2>&1{% if not loop.last %};{% endif %} -{% endfor -%} diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 30b2c78e1..8ccb7cf2d 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -17,12 +17,6 @@ docker: port_bindings: - 0.0.0.0:9200:9200/tcp - 0.0.0.0:9300:9300/tcp - 'so-filebeat': - final_octet: 23 - port_bindings: - - 0.0.0.0:514:514/udp - - 0.0.0.0:514:514/tcp - - 0.0.0.0:5066:5066/tcp 'so-idstools': final_octet: 25 'so-influxdb': diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 70a676e89..f1e93ddea 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -32,7 +32,6 @@ 'so-elasticsearch', 'so-elastic-fleet', 'so-elastic-fleet-package-registry', - 'so-filebeat', 'so-influxdb', 'so-kibana', 'so-kratos', @@ -54,7 +53,6 @@ {% if GLOBALS.role == 'so-searchnode' %} {% set NODE_CONTAINERS = [ 'so-elasticsearch', - 'so-filebeat', 'so-logstash', 'so-nginx' ] %} @@ -64,7 +62,6 @@ {% set NODE_CONTAINERS = [ 'so-curator', 'so-elasticsearch', - 'so-filebeat', 'so-logstash', 'so-nginx', 'so-redis', @@ -83,7 +80,6 @@ 'so-elasticsearch', 'so-elastic-fleet', 'so-elastic-fleet-package-registry', - 'so-filebeat', 'so-influxdb', 'so-kibana', 'so-kratos', @@ -94,7 +90,6 @@ {% if GLOBALS.role == 'so-receiver' %} {% set NODE_CONTAINERS = [ - 'so-filebeat', 'so-logstash', 'so-redis', ] %} diff --git a/setup/so-functions b/setup/so-functions index 78b87a113..64248c30f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1544,7 +1544,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni curator soc soctopus docker zeek suricata nginx telegraf filebeat logstash soc manager kratos idstools idh elastalert;do + for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni curator soc soctopus docker zeek suricata nginx telegraf logstash soc manager kratos idstools idh elastalert;do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls diff --git a/setup/so-variables b/setup/so-variables index 0caf10ee7..6a3861e92 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -136,12 +136,6 @@ export suricata_pillar_file adv_suricata_pillar_file="$local_salt_dir/pillar/suricata/adv_suricata.sls" export adv_suricata_pillar_file -filebeat_pillar_file="$local_salt_dir/pillar/filebeat/soc_filebeat.sls" -export filebeat_pillar_file - -adv_filebeat_pillar_file="$local_salt_dir/pillar/filebeat/adv_filebeat.sls" -export adv_filebeat_pillar_file - logstash_pillar_file="$local_salt_dir/pillar/logstash/soc_logstash.sls" export logstash_pillar_file @@ -200,4 +194,4 @@ influxdb_pillar_file="$local_salt_dir/pillar/influxdb/soc_influxdb.sls" export influxdb_pillar_file adv_influxdb_pillar_file="$local_salt_dir/pillar/influxdb/adv_influxdb.sls" -export adv_influxdb_pillar_file \ No newline at end of file +export adv_influxdb_pillar_file