diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls new file mode 100644 index 000000000..654afd2b3 --- /dev/null +++ b/pillar/logstash/eval.sls @@ -0,0 +1,4 @@ +logstash: + pipelines: + eval: + config: "/usr/share/logstash/pipelines/eval/*.conf" diff --git a/pillar/logstash/mastersearch.sls b/pillar/logstash/search.sls similarity index 56% rename from pillar/logstash/mastersearch.sls rename to pillar/logstash/search.sls index 2fbc5be5f..0eca8571f 100644 --- a/pillar/logstash/mastersearch.sls +++ b/pillar/logstash/search.sls @@ -1,6 +1,4 @@ logstash: pipelines: - master: - config: "/usr/share/logstash/pipelines/master/*.conf" search: config: "/usr/share/logstash/pipelines/search/*.conf" diff --git a/pillar/top.sls b/pillar/top.sls index 8e8c22de3..8b604283e 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -4,7 +4,8 @@ base: 'G@role:so-mastersearch or G@role:so-heavynode': - match: compound - - logstash.mastersearch + - logstash.master + - logstash.search 'G@role:so-sensor': - static @@ -29,6 +30,7 @@ base: - data.* - brologs - auth + - logstash.eval - minions.{{ grains.id }} 'G@role:so-node': diff --git a/salt/auth/init.sls b/salt/auth/init.sls index bed7d18d5..18850d534 100644 --- a/salt/auth/init.sls +++ b/salt/auth/init.sls @@ -1,3 +1,6 @@ +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} + so-auth-api-dir: file.directory: - name: /opt/so/conf/auth/api @@ -5,19 +8,9 @@ so-auth-api-dir: - group: 939 - makedirs: True -so-auth-api-image: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-auth-api:HH1.1.4 - -so-auth-ui-image: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-auth-ui:HH1.1.4 - so-auth-api: docker_container.running: - - require: - - so-auth-api-image - - image: docker.io/soshybridhunter/so-auth-api:HH1.1.4 + - image: {{ MASTER }}:5000/soshybridhunter/so-auth-api:{{ VERSION }} - hostname: so-auth-api - name: so-auth-api - environment: @@ -29,9 +22,7 @@ so-auth-api: so-auth-ui: docker_container.running: - - require: - - so-auth-ui-image - - image: docker.io/soshybridhunter/so-auth-ui:HH1.1.4 + - image: {{ MASTER }}:5000/soshybridhunter/so-auth-ui:{{ VERSION }} - hostname: so-auth-ui - name: so-auth-ui - port_bindings: diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 45936c180..2eb2092f4 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -1,5 +1,5 @@ {%- if grains.role == 'so-heavynode' %} -{%- set MASTER = grains.host %} +{%- set MASTER = salt['pillar.get']('sensor:mainip' '') %} {%- else %} {%- set MASTER = grains['master'] %} {%- endif %} diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 7dd59ef01..b058f1408 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -13,14 +13,13 @@ # along with this program. If not, see . {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} {% set MASTER = salt['grains.get']('master') %} -{%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} +{% set MASTERIP = salt['pillar.get']('static:masterip', '') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} {% if FEATURES %} {% set FEATURES = "-features" %} {% else %} {% set FEATURES = '' %} {% endif %} -# Filebeat Setup filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc diff --git a/salt/logstash/conf/pipelines/eval/0010_input_hhbeats.conf b/salt/logstash/conf/pipelines/eval/0010_input_hhbeats.conf deleted file mode 100644 index 6b7667f5c..000000000 --- a/salt/logstash/conf/pipelines/eval/0010_input_hhbeats.conf +++ /dev/null @@ -1,40 +0,0 @@ -input { - beats { - port => "5644" - ssl => true - ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] - ssl_certificate => "/usr/share/logstash/filebeat.crt" - ssl_key => "/usr/share/logstash/filebeat.key" - tags => [ "beat" ] - } -} -filter { - if [type] == "ids" or [type] =~ "bro" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "sensor_name" => "%{[beat][name]}" } - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] =~ "ossec" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] == "osquery" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_tag => ["osquery"] - } - json { - source => "message" - target => "osquery" - } - } -} diff --git a/salt/logstash/conf/pipelines/eval/templates/0800_input_eval.conf b/salt/logstash/conf/pipelines/eval/templates/0800_input_eval.conf index 08237884f..e0f2e132e 100644 --- a/salt/logstash/conf/pipelines/eval/templates/0800_input_eval.conf +++ b/salt/logstash/conf/pipelines/eval/templates/0800_input_eval.conf @@ -9,173 +9,173 @@ input { } file { path => "/nsm/zeek/logs/current/conn*.log" - type => "zeek_conn" - tags => ["zeek"] + type => "bro_conn" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/dce_rpc*.log" - type => "zeek_dce_rpc" - tags => ["zeek"] + type => "bro_dce_rpc" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/dhcp*.log" - type => "zeek_dhcp" - tags => ["zeek"] + type => "bro_dhcp" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/dnp3*.log" - type => "zeek_dnp3" - tags => ["zeek"] + type => "bro_dnp3" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/dns*.log" - type => "zeek_dns" - tags => ["zeek"] + type => "bro_dns" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/dpd*.log" - type => "zeek_dpd" - tags => ["zeek"] + type => "bro_dpd" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/files*.log" - type => "zeek_files" - tags => ["zeek"] + type => "bro_files" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/ftp*.log" - type => "zeek_ftp" - tags => ["zeek"] + type => "bro_ftp" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/http*.log" - type => "zeek_http" - tags => ["zeek"] + type => "bro_http" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/intel*.log" - type => "zeek_intel" - tags => ["zeek"] + type => "bro_intel" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/irc*.log" - type => "zeek_irc" - tags => ["zeek"] + type => "bro_irc" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/kerberos*.log" - type => "zeek_kerberos" - tags => ["zeek"] + type => "bro_kerberos" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/modbus*.log" - type => "zeek_modbus" - tags => ["zeek"] + type => "bro_modbus" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/mysql*.log" - type => "zeek_mysql" - tags => ["zeek"] + type => "bro_mysql" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/notice*.log" - type => "zeek_notice" - tags => ["zeek"] + type => "bro_notice" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/ntlm*.log" - type => "zeek_ntlm" - tags => ["zeek"] + type => "bro_ntlm" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/pe*.log" - type => "zeek_pe" - tags => ["zeek"] + type => "bro_pe" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/radius*.log" - type => "zeek_radius" - tags => ["zeek"] + type => "bro_radius" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/rdp*.log" - type => "zeek_rdp" - tags => ["zeek"] + type => "bro_rdp" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/rfb*.log" - type => "zeek_rfb" - tags => ["zeek"] + type => "bro_rfb" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/signatures*.log" - type => "zeek_signatures" - tags => ["zeek"] + type => "bro_signatures" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/sip*.log" - type => "zeek_sip" - tags => ["zeek"] + type => "bro_sip" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/smb_files*.log" - type => "zeek_smb_files" - tags => ["zeek"] + type => "bro_smb_files" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/smb_mapping*.log" - type => "zeek_smb_mapping" - tags => ["zeek"] + type => "bro_smb_mapping" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/smtp*.log" - type => "zeek_smtp" - tags => ["zeek"] + type => "bro_smtp" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/snmp*.log" - type => "zeek_snmp" - tags => ["zeek"] + type => "bro_snmp" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/socks*.log" - type => "zeek_socks" - tags => ["zeek"] + type => "bro_socks" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/software*.log" - type => "zeek_software" - tags => ["zeek"] + type => "bro_software" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/ssh*.log" - type => "zeek_ssh" - tags => ["zeek"] + type => "bro_ssh" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/ssl*.log" - type => "zeek_ssl" - tags => ["zeek"] + type => "bro_ssl" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/syslog*.log" - type => "zeek_syslog" - tags => ["zeek"] + type => "bro_syslog" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/tunnel*.log" - type => "zeek_tunnels" - tags => ["zeek"] + type => "bro_tunnels" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/weird*.log" - type => "zeek_weird" - tags => ["zeek"] + type => "bro_weird" + tags => ["bro"] } file { path => "/nsm/zeek/logs/current/x509*.log" - type => "zeek_x509" - tags => ["zeek"] + type => "bro_x509" + tags => ["bro"] } file { path => "/wazuh/alerts/alerts.json" diff --git a/salt/logstash/files/dynamic/0008_input_eval.conf b/salt/logstash/files/dynamic/0008_input_eval.conf new file mode 100644 index 000000000..b02f9d516 --- /dev/null +++ b/salt/logstash/files/dynamic/0008_input_eval.conf @@ -0,0 +1,203 @@ +# Updated by: Mike Reeves +# Last Update: 11/1/2018 + +input { + file { + path => "/suricata/eve.json" + type => "ids" + add_field => { "engine" => "suricata" } + } + file { + path => "/nsm/bro/logs/current/conn*.log" + type => "bro_conn" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/dce_rpc*.log" + type => "bro_dce_rpc" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/dhcp*.log" + type => "bro_dhcp" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/dnp3*.log" + type => "bro_dnp3" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/dns*.log" + type => "bro_dns" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/dpd*.log" + type => "bro_dpd" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/files*.log" + type => "bro_files" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/ftp*.log" + type => "bro_ftp" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/http*.log" + type => "bro_http" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/intel*.log" + type => "bro_intel" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/irc*.log" + type => "bro_irc" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/kerberos*.log" + type => "bro_kerberos" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/modbus*.log" + type => "bro_modbus" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/mysql*.log" + type => "bro_mysql" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/notice*.log" + type => "bro_notice" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/ntlm*.log" + type => "bro_ntlm" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/pe*.log" + type => "bro_pe" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/radius*.log" + type => "bro_radius" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/rdp*.log" + type => "bro_rdp" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/rfb*.log" + type => "bro_rfb" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/signatures*.log" + type => "bro_signatures" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/sip*.log" + type => "bro_sip" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/smb_files*.log" + type => "bro_smb_files" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/smb_mapping*.log" + type => "bro_smb_mapping" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/smtp*.log" + type => "bro_smtp" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/snmp*.log" + type => "bro_snmp" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/socks*.log" + type => "bro_socks" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/software*.log" + type => "bro_software" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/ssh*.log" + type => "bro_ssh" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/ssl*.log" + type => "bro_ssl" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/syslog*.log" + type => "bro_syslog" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/tunnel*.log" + type => "bro_tunnels" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/weird*.log" + type => "bro_weird" + tags => ["bro"] + } + file { + path => "/nsm/bro/logs/current/x509*.log" + type => "bro_x509" + tags => ["bro"] + } + file { + path => "/wazuh/alerts/alerts.json" + type => "ossec" + } + file { + path => "/wazuh/archives/archive.json" + type => "ossec_archive" + } + file { + path => "/osquery/logs/result.log" + type => "osquery" + } + file { + path => "/strelka/strelka.log" + type => "strelka" + } +} +filter { + if "import" in [tags] { + mutate { + #add_tag => [ "conf_file_0007"] + } + } +} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 4598ae53c..c61bee921 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -130,7 +130,7 @@ lspipelinesyml: - name: /opt/so/conf/logstash/etc/pipelines.yml - source: salt://logstash/etc/pipelines.yml.jinja - template: jinja - - defaults: + - defaults: pipelines: {{ pipelines }} # Copy down all the configs including custom - TODO add watch restart @@ -166,7 +166,7 @@ lsconfsync: - source: salt://logstash/conf/conf.enabled.txt.so-master {% else %} - source: salt://logstash/conf/conf.enabled.txt.{{ nodetype }} -{% endif %} +{% endif %} - user: 931 - group: 939 - template: jinja diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 57579f6ca..83c7c92e4 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -2,7 +2,7 @@ {% set master_minion_id = master.split(".")[0] %} {%- set masterip = salt['pillar.get']('static:masterip', '') -%} -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-heavynode' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} {% set trusttheca_text = salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %} {% set ca_server = grains.id %} {% else %} @@ -41,7 +41,7 @@ m2cryptopkgs: bits: 4096 backup: True -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-heavynode' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %} # Request a cert and drop it where it needs to go to be distributed /etc/pki/filebeat.crt: diff --git a/setup/functions.sh b/setup/functions.sh index 0872d3f31..48060847b 100644 --- a/setup/functions.sh +++ b/setup/functions.sh @@ -651,7 +651,7 @@ install_master() { ls_heapsize() { # Determine LS Heap Size - if [ $TOTAL_MEM -ge 32000 ] ; then + if [ $TOTAL_MEM -ge 32000 ] || [ $INSTALLTYPE == 'EVALMODE' ]; then LS_HEAP_SIZE="1000m" else # If minimal RAM, then set minimal heap diff --git a/setup/so-setup.sh b/setup/so-setup.sh index dd1970191..2313d7786 100644 --- a/setup/so-setup.sh +++ b/setup/so-setup.sh @@ -843,7 +843,7 @@ if (whiptail_you_sure) ; then whiptail_log_size_limit else NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE - NODE_LS_HEAP_SIZE=1000m + NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE LSPIPELINEWORKERS=$CPUCORES LSPIPELINEBATCH=125 LSINPUTTHREADS=1