diff --git a/pillar/logstash/fleet.sls b/pillar/logstash/fleet.sls deleted file mode 100644 index fb70e7f0d..000000000 --- a/pillar/logstash/fleet.sls +++ /dev/null @@ -1,6 +0,0 @@ -logstash: - pipelines: - fleet: - config: - - so/0012_input_elastic_agent.conf - - so/9806_output_lumberjack_fleet.conf.jinja \ No newline at end of file diff --git a/pillar/logstash/helix.sls b/pillar/logstash/helix.sls deleted file mode 100644 index ddc1c745b..000000000 --- a/pillar/logstash/helix.sls +++ /dev/null @@ -1,42 +0,0 @@ -logstash: - pipelines: - helix: - config: - - so/0010_input_hhbeats.conf - - so/1033_preprocess_snort.conf - - so/1100_preprocess_bro_conn.conf - - so/1101_preprocess_bro_dhcp.conf - - so/1102_preprocess_bro_dns.conf - - so/1103_preprocess_bro_dpd.conf - - so/1104_preprocess_bro_files.conf - - so/1105_preprocess_bro_ftp.conf - - so/1106_preprocess_bro_http.conf - - so/1107_preprocess_bro_irc.conf - - so/1108_preprocess_bro_kerberos.conf - - so/1109_preprocess_bro_notice.conf - - so/1110_preprocess_bro_rdp.conf - - so/1111_preprocess_bro_signatures.conf - - so/1112_preprocess_bro_smtp.conf - - so/1113_preprocess_bro_snmp.conf - - so/1114_preprocess_bro_software.conf - - so/1115_preprocess_bro_ssh.conf - - so/1116_preprocess_bro_ssl.conf - - so/1117_preprocess_bro_syslog.conf - - so/1118_preprocess_bro_tunnel.conf - - so/1119_preprocess_bro_weird.conf - - so/1121_preprocess_bro_mysql.conf - - so/1122_preprocess_bro_socks.conf - - so/1123_preprocess_bro_x509.conf - - so/1124_preprocess_bro_intel.conf - - so/1125_preprocess_bro_modbus.conf - - so/1126_preprocess_bro_sip.conf - - so/1127_preprocess_bro_radius.conf - - so/1128_preprocess_bro_pe.conf - - so/1129_preprocess_bro_rfb.conf - - so/1130_preprocess_bro_dnp3.conf - - so/1131_preprocess_bro_smb_files.conf - - so/1132_preprocess_bro_smb_mapping.conf - - so/1133_preprocess_bro_ntlm.conf - - so/1134_preprocess_bro_dce_rpc.conf - - so/8001_postprocess_common_ip_augmentation.conf - - so/9997_output_helix.conf.jinja diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls deleted file mode 100644 index cee8eec02..000000000 --- a/pillar/logstash/manager.sls +++ /dev/null @@ -1,8 +0,0 @@ -logstash: - pipelines: - manager: - config: - - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/0013_input_lumberjack_fleet.conf - - so/9999_output_redis.conf.jinja \ No newline at end of file diff --git a/pillar/logstash/receiver.sls b/pillar/logstash/receiver.sls deleted file mode 100644 index 4d0637dde..000000000 --- a/pillar/logstash/receiver.sls +++ /dev/null @@ -1,8 +0,0 @@ -logstash: - pipelines: - receiver: - config: - - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/9999_output_redis.conf.jinja - diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls deleted file mode 100644 index 0b660b7ef..000000000 --- a/pillar/logstash/search.sls +++ /dev/null @@ -1,7 +0,0 @@ -logstash: - pipelines: - search: - config: - - so/0900_input_redis.conf.jinja - - so/9805_output_elastic_agent.conf.jinja - - so/9900_output_endgame.conf.jinja diff --git a/pillar/top.sls b/pillar/top.sls index ebcd6bbb2..ac46bfc12 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -1,47 +1,26 @@ base: '*': - - patch.needs_restarting - - ntp.soc_ntp - - ntp.adv_ntp - - logrotate + - global.soc_global + - global.adv_global - docker.soc_docker - docker.adv_docker - firewall.soc_firewall - firewall.adv_firewall + - influxdb.token + - logrotate.soc_logrotate + - logrotate.adv_logrotate + - nginx.soc_nginx + - nginx.adv_nginx + - node_data.ips + - ntp.soc_ntp + - ntp.adv_ntp + - patch.needs_restarting + - patch.soc_patch + - patch.adv_patch - sensoroni.soc_sensoroni - sensoroni.adv_sensoroni - telegraf.soc_telegraf - telegraf.adv_telegraf - - influxdb.token - - node_data.ips - - '* and not *_eval and not *_import': - - logstash.nodes - - '*_eval or *_heavynode or *_sensor or *_standalone or *_import': - - match: compound - - zeek.soc_zeek - - zeek.adv_zeek - - bpf.soc_bpf - - bpf.adv_bpf - - '*_managersearch or *_heavynode': - - match: compound - - logstash - - logstash.manager - - logstash.search - - logstash.soc_logstash - - logstash.adv_logstash - - elasticsearch.index_templates - - elasticsearch.soc_elasticsearch - - elasticsearch.adv_elasticsearch - - '*_manager': - - logstash - - logstash.manager - - logstash.soc_logstash - - logstash.adv_logstash - - elasticsearch.index_templates '*_manager or *_managersearch': - match: compound @@ -52,14 +31,19 @@ base: - kibana.secrets {% endif %} - secrets - - global.soc_global - - global.adv_global - manager.soc_manager - manager.adv_manager - idstools.soc_idstools - idstools.adv_idstools + - logstash.nodes + - logstash.soc_logstash + - logstash.adv_logstash - soc.soc_soc - soc.adv_soc + - soctopus.soc_soctopus + - soctopus.adv_soctopus + - kibana.soc_kibana + - kibana.adv_kibana - kratos.soc_kratos - kratos.adv_kratos - redis.soc_redis @@ -68,15 +52,29 @@ base: - influxdb.adv_influxdb - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elastalert.soc_elastalert + - elastalert.adv_elastalert - backup.soc_backup - backup.adv_backup + - curator.soc_curator + - curator.adv_curator + - soctopus.soc_soctopus + - soctopus.adv_soctopus - minions.{{ grains.id }} - minions.adv_{{ grains.id }} '*_sensor': - healthcheck.sensor - - global.soc_global - - global.adv_global + - strelka.soc_strelka + - strelka.adv_strelka + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -90,16 +88,23 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %} - kibana.secrets {% endif %} - - global.soc_global - - global.adv_global - kratos.soc_kratos - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elastalert.soc_elastalert + - elastalert.adv_elastalert - manager.soc_manager - manager.adv_manager - idstools.soc_idstools - idstools.adv_idstools - soc.soc_soc + - soc.adv_soc + - soctopus.soc_soctopus + - soctopus.adv_soctopus + - strelka.soc_strelka + - strelka.adv_strelka + - curator.soc_curator + - curator.adv_curator - kratos.soc_kratos - kratos.adv_kratos - redis.soc_redis @@ -108,13 +113,19 @@ base: - influxdb.adv_influxdb - backup.soc_backup - backup.adv_backup + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata - minions.{{ grains.id }} - minions.adv_{{ grains.id }} '*_standalone': - - logstash - - logstash.manager - - logstash.search + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.index_templates @@ -126,8 +137,6 @@ base: {% endif %} - secrets - healthcheck.standalone - - global.soc_global - - global.adv_global - idstools.soc_idstools - idstools.adv_idstools - kratos.soc_kratos @@ -138,50 +147,77 @@ base: - influxdb.adv_influxdb - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elastalert.soc_elastalert + - elastalert.adv_elastalert - manager.soc_manager - manager.adv_manager - soc.soc_soc + - soc.adv_soc + - soctopus.soc_soctopus + - soctopus.adv_soctopus + - strelka.soc_strelka + - strelka.adv_strelka + - curator.soc_curator + - curator.adv_curator - backup.soc_backup - backup.adv_backup + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata - minions.{{ grains.id }} - minions.adv_{{ grains.id }} '*_heavynode': - elasticsearch.auth - - global.soc_global - - global.adv_global + - logstash.nodes + - logstash.soc_logstash + - logstash.adv_logstash + - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch + - curator.soc_curator + - curator.adv_curator - redis.soc_redis + - redis.adv_redis + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata + - strelka.soc_strelka + - strelka.adv_strelka - minions.{{ grains.id }} - minions.adv_{{ grains.id }} '*_idh': - - global.soc_global - - global.adv_global - idh.soc_idh - idh.adv_idh - minions.{{ grains.id }} - minions.adv_{{ grains.id }} '*_searchnode': - - logstash - - logstash.search + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - - elasticsearch.index_templates - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} - redis.soc_redis - - global.soc_global - - global.adv_global + - redis.adv_redis - minions.{{ grains.id }} - minions.adv_{{ grains.id }} '*_receiver': - - logstash - - logstash.receiver + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} @@ -189,8 +225,6 @@ base: {% endif %} - redis.soc_redis - redis.adv_redis - - global.soc_global - - global.adv_global - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -206,11 +240,16 @@ base: - kratos.soc_kratos - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elastalert.soc_elastalert + - elastalert.adv_elastalert - manager.soc_manager - manager.adv_manager - soc.soc_soc - - global.soc_global - - global.adv_global + - soc.adv_soc + - soctopus.soc_soctopus + - soctopus.adv_soctopus + - curator.soc_curator + - curator.adv_curator - backup.soc_backup - backup.adv_backup - kratos.soc_kratos @@ -219,21 +258,28 @@ base: - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata + - strelka.soc_strelka + - strelka.adv_strelka - minions.{{ grains.id }} - minions.adv_{{ grains.id }} '*_fleet': - - global.soc_global - - global.adv_global - backup.soc_backup - backup.adv_backup - - logstash - - logstash.fleet + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - '*_workstation': + '*_desktop': - minions.{{ grains.id }} - minions.adv_{{ grains.id }} diff --git a/salt/common/init.sls b/salt/common/init.sls index f23a05757..8723cc3c5 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -49,13 +49,12 @@ so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - unless: ls /opt/so/conf/so-status/so-status.conf -sosaltstackperms: +socore_opso_perms: file.directory: - - name: /opt/so/saltstack + - name: /opt/so - user: 939 - group: 939 - - dir_mode: 770 - + so_log_perms: file.directory: - name: /opt/so/log @@ -112,21 +111,23 @@ elastic_curl_config: {% endif %} {% endif %} -# Sync some Utilities -utilsyncscripts: + +common_sbin: file.recurse: - name: /usr/sbin - - user: root - - group: root + - source: salt://common/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +common_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://common/tools/sbin_jinja + - user: 939 + - group: 939 - file_mode: 755 - template: jinja - - source: salt://common/tools/sbin - - exclude_pat: - - so-common - - so-firewall - - so-image-common - - soup - - so-status so-status_script: file.managed: diff --git a/salt/common/tools/sbin/so-elastic-restart b/salt/common/tools/sbin/so-elastic-restart deleted file mode 100755 index 67988193f..000000000 --- a/salt/common/tools/sbin/so-elastic-restart +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} -/usr/sbin/so-restart elasticsearch $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} -/usr/sbin/so-restart kibana $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-restart logstash $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-restart curator $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} -/usr/sbin/so-restart elastalert $1 -{%- endif %} diff --git a/salt/common/tools/sbin/so-elastic-start b/salt/common/tools/sbin/so-elastic-start deleted file mode 100755 index fd78d1859..000000000 --- a/salt/common/tools/sbin/so-elastic-start +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} -/usr/sbin/so-start elasticsearch $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} -/usr/sbin/so-start kibana $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-start logstash $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-start curator $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} -/usr/sbin/so-start elastalert $1 -{%- endif %} diff --git a/salt/common/tools/sbin/so-elastic-stop b/salt/common/tools/sbin/so-elastic-stop deleted file mode 100755 index 88350a8fe..000000000 --- a/salt/common/tools/sbin/so-elastic-stop +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} -/usr/sbin/so-stop elasticsearch $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} -/usr/sbin/so-stop kibana $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-stop logstash $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-stop curator $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} -/usr/sbin/so-stop elastalert $1 -{%- endif %} diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/common/tools/sbin/so-elasticsearch-indices-rw deleted file mode 100755 index 724dd9dcf..000000000 --- a/salt/common/tools/sbin/so-elasticsearch-indices-rw +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# -# -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -IP={{ salt['grains.get']('ip_interfaces').get(salt['pillar.get']('sensor:mainint', salt['pillar.get']('manager:mainint', salt['pillar.get']('elasticsearch:mainint', salt['pillar.get']('host:mainint')))))[0] }} -ESPORT=9200 - -echo "Removing read only attributes for indices..." -echo -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; diff --git a/salt/common/tools/sbin/so-helix-apikey b/salt/common/tools/sbin/so-helix-apikey deleted file mode 100755 index c58d2ad89..000000000 --- a/salt/common/tools/sbin/so-helix-apikey +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -local_salt_dir=/opt/so/saltstack/local - -got_root() { - - # Make sure you are root - if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" - exit 1 - fi - -} - -got_root -if [ ! -f $local_salt_dir/pillar/fireeye/init.sls ]; then - echo "This is nto configured for Helix Mode. Please re-install." - exit -else - echo "Enter your Helix API Key: " - read APIKEY - sed -i "s/^ api_key.*/ api_key: $APIKEY/g" $local_salt_dir/pillar/fireeye/init.sls - docker stop so-logstash - docker rm so-logstash - echo "Restarting Logstash for updated key" - salt-call state.apply logstash queue=True -fi diff --git a/salt/common/tools/sbin/so-logstash-events b/salt/common/tools/sbin/so-logstash-events deleted file mode 100755 index 5ea34ad80..000000000 --- a/salt/common/tools/sbin/so-logstash-events +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% set MAININT = salt['pillar.get']('host:mainint') -%} -{% set NODEIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] -%} - -. /usr/sbin/so-common - -if [ "$1" == "" ]; then - for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done -else - curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events -fi diff --git a/salt/common/tools/sbin/so-logstash-get-parsed b/salt/common/tools/sbin/so-logstash-get-parsed deleted file mode 100755 index 1575010ac..000000000 --- a/salt/common/tools/sbin/so-logstash-get-parsed +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -docker exec -it so-redis redis-cli llen logstash:unparsed diff --git a/salt/common/tools/sbin/so-nodered-start b/salt/common/tools/sbin/so-nodered-start deleted file mode 100755 index f5ab36c80..000000000 --- a/salt/common/tools/sbin/so-nodered-start +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-start nodered $1 - diff --git a/salt/common/tools/sbin/so-nodered-stop b/salt/common/tools/sbin/so-nodered-stop deleted file mode 100755 index 0286a175c..000000000 --- a/salt/common/tools/sbin/so-nodered-stop +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-stop nodered $1 diff --git a/salt/common/tools/sbin/so-analyst-install b/salt/common/tools/sbin_jinja/so-analyst-install similarity index 100% rename from salt/common/tools/sbin/so-analyst-install rename to salt/common/tools/sbin_jinja/so-analyst-install diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx similarity index 100% rename from salt/common/tools/sbin/so-import-evtx rename to salt/common/tools/sbin_jinja/so-import-evtx diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin_jinja/so-import-pcap similarity index 100% rename from salt/common/tools/sbin/so-import-pcap rename to salt/common/tools/sbin_jinja/so-import-pcap diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin_jinja/so-raid-status similarity index 100% rename from salt/common/tools/sbin/so-raid-status rename to salt/common/tools/sbin_jinja/so-raid-status diff --git a/salt/curator/init.sls b/salt/curator/init.sls index d1e4276e1..eaa5639ff 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -60,30 +60,21 @@ curconf: - template: jinja - show_changes: False -curclusterclose: - file.managed: - - name: /usr/sbin/so-curator-cluster-close - - source: salt://curator/files/bin/so-curator-cluster-close +curator_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://curator/tools/sbin - user: 934 - group: 939 - - mode: 755 - - template: jinja + - file_mode: 755 -curclusterdelete: - file.managed: - - name: /usr/sbin/so-curator-cluster-delete - - source: salt://curator/files/bin/so-curator-cluster-delete +curator_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://curator/tools/sbin_jinja - user: 934 - - group: 939 - - mode: 755 - -curclusterdeletedelete: - file.managed: - - name: /usr/sbin/so-curator-cluster-delete-delete - - source: salt://curator/files/bin/so-curator-cluster-delete-delete - - user: 934 - - group: 939 - - mode: 755 + - group: 939 + - file_mode: 755 - template: jinja so-curator: diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/tools/sbin/so-curator-close similarity index 100% rename from salt/curator/files/bin/so-curator-close rename to salt/curator/tools/sbin/so-curator-close diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/tools/sbin/so-curator-cluster-close similarity index 100% rename from salt/curator/files/bin/so-curator-cluster-close rename to salt/curator/tools/sbin/so-curator-cluster-close diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/tools/sbin/so-curator-cluster-delete similarity index 100% rename from salt/curator/files/bin/so-curator-cluster-delete rename to salt/curator/tools/sbin/so-curator-cluster-delete diff --git a/salt/curator/files/bin/so-curator-delete b/salt/curator/tools/sbin/so-curator-delete similarity index 100% rename from salt/curator/files/bin/so-curator-delete rename to salt/curator/tools/sbin/so-curator-delete diff --git a/salt/common/tools/sbin/so-curator-restart b/salt/curator/tools/sbin/so-curator-restart old mode 100755 new mode 100644 similarity index 100% rename from salt/common/tools/sbin/so-curator-restart rename to salt/curator/tools/sbin/so-curator-restart diff --git a/salt/common/tools/sbin/so-curator-start b/salt/curator/tools/sbin/so-curator-start old mode 100755 new mode 100644 similarity index 100% rename from salt/common/tools/sbin/so-curator-start rename to salt/curator/tools/sbin/so-curator-start diff --git a/salt/common/tools/sbin/so-curator-stop b/salt/curator/tools/sbin/so-curator-stop old mode 100755 new mode 100644 similarity index 100% rename from salt/common/tools/sbin/so-curator-stop rename to salt/curator/tools/sbin/so-curator-stop diff --git a/salt/curator/files/bin/so-curator-cluster-delete-delete b/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete similarity index 100% rename from salt/curator/files/bin/so-curator-cluster-delete-delete rename to salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete diff --git a/salt/docker/init.sls b/salt/docker/init.sls index f2a4c80a9..36530c9c3 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -26,10 +26,10 @@ dockerheldpackages: dockerheldpackages: pkg.installed: - pkgs: - - containerd.io: 1.6.18-3.1.el9 - - docker-ce: 23.0.1-1.el9 - - docker-ce-cli: 23.0.1-1.el9 - - docker-ce-rootless-extras: 23.0.1-1.el9 + - containerd.io: 1.6.20-3.1.el9 + - docker-ce: 23.0.5-1.el9 + - docker-ce-cli: 23.0.5-1.el9 + - docker-ce-rootless-extras: 23.0.5-1.el9 - hold: True - update_holds: True {% endif %} diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index b04fe1147..148fe7e1b 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -29,6 +29,23 @@ elastalogdir: - group: 933 - makedirs: True +elastalert_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://elastalert/tools/sbin + - user: 933 + - group: 939 + - file_mode: 755 + +#elastalert_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://elastalert/tools/sbin_jinja +# - user: 933 +# - group: 939 +# - file_mode: 755 +# - template: jinja + elastarules: file.directory: - name: /opt/so/rules/elastalert diff --git a/salt/common/tools/sbin/so-elastalert-create b/salt/elastalert/tools/sbin/so-elastalert-create similarity index 100% rename from salt/common/tools/sbin/so-elastalert-create rename to salt/elastalert/tools/sbin/so-elastalert-create diff --git a/salt/common/tools/sbin/so-elastalert-restart b/salt/elastalert/tools/sbin/so-elastalert-restart similarity index 100% rename from salt/common/tools/sbin/so-elastalert-restart rename to salt/elastalert/tools/sbin/so-elastalert-restart diff --git a/salt/common/tools/sbin/so-elastalert-start b/salt/elastalert/tools/sbin/so-elastalert-start similarity index 100% rename from salt/common/tools/sbin/so-elastalert-start rename to salt/elastalert/tools/sbin/so-elastalert-start diff --git a/salt/common/tools/sbin/so-elastalert-stop b/salt/elastalert/tools/sbin/so-elastalert-stop similarity index 100% rename from salt/common/tools/sbin/so-elastalert-stop rename to salt/elastalert/tools/sbin/so-elastalert-stop diff --git a/salt/common/tools/sbin/so-elastalert-test b/salt/elastalert/tools/sbin/so-elastalert-test similarity index 100% rename from salt/common/tools/sbin/so-elastalert-test rename to salt/elastalert/tools/sbin/so-elastalert-test diff --git a/salt/elasticfleet/init.sls b/salt/elasticfleet/init.sls index da735ffac..9476c3b94 100644 --- a/salt/elasticfleet/init.sls +++ b/salt/elasticfleet/init.sls @@ -25,6 +25,23 @@ elastic-agent: - home: /opt/so/conf/elastic-fleet - createhome: False +elasticfleet_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://elasticfleet/tools/sbin + - user: 947 + - group: 939 + - file_mode: 755 + +elasticfleet_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://elasticfleet/tools/sbin_jinja + - user: 947 + - group: 939 + - file_mode: 755 + - template: jinja + eaconfdir: file.directory: - name: /opt/so/conf/elastic-fleet diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete b/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-delete similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-delete diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list b/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-list similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-agent-policy-list rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-list diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view b/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-view similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-agent-policy-view rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-view diff --git a/salt/common/tools/sbin/so-elastic-fleet-data-streams-list b/salt/elasticfleet/tools/sbin/so-elastic-fleet-data-streams-list similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-data-streams-list rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-data-streams-list diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-delete similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-delete diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-list similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-integration-policy-list rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-list diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-integration-policy-load rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load diff --git a/salt/common/tools/sbin/so-elastic-fleet-restart b/salt/elasticfleet/tools/sbin/so-elastic-fleet-restart similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-restart rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-restart diff --git a/salt/common/tools/sbin/so-elastic-fleet-start b/salt/elasticfleet/tools/sbin/so-elastic-fleet-start similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-start rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-start diff --git a/salt/common/tools/sbin/so-elastic-fleet-stop b/salt/elasticfleet/tools/sbin/so-elastic-fleet-stop similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-stop rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-stop diff --git a/salt/common/tools/sbin/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers similarity index 100% rename from salt/common/tools/sbin/so-elastic-agent-gen-installers rename to salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-setup rename to salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index bfe288b8e..5e2ffae9d 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -21,12 +21,33 @@ vm.max_map_count: sysctl.present: - value: 262144 +# Add ES Group +elasticsearchgroup: + group.present: + - name: elasticsearch + - gid: 930 + +esconfdir: + file.directory: + - name: /opt/so/conf/elasticsearch + - user: 930 + - group: 939 + - makedirs: True + +# Add ES user +elasticsearch: + user.present: + - uid: 930 + - gid: 930 + - home: /opt/so/conf/elasticsearch + - createhome: False + {% if GLOBALS.is_manager %} # We have to add the Manager CA to the CA list cascriptsync: file.managed: - name: /usr/sbin/so-catrust - - source: salt://elasticsearch/tools/sbin/so-catrust + - source: salt://elasticsearch/tools/sbin_jinja/so-catrust - user: 939 - group: 939 - mode: 750 @@ -42,25 +63,34 @@ cascriptfun: - file: cascriptsync {% endif %} -# Sync some es scripts -es_sync_scripts: +elasticsearch_sbin: file.recurse: - name: /usr/sbin - - user: root - - group: root + - source: salt://elasticsearch/tools/sbin + - user: 930 + - group: 939 + - file_mode: 755 + - exclude_pat: + - so-catrust + - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state + +elasticsearch_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://elasticsearch/tools/sbin_jinja + - user: 939 + - group: 939 - file_mode: 755 - template: jinja - - source: salt://elasticsearch/tools/sbin - exclude_pat: - - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state - - so-elasticsearch-ilm-policy-load + - so-elasticsearch-ilm-policy-load # exclude this because we need to watch it for changes, we sync it in another state - defaults: GLOBALS: {{ GLOBALS }} so-elasticsearch-ilm-policy-load-script: file.managed: - name: /usr/sbin/so-elasticsearch-ilm-policy-load - - source: salt://elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load + - source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load - user: 930 - group: 939 - mode: 754 @@ -96,29 +126,6 @@ capemz: - user: 939 - group: 939 - - -# Add ES Group -elasticsearchgroup: - group.present: - - name: elasticsearch - - gid: 930 - -# Add ES user -elasticsearch: - user.present: - - uid: 930 - - gid: 930 - - home: /opt/so/conf/elasticsearch - - createhome: False - -esconfdir: - file.directory: - - name: /opt/so/conf/elasticsearch - - user: 930 - - group: 939 - - makedirs: True - esingestdir: file.directory: - name: /opt/so/conf/elasticsearch/ingest @@ -374,7 +381,7 @@ so-es-cluster-settings: - template: jinja - require: - docker_container: so-elasticsearch - - file: es_sync_scripts + - file: elasticsearch_sbin_jinja so-elasticsearch-ilm-policy-load: cmd.run: @@ -393,7 +400,7 @@ so-elasticsearch-templates: - template: jinja - require: - docker_container: so-elasticsearch - - file: es_sync_scripts + - file: elasticsearch_sbin_jinja so-elasticsearch-pipelines: cmd.run: @@ -409,7 +416,7 @@ so-elasticsearch-roles-load: - template: jinja - require: - docker_container: so-elasticsearch - - file: es_sync_scripts + - file: elasticsearch_sbin_jinja {% endif %} {% else %} diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/elasticsearch/tools/sbin/so-elastic-clear similarity index 97% rename from salt/common/tools/sbin/so-elastic-clear rename to salt/elasticsearch/tools/sbin/so-elastic-clear index f491fb62f..3b4f5fc62 100755 --- a/salt/common/tools/sbin/so-elastic-clear +++ b/salt/elasticsearch/tools/sbin/so-elastic-clear @@ -5,7 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common SKIP=0 @@ -59,7 +58,7 @@ done if [ $SKIP -ne 1 ]; then # List indices echo - curl -K /opt/so/conf/elasticsearch/curl.config -k -L https://{{ NODEIP }}:9200/_cat/indices?v + curl -K /opt/so/conf/elasticsearch/curl.config -k -L https://localhost:9200/_cat/indices?v echo # Inform user we are about to delete all data echo diff --git a/salt/common/tools/sbin/so-elastic-diagnose b/salt/elasticsearch/tools/sbin/so-elastic-diagnose similarity index 100% rename from salt/common/tools/sbin/so-elastic-diagnose rename to salt/elasticsearch/tools/sbin/so-elastic-diagnose diff --git a/salt/common/tools/sbin/so-elasticsearch-component-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list similarity index 72% rename from salt/common/tools/sbin/so-elasticsearch-component-templates-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list index 976499574..2fccce9cb 100755 --- a/salt/common/tools/sbin/so-elasticsearch-component-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list @@ -5,10 +5,9 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_component_template | jq '.component_templates[] |.name'| sort + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_component_template/$1 | jq + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq fi diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-lifecycle-status similarity index 75% rename from salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-lifecycle-status index 130a7cf16..db72f8078 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-lifecycle-status @@ -6,10 +6,8 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_all/_ilm/explain | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_all/_ilm/explain | jq . else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/$1/_ilm/explain | jq .[] + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/$1/_ilm/explain | jq .[] fi diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-delete similarity index 80% rename from salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-delete index 2be9dabb2..ef936b742 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-delete @@ -6,6 +6,4 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X DELETE https://{{ NODEIP }}:9200/_ilm/policy/$1 +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X DELETE https://localhost:9200/_ilm/policy/$1 diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load deleted file mode 100755 index 26ce487a7..000000000 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -. /usr/sbin/so-common - -{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - -{%- for index, settings in ES_INDEX_SETTINGS.items() %} - {%- if settings.policy is defined %} -echo -echo "Setting up {{ index }}-logs policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' -echo - {%- endif %} -{%- endfor %} -echo diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-view similarity index 76% rename from salt/common/tools/sbin/so-elasticsearch-ilm-policy-view rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-view index 426b6938d..f488bab87 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-view @@ -6,10 +6,9 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/policy | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ilm/policy | jq . else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/policy/$1 | jq .[] + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ilm/policy/$1 | jq .[] fi diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-restart b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-restart similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-ilm-restart rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-restart diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-start b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start similarity index 81% rename from salt/common/tools/sbin/so-elasticsearch-ilm-start rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start index 98dd38e9e..d9c63f8ea 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-start +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start @@ -6,7 +6,6 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} echo "Starting ILM..." -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://{{ NODEIP }}:9200/_ilm/start +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/start diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-status b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-status similarity index 78% rename from salt/common/tools/sbin/so-elasticsearch-ilm-status rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-status index 8d78adc5b..7ba0201a4 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-status +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-status @@ -1,4 +1,4 @@ -/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the @@ -6,6 +6,4 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/status | jq . +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ilm/status | jq . diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-stop b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop similarity index 81% rename from salt/common/tools/sbin/so-elasticsearch-ilm-stop rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop index 4868fd86d..034082699 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-stop +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop @@ -6,7 +6,5 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - echo "Stopping ILM..." -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://{{ NODEIP }}:9200/_ilm/stop +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/stop diff --git a/salt/common/tools/sbin/so-elasticsearch-index-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-templates-list similarity index 73% rename from salt/common/tools/sbin/so-elasticsearch-index-templates-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-index-templates-list index bea975c93..6df836c1d 100755 --- a/salt/common/tools/sbin/so-elasticsearch-index-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-templates-list @@ -5,10 +5,9 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_index_template | jq '.index_templates[] |.name'| sort + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_index_template | jq '.index_templates[] |.name'| sort else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_index_template/$1 | jq + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_index_template/$1 | jq fi diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-list similarity index 79% rename from salt/common/tools/sbin/so-elasticsearch-indices-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-indices-list index da8ea4cca..57cc5e799 100755 --- a/salt/common/tools/sbin/so-elasticsearch-indices-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-list @@ -5,8 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://{{ NODEIP }}:9200/_cat/indices?pretty&v&s=index" +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://localhost:9200/_cat/indices?pretty&v&s=index" diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-rw b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-rw new file mode 100755 index 000000000..5e6bf71a5 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-rw @@ -0,0 +1,13 @@ +#!/bin/bash +# +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +echo "Removing read only attributes for indices..." +echo +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -H "Content-Type: application/json" -L https://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-stats similarity index 68% rename from salt/common/tools/sbin/so-elasticsearch-pipeline-stats rename to salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-stats index 952773cda..fd06eeb78 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-stats @@ -5,12 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" fi diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-view similarity index 75% rename from salt/common/tools/sbin/so-elasticsearch-pipeline-view rename to salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-view index 32a26b948..8de82f901 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipeline-view +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-view @@ -5,12 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/* | jq . else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .[] + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/$1 | jq .[] fi diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines-list similarity index 76% rename from salt/common/tools/sbin/so-elasticsearch-pipelines-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines-list index b690d5846..feeecb68b 100755 --- a/salt/common/tools/sbin/so-elasticsearch-pipelines-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines-list @@ -5,10 +5,9 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/* | jq 'keys' else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/$1 | jq fi diff --git a/salt/common/tools/sbin/so-elasticsearch-query b/salt/elasticsearch/tools/sbin/so-elasticsearch-query similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-query rename to salt/elasticsearch/tools/sbin/so-elasticsearch-query diff --git a/salt/common/tools/sbin/so-elasticsearch-restart b/salt/elasticsearch/tools/sbin/so-elasticsearch-restart similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-restart rename to salt/elasticsearch/tools/sbin/so-elasticsearch-restart diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load index 17265a7c4..90b262989 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load @@ -7,8 +7,6 @@ . /usr/sbin/so-common default_conf_dir=/opt/so/conf -ELASTICSEARCH_HOST="{{ GLOBALS.node_ip }}" -ELASTICSEARCH_PORT=9200 # Define a default directory to load roles from ELASTICSEARCH_ROLES="$default_conf_dir/elasticsearch/roles/" @@ -18,7 +16,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://localhost:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" diff --git a/salt/common/tools/sbin/so-elasticsearch-shards-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-shards-list similarity index 81% rename from salt/common/tools/sbin/so-elasticsearch-shards-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-shards-list index 8865e05ac..cd6410b99 100755 --- a/salt/common/tools/sbin/so-elasticsearch-shards-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-shards-list @@ -5,8 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_cat/shards?pretty diff --git a/salt/common/tools/sbin/so-elasticsearch-start b/salt/elasticsearch/tools/sbin/so-elasticsearch-start similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-start rename to salt/elasticsearch/tools/sbin/so-elasticsearch-start diff --git a/salt/common/tools/sbin/so-elasticsearch-stop b/salt/elasticsearch/tools/sbin/so-elasticsearch-stop similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-stop rename to salt/elasticsearch/tools/sbin/so-elasticsearch-stop diff --git a/salt/common/tools/sbin/so-elasticsearch-template-remove b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-remove similarity index 80% rename from salt/common/tools/sbin/so-elasticsearch-template-remove rename to salt/elasticsearch/tools/sbin/so-elasticsearch-template-remove index f69495152..7d5ae5b3e 100755 --- a/salt/common/tools/sbin/so-elasticsearch-template-remove +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-remove @@ -5,8 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1 +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -XDELETE https://localhost:9200/_template/$1 diff --git a/salt/common/tools/sbin/so-elasticsearch-template-view b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-view similarity index 77% rename from salt/common/tools/sbin/so-elasticsearch-template-view rename to salt/elasticsearch/tools/sbin/so-elasticsearch-template-view index c56127703..cc2678582 100755 --- a/salt/common/tools/sbin/so-elasticsearch-template-view +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-view @@ -5,12 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/* | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/* | jq . else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/$1 | jq . fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-list similarity index 77% rename from salt/common/tools/sbin/so-elasticsearch-templates-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-templates-list index 7db4fdeff..28f23c6e1 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-list @@ -5,10 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common + if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/* | jq 'keys' else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/$1 | jq fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index 386026f0c..bce8af1ff 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -7,9 +7,6 @@ . /usr/sbin/so-common default_conf_dir=/opt/so/conf -ELASTICSEARCH_HOST="{{ GLOBALS.node_ip }}" -ELASTICSEARCH_PORT=9200 -#ELASTICSEARCH_AUTH="" # Define a default directory to load pipelines from ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" diff --git a/salt/common/tools/sbin/so-elasticsearch-wait b/salt/elasticsearch/tools/sbin/so-elasticsearch-wait similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-wait rename to salt/elasticsearch/tools/sbin/so-elasticsearch-wait diff --git a/salt/common/tools/sbin/so-index-list b/salt/elasticsearch/tools/sbin/so-index-list similarity index 100% rename from salt/common/tools/sbin/so-index-list rename to salt/elasticsearch/tools/sbin/so-index-list diff --git a/salt/elasticsearch/tools/sbin/so-catrust b/salt/elasticsearch/tools/sbin_jinja/so-catrust similarity index 100% rename from salt/elasticsearch/tools/sbin/so-catrust rename to salt/elasticsearch/tools/sbin_jinja/so-catrust diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elastic-restart b/salt/elasticsearch/tools/sbin_jinja/so-elastic-restart new file mode 100755 index 000000000..1b5e9bf03 --- /dev/null +++ b/salt/elasticsearch/tools/sbin_jinja/so-elastic-restart @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} +/usr/sbin/so-restart elasticsearch $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} +/usr/sbin/so-restart kibana $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-restart logstash $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-restart curator $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} +/usr/sbin/so-restart elastalert $1 +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elastic-start b/salt/elasticsearch/tools/sbin_jinja/so-elastic-start new file mode 100755 index 000000000..6be969e9d --- /dev/null +++ b/salt/elasticsearch/tools/sbin_jinja/so-elastic-start @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} +/usr/sbin/so-start elasticsearch $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} +/usr/sbin/so-start kibana $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-start logstash $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-start curator $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} +/usr/sbin/so-start elastalert $1 +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elastic-stop b/salt/elasticsearch/tools/sbin_jinja/so-elastic-stop new file mode 100755 index 000000000..b6ea04964 --- /dev/null +++ b/salt/elasticsearch/tools/sbin_jinja/so-elastic-stop @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} +/usr/sbin/so-stop elasticsearch $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} +/usr/sbin/so-stop kibana $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-stop logstash $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-stop curator $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} +/usr/sbin/so-stop elastalert $1 +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings similarity index 100% rename from salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings diff --git a/salt/common/tools/sbin/so-elasticsearch-cluster-space-total b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-cluster-space-total rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total diff --git a/salt/common/tools/sbin/so-elasticsearch-cluster-space-used b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-cluster-space-used rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load similarity index 77% rename from salt/common/tools/sbin/so-elasticsearch-ilm-policy-load rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 26ce487a7..afeddfa01 100755 --- a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -8,13 +8,12 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} echo echo "Setting up {{ index }}-logs policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' echo {%- endif %} {%- endfor %} diff --git a/salt/idh/init.sls b/salt/idh/init.sls index 20a6412ce..895cd61ac 100644 --- a/salt/idh/init.sls +++ b/salt/idh/init.sls @@ -60,6 +60,23 @@ opencanary_config: - defaults: OPENCANARYCONFIG: {{ OPENCANARYCONFIG }} +idh_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://idh/tools/sbin + - user: 934 + - group: 939 + - file_mode: 755 + +#idh_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://idh/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + so-idh: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idh:{{ GLOBALS.so_version }} diff --git a/salt/common/tools/sbin/so-idh-restart b/salt/idh/tools/sbin/so-idh-restart similarity index 100% rename from salt/common/tools/sbin/so-idh-restart rename to salt/idh/tools/sbin/so-idh-restart diff --git a/salt/common/tools/sbin/so-idh-start b/salt/idh/tools/sbin/so-idh-start similarity index 100% rename from salt/common/tools/sbin/so-idh-start rename to salt/idh/tools/sbin/so-idh-start diff --git a/salt/common/tools/sbin/so-idh-stop b/salt/idh/tools/sbin/so-idh-stop similarity index 100% rename from salt/common/tools/sbin/so-idh-stop rename to salt/idh/tools/sbin/so-idh-stop diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls index 5ec9464cc..7ad22e58b 100644 --- a/salt/idstools/init.sls +++ b/salt/idstools/init.sls @@ -20,6 +20,23 @@ idstoolslogdir: - group: 939 - makedirs: True +idstools_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://idstools/tools/sbin + - user: 934 + - group: 939 + - file_mode: 755 + +#idstools_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://idstools/tools/sbin_jinja +# - user: 934 +# - group: 939 +# - file_mode: 755 +# - template: jinja + so-rule-update: cron.present: - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1 diff --git a/salt/common/tools/sbin/so-idstools-restart b/salt/idstools/tools/sbin/so-idstools-restart similarity index 100% rename from salt/common/tools/sbin/so-idstools-restart rename to salt/idstools/tools/sbin/so-idstools-restart diff --git a/salt/common/tools/sbin/so-idstools-start b/salt/idstools/tools/sbin/so-idstools-start similarity index 100% rename from salt/common/tools/sbin/so-idstools-start rename to salt/idstools/tools/sbin/so-idstools-start diff --git a/salt/common/tools/sbin/so-idstools-stop b/salt/idstools/tools/sbin/so-idstools-stop similarity index 100% rename from salt/common/tools/sbin/so-idstools-stop rename to salt/idstools/tools/sbin/so-idstools-stop diff --git a/salt/common/tools/sbin/so-rule b/salt/idstools/tools/sbin/so-rule similarity index 100% rename from salt/common/tools/sbin/so-rule rename to salt/idstools/tools/sbin/so-rule diff --git a/salt/common/tools/sbin/so-rule-update b/salt/idstools/tools/sbin/so-rule-update similarity index 100% rename from salt/common/tools/sbin/so-rule-update rename to salt/idstools/tools/sbin/so-rule-update diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index b4824825b..7e10a6798 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -31,6 +31,23 @@ influxdbdir: - name: /nsm/influxdb - makedirs: True +influxdb_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://influxdb/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#influxdb_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://influxdb/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + influxdbconf: file.managed: - name: /opt/so/conf/influxdb/config.yaml diff --git a/salt/common/tools/sbin/so-influxdb-manage b/salt/influxdb/tools/sbin/so-influxdb-manage similarity index 100% rename from salt/common/tools/sbin/so-influxdb-manage rename to salt/influxdb/tools/sbin/so-influxdb-manage diff --git a/salt/common/tools/sbin/so-influxdb-restart b/salt/influxdb/tools/sbin/so-influxdb-restart similarity index 100% rename from salt/common/tools/sbin/so-influxdb-restart rename to salt/influxdb/tools/sbin/so-influxdb-restart diff --git a/salt/common/tools/sbin/so-influxdb-start b/salt/influxdb/tools/sbin/so-influxdb-start similarity index 100% rename from salt/common/tools/sbin/so-influxdb-start rename to salt/influxdb/tools/sbin/so-influxdb-start diff --git a/salt/common/tools/sbin/so-influxdb-stop b/salt/influxdb/tools/sbin/so-influxdb-stop similarity index 100% rename from salt/common/tools/sbin/so-influxdb-stop rename to salt/influxdb/tools/sbin/so-influxdb-stop diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index a974dcf48..a9d3c6da9 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -34,6 +34,25 @@ kibanaconfdir: - group: 939 - makedirs: True +kibana_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://kibana/tools/sbin + - user: 932 + - group: 939 + - file_mode: 755 + +kibana_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://kibana/tools/sbin_jinja + - user: 932 + - group: 939 + - file_mode: 755 + - template: jinja + - defaults: + GLOBALS: {{ GLOBALS }} + kibanaconfig: file.managed: - name: /opt/so/conf/kibana/etc/kibana.yml @@ -67,15 +86,6 @@ synckibanacustom: - user: 932 - group: 939 -kibanabin: - file.managed: - - name: /usr/sbin/so-kibana-config-load - - source: salt://kibana/bin/so-kibana-config-load - - mode: 755 - - template: jinja - - defaults: - GLOBALS: {{ GLOBALS }} - # Start the kibana docker so-kibana: docker_container.running: diff --git a/salt/common/tools/sbin/so-kibana-restart b/salt/kibana/tools/sbin/so-kibana-restart similarity index 100% rename from salt/common/tools/sbin/so-kibana-restart rename to salt/kibana/tools/sbin/so-kibana-restart diff --git a/salt/common/tools/sbin/so-kibana-savedobjects-defaults b/salt/kibana/tools/sbin/so-kibana-savedobjects-defaults similarity index 100% rename from salt/common/tools/sbin/so-kibana-savedobjects-defaults rename to salt/kibana/tools/sbin/so-kibana-savedobjects-defaults diff --git a/salt/common/tools/sbin/so-kibana-start b/salt/kibana/tools/sbin/so-kibana-start similarity index 100% rename from salt/common/tools/sbin/so-kibana-start rename to salt/kibana/tools/sbin/so-kibana-start diff --git a/salt/common/tools/sbin/so-kibana-stop b/salt/kibana/tools/sbin/so-kibana-stop similarity index 100% rename from salt/common/tools/sbin/so-kibana-stop rename to salt/kibana/tools/sbin/so-kibana-stop diff --git a/salt/common/tools/sbin/so-kibana-config-export b/salt/kibana/tools/sbin_jinja/so-kibana-config-export similarity index 100% rename from salt/common/tools/sbin/so-kibana-config-export rename to salt/kibana/tools/sbin_jinja/so-kibana-config-export diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load similarity index 100% rename from salt/kibana/bin/so-kibana-config-load rename to salt/kibana/tools/sbin_jinja/so-kibana-config-load diff --git a/salt/common/tools/sbin/so-kibana-space-defaults b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults similarity index 100% rename from salt/common/tools/sbin/so-kibana-space-defaults rename to salt/kibana/tools/sbin_jinja/so-kibana-space-defaults diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml new file mode 100644 index 000000000..d253a6b51 --- /dev/null +++ b/salt/logstash/defaults.yaml @@ -0,0 +1,63 @@ +logstash: + assigned_pipelines: + roles: + standalone: + - manager + - search + receiver: + - receiver + heavynode: + - search + searchnode: + - search + manager: + - manager + managersearch: + - manager + - search + fleet: + - fleet + defined_pipelines: + fleet: + - so/0012_input_elastic_agent.conf + - so/9806_output_lumberjack_fleet.conf.jinja + manager: + - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf + - so/0013_input_lumberjack_fleet.conf + - so/9999_output_redis.conf.jinja + receiver: + - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf + - so/9999_output_redis.conf.jinja + search: + - so/0900_input_redis.conf.jinja + - so/9805_output_elastic_agent.conf.jinja + - so/9900_output_endgame.conf.jinja + custom0: [] + custom1: [] + custom2: [] + custom3: [] + custom4: [] + docker_options: + port_bindings: + - 0.0.0.0:3765:3765 + - 0.0.0.0:5044:5044 + - 0.0.0.0:5055:5055 + - 0.0.0.0:5056:5056 + - 0.0.0.0:5644:5644 + - 0.0.0.0:6050:6050 + - 0.0.0.0:6051:6051 + - 0.0.0.0:6052:6052 + - 0.0.0.0:6053:6053 + - 0.0.0.0:9600:9600 + settings: + lsheap: 500m + config: + http_x_host: 0.0.0.0 + path_x_logs: /var/log/logstash + pipeline_x_workers: 1 + pipeline_x_batch_x_size: 125 + pipeline_x_ecs_compatibility: disabled + dmz_nodes: [] + diff --git a/salt/logstash/dmz_nodes.yaml b/salt/logstash/dmz_nodes.yaml deleted file mode 100644 index 460088a7d..000000000 --- a/salt/logstash/dmz_nodes.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -# Do not edit this file. Copy it to /opt/so/saltstack/local/salt/logstash/ and make changes there. It should be formatted as a list. -# logstash: -# dmz_nodes: -# - mydmznodehostname1 -# - mydmznodehostname2 -# - mydmznodehostname3 - -logstash: - dmz_nodes: \ No newline at end of file diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index 2a952c754..973b2ab10 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -1,226 +1 @@ -# Settings file in YAML -# -# Settings can be specified either in hierarchical form, e.g.: -# -# pipeline: -# batch: -# size: 125 -# delay: 5 -# -# Or as flat keys: -# -# pipeline.batch.size: 125 -# pipeline.batch.delay: 5 -# -# ------------ Node identity ------------ -# -# Use a descriptive name for the node: -# -# node.name: test -# -# If omitted the node name will default to the machine's host name -# -# ------------ Data path ------------------ -# -# Which directory should be used by logstash and its plugins -# for any persistent needs. Defaults to LOGSTASH_HOME/data -# -# path.data: -# -# ------------ Pipeline Settings -------------- -# -# The ID of the pipeline. -# -# pipeline.id: main -# -# Set the number of workers that will, in parallel, execute the filters+outputs -# stage of the pipeline. -# -# This defaults to the number of the host's CPU cores. -# -# pipeline.workers: 2 -# -# How many events to retrieve from inputs before sending to filters+workers -# -# pipeline.batch.size: 125 -# -# How long to wait in milliseconds while polling for the next event -# before dispatching an undersized batch to filters+outputs -# -# pipeline.batch.delay: 50 -# -# Force Logstash to exit during shutdown even if there are still inflight -# events in memory. By default, logstash will refuse to quit until all -# received events have been pushed to the outputs. -# -# WARNING: enabling this can lead to data loss during shutdown -# -# pipeline.unsafe_shutdown: false -# -# ------------ Pipeline Configuration Settings -------------- -# -# Where to fetch the pipeline configuration for the main pipeline -# -# path.config: -# /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image -# Special Docker path -# path.config: /usr/share/logstash/pipeline - -# -# Pipeline configuration string for the main pipeline -# -# config.string: -# -# At startup, test if the configuration is valid and exit (dry run) -# -# config.test_and_exit: false -# -# Periodically check if the configuration has changed and reload the pipeline -# This can also be triggered manually through the SIGHUP signal -# -# config.reload.automatic: false -# -# How often to check if the pipeline configuration has changed (in seconds) -# -# config.reload.interval: 3s -# -# Show fully compiled configuration as debug log message -# NOTE: --log.level must be 'debug' -# -# config.debug: false -# -# When enabled, process escaped characters such as \n and \" in strings in the -# pipeline configuration files. -# -# config.support_escapes: false -# -# ------------ Module Settings --------------- -# Define modules here. Modules definitions must be defined as an array. -# The simple way to see this is to prepend each `name` with a `-`, and keep -# all associated variables under the `name` they are associated with, and -# above the next, like this: -# -# modules: -# - name: MODULE_NAME -# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE -# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE -# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE -# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE -# -# Module variable names must be in the format of -# -# var.PLUGIN_TYPE.PLUGIN_NAME.KEY -# -# modules: -# -# ------------ Cloud Settings --------------- -# Define Elastic Cloud settings here. -# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy -# and it may have an label prefix e.g. staging:dXMtZ... -# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host' -# cloud.id: -# -# Format of cloud.auth is: : -# This is optional -# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password' -# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password' -# cloud.auth: elastic: -# -# ------------ Queuing Settings -------------- -# -# Internal queuing model, "memory" for legacy in-memory based queuing and -# "persisted" for disk-based acked queueing. Defaults is memory -# -# queue.type: memory -# -# If using queue.type: persisted, the directory path where the data files will be stored. -# Default is path.data/queue -# -# path.queue: -# -# If using queue.type: persisted, the page data files size. The queue data consists of -# append-only data files separated into pages. Default is 64mb -# -# queue.page_capacity: 64mb -# -# If using queue.type: persisted, the maximum number of unread events in the queue. -# Default is 0 (unlimited) -# -# queue.max_events: 0 -# -# If using queue.type: persisted, the total capacity of the queue in number of bytes. -# If you would like more unacked events to be buffered in Logstash, you can increase the -# capacity using this setting. Please make sure your disk drive has capacity greater than -# the size specified here. If both max_bytes and max_events are specified, Logstash will pick -# whichever criteria is reached first -# Default is 1024mb or 1gb -# -# queue.max_bytes: 1024mb -# -# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint -# Default is 1024, 0 for unlimited -# -# queue.checkpoint.acks: 1024 -# -# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint -# Default is 1024, 0 for unlimited -# -# queue.checkpoint.writes: 1024 -# -# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page -# Default is 1000, 0 for no periodic checkpoint. -# -# queue.checkpoint.interval: 1000 -# -# ------------ Dead-Letter Queue Settings -------------- -# Flag to turn on dead-letter queue. -# -# dead_letter_queue.enable: false - -# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries -# will be dropped if they would increase the size of the dead letter queue beyond this setting. -# Default is 1024mb -# dead_letter_queue.max_bytes: 1024mb - -# If using dead_letter_queue.enable: true, the directory path where the data files will be stored. -# Default is path.data/dead_letter_queue -# -# path.dead_letter_queue: -# -# ------------ Metrics Settings -------------- -# -# Bind address for the metrics REST endpoint -# -# http.host: "127.0.0.1" -http.host: 0.0.0.0 -# -# Bind port for the metrics REST endpoint, this option also accept a range -# (9600-9700) and logstash will pick up the first available ports. -# -# http.port: 9600-9700 -# -# ------------ Debugging Settings -------------- -# -# Options for log.level: -# * fatal -# * error -# * warn -# * info (default) -# * debug -# * trace -# -# log.level: info -# path.logs: -path.logs: /var/log/logstash -# -# ------------ Other Settings -------------- -# -# Where to find custom plugins -# path.plugins: [] -{% set pipeline_workers = salt['pillar.get']('logstash_settings:ls_pipeline_workers', '1') %} -{% set pipeline_batch = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', '125') %} -{% set pipeline_ecs_compatibility = salt['pillar.get']('logstash_settings:ls_ecs_compatibility', 'disabled') %} - -pipeline.workers: {{ pipeline_workers }} -pipeline.batch.size: {{ pipeline_batch }} -pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }} +{{ LOGSTASH_MERGED.config | yaml(False) | replace("_x_", ".") }} diff --git a/salt/logstash/etc/pipelines.yml.jinja b/salt/logstash/etc/pipelines.yml.jinja index 3ee7a0d3b..427cc9f14 100644 --- a/salt/logstash/etc/pipelines.yml.jinja +++ b/salt/logstash/etc/pipelines.yml.jinja @@ -1,4 +1,4 @@ -{%- for pl in pipelines %} -- pipeline.id: {{ pl }} - path.config: "/usr/share/logstash/pipelines/{{ pl }}/" +{%- for assigned_pipeline in ASSIGNED_PIPELINES %} +- pipeline.id: {{ assigned_pipeline }} + path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/" {% endfor -%} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 7f3aef0aa..7072ed46d 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -6,19 +6,19 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% from 'docker/docker.map.jinja' import DOCKER %} -{% from 'logstash/map.jinja' import REDIS_NODES with context %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'logstash/map.jinja' import REDIS_NODES %} +{% from 'logstash/map.jinja' import LOGSTASH_MERGED %} # Logstash Section - Decide which pillar to use -{% set lsheap = salt['pillar.get']('logstash_settings:lsheap') %} +{% set lsheap = LOGSTASH_MERGED.settings.lsheap %} {% if GLOBALS.role in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} {% set nodetype = GLOBALS.role %} {% endif %} -{% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %} -{% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} +{% set ASSIGNED_PIPELINES = LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} +{% set DOCKER_OPTIONS = LOGSTASH_MERGED.docker_options %} include: - ssl @@ -43,6 +43,23 @@ lslibdir: file.absent: - name: /opt/so/conf/logstash/lib +logstash_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://logstash/tools/sbin + - user: 931 + - group: 939 + - file_mode: 755 + +#logstash_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://logstash/tools/sbin_jinja +# - user: 931 +# - group: 939 +# - file_mode: 755 +# - template: jinja + lsetcdir: file.directory: - name: /opt/so/conf/logstash/etc @@ -56,20 +73,22 @@ lspipelinedir: - user: 931 - group: 939 - {% for PL in PIPELINES %} - {% for CONFIGFILE in PIPELINES[PL].config %} -ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: +{% for assigned_pipeline in ASSIGNED_PIPELINES %} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} +ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: file.managed: - source: salt://logstash/pipelines/config/{{CONFIGFILE}} {% if 'jinja' in CONFIGFILE.split('.')[-1] %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} - template: jinja - defaults: GLOBALS: {{ GLOBALS }} ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}" ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}" + THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }} + BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }} {% else %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}} {% endif %} - user: 931 - group: 939 @@ -78,28 +97,27 @@ ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: - show_changes: False {% endfor %} -ls_pipeline_{{PL}}: +ls_pipeline_{{assigned_pipeline}}: file.directory: - - name: /opt/so/conf/logstash/pipelines/{{PL}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}} - user: 931 - group: 939 - require: - {% for CONFIGFILE in PIPELINES[PL].config %} - - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} + - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} - clean: True +{% endfor %} - {% endfor %} - +# Copy down all the configs lspipelinesyml: file.managed: - name: /opt/so/conf/logstash/etc/pipelines.yml - source: salt://logstash/etc/pipelines.yml.jinja - template: jinja - defaults: - pipelines: {{ PIPELINES }} + ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }} -# Copy down all the configs lsetcsync: file.recurse: - name: /opt/so/conf/logstash/etc @@ -109,6 +127,8 @@ lsetcsync: - template: jinja - clean: True - exclude_pat: pipelines* + - defaults: + LOGSTASH_MERGED: {{ LOGSTASH_MERGED }} # Create the import directory importdir: @@ -185,10 +205,10 @@ so-logstash: {%- endif %} - watch: - file: lsetcsync - {% for PL in PIPELINES %} - - file: ls_pipeline_{{PL}} - {% for CONFIGFILE in PIPELINES[PL].config %} - - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} + {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} + - file: ls_pipeline_{{assigned_pipeline}} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} + - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} {% endfor %} - require: diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index e23f944a2..c4ad5d96a 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -1,4 +1,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %} +{% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} + {% set REDIS_NODES = [] %} {% set LOGSTASH_NODES = [] %} {% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index b4251b81a..661bc0b73 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,5 +1,3 @@ -{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} -{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} {%- from 'logstash/map.jinja' import REDIS_NODES with context %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 6b8b8503f..0d3b3324b 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set HOST = GLOBALS.manager %} {%- endif %} -{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} output { diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml new file mode 100644 index 000000000..e41ff000f --- /dev/null +++ b/salt/logstash/soc_logstash.yaml @@ -0,0 +1,71 @@ +logstash: + assigned_pipelines: + roles: + standalone: &assigned_pipelines + description: List of defined pipelines to add to this role. + advanced: True + helpLink: logstash.html + multiline: True + forcedType: "[]string" + receiver: *assigned_pipelines + heavynode: *assigned_pipelines + searchnode: *assigned_pipelines + manager: *assigned_pipelines + managersearch: *assigned_pipelines + fleet: *assigned_pipelines + defined_pipelines: + receiver: &defined_pipelines + description: List of pipeline configurations assign to this group. + advanced: True + helpLink: logstash.html + multiline: True + forcedType: "[]string" + fleet: *defined_pipelines + manager: *defined_pipelines + search: *defined_pipelines + custom0: *defined_pipelines + custom1: *defined_pipelines + custom2: *defined_pipelines + custom3: *defined_pipelines + custom4: *defined_pipelines + settings: + lsheap: + description: Heap size to use for logstash + helpLink: logstash.html + global: False + config: + http_x_host: + description: Host interface to listen to connections. + helpLink: logstash.html + readonly: True + advanced: True + path_x_logs: + description: Path inside the container to wrote logs. + helpLink: logstash.html + readonly: True + advanced: True + pipeline_x_workers: + description: Number of worker threads to process events in logstash. + helpLink: logstash.html + global: False + pipeline_x_batch_x_size: + description: Logstash batch size. + helpLink: logstash.html + global: False + pipeline_x_ecs_compatibility: + description: Sets ECS compatibility. This is set per pipeline so you should never need to change this. + helpLink: logstash.html + readonly: True + advanced: True + docker_options: + port_bindings: + description: List of ports to open to the logstash docker container. Firewall ports will still need to be added to the firewall configuration. + helpLink: logstash.html + advanced: True + multiline: True + dmz_nodes: + description: "List of receiver nodes in DMZs. Prevents sensors from sending to these receivers. Primarily used for external Elastic agents." + helpLink: logstash.html + multiline: True + advanced: True + forcedType: "[]string" diff --git a/salt/common/tools/sbin/so-logstash-pipeline-stats b/salt/logstash/tools/sbin/so-logstash-events similarity index 56% rename from salt/common/tools/sbin/so-logstash-pipeline-stats rename to salt/logstash/tools/sbin/so-logstash-events index 4ad58e5b3..60d02e8d9 100755 --- a/salt/common/tools/sbin/so-logstash-pipeline-stats +++ b/salt/logstash/tools/sbin/so-logstash-events @@ -5,13 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{% set MAININT = salt['pillar.get']('host:mainint') -%} -{% set NODEIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] -%} - . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines + for i in $(curl -s -L http://localhost:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done else - curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1 + curl -s -L http://localhost:9600/_node/stats | jq .pipelines.$1.events fi diff --git a/salt/common/tools/sbin/so-logstash-get-unparsed b/salt/logstash/tools/sbin/so-logstash-get-unparsed similarity index 100% rename from salt/common/tools/sbin/so-logstash-get-unparsed rename to salt/logstash/tools/sbin/so-logstash-get-unparsed diff --git a/salt/common/tools/sbin/so-nodered-restart b/salt/logstash/tools/sbin/so-logstash-pipeline-stats similarity index 66% rename from salt/common/tools/sbin/so-nodered-restart rename to salt/logstash/tools/sbin/so-logstash-pipeline-stats index 06060b764..badcddf72 100755 --- a/salt/common/tools/sbin/so-nodered-restart +++ b/salt/logstash/tools/sbin/so-logstash-pipeline-stats @@ -1,12 +1,15 @@ #!/bin/bash - +# # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - . /usr/sbin/so-common -/usr/sbin/so-restart nodered $1 +if [ "$1" == "" ]; then + curl -s -L http://localhost:9600/_node/stats | jq .pipelines +else + curl -s -L http://localhost:9600/_node/stats | jq .pipelines.$1 +fi diff --git a/salt/common/tools/sbin/so-logstash-restart b/salt/logstash/tools/sbin/so-logstash-restart similarity index 100% rename from salt/common/tools/sbin/so-logstash-restart rename to salt/logstash/tools/sbin/so-logstash-restart diff --git a/salt/common/tools/sbin/so-logstash-start b/salt/logstash/tools/sbin/so-logstash-start similarity index 100% rename from salt/common/tools/sbin/so-logstash-start rename to salt/logstash/tools/sbin/so-logstash-start diff --git a/salt/common/tools/sbin/so-logstash-stop b/salt/logstash/tools/sbin/so-logstash-stop similarity index 100% rename from salt/common/tools/sbin/so-logstash-stop rename to salt/logstash/tools/sbin/so-logstash-stop diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 9973dcb41..47867edaf 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -43,13 +43,22 @@ repo_dir: - user - group -repo_sync_script: - file.managed: - - name: /usr/sbin/so-repo-sync - - source: salt://manager/files/so-repo-sync - - user: root - - group: root - - mode: 755 +manager_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://manager/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#manager_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://manager/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja so-repo-sync: {% if MANAGERMERGED.reposync.enabled %} diff --git a/salt/common/tools/sbin/so-allow b/salt/manager/tools/sbin/so-allow similarity index 100% rename from salt/common/tools/sbin/so-allow rename to salt/manager/tools/sbin/so-allow diff --git a/salt/common/tools/sbin/so-allow-view b/salt/manager/tools/sbin/so-allow-view similarity index 100% rename from salt/common/tools/sbin/so-allow-view rename to salt/manager/tools/sbin/so-allow-view diff --git a/salt/common/tools/sbin/so-deny b/salt/manager/tools/sbin/so-deny similarity index 100% rename from salt/common/tools/sbin/so-deny rename to salt/manager/tools/sbin/so-deny diff --git a/salt/common/tools/sbin/so-docker-refresh b/salt/manager/tools/sbin/so-docker-refresh similarity index 100% rename from salt/common/tools/sbin/so-docker-refresh rename to salt/manager/tools/sbin/so-docker-refresh diff --git a/salt/common/tools/sbin/so-elastic-auth-password-reset b/salt/manager/tools/sbin/so-elastic-auth-password-reset similarity index 100% rename from salt/common/tools/sbin/so-elastic-auth-password-reset rename to salt/manager/tools/sbin/so-elastic-auth-password-reset diff --git a/salt/common/tools/sbin/so-firewall b/salt/manager/tools/sbin/so-firewall similarity index 99% rename from salt/common/tools/sbin/so-firewall rename to salt/manager/tools/sbin/so-firewall index 94302b5b2..6c47a3719 100755 --- a/salt/common/tools/sbin/so-firewall +++ b/salt/manager/tools/sbin/so-firewall @@ -144,4 +144,4 @@ def main(): sys.exit(code) if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/salt/common/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion similarity index 99% rename from salt/common/tools/sbin/so-firewall-minion rename to salt/manager/tools/sbin/so-firewall-minion index 610d0fc3a..4834f0e41 100755 --- a/salt/common/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -79,4 +79,4 @@ fi 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; - esac + esac \ No newline at end of file diff --git a/salt/common/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion similarity index 98% rename from salt/common/tools/sbin/so-minion rename to salt/manager/tools/sbin/so-minion index 541104c4d..d5d7bb770 100755 --- a/salt/common/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -163,12 +163,11 @@ function add_idh_to_minion() { function add_logstash_to_minion() { # Create the logstash advanced pillar printf '%s\n'\ - "logstash_settings:"\ - " ls_host: '$LSHOSTNAME'"\ - " ls_pipeline_batch_size: 125"\ - " ls_input_threads: 1"\ - " lsheap: $LSHEAP"\ - " ls_pipeline_workers: $CPUCORES"\ + "logstash:"\ + " config:"\ + " pipeline_x_workers: $CPUCORES"\ + " settings:"\ + " lsheap: $LSHEAP"\ " " >> $PILLARFILE } diff --git a/salt/manager/files/so-repo-sync b/salt/manager/tools/sbin/so-repo-sync similarity index 100% rename from salt/manager/files/so-repo-sync rename to salt/manager/tools/sbin/so-repo-sync diff --git a/salt/common/tools/sbin/so-saltstack-update b/salt/manager/tools/sbin/so-saltstack-update similarity index 100% rename from salt/common/tools/sbin/so-saltstack-update rename to salt/manager/tools/sbin/so-saltstack-update diff --git a/salt/common/tools/sbin/so-user b/salt/manager/tools/sbin/so-user similarity index 99% rename from salt/common/tools/sbin/so-user rename to salt/manager/tools/sbin/so-user index 3c712491a..8234f7ae5 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/manager/tools/sbin/so-user @@ -5,10 +5,13 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +if [[ -f /usr/sbin/so-common ]]; then + source /usr/sbin/so-common +else + source $(dirname $0)/../../../common/tools/sbin/so-common +fi -source $(dirname $0)/so-common - DEFAULT_ROLE=analyst function usage() { diff --git a/salt/common/tools/sbin/so-user-add b/salt/manager/tools/sbin/so-user-add similarity index 100% rename from salt/common/tools/sbin/so-user-add rename to salt/manager/tools/sbin/so-user-add diff --git a/salt/common/tools/sbin/so-user-disable b/salt/manager/tools/sbin/so-user-disable similarity index 100% rename from salt/common/tools/sbin/so-user-disable rename to salt/manager/tools/sbin/so-user-disable diff --git a/salt/common/tools/sbin/so-user-enable b/salt/manager/tools/sbin/so-user-enable similarity index 100% rename from salt/common/tools/sbin/so-user-enable rename to salt/manager/tools/sbin/so-user-enable diff --git a/salt/common/tools/sbin/so-user-list b/salt/manager/tools/sbin/so-user-list similarity index 100% rename from salt/common/tools/sbin/so-user-list rename to salt/manager/tools/sbin/so-user-list diff --git a/salt/common/tools/sbin/soup b/salt/manager/tools/sbin/soup similarity index 100% rename from salt/common/tools/sbin/soup rename to salt/manager/tools/sbin/soup diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 44e6789af..1c0ca70c0 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -69,6 +69,23 @@ mysqldatadir: - group: 939 - makedirs: True +mysql_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://mysql/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#mysql_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://mysql/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + {% if MYSQLPASS == None %} mysql_password_none: diff --git a/salt/common/tools/sbin/so-mysql-restart b/salt/mysql/tools/sbin/so-mysql-restart similarity index 100% rename from salt/common/tools/sbin/so-mysql-restart rename to salt/mysql/tools/sbin/so-mysql-restart diff --git a/salt/common/tools/sbin/so-mysql-start b/salt/mysql/tools/sbin/so-mysql-start similarity index 100% rename from salt/common/tools/sbin/so-mysql-start rename to salt/mysql/tools/sbin/so-mysql-start diff --git a/salt/common/tools/sbin/so-mysql-stop b/salt/mysql/tools/sbin/so-mysql-stop similarity index 100% rename from salt/common/tools/sbin/so-mysql-stop rename to salt/mysql/tools/sbin/so-mysql-stop diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index 833bda98a..c66af0837 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -81,6 +81,23 @@ navigatorenterpriseattack: - makedirs: True - replace: False +nginx_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://nginx/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#nginx_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://nginx/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + so-nginx: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }} diff --git a/salt/common/tools/sbin/so-nginx-restart b/salt/nginx/tools/sbin/so-nginx-restart similarity index 100% rename from salt/common/tools/sbin/so-nginx-restart rename to salt/nginx/tools/sbin/so-nginx-restart diff --git a/salt/common/tools/sbin/so-nginx-start b/salt/nginx/tools/sbin/so-nginx-start similarity index 100% rename from salt/common/tools/sbin/so-nginx-start rename to salt/nginx/tools/sbin/so-nginx-start diff --git a/salt/common/tools/sbin/so-nginx-stop b/salt/nginx/tools/sbin/so-nginx-stop similarity index 100% rename from salt/common/tools/sbin/so-nginx-stop rename to salt/nginx/tools/sbin/so-nginx-stop diff --git a/salt/pcap/config.sls b/salt/pcap/config.sls new file mode 100644 index 000000000..c83abfe0f --- /dev/null +++ b/salt/pcap/config.sls @@ -0,0 +1,116 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from "pcap/config.map.jinja" import PCAPMERGED with context %} +{% from 'bpf/pcap.map.jinja' import PCAPBPF %} + +{% set BPF_COMPILED = "" %} + +# PCAP Section + +stenographergroup: + group.present: + - name: stenographer + - gid: 941 + +stenographer: + user.present: + - uid: 941 + - gid: 941 + - home: /opt/so/conf/steno + +stenoconfdir: + file.directory: + - name: /opt/so/conf/steno + - user: 941 + - group: 939 + - makedirs: True + +pcap_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://pcap/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +{% if PCAPBPF %} + {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %} + {% if BPF_CALC['stderr'] == "" %} + {% set BPF_COMPILED = ",\\\"--filter=" + BPF_CALC['stdout'] + "\\\"" %} + {% else %} + +bpfcompilationfailure: + test.configurable_test_state: + - changes: False + - result: False + - comment: "BPF Compilation Failed - Discarding Specified BPF" + {% endif %} +{% endif %} + +stenoconf: + file.managed: + - name: /opt/so/conf/steno/config + - source: salt://pcap/files/config.jinja + - user: stenographer + - group: stenographer + - mode: 644 + - template: jinja + - defaults: + PCAPMERGED: {{ PCAPMERGED }} + BPF_COMPILED: "{{ BPF_COMPILED }}" + +stenoca: + file.directory: + - name: /opt/so/conf/steno/certs + - user: 941 + - group: 939 + +pcapdir: + file.directory: + - name: /nsm/pcap + - user: 941 + - group: 941 + - makedirs: True + +pcaptmpdir: + file.directory: + - name: /nsm/pcaptmp + - user: 941 + - group: 941 + - makedirs: True + +pcapoutdir: + file.directory: + - name: /nsm/pcapout + - user: 939 + - group: 939 + - makedirs: True + +pcapindexdir: + file.directory: + - name: /nsm/pcapindex + - user: 941 + - group: 941 + - makedirs: True + +stenolog: + file.directory: + - name: /opt/so/log/stenographer + - user: 941 + - group: 941 + - makedirs: True + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/pcap/disabled.sls b/salt/pcap/disabled.sls index b9afd6e15..eaa227303 100644 --- a/salt/pcap/disabled.sls +++ b/salt/pcap/disabled.sls @@ -1,6 +1,9 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} +include: + - pcap.sostatus + so-steno: docker_container.absent: - force: True diff --git a/salt/pcap/enabled.sls b/salt/pcap/enabled.sls index 803c31e3a..6d2e79b61 100644 --- a/salt/pcap/enabled.sls +++ b/salt/pcap/enabled.sls @@ -6,98 +6,9 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% from "pcap/config.map.jinja" import PCAPMERGED with context %} -{% from 'bpf/pcap.map.jinja' import PCAPBPF %} - -{% set BPF_COMPILED = "" %} - -# PCAP Section - -stenographergroup: - group.present: - - name: stenographer - - gid: 941 - -stenographer: - user.present: - - uid: 941 - - gid: 941 - - home: /opt/so/conf/steno - -stenoconfdir: - file.directory: - - name: /opt/so/conf/steno - - user: 941 - - group: 939 - - makedirs: True - -{% if PCAPBPF %} - {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %} - {% if BPF_CALC['stderr'] == "" %} - {% set BPF_COMPILED = ",\\\"--filter=" + BPF_CALC['stdout'] + "\\\"" %} - {% else %} - -bpfcompilationfailure: - test.configurable_test_state: - - changes: False - - result: False - - comment: "BPF Compilation Failed - Discarding Specified BPF" - {% endif %} -{% endif %} - -stenoconf: - file.managed: - - name: /opt/so/conf/steno/config - - source: salt://pcap/files/config.jinja - - user: stenographer - - group: stenographer - - mode: 644 - - template: jinja - - defaults: - PCAPMERGED: {{ PCAPMERGED }} - BPF_COMPILED: "{{ BPF_COMPILED }}" - -stenoca: - file.directory: - - name: /opt/so/conf/steno/certs - - user: 941 - - group: 939 - -pcapdir: - file.directory: - - name: /nsm/pcap - - user: 941 - - group: 941 - - makedirs: True - -pcaptmpdir: - file.directory: - - name: /nsm/pcaptmp - - user: 941 - - group: 941 - - makedirs: True - -pcapoutdir: - file.directory: - - name: /nsm/pcapout - - user: 939 - - group: 939 - - makedirs: True - -pcapindexdir: - file.directory: - - name: /nsm/pcapindex - - user: 941 - - group: 941 - - makedirs: True - -stenolog: - file.directory: - - name: /opt/so/log/stenographer - - user: 941 - - group: 941 - - makedirs: True +include: + - pcap.config + - pcap.sostatus so-steno: docker_container.running: diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 31ac4dd31..46ad04e95 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -1,9 +1,12 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'pcap/config.map.jinja' import PCAPMERGED %} include: - - pcap.sostatus -{% if PCAPMERGED.enabled %} +{% if PCAPMERGED.enabled and GLOBALS.role != 'so-import'%} - pcap.enabled +{% elif GLOBALS.role == 'so-import' %} + - pcap.config + - pcap.disabled {% else %} - pcap.disabled {% endif %} diff --git a/salt/pcap/sostatus.sls b/salt/pcap/sostatus.sls index 9e23892c9..d7380e804 100644 --- a/salt/pcap/sostatus.sls +++ b/salt/pcap/sostatus.sls @@ -1,5 +1,16 @@ +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states %} + append_so-steno_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - text: so-steno - unless: grep -q so-steno /opt/so/conf/so-status/so-status.conf + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/common/tools/sbin/so-pcap-export b/salt/pcap/tools/sbin/so-pcap-export similarity index 100% rename from salt/common/tools/sbin/so-pcap-export rename to salt/pcap/tools/sbin/so-pcap-export diff --git a/salt/common/tools/sbin/so-pcap-restart b/salt/pcap/tools/sbin/so-pcap-restart similarity index 100% rename from salt/common/tools/sbin/so-pcap-restart rename to salt/pcap/tools/sbin/so-pcap-restart diff --git a/salt/common/tools/sbin/so-pcap-start b/salt/pcap/tools/sbin/so-pcap-start similarity index 100% rename from salt/common/tools/sbin/so-pcap-start rename to salt/pcap/tools/sbin/so-pcap-start diff --git a/salt/common/tools/sbin/so-pcap-stop b/salt/pcap/tools/sbin/so-pcap-stop similarity index 100% rename from salt/common/tools/sbin/so-pcap-stop rename to salt/pcap/tools/sbin/so-pcap-stop diff --git a/salt/playbook/init.sls b/salt/playbook/init.sls index f76292333..930c3b9ec 100644 --- a/salt/playbook/init.sls +++ b/salt/playbook/init.sls @@ -66,6 +66,23 @@ query_updatepluginurls: - connection_user: root - connection_pass: {{ MYSQLPASS }} +playbook_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://playbook/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#playbook_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://playbook/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + playbooklogdir: file.directory: - name: /opt/so/log/playbook diff --git a/salt/common/tools/sbin/so-playbook-import b/salt/playbook/tools/sbin/so-playbook-import similarity index 100% rename from salt/common/tools/sbin/so-playbook-import rename to salt/playbook/tools/sbin/so-playbook-import diff --git a/salt/common/tools/sbin/so-playbook-reset b/salt/playbook/tools/sbin/so-playbook-reset similarity index 100% rename from salt/common/tools/sbin/so-playbook-reset rename to salt/playbook/tools/sbin/so-playbook-reset diff --git a/salt/common/tools/sbin/so-playbook-restart b/salt/playbook/tools/sbin/so-playbook-restart similarity index 100% rename from salt/common/tools/sbin/so-playbook-restart rename to salt/playbook/tools/sbin/so-playbook-restart diff --git a/salt/common/tools/sbin/so-playbook-ruleupdate b/salt/playbook/tools/sbin/so-playbook-ruleupdate similarity index 100% rename from salt/common/tools/sbin/so-playbook-ruleupdate rename to salt/playbook/tools/sbin/so-playbook-ruleupdate diff --git a/salt/common/tools/sbin/so-playbook-sigma-refresh b/salt/playbook/tools/sbin/so-playbook-sigma-refresh similarity index 100% rename from salt/common/tools/sbin/so-playbook-sigma-refresh rename to salt/playbook/tools/sbin/so-playbook-sigma-refresh diff --git a/salt/common/tools/sbin/so-playbook-start b/salt/playbook/tools/sbin/so-playbook-start similarity index 100% rename from salt/common/tools/sbin/so-playbook-start rename to salt/playbook/tools/sbin/so-playbook-start diff --git a/salt/common/tools/sbin/so-playbook-stop b/salt/playbook/tools/sbin/so-playbook-stop similarity index 100% rename from salt/common/tools/sbin/so-playbook-stop rename to salt/playbook/tools/sbin/so-playbook-stop diff --git a/salt/common/tools/sbin/so-playbook-sync b/salt/playbook/tools/sbin/so-playbook-sync similarity index 100% rename from salt/common/tools/sbin/so-playbook-sync rename to salt/playbook/tools/sbin/so-playbook-sync diff --git a/salt/redis/init.sls b/salt/redis/init.sls index ebaad842b..5806d99f3 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -41,6 +41,23 @@ redisconf: - group: 939 - template: jinja +redis_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://redis/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +redis_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://redis/tools/sbin_jinja + - user: 939 + - group: 939 + - file_mode: 755 + - template: jinja + so-redis: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-redis:{{ GLOBALS.so_version }} diff --git a/salt/common/tools/sbin/so-redis-restart b/salt/redis/tools/sbin/so-redis-restart similarity index 100% rename from salt/common/tools/sbin/so-redis-restart rename to salt/redis/tools/sbin/so-redis-restart diff --git a/salt/common/tools/sbin/so-redis-start b/salt/redis/tools/sbin/so-redis-start similarity index 100% rename from salt/common/tools/sbin/so-redis-start rename to salt/redis/tools/sbin/so-redis-start diff --git a/salt/common/tools/sbin/so-redis-stop b/salt/redis/tools/sbin/so-redis-stop similarity index 100% rename from salt/common/tools/sbin/so-redis-stop rename to salt/redis/tools/sbin/so-redis-stop diff --git a/salt/common/tools/sbin/so-redis-count b/salt/redis/tools/sbin_jinja/so-redis-count similarity index 100% rename from salt/common/tools/sbin/so-redis-count rename to salt/redis/tools/sbin_jinja/so-redis-count diff --git a/salt/sensoroni/init.sls b/salt/sensoroni/init.sls index c410a6fd9..df6b99948 100644 --- a/salt/sensoroni/init.sls +++ b/salt/sensoroni/init.sls @@ -39,6 +39,23 @@ analyzerscripts: - template: jinja - source: salt://sensoroni/files/analyzers +sensoroni_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://sensoroni/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#sensoroni_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://sensoroni/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + so-sensoroni: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-soc:{{ GLOBALS.so_version }} diff --git a/salt/common/tools/sbin/so-sensoroni-restart b/salt/sensoroni/tools/sbin/so-sensoroni-restart similarity index 100% rename from salt/common/tools/sbin/so-sensoroni-restart rename to salt/sensoroni/tools/sbin/so-sensoroni-restart diff --git a/salt/common/tools/sbin/so-sensoroni-start b/salt/sensoroni/tools/sbin/so-sensoroni-start similarity index 100% rename from salt/common/tools/sbin/so-sensoroni-start rename to salt/sensoroni/tools/sbin/so-sensoroni-start diff --git a/salt/common/tools/sbin/so-sensoroni-stop b/salt/sensoroni/tools/sbin/so-sensoroni-stop similarity index 100% rename from salt/common/tools/sbin/so-sensoroni-stop rename to salt/sensoroni/tools/sbin/so-sensoroni-stop diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 9460eeac2..8c3ed5104 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -64,6 +64,23 @@ socbanner: - mode: 600 - template: jinja +soc_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://soc/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#soc_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://soc/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + soccustom: file.managed: - name: /opt/so/conf/soc/custom.js diff --git a/salt/common/tools/sbin/so-soc-restart b/salt/soc/tools/sbin/so-soc-restart similarity index 100% rename from salt/common/tools/sbin/so-soc-restart rename to salt/soc/tools/sbin/so-soc-restart diff --git a/salt/common/tools/sbin/so-soc-start b/salt/soc/tools/sbin/so-soc-start similarity index 100% rename from salt/common/tools/sbin/so-soc-start rename to salt/soc/tools/sbin/so-soc-start diff --git a/salt/common/tools/sbin/so-soc-stop b/salt/soc/tools/sbin/so-soc-stop similarity index 100% rename from salt/common/tools/sbin/so-soc-stop rename to salt/soc/tools/sbin/so-soc-stop diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index 203950bb4..6470d1163 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -58,6 +58,23 @@ playbookrulessync: - defaults: GLOBALS: {{ GLOBALS }} +soctopus_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://soctopus/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#soctopus_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://soctopus/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + so-soctopus: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-soctopus:{{ GLOBALS.so_version }} diff --git a/salt/common/tools/sbin/so-soctopus-restart b/salt/soctopus/tools/sbin/so-soctopus-restart similarity index 100% rename from salt/common/tools/sbin/so-soctopus-restart rename to salt/soctopus/tools/sbin/so-soctopus-restart diff --git a/salt/common/tools/sbin/so-soctopus-start b/salt/soctopus/tools/sbin/so-soctopus-start similarity index 100% rename from salt/common/tools/sbin/so-soctopus-start rename to salt/soctopus/tools/sbin/so-soctopus-start diff --git a/salt/common/tools/sbin/so-soctopus-stop b/salt/soctopus/tools/sbin/so-soctopus-stop similarity index 100% rename from salt/common/tools/sbin/so-soctopus-stop rename to salt/soctopus/tools/sbin/so-soctopus-stop diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 081f2ebd1..6b7a2bbd2 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -57,6 +57,23 @@ backend_passwords: - defaults: PASSWORDS: {{ STRELKAMERGED.config.backend.passwords }} +strelka_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://strelka/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#strelka_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://strelka/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + backend_taste: file.managed: - name: /opt/so/conf/strelka/backend/taste/taste.yara diff --git a/salt/common/tools/sbin/so-strelka-restart b/salt/strelka/tools/sbin/so-strelka-restart similarity index 100% rename from salt/common/tools/sbin/so-strelka-restart rename to salt/strelka/tools/sbin/so-strelka-restart diff --git a/salt/common/tools/sbin/so-strelka-start b/salt/strelka/tools/sbin/so-strelka-start similarity index 100% rename from salt/common/tools/sbin/so-strelka-start rename to salt/strelka/tools/sbin/so-strelka-start diff --git a/salt/common/tools/sbin/so-strelka-stop b/salt/strelka/tools/sbin/so-strelka-stop similarity index 100% rename from salt/common/tools/sbin/so-strelka-stop rename to salt/strelka/tools/sbin/so-strelka-stop diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 159e59f4f..7788fa94a 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -38,6 +38,23 @@ socoregroupwithsuricata: - addusers: - suricata +suricata_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://suricata/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +suricata_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://suricata/tools/sbin_jinja + - user: 939 + - group: 939 + - file_mode: 755 + - template: jinja + suridir: file.directory: - name: /opt/so/conf/suricata diff --git a/salt/common/tools/sbin/so-suricata-restart b/salt/suricata/tools/sbin/so-suricata-restart similarity index 100% rename from salt/common/tools/sbin/so-suricata-restart rename to salt/suricata/tools/sbin/so-suricata-restart diff --git a/salt/common/tools/sbin/so-suricata-start b/salt/suricata/tools/sbin/so-suricata-start similarity index 100% rename from salt/common/tools/sbin/so-suricata-start rename to salt/suricata/tools/sbin/so-suricata-start diff --git a/salt/common/tools/sbin/so-suricata-stop b/salt/suricata/tools/sbin/so-suricata-stop similarity index 100% rename from salt/common/tools/sbin/so-suricata-stop rename to salt/suricata/tools/sbin/so-suricata-stop diff --git a/salt/common/tools/sbin/so-suricata-testrule b/salt/suricata/tools/sbin_jinja/so-suricata-testrule similarity index 100% rename from salt/common/tools/sbin/so-suricata-testrule rename to salt/suricata/tools/sbin_jinja/so-suricata-testrule diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index e5574e7d1..f14ef14e4 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -39,6 +39,23 @@ tgrafsyncscripts: - exclude_pat: zeekcaptureloss.sh {% endif %} +telegraf_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://telegraf/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#telegraf_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://telegraf/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + tgrafconf: file.managed: - name: /opt/so/conf/telegraf/etc/telegraf.conf diff --git a/salt/common/tools/sbin/so-telegraf-restart b/salt/telegraf/tools/sbin/so-telegraf-restart similarity index 100% rename from salt/common/tools/sbin/so-telegraf-restart rename to salt/telegraf/tools/sbin/so-telegraf-restart diff --git a/salt/common/tools/sbin/so-telegraf-start b/salt/telegraf/tools/sbin/so-telegraf-start similarity index 100% rename from salt/common/tools/sbin/so-telegraf-start rename to salt/telegraf/tools/sbin/so-telegraf-start diff --git a/salt/common/tools/sbin/so-telegraf-stop b/salt/telegraf/tools/sbin/so-telegraf-stop similarity index 100% rename from salt/common/tools/sbin/so-telegraf-stop rename to salt/telegraf/tools/sbin/so-telegraf-stop diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 3b8390a77..ce5996888 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -92,6 +92,23 @@ zeekstatedbownership: - replace: False - create: False +zeek_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://zeek/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#zeek_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://zeek/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + # Sync Intel zeekintelloadsync: file.managed: diff --git a/salt/common/tools/sbin/so-zeek-restart b/salt/zeek/tools/sbin/so-zeek-restart similarity index 100% rename from salt/common/tools/sbin/so-zeek-restart rename to salt/zeek/tools/sbin/so-zeek-restart diff --git a/salt/common/tools/sbin/so-zeek-start b/salt/zeek/tools/sbin/so-zeek-start similarity index 100% rename from salt/common/tools/sbin/so-zeek-start rename to salt/zeek/tools/sbin/so-zeek-start diff --git a/salt/common/tools/sbin/so-zeek-stats b/salt/zeek/tools/sbin/so-zeek-stats similarity index 100% rename from salt/common/tools/sbin/so-zeek-stats rename to salt/zeek/tools/sbin/so-zeek-stats diff --git a/salt/common/tools/sbin/so-zeek-stop b/salt/zeek/tools/sbin/so-zeek-stop similarity index 100% rename from salt/common/tools/sbin/so-zeek-stop rename to salt/zeek/tools/sbin/so-zeek-stop diff --git a/setup/so-functions b/setup/so-functions index 4fbd26c41..1a96d4bd0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -908,6 +908,8 @@ create_manager_pillars() { soc_pillar idh_pillar influxdb_pillar + logrotate_pillar + patch_pillar } @@ -1238,11 +1240,6 @@ firewall_generate_templates() { logCmd "cp -r ../files/firewall/* /opt/so/saltstack/local/salt/firewall/" - # i think this can be commented out for 2.4 - #for i in analyst beats_endpoint endgame sensors manager managersearch elastic_agent_endpoint searchnodes; do - # $default_salt_dir/salt/common/tools/sbin/so-firewall --role="$i" --ip=127.0.0.1 - #done - } generate_ca() { @@ -1349,6 +1346,21 @@ idh_pillar() { touch $adv_idh_pillar_file } +kibana_pillar() { + touch $adv_kibana_pillar_file + touch $kibana_pillar_file +} + +logrotate_pillar() { + touch $adv_logrotate_pillar_file + touch $logrotate_pillar_file +} + +patch_pillar() { + touch $adv_patch_pillar_file + touch $patch_pillar_file +} + logstash_pillar() { # Create the logstash advanced pillar touch $adv_logstash_pillar_file @@ -1658,51 +1670,6 @@ parse_install_username() { INSTALLUSERNAME=${SUDO_USER:-${USER}} } -patch_pillar() { - title "Create the patch pillar file" - local pillar_file=$local_salt_dir/pillar/minions/$MINION_ID.sls - - - if [[ $MANAGERUPDATES == 1 ]]; then - local source="manager" - else - local source="direct" - fi - - printf '%s\n'\ - "patch:"\ - " os:"\ - " source: '$source'"\ - " schedule_name: '$PATCHSCHEDULENAME'"\ - " enabled: True"\ - " splay: 300"\ - "" > "$pillar_file" - -} - -patch_schedule_os_new() { - title "Create the patch schedule" - local OSPATCHSCHEDULEDIR="$temp_install_dir/salt/patch/os/schedules" - local OSPATCHSCHEDULE="$OSPATCHSCHEDULEDIR/$PATCHSCHEDULENAME.yml" - - logCmd "mkdir -p $OSPATCHSCHEDULEDIR" - - printf '%s\n'\ - "patch:"\ - " os:"\ - " schedule:"> "$OSPATCHSCHEDULE" - for psd in "${PATCHSCHEDULEDAYS[@]}";do - psd="${psd//\"/}" - echo " - $psd:" >> "$OSPATCHSCHEDULE" - for psh in "${PATCHSCHEDULEHOURS[@]}" - do - psh="${psh//\"/}" - echo " - '$psh'" >> "$OSPATCHSCHEDULE" - done - done - -} - print_salt_state_apply() { local state=$1 @@ -1992,8 +1959,8 @@ repo_sync_local() { info "Backing up old repos" mkdir -p /nsm/repo mkdir -p /opt/so/conf/reposync/cache - echo "https://repo.securityonion.net/file/so-repo/2.4/" > /opt/so/conf/reposync/mirror.txt - echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/2.4/" >> /opt/so/conf/reposync/mirror.txt + echo "https://repo.securityonion.net/file/so-repo/prod/2.4/rocky/9" > /opt/so/conf/reposync/mirror.txt + echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/prod/2.4/rocky/9" >> /opt/so/conf/reposync/mirror.txt echo "[main]" > /opt/so/conf/reposync/repodownload.conf echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf @@ -2014,8 +1981,6 @@ repo_sync_local() { # TODO Add if for ISO install curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install logCmd "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" - # Run it again and make sure we got allt he things - logCmd "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" # After the download is complete run createrepo create_repo @@ -2307,23 +2272,20 @@ set_hostname() { } set_initial_firewall_policy() { - title "Setting Initial Firewall Policy" - if [ -f $default_salt_dir/salt/common/tools/sbin/so-firewall ]; then chmod +x $default_salt_dir/salt/common/tools/sbin/so-firewall; fi - case "$install_type" in 'EVAL' | 'MANAGER' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT') - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost $minion_type $MAINIP --apply + so-firewall includehost $minion_type $MAINIP --apply ;; esac } set_initial_firewall_access() { if [[ ! -z "$ALLOW_CIDR" ]]; then - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost analyst $ALLOW_CIDR --apply + so-firewall includehost analyst $ALLOW_CIDR --apply fi if [[ ! -z "$MINION_CIDR" ]]; then - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost sensors $MINION_CIDR - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost searchnodes $MINION_CIDR --apply + so-firewall includehost sensors $MINION_CIDR + so-firewall includehost searchnodes $MINION_CIDR --apply fi } @@ -2399,19 +2361,6 @@ update_sudoers_for_testing() { fi } -update_sudoers() { - - if ! grep -qE '^soremote\ ALL=\(ALL\)\ NOPASSWD:(\/usr\/bin\/salt\-key|\/opt\/so\/saltstack)' /etc/sudoers; then - # Update Sudoers so that soremote can accept keys without a password - echo "soremote ALL=(ALL) NOPASSWD:/usr/bin/salt-key" | tee -a /etc/sudoers - echo "soremote ALL=(ALL) NOPASSWD:$default_salt_dir/salt/common/tools/sbin/so-firewall" | tee -a /etc/sudoers - echo "soremote ALL=(ALL) NOPASSWD:$default_salt_dir/pillar/data/addtotab.sh" | tee -a /etc/sudoers - echo "soremote ALL=(ALL) NOPASSWD:$default_salt_dir/salt/manager/files/add_minion.sh" | tee -a /etc/sudoers - else - info "User soremote already granted sudo privileges" - fi -} - update_packages() { if [[ $is_rocky ]]; then logCmd "dnf repolist" diff --git a/setup/so-setup b/setup/so-setup index 72549d79d..b1c4ce42b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -239,7 +239,8 @@ esac # Allow execution of SO tools during setup local_sbin="$(pwd)/../salt/common/tools/sbin" -export PATH=$PATH:$local_sbin +manager_sbin="$(pwd)/../salt/manager/tools/sbin" +export PATH=$PATH:$local_sbin:$manager_sbin # Ubuntu whiptail pallete to make it look the same as CentOS and Rocky. set_palette >> $setup_log 2>&1 @@ -607,9 +608,6 @@ if ! [[ -f $install_opt_file ]]; then securityonion_repo # Update existing packages update_packages - # Download Elastic Agent Artifacts - title "Downloading Elastic Agent Artifacts" - download_elastic_agent_artifacts # Install salt saltify # Start the master service @@ -626,6 +624,9 @@ if ! [[ -f $install_opt_file ]]; then logCmd "salt-call state.apply docker" firewall_generate_templates set_initial_firewall_policy + # Download Elastic Agent Artifacts + title "Downloading Elastic Agent Artifacts" + download_elastic_agent_artifacts generate_ca generate_ssl diff --git a/setup/so-variables b/setup/so-variables index 98ecb2b4f..b2e439a5c 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -154,6 +154,12 @@ export manager_pillar_file adv_manager_pillar_file="$local_salt_dir/pillar/manager/adv_manager.sls" export adv_manager_pillar_file +kibana_pillar_file="$local_salt_dir/pillar/kibana/soc_kibana.sls" +export kibana_pillar_file + +adv_kibana_pillar_file="$local_salt_dir/pillar/kibana/adv_kibana.sls" +export adv_kibana_pillar_file + kratos_pillar_file="$local_salt_dir/pillar/kratos/soc_kratos.sls" export kratos_pillar_file @@ -195,3 +201,15 @@ export influxdb_pillar_file adv_influxdb_pillar_file="$local_salt_dir/pillar/influxdb/adv_influxdb.sls" export adv_influxdb_pillar_file + +logrotate_pillar_file="$local_salt_dir/pillar/logrotate/soc_logrotate.sls" +export logrotate_pillar_file + +adv_logrotate_pillar_file="$local_salt_dir/pillar/logrotate/adv_logrotate.sls" +export adv_logrotate_pillar_file + +patch_pillar_file="$local_salt_dir/pillar/patch/soc_patch.sls" +export patch_pillar_file + +adv_patch_pillar_file="$local_salt_dir/pillar/patch/adv_patch.sls" +export adv_patch_pillar_file \ No newline at end of file