From 4b73f859d115236ec7a525c98c87afda8676ef4c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 Apr 2023 15:33:08 -0400 Subject: [PATCH 01/41] don't sync the repo 2x --- setup/so-functions | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index c02b93fcb..d2506cac5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2001,8 +2001,6 @@ repo_sync_local() { # TODO Add if for ISO install curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install logCmd "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" - # Run it again and make sure we got allt he things - logCmd "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" # After the download is complete run createrepo create_repo From b3f94961eaaceec597d4c10e7edd2183a1575e2c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 Apr 2023 13:30:23 -0400 Subject: [PATCH 02/41] Fix Kibana and friends --- pillar/top.sls | 19 ++++++++++++++++++- setup/so-functions | 5 +++++ setup/so-variables | 6 ++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pillar/top.sls b/pillar/top.sls index 1acc5d030..9e65257d0 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -18,9 +18,12 @@ base: '*_eval or *_heavynode or *_sensor or *_standalone or *_import': - match: compound - - zeek + - zeek.soc_zeek + - zeek.adv_zeek - bpf.soc_bpf - bpf.adv_bpf + - suricata.soc_suricata + - suricata.adv_suricata '*_managersearch or *_heavynode': - match: compound @@ -32,6 +35,8 @@ base: - elasticsearch.index_templates - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - curator.soc_curator + - curator.adv_curator '*_manager': - logstash @@ -39,6 +44,8 @@ base: - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.index_templates + - curator.soc_curator + - curator.adv_curator '*_manager or *_managersearch': - match: compound @@ -57,6 +64,8 @@ base: - idstools.adv_idstools - soc.soc_soc - soc.adv_soc + - kibana.soc_kibana + - kibana.adv_kibana - kratos.soc_kratos - kratos.adv_kratos - redis.soc_redis @@ -65,6 +74,8 @@ base: - influxdb.adv_influxdb - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elastalert.soc_elastalert + - elastalert.adv_elastalert - backup.soc_backup - backup.adv_backup - firewall.soc_firewall @@ -94,6 +105,8 @@ base: - kratos.soc_kratos - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elastalert.soc_elastalert + - elastalert.adv_elastalert - manager.soc_manager - manager.adv_manager - idstools.soc_idstools @@ -139,6 +152,8 @@ base: - influxdb.adv_influxdb - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elastalert.soc_elastalert + - elastalert.adv_elastalert - manager.soc_manager - manager.adv_manager - soc.soc_soc @@ -209,6 +224,8 @@ base: - kratos.soc_kratos - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elastalert.soc_elastalert + - elastalert.adv_elastalert - manager.soc_manager - manager.adv_manager - soc.soc_soc diff --git a/setup/so-functions b/setup/so-functions index d2506cac5..3d7017d8e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1336,6 +1336,11 @@ idh_pillar() { touch $adv_idh_pillar_file } +kibana_pillar() { + touch $adv_kibana_pillar_file + touch $kibana_pillar_file +} + logstash_pillar() { # Create the logstash advanced pillar touch $adv_logstash_pillar_file diff --git a/setup/so-variables b/setup/so-variables index 98ecb2b4f..3d599afb4 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -154,6 +154,12 @@ export manager_pillar_file adv_manager_pillar_file="$local_salt_dir/pillar/manager/adv_manager.sls" export adv_manager_pillar_file +kibana_pillar_file="$local_salt_dir/pillar/kibana/soc_kibana.sls" +export kibana_pillar_file + +adv_kibana_pillar_file="$local_salt_dir/pillar/kibana/adv_kibana.sls" +export adv_kibana_pillar_file + kratos_pillar_file="$local_salt_dir/pillar/kratos/soc_kratos.sls" export kratos_pillar_file From 868cb8183c76ff58f0ad11097b2a47aa65eb32f8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 Apr 2023 14:15:34 -0400 Subject: [PATCH 03/41] Fix the top file --- pillar/top.sls | 141 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 92 insertions(+), 49 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 9e65257d0..e72e40e46 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -1,51 +1,22 @@ base: '*': - - patch.needs_restarting - - ntp.soc_ntp - - ntp.adv_ntp - - logrotate - docker.soc_docker - docker.adv_docker + - influxdb.token + - logrotate.soc_logrotate + - logrotate.adv_logrotate + - nginx.soc_nginx + - nginx.adv_nginx + - node_data.ips + - ntp.soc_ntp + - ntp.adv_ntp + - patch.needs_restarting + - patch.soc_patch + - patch.adv_patch - sensoroni.soc_sensoroni - sensoroni.adv_sensoroni - telegraf.soc_telegraf - telegraf.adv_telegraf - - influxdb.token - - node_data.ips - - '* and not *_eval and not *_import': - - logstash.nodes - - '*_eval or *_heavynode or *_sensor or *_standalone or *_import': - - match: compound - - zeek.soc_zeek - - zeek.adv_zeek - - bpf.soc_bpf - - bpf.adv_bpf - - suricata.soc_suricata - - suricata.adv_suricata - - '*_managersearch or *_heavynode': - - match: compound - - logstash - - logstash.manager - - logstash.search - - logstash.soc_logstash - - logstash.adv_logstash - - elasticsearch.index_templates - - elasticsearch.soc_elasticsearch - - elasticsearch.adv_elasticsearch - - curator.soc_curator - - curator.adv_curator - - '*_manager': - - logstash - - logstash.manager - - logstash.soc_logstash - - logstash.adv_logstash - - elasticsearch.index_templates - - curator.soc_curator - - curator.adv_curator '*_manager or *_managersearch': - match: compound @@ -62,8 +33,12 @@ base: - manager.adv_manager - idstools.soc_idstools - idstools.adv_idstools + - logstash.soc_logstash + - logstash.adv_logstash - soc.soc_soc - soc.adv_soc + - soctopus.soc_soctopus + - soctopus.adv_soctopus - kibana.soc_kibana - kibana.adv_kibana - kratos.soc_kratos @@ -80,6 +55,10 @@ base: - backup.adv_backup - firewall.soc_firewall - firewall.adv_firewall + - curator.soc_curator + - curator.adv_curator + - soctopus.soc_soctopus + - soctopus.adv_soctopus - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -87,6 +66,16 @@ base: - healthcheck.sensor - global.soc_global - global.adv_global + - strelka.soc_strelka + - strelka.adv_strelka + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -112,6 +101,13 @@ base: - idstools.soc_idstools - idstools.adv_idstools - soc.soc_soc + - soc.adv_soc + - soctopus.soc_soctopus + - soctopus.adv_soctopus + - strelka.soc_strelka + - strelka.adv_strelka + - curator.soc_curator + - curator.adv_curator - kratos.soc_kratos - kratos.adv_kratos - redis.soc_redis @@ -122,6 +118,14 @@ base: - backup.adv_backup - firewall.soc_firewall - firewall.adv_firewall + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -157,18 +161,50 @@ base: - manager.soc_manager - manager.adv_manager - soc.soc_soc + - soc.adv_soc + - soctopus.soc_soctopus + - soctopus.adv_soctopus + - strelka.soc_strelka + - strelka.adv_strelka + - curator.soc_curator + - curator.adv_curator - backup.soc_backup - backup.adv_backup - firewall.soc_firewall - firewall.adv_firewall + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata - minions.{{ grains.id }} - minions.adv_{{ grains.id }} '*_heavynode': - elasticsearch.auth + - logstash.soc_logstash + - logstash.adv_logstash + - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch + - curator.soc_curator + - curator.adv_curator - global.soc_global - global.adv_global - redis.soc_redis + - redis.adv_redis + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata + - strelka.soc_strelka + - strelka.adv_strelka - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -181,25 +217,19 @@ base: - minions.adv_{{ grains.id }} '*_searchnode': - - logstash - - logstash.search - logstash.soc_logstash - logstash.adv_logstash - - elasticsearch.index_templates - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} - - redis.soc_redis - global.soc_global - global.adv_global - minions.{{ grains.id }} - minions.adv_{{ grains.id }} '*_receiver': - - logstash - - logstash.receiver - logstash.soc_logstash - logstash.adv_logstash {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} @@ -229,6 +259,11 @@ base: - manager.soc_manager - manager.adv_manager - soc.soc_soc + - soc.adv_soc + - soctopus.soc_soctopus + - soctopus.adv_soctopus + - curator.soc_curator + - curator.adv_curator - global.soc_global - global.adv_global - backup.soc_backup @@ -241,6 +276,16 @@ base: - influxdb.adv_influxdb - firewall.soc_firewall - firewall.adv_firewall + - zeek.soc_zeek + - zeek.adv_zeek + - bpf.soc_bpf + - bpf.adv_bpf + - pcap.soc_pcap + - pcap.adv_pcap + - suricata.soc_suricata + - suricata.adv_suricata + - strelka.soc_strelka + - strelka.adv_strelka - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -249,13 +294,11 @@ base: - global.adv_global - backup.soc_backup - backup.adv_backup - - logstash - - logstash.fleet - logstash.soc_logstash - logstash.adv_logstash - minions.{{ grains.id }} - minions.adv_{{ grains.id }} - '*_workstation': + '*_desktop': - minions.{{ grains.id }} - minions.adv_{{ grains.id }} From e799edaf491d7ff63911ac47321121c7fdf3dfb7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 26 Apr 2023 15:54:16 -0400 Subject: [PATCH 04/41] Fix globals order --- pillar/top.sls | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index e72e40e46..90b0a41b9 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -1,5 +1,7 @@ base: '*': + - global.soc_global + - global.adv_global - docker.soc_docker - docker.adv_docker - influxdb.token @@ -27,8 +29,6 @@ base: - kibana.secrets {% endif %} - secrets - - global.soc_global - - global.adv_global - manager.soc_manager - manager.adv_manager - idstools.soc_idstools @@ -64,8 +64,6 @@ base: '*_sensor': - healthcheck.sensor - - global.soc_global - - global.adv_global - strelka.soc_strelka - strelka.adv_strelka - zeek.soc_zeek @@ -89,8 +87,6 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %} - kibana.secrets {% endif %} - - global.soc_global - - global.adv_global - kratos.soc_kratos - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch @@ -144,8 +140,6 @@ base: {% endif %} - secrets - healthcheck.standalone - - global.soc_global - - global.adv_global - idstools.soc_idstools - idstools.adv_idstools - kratos.soc_kratos @@ -191,8 +185,6 @@ base: - elasticsearch.adv_elasticsearch - curator.soc_curator - curator.adv_curator - - global.soc_global - - global.adv_global - redis.soc_redis - redis.adv_redis - zeek.soc_zeek @@ -209,8 +201,6 @@ base: - minions.adv_{{ grains.id }} '*_idh': - - global.soc_global - - global.adv_global - idh.soc_idh - idh.adv_idh - minions.{{ grains.id }} @@ -224,8 +214,6 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} - - global.soc_global - - global.adv_global - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -237,8 +225,6 @@ base: {% endif %} - redis.soc_redis - redis.adv_redis - - global.soc_global - - global.adv_global - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -264,8 +250,6 @@ base: - soctopus.adv_soctopus - curator.soc_curator - curator.adv_curator - - global.soc_global - - global.adv_global - backup.soc_backup - backup.adv_backup - kratos.soc_kratos @@ -290,8 +274,6 @@ base: - minions.adv_{{ grains.id }} '*_fleet': - - global.soc_global - - global.adv_global - backup.soc_backup - backup.adv_backup - logstash.soc_logstash From 3d7f2bc691f4b251dde2b8093b75877e06b99052 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 27 Apr 2023 13:23:53 -0400 Subject: [PATCH 05/41] Fix annotations and file locations --- salt/common/init.sls | 7 +- salt/common/tools/sbin/so-helix-apikey | 27 -- .../files/bin}/so-curator-restart | 0 .../files/bin}/so-curator-start | 0 .../files/bin}/so-curator-stop | 0 .../bin}/so-elastalert-create | 0 .../bin}/so-elastalert-restart | 0 .../bin}/so-elastalert-start | 0 .../bin}/so-elastalert-stop | 0 .../bin}/so-elastalert-test | 0 .../sbin/so-elasticsearch-cluster-space-total | 0 .../sbin/so-elasticsearch-cluster-space-used | 0 .../so-elasticsearch-component-templates-list | 0 .../so-elasticsearch-ilm-lifecycle-status | 0 .../sbin/so-elasticsearch-ilm-policy-delete | 0 .../so-elasticsearch-ilm-policy-load copy} | 0 .../sbin/so-elasticsearch-ilm-policy-view | 0 .../tools/sbin/so-elasticsearch-ilm-restart | 0 .../tools/sbin/so-elasticsearch-ilm-start | 0 .../tools/sbin/so-elasticsearch-ilm-status | 0 .../tools/sbin/so-elasticsearch-ilm-stop | 0 .../so-elasticsearch-index-templates-list | 0 .../tools/sbin/so-elasticsearch-indices-list | 0 .../tools/sbin/so-elasticsearch-indices-rw | 0 .../sbin/so-elasticsearch-pipeline-stats | 0 .../tools/sbin/so-elasticsearch-pipeline-view | 0 .../sbin/so-elasticsearch-pipelines-list | 0 .../tools/sbin/so-elasticsearch-query | 0 .../tools/sbin/so-elasticsearch-restart | 0 .../tools/sbin/so-elasticsearch-shards-list | 0 .../tools/sbin/so-elasticsearch-start | 0 .../tools/sbin/so-elasticsearch-stop | 0 .../sbin/so-elasticsearch-template-remove | 0 .../tools/sbin/so-elasticsearch-template-view | 0 .../sbin/so-elasticsearch-templates-list | 0 .../tools/sbin/so-elasticsearch-wait | 0 salt/firewall/soc_firewall.yaml | 413 ++++++++++++++++++ .../sbin => idstools/bin}/so-idstools-restart | 0 .../sbin => idstools/bin}/so-idstools-start | 0 .../sbin => idstools/bin}/so-idstools-stop | 0 salt/manager/{files => sbin}/so-repo-sync | 0 salt/manager/sbin/so-saltstack-update | 53 +++ .../tools/sbin => zeek/bin}/so-zeek-restart | 0 .../tools/sbin => zeek/bin}/so-zeek-start | 0 .../tools/sbin => zeek/bin}/so-zeek-stats | 0 .../tools/sbin => zeek/bin}/so-zeek-stop | 0 setup/so-functions | 10 + setup/so-variables | 12 + 48 files changed, 491 insertions(+), 31 deletions(-) delete mode 100755 salt/common/tools/sbin/so-helix-apikey rename salt/{common/tools/sbin => curator/files/bin}/so-curator-restart (100%) mode change 100755 => 100644 rename salt/{common/tools/sbin => curator/files/bin}/so-curator-start (100%) mode change 100755 => 100644 rename salt/{common/tools/sbin => curator/files/bin}/so-curator-stop (100%) mode change 100755 => 100644 rename salt/{common/tools/sbin => elastalert/bin}/so-elastalert-create (100%) rename salt/{common/tools/sbin => elastalert/bin}/so-elastalert-restart (100%) rename salt/{common/tools/sbin => elastalert/bin}/so-elastalert-start (100%) rename salt/{common/tools/sbin => elastalert/bin}/so-elastalert-stop (100%) rename salt/{common/tools/sbin => elastalert/bin}/so-elastalert-test (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-cluster-space-total (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-cluster-space-used (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-component-templates-list (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-ilm-lifecycle-status (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-ilm-policy-delete (100%) rename salt/{common/tools/sbin/so-elasticsearch-ilm-policy-load => elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load copy} (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-ilm-policy-view (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-ilm-restart (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-ilm-start (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-ilm-status (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-ilm-stop (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-index-templates-list (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-indices-list (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-indices-rw (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-pipeline-stats (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-pipeline-view (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-pipelines-list (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-query (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-restart (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-shards-list (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-start (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-stop (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-template-remove (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-template-view (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-templates-list (100%) rename salt/{common => elasticsearch}/tools/sbin/so-elasticsearch-wait (100%) create mode 100644 salt/firewall/soc_firewall.yaml rename salt/{common/tools/sbin => idstools/bin}/so-idstools-restart (100%) rename salt/{common/tools/sbin => idstools/bin}/so-idstools-start (100%) rename salt/{common/tools/sbin => idstools/bin}/so-idstools-stop (100%) rename salt/manager/{files => sbin}/so-repo-sync (100%) create mode 100755 salt/manager/sbin/so-saltstack-update rename salt/{common/tools/sbin => zeek/bin}/so-zeek-restart (100%) rename salt/{common/tools/sbin => zeek/bin}/so-zeek-start (100%) rename salt/{common/tools/sbin => zeek/bin}/so-zeek-stats (100%) rename salt/{common/tools/sbin => zeek/bin}/so-zeek-stop (100%) diff --git a/salt/common/init.sls b/salt/common/init.sls index f23a05757..2feee941c 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -49,13 +49,12 @@ so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - unless: ls /opt/so/conf/so-status/so-status.conf -sosaltstackperms: +socore_opso_perms: file.directory: - - name: /opt/so/saltstack + - name: /opt/so - user: 939 - group: 939 - - dir_mode: 770 - + so_log_perms: file.directory: - name: /opt/so/log diff --git a/salt/common/tools/sbin/so-helix-apikey b/salt/common/tools/sbin/so-helix-apikey deleted file mode 100755 index c58d2ad89..000000000 --- a/salt/common/tools/sbin/so-helix-apikey +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -local_salt_dir=/opt/so/saltstack/local - -got_root() { - - # Make sure you are root - if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" - exit 1 - fi - -} - -got_root -if [ ! -f $local_salt_dir/pillar/fireeye/init.sls ]; then - echo "This is nto configured for Helix Mode. Please re-install." - exit -else - echo "Enter your Helix API Key: " - read APIKEY - sed -i "s/^ api_key.*/ api_key: $APIKEY/g" $local_salt_dir/pillar/fireeye/init.sls - docker stop so-logstash - docker rm so-logstash - echo "Restarting Logstash for updated key" - salt-call state.apply logstash queue=True -fi diff --git a/salt/common/tools/sbin/so-curator-restart b/salt/curator/files/bin/so-curator-restart old mode 100755 new mode 100644 similarity index 100% rename from salt/common/tools/sbin/so-curator-restart rename to salt/curator/files/bin/so-curator-restart diff --git a/salt/common/tools/sbin/so-curator-start b/salt/curator/files/bin/so-curator-start old mode 100755 new mode 100644 similarity index 100% rename from salt/common/tools/sbin/so-curator-start rename to salt/curator/files/bin/so-curator-start diff --git a/salt/common/tools/sbin/so-curator-stop b/salt/curator/files/bin/so-curator-stop old mode 100755 new mode 100644 similarity index 100% rename from salt/common/tools/sbin/so-curator-stop rename to salt/curator/files/bin/so-curator-stop diff --git a/salt/common/tools/sbin/so-elastalert-create b/salt/elastalert/bin/so-elastalert-create similarity index 100% rename from salt/common/tools/sbin/so-elastalert-create rename to salt/elastalert/bin/so-elastalert-create diff --git a/salt/common/tools/sbin/so-elastalert-restart b/salt/elastalert/bin/so-elastalert-restart similarity index 100% rename from salt/common/tools/sbin/so-elastalert-restart rename to salt/elastalert/bin/so-elastalert-restart diff --git a/salt/common/tools/sbin/so-elastalert-start b/salt/elastalert/bin/so-elastalert-start similarity index 100% rename from salt/common/tools/sbin/so-elastalert-start rename to salt/elastalert/bin/so-elastalert-start diff --git a/salt/common/tools/sbin/so-elastalert-stop b/salt/elastalert/bin/so-elastalert-stop similarity index 100% rename from salt/common/tools/sbin/so-elastalert-stop rename to salt/elastalert/bin/so-elastalert-stop diff --git a/salt/common/tools/sbin/so-elastalert-test b/salt/elastalert/bin/so-elastalert-test similarity index 100% rename from salt/common/tools/sbin/so-elastalert-test rename to salt/elastalert/bin/so-elastalert-test diff --git a/salt/common/tools/sbin/so-elasticsearch-cluster-space-total b/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-space-total similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-cluster-space-total rename to salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-space-total diff --git a/salt/common/tools/sbin/so-elasticsearch-cluster-space-used b/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-space-used similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-cluster-space-used rename to salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-space-used diff --git a/salt/common/tools/sbin/so-elasticsearch-component-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-component-templates-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-lifecycle-status similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-ilm-lifecycle-status rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-lifecycle-status diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-delete similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-ilm-policy-delete rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-delete diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load copy similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-ilm-policy-load rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load copy diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-policy-view b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-view similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-ilm-policy-view rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-view diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-restart b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-restart similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-ilm-restart rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-restart diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-start b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-ilm-start rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-status b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-status similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-ilm-status rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-status diff --git a/salt/common/tools/sbin/so-elasticsearch-ilm-stop b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-ilm-stop rename to salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop diff --git a/salt/common/tools/sbin/so-elasticsearch-index-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-templates-list similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-index-templates-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-index-templates-list diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-list similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-indices-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-indices-list diff --git a/salt/common/tools/sbin/so-elasticsearch-indices-rw b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-rw similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-indices-rw rename to salt/elasticsearch/tools/sbin/so-elasticsearch-indices-rw diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-stats b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-stats similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-pipeline-stats rename to salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-stats diff --git a/salt/common/tools/sbin/so-elasticsearch-pipeline-view b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-view similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-pipeline-view rename to salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-view diff --git a/salt/common/tools/sbin/so-elasticsearch-pipelines-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines-list similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-pipelines-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines-list diff --git a/salt/common/tools/sbin/so-elasticsearch-query b/salt/elasticsearch/tools/sbin/so-elasticsearch-query similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-query rename to salt/elasticsearch/tools/sbin/so-elasticsearch-query diff --git a/salt/common/tools/sbin/so-elasticsearch-restart b/salt/elasticsearch/tools/sbin/so-elasticsearch-restart similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-restart rename to salt/elasticsearch/tools/sbin/so-elasticsearch-restart diff --git a/salt/common/tools/sbin/so-elasticsearch-shards-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-shards-list similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-shards-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-shards-list diff --git a/salt/common/tools/sbin/so-elasticsearch-start b/salt/elasticsearch/tools/sbin/so-elasticsearch-start similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-start rename to salt/elasticsearch/tools/sbin/so-elasticsearch-start diff --git a/salt/common/tools/sbin/so-elasticsearch-stop b/salt/elasticsearch/tools/sbin/so-elasticsearch-stop similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-stop rename to salt/elasticsearch/tools/sbin/so-elasticsearch-stop diff --git a/salt/common/tools/sbin/so-elasticsearch-template-remove b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-remove similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-template-remove rename to salt/elasticsearch/tools/sbin/so-elasticsearch-template-remove diff --git a/salt/common/tools/sbin/so-elasticsearch-template-view b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-view similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-template-view rename to salt/elasticsearch/tools/sbin/so-elasticsearch-template-view diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-list similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-templates-list rename to salt/elasticsearch/tools/sbin/so-elasticsearch-templates-list diff --git a/salt/common/tools/sbin/so-elasticsearch-wait b/salt/elasticsearch/tools/sbin/so-elasticsearch-wait similarity index 100% rename from salt/common/tools/sbin/so-elasticsearch-wait rename to salt/elasticsearch/tools/sbin/so-elasticsearch-wait diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml new file mode 100644 index 000000000..452c3c26f --- /dev/null +++ b/salt/firewall/soc_firewall.yaml @@ -0,0 +1,413 @@ +firewall: + hostgroups: + analyst: &hostgroupsettings + description: List of IP or CIDR blocks to allow access to for this hostgroup. + helplink: firewall.html + multiline: True + regex: ^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$ + regexFailureMessage: You must enter a properly formatted IP address or CIDR. + anywhere: *hostgroupsettings + beats_endpoint: *hostgroupsettings + beats_endpoint_ssl: *hostgroupsettings + dockernet: *hostgroupsettings + elastic_agent_endpoint: *hostgroupsettings + elasticsearch_rest: *hostgroupsettings + endgame: *hostgroupsettings + eval: *hostgroupsettings + fleet: *hostgroupsettings + heavynodes: *hostgroupsettings + idh: *hostgroupsettings + localhost: *hostgroupsettings + manager: *hostgroupsettings + receivers: *hostgroupsettings + searchnodes: *hostgroupsettings + securityonion_desktops: *hostgroupsettings + self: *hostgroupsettings + sensors: *hostgroupsettings + standalone: *hostgroupsettings + strelka_frontend: *hostgroupsettings + syslog: *hostgroupsettings + portgroups: + all: + tcp: + udp: + agrules: + tcp: + udp: + beats_5044: + tcp: + udp: + beats_5644: + tcp: + udp: + beats_5066: + tcp: + udp: + beats_5056: + tcp: + udp: + docker_registry: + tcp: + udp: + elasticsearch_node: + tcp: + udp: + elasticsearch_rest: + tcp: + udp: + elastic_agent_control: + tcp: + udp: + elastic_agent_data: + tcp: + udp: + endgame: + tcp: + udp: + influxdb: + tcp: + udp: + kibana: + tcp: + udp: + mysql: + tcp: + udp: + nginx: + tcp: + udp: + playbook: + tcp: + udp: + redis: + tcp: + udp: + salt_manager: + tcp: + udp: + sensoroni: + tcp: + udp: + ssh: + tcp: + udp: + strelka_frontend: + tcp: + udp: + syslog: + tcp: + udp: + yum: + tcp: + udp: + role: + eval: + chain: + DOCKER-USER: + hostgroups: + eval: + portgroups: + sensors: + portgroups: + searchnodes: + portgroups: + heavynodes: + portgroups: + self: + portgroups: + beats_endpoint: + portgroups: + beats_endpoint_ssl: + portgroups: + elasticsearch_rest: + portgroups: + elastic_agent_endpoint: + portgroups: + strelka_frontend: + portgroups: + syslog: + portgroups: + analyst: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + localhost: + portgroups: + fleet: + chain: + DOCKER-USER: + hostgroups: + sensors: + portgroups: + elastic_agent_endpoint: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + localhost: + portgroups: + standalone: + portgroups: + sensors: + portgroups: + searchnodes: + portgroups: + heavynodes: + portgroups: + manager: + chain: + DOCKER-USER: + hostgroups: + manager: + portgroups: + sensors: + portgroups: + searchnodes: + portgroups: + heavynodes: + portgroups: + self: + portgroups: + syslog: + portgroups: + beats_endpoint: + portgroups: + beats_endpoint_ssl: + portgroups: + elasticsearch_rest: + portgroups: + elastic_agent_endpoint: + portgroups: + endgame: + portgroups: + analyst: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + localhost: + portgroups: + sensors: + portgroups: + searchnodes: + portgroups: + heavynodes: + portgroups: + managersearch: + chain: + DOCKER-USER: + hostgroups: + managersearch: + portgroups: + sensors: + portgroups: + searchnodes: + portgroups: + heavynodes: + portgroups: + self: + portgroups: + beats_endpoint: + portgroups: + beats_endpoint_ssl: + portgroups: + elasticsearch_rest: + portgroups: + elastic_agent_endpoint: + portgroups: + endgame: + portgroups: + syslog: + portgroups: + analyst: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + localhost: + portgroups: + sensors: + portgroups: + searchnodes: + portgroups: + heavynodes: + portgroups: + standalone: + chain: + DOCKER-USER: + hostgroups: + localhost: + portgroups: + standalone: + portgroups: + fleet: + portgroups: + sensors: + portgroups: + searchnodes: + portgroups: + heavynodes: + portgroups: + self: + portgroups: + beats_endpoint: + portgroups: + beats_endpoint_ssl: + portgroups: + elasticsearch_rest: + portgroups: + elastic_agent_endpoint: + portgroups: + endgame: + portgroups: + strelka_frontend: + portgroups: + syslog: + portgroups: + analyst: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + fleet: + portgroups: + localhost: + portgroups: + standalone: + portgroups: + sensors: + portgroups: + searchnodes: + portgroups: + heavynodes: + portgroups: + searchnode: + chain: + DOCKER-USER: + hostgroups: + manager: + portgroups: + dockernet: + portgroups: + elasticsearch_rest: + portgroups: + searchnodes: + portgroups: + self: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + localhost: + portgroups: + sensor: + chain: + DOCKER-USER: + hostgroups: + self: + portgroups: + strelka_frontend: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + localhost: + portgroups: + heavynode: + chain: + DOCKER-USER: + hostgroups: + manager: + portgroups: + dockernet: + portgroups: + elasticsearch_rest: + portgroups: + self: + portgroups: + strelka_frontend: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + localhost: + portgroups: + import: + chain: + DOCKER-USER: + hostgroups: + manager: + portgroups: + sensors: + portgroups: + searchnodes: + portgroups: + beats_endpoint: + portgroups: + beats_endpoint_ssl: + portgroups: + elasticsearch_rest: + portgroups: + elastic_agent_endpoint: + portgroups: + analyst: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + localhost: + portgroups: + receiver: + chain: + DOCKER-USER: + hostgroups: + sensors: + portgroups: + searchnodes: + portgroups: + self: + portgroups: + syslog: + portgroups: + beats_endpoint: + portgroups: + beats_endpoint_ssl: + portgroups: + endgame: + portgroups: + INPUT: + hostgroups: + anywhere: + portgroups: + dockernet: + portgroups: + localhost: + portgroups: diff --git a/salt/common/tools/sbin/so-idstools-restart b/salt/idstools/bin/so-idstools-restart similarity index 100% rename from salt/common/tools/sbin/so-idstools-restart rename to salt/idstools/bin/so-idstools-restart diff --git a/salt/common/tools/sbin/so-idstools-start b/salt/idstools/bin/so-idstools-start similarity index 100% rename from salt/common/tools/sbin/so-idstools-start rename to salt/idstools/bin/so-idstools-start diff --git a/salt/common/tools/sbin/so-idstools-stop b/salt/idstools/bin/so-idstools-stop similarity index 100% rename from salt/common/tools/sbin/so-idstools-stop rename to salt/idstools/bin/so-idstools-stop diff --git a/salt/manager/files/so-repo-sync b/salt/manager/sbin/so-repo-sync similarity index 100% rename from salt/manager/files/so-repo-sync rename to salt/manager/sbin/so-repo-sync diff --git a/salt/manager/sbin/so-saltstack-update b/salt/manager/sbin/so-saltstack-update new file mode 100755 index 000000000..73c9c7791 --- /dev/null +++ b/salt/manager/sbin/so-saltstack-update @@ -0,0 +1,53 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + +default_salt_dir=/opt/so/saltstack/default +clone_to_tmp() { + + # Make a temp location for the files + mkdir /tmp/sogh + cd /tmp/sogh + git clone https://github.com/Security-Onion-Solutions/securityonion.git + cd /tmp + +} + +copy_new_files() { + + # Copy new files over to the salt dir + cd /tmp/sogh/securityonion + git checkout $BRANCH + VERSION=$(cat VERSION) + # We need to overwrite if there is a repo file + if [ -d /opt/so/repo ]; then + tar -czf /opt/so/repo/"$VERSION".tar.gz -C "$(pwd)/.." . + fi + rsync -a salt $default_salt_dir/ + rsync -a pillar $default_salt_dir/ + chown -R socore:socore $default_salt_dir/salt + chown -R socore:socore $default_salt_dir/pillar + chmod 755 $default_salt_dir/pillar/firewall/addfirewall.sh + + rm -rf /tmp/sogh +} + +got_root(){ + if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run using sudo!" + exit 1 + fi +} + +got_root +if [ $# -ne 1 ] ; then + BRANCH=2.4/main +else + BRANCH=$1 +fi +clone_to_tmp +copy_new_files diff --git a/salt/common/tools/sbin/so-zeek-restart b/salt/zeek/bin/so-zeek-restart similarity index 100% rename from salt/common/tools/sbin/so-zeek-restart rename to salt/zeek/bin/so-zeek-restart diff --git a/salt/common/tools/sbin/so-zeek-start b/salt/zeek/bin/so-zeek-start similarity index 100% rename from salt/common/tools/sbin/so-zeek-start rename to salt/zeek/bin/so-zeek-start diff --git a/salt/common/tools/sbin/so-zeek-stats b/salt/zeek/bin/so-zeek-stats similarity index 100% rename from salt/common/tools/sbin/so-zeek-stats rename to salt/zeek/bin/so-zeek-stats diff --git a/salt/common/tools/sbin/so-zeek-stop b/salt/zeek/bin/so-zeek-stop similarity index 100% rename from salt/common/tools/sbin/so-zeek-stop rename to salt/zeek/bin/so-zeek-stop diff --git a/setup/so-functions b/setup/so-functions index 92c47211f..a3f1fe0d5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1341,6 +1341,16 @@ kibana_pillar() { touch $kibana_pillar_file } +logrotate_pillar() { + touch $adv_logrotate_pillar_file + touch $logrotate_pillar_file +} + +patch_pillar() { + touch $adv_patch_pillar_file + touch $patch_pillar_file +} + logstash_pillar() { # Create the logstash advanced pillar touch $adv_logstash_pillar_file diff --git a/setup/so-variables b/setup/so-variables index 3d599afb4..b2e439a5c 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -201,3 +201,15 @@ export influxdb_pillar_file adv_influxdb_pillar_file="$local_salt_dir/pillar/influxdb/adv_influxdb.sls" export adv_influxdb_pillar_file + +logrotate_pillar_file="$local_salt_dir/pillar/logrotate/soc_logrotate.sls" +export logrotate_pillar_file + +adv_logrotate_pillar_file="$local_salt_dir/pillar/logrotate/adv_logrotate.sls" +export adv_logrotate_pillar_file + +patch_pillar_file="$local_salt_dir/pillar/patch/soc_patch.sls" +export patch_pillar_file + +adv_patch_pillar_file="$local_salt_dir/pillar/patch/adv_patch.sls" +export adv_patch_pillar_file \ No newline at end of file From b8f9a9a311dc8a3e340974cde45a68013066670d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 27 Apr 2023 15:08:01 -0400 Subject: [PATCH 06/41] regex for hostgroups --- salt/firewall/soc_firewall.yaml | 184 ++++++++++++++++++++------------ 1 file changed, 113 insertions(+), 71 deletions(-) diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 452c3c26f..b1faed41c 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -1,138 +1,176 @@ firewall: hostgroups: analyst: &hostgroupsettings - description: List of IP or CIDR blocks to allow access to for this hostgroup. + description: List of IP or CIDR blocks to allow access to this hostgroup. helplink: firewall.html multiline: True - regex: ^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$ - regexFailureMessage: You must enter a properly formatted IP address or CIDR. - anywhere: *hostgroupsettings + regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ + regexFailureMessage: You must enter a valid IP address or CIDR. + anywhere: &hostgroupsettingsadv + description: List of IP or CIDR blocks to allow access to this hostgroup. + helplink: firewall.html + multiline: True + advanced: True + regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ + regexFailureMessage: You must enter a valid IP address or CIDR. beats_endpoint: *hostgroupsettings beats_endpoint_ssl: *hostgroupsettings - dockernet: *hostgroupsettings + dockernet: *hostgroupsettingsadv elastic_agent_endpoint: *hostgroupsettings - elasticsearch_rest: *hostgroupsettings - endgame: *hostgroupsettings + elasticsearch_rest: *hostgroupsettingsadv + endgame: *hostgroupsettingsadv eval: *hostgroupsettings fleet: *hostgroupsettings heavynodes: *hostgroupsettings idh: *hostgroupsettings - localhost: *hostgroupsettings + localhost: *hostgroupsettingsadv manager: *hostgroupsettings receivers: *hostgroupsettings searchnodes: *hostgroupsettings securityonion_desktops: *hostgroupsettings - self: *hostgroupsettings + self: *hostgroupsettingsadv sensors: *hostgroupsettings standalone: *hostgroupsettings strelka_frontend: *hostgroupsettings syslog: *hostgroupsettings + customhostgroup1: &customhostgroupsettings + description: List of IP or CIDR blocks to allow to this hostgroup. + helpLink: firewall.html + advanced: True + regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ + regexFailureMessage: You must enter a valid IP address or CIDR. + customhostgroup2: *customhostgroupsettings + customhostgroup3: *customhostgroupsettings + customhostgroup4: *customhostgroupsettings + customhostgroup5: *customhostgroupsettings + customhostgroup6: *customhostgroupsettings + customhostgroup7: *customhostgroupsettings + customhostgroup8: *customhostgroupsettings + customhostgroup9: *customhostgroupsettings + customhostgroup10: *customhostgroupsettings + portgroups: all: - tcp: - udp: + tcp: &tcpsettings + description: List of TCP ports for this port group. + helplink: firewall.html + advanced: True + multiline: True + udp: &udpsettings + description: List of UDP ports for this port group. + helplink: firewall.html + advanced: True + multiline: True agrules: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings beats_5044: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings beats_5644: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings beats_5066: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings beats_5056: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings docker_registry: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings elasticsearch_node: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings elasticsearch_rest: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings elastic_agent_control: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings elastic_agent_data: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings endgame: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings influxdb: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings kibana: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings mysql: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings nginx: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings playbook: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings redis: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings salt_manager: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings sensoroni: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings ssh: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings strelka_frontend: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings syslog: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings yum: - tcp: - udp: + tcp: *tcpsettings + udp: *udpsettings role: eval: chain: DOCKER-USER: hostgroups: eval: - portgroups: + portgroups: &portgroupsdocker + description: Portgroups to add access to the docker containers for this role. + advanced: True + multiline: True + helpLink: firewall.html sensors: - portgroups: + portgroups: *portgroupsdocker searchnodes: - portgroups: + portgroups: *portgroupsdocker heavynodes: - portgroups: + portgroups: *portgroupsdocker self: - portgroups: - beats_endpoint: - portgroups: + portgroups: *portgroupsdocker + beats_endpoint: + portgroups: *portgroupsdocker beats_endpoint_ssl: - portgroups: + portgroups: *portgroupsdocker elasticsearch_rest: - portgroups: + portgroups: *portgroupsdocker elastic_agent_endpoint: - portgroups: + portgroups: *portgroupsdocker strelka_frontend: - portgroups: + portgroups: *portgroupsdocker syslog: - portgroups: + portgroups: *portgroupsdocker analyst: - portgroups: + portgroups: *portgroupsdocker INPUT: hostgroups: anywhere: - portgroups: + portgroups: &portgroupshost + description: Portgroups to add access to the host. + advacned: True + multiline: True + helpLink dockernet: portgroups: localhost: @@ -189,6 +227,10 @@ firewall: portgroups: analyst: portgroups: + custom1: + portgroups: + custom2: + INPUT: hostgroups: anywhere: From e4375a656851dc002c3cf3f7c3c5cbea2efb789c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 27 Apr 2023 15:30:59 -0400 Subject: [PATCH 07/41] regex for hostgroups --- salt/firewall/soc_firewall.yaml | 1 + salt/top.sls | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index b1faed41c..4eb297c78 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -37,6 +37,7 @@ firewall: description: List of IP or CIDR blocks to allow to this hostgroup. helpLink: firewall.html advanced: True + multiline: True regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ regexFailureMessage: You must enter a valid IP address or CIDR. customhostgroup2: *customhostgroupsettings diff --git a/salt/top.sls b/salt/top.sls index 372c64115..0459a6da4 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -74,7 +74,6 @@ base: - telegraf - influxdb - soc - - firewall.soc - kratos - firewall - idstools @@ -119,7 +118,6 @@ base: - telegraf - influxdb - soc - - firewall.soc - kratos - firewall - manager @@ -162,7 +160,6 @@ base: - telegraf - influxdb - soc - - firewall.soc - kratos - firewall - idstools @@ -226,7 +223,6 @@ base: - telegraf - influxdb - soc - - firewall.soc - kratos - firewall - manager @@ -296,7 +292,6 @@ base: - telegraf - influxdb - soc - - firewall.soc - kratos - firewall - idstools From 38629a7676010e901e12594f6e2034a9891853da Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 1 May 2023 09:55:16 -0400 Subject: [PATCH 08/41] fix defaults for logstash --- salt/common/tools/sbin/so-minion | 11 +- salt/firewall/soc_firewall.yaml | 456 ------------------------------- salt/logstash/defaults.yaml | 28 ++ salt/logstash/init.sls | 2 +- salt/logstash/soc_logstash.yaml | 39 +++ 5 files changed, 73 insertions(+), 463 deletions(-) delete mode 100644 salt/firewall/soc_firewall.yaml create mode 100644 salt/logstash/defaults.yaml create mode 100644 salt/logstash/soc_logstash.yaml diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion index 2f506863d..4145b16b1 100755 --- a/salt/common/tools/sbin/so-minion +++ b/salt/common/tools/sbin/so-minion @@ -163,12 +163,11 @@ function add_idh_to_minion() { function add_logstash_to_minion() { # Create the logstash advanced pillar printf '%s\n'\ - "logstash_settings:"\ - " ls_host: '$LSHOSTNAME'"\ - " ls_pipeline_batch_size: 125"\ - " ls_input_threads: 1"\ - " lsheap: $LSHEAP"\ - " ls_pipeline_workers: $CPUCORES"\ + "logstash:"\ + " config:"\ + " pipeline_x_workers: $CPUCORES"\ + " settings:"\ + " lsheap: $LSHEAP"\ " " >> $PILLARFILE } diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml deleted file mode 100644 index 4eb297c78..000000000 --- a/salt/firewall/soc_firewall.yaml +++ /dev/null @@ -1,456 +0,0 @@ -firewall: - hostgroups: - analyst: &hostgroupsettings - description: List of IP or CIDR blocks to allow access to this hostgroup. - helplink: firewall.html - multiline: True - regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ - regexFailureMessage: You must enter a valid IP address or CIDR. - anywhere: &hostgroupsettingsadv - description: List of IP or CIDR blocks to allow access to this hostgroup. - helplink: firewall.html - multiline: True - advanced: True - regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ - regexFailureMessage: You must enter a valid IP address or CIDR. - beats_endpoint: *hostgroupsettings - beats_endpoint_ssl: *hostgroupsettings - dockernet: *hostgroupsettingsadv - elastic_agent_endpoint: *hostgroupsettings - elasticsearch_rest: *hostgroupsettingsadv - endgame: *hostgroupsettingsadv - eval: *hostgroupsettings - fleet: *hostgroupsettings - heavynodes: *hostgroupsettings - idh: *hostgroupsettings - localhost: *hostgroupsettingsadv - manager: *hostgroupsettings - receivers: *hostgroupsettings - searchnodes: *hostgroupsettings - securityonion_desktops: *hostgroupsettings - self: *hostgroupsettingsadv - sensors: *hostgroupsettings - standalone: *hostgroupsettings - strelka_frontend: *hostgroupsettings - syslog: *hostgroupsettings - customhostgroup1: &customhostgroupsettings - description: List of IP or CIDR blocks to allow to this hostgroup. - helpLink: firewall.html - advanced: True - multiline: True - regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$ - regexFailureMessage: You must enter a valid IP address or CIDR. - customhostgroup2: *customhostgroupsettings - customhostgroup3: *customhostgroupsettings - customhostgroup4: *customhostgroupsettings - customhostgroup5: *customhostgroupsettings - customhostgroup6: *customhostgroupsettings - customhostgroup7: *customhostgroupsettings - customhostgroup8: *customhostgroupsettings - customhostgroup9: *customhostgroupsettings - customhostgroup10: *customhostgroupsettings - - portgroups: - all: - tcp: &tcpsettings - description: List of TCP ports for this port group. - helplink: firewall.html - advanced: True - multiline: True - udp: &udpsettings - description: List of UDP ports for this port group. - helplink: firewall.html - advanced: True - multiline: True - agrules: - tcp: *tcpsettings - udp: *udpsettings - beats_5044: - tcp: *tcpsettings - udp: *udpsettings - beats_5644: - tcp: *tcpsettings - udp: *udpsettings - beats_5066: - tcp: *tcpsettings - udp: *udpsettings - beats_5056: - tcp: *tcpsettings - udp: *udpsettings - docker_registry: - tcp: *tcpsettings - udp: *udpsettings - elasticsearch_node: - tcp: *tcpsettings - udp: *udpsettings - elasticsearch_rest: - tcp: *tcpsettings - udp: *udpsettings - elastic_agent_control: - tcp: *tcpsettings - udp: *udpsettings - elastic_agent_data: - tcp: *tcpsettings - udp: *udpsettings - endgame: - tcp: *tcpsettings - udp: *udpsettings - influxdb: - tcp: *tcpsettings - udp: *udpsettings - kibana: - tcp: *tcpsettings - udp: *udpsettings - mysql: - tcp: *tcpsettings - udp: *udpsettings - nginx: - tcp: *tcpsettings - udp: *udpsettings - playbook: - tcp: *tcpsettings - udp: *udpsettings - redis: - tcp: *tcpsettings - udp: *udpsettings - salt_manager: - tcp: *tcpsettings - udp: *udpsettings - sensoroni: - tcp: *tcpsettings - udp: *udpsettings - ssh: - tcp: *tcpsettings - udp: *udpsettings - strelka_frontend: - tcp: *tcpsettings - udp: *udpsettings - syslog: - tcp: *tcpsettings - udp: *udpsettings - yum: - tcp: *tcpsettings - udp: *udpsettings - role: - eval: - chain: - DOCKER-USER: - hostgroups: - eval: - portgroups: &portgroupsdocker - description: Portgroups to add access to the docker containers for this role. - advanced: True - multiline: True - helpLink: firewall.html - sensors: - portgroups: *portgroupsdocker - searchnodes: - portgroups: *portgroupsdocker - heavynodes: - portgroups: *portgroupsdocker - self: - portgroups: *portgroupsdocker - beats_endpoint: - portgroups: *portgroupsdocker - beats_endpoint_ssl: - portgroups: *portgroupsdocker - elasticsearch_rest: - portgroups: *portgroupsdocker - elastic_agent_endpoint: - portgroups: *portgroupsdocker - strelka_frontend: - portgroups: *portgroupsdocker - syslog: - portgroups: *portgroupsdocker - analyst: - portgroups: *portgroupsdocker - INPUT: - hostgroups: - anywhere: - portgroups: &portgroupshost - description: Portgroups to add access to the host. - advacned: True - multiline: True - helpLink - dockernet: - portgroups: - localhost: - portgroups: - fleet: - chain: - DOCKER-USER: - hostgroups: - sensors: - portgroups: - elastic_agent_endpoint: - portgroups: - INPUT: - hostgroups: - anywhere: - portgroups: - dockernet: - portgroups: - localhost: - portgroups: - standalone: - portgroups: - sensors: - portgroups: - searchnodes: - portgroups: - heavynodes: - portgroups: - manager: - chain: - DOCKER-USER: - hostgroups: - manager: - portgroups: - sensors: - portgroups: - searchnodes: - portgroups: - heavynodes: - portgroups: - self: - portgroups: - syslog: - portgroups: - beats_endpoint: - portgroups: - beats_endpoint_ssl: - portgroups: - elasticsearch_rest: - portgroups: - elastic_agent_endpoint: - portgroups: - endgame: - portgroups: - analyst: - portgroups: - custom1: - portgroups: - custom2: - - INPUT: - hostgroups: - anywhere: - portgroups: - dockernet: - portgroups: - localhost: - portgroups: - sensors: - portgroups: - searchnodes: - portgroups: - heavynodes: - portgroups: - managersearch: - chain: - DOCKER-USER: - hostgroups: - managersearch: - portgroups: - sensors: - portgroups: - searchnodes: - portgroups: - heavynodes: - portgroups: - self: - portgroups: - beats_endpoint: - portgroups: - beats_endpoint_ssl: - portgroups: - elasticsearch_rest: - portgroups: - elastic_agent_endpoint: - portgroups: - endgame: - portgroups: - syslog: - portgroups: - analyst: - portgroups: - INPUT: - hostgroups: - anywhere: - portgroups: - dockernet: - portgroups: - localhost: - portgroups: - sensors: - portgroups: - searchnodes: - portgroups: - heavynodes: - portgroups: - standalone: - chain: - DOCKER-USER: - hostgroups: - localhost: - portgroups: - standalone: - portgroups: - fleet: - portgroups: - sensors: - portgroups: - searchnodes: - portgroups: - heavynodes: - portgroups: - self: - portgroups: - beats_endpoint: - portgroups: - beats_endpoint_ssl: - portgroups: - elasticsearch_rest: - portgroups: - elastic_agent_endpoint: - portgroups: - endgame: - portgroups: - strelka_frontend: - portgroups: - syslog: - portgroups: - analyst: - portgroups: - INPUT: - hostgroups: - anywhere: - portgroups: - dockernet: - portgroups: - fleet: - portgroups: - localhost: - portgroups: - standalone: - portgroups: - sensors: - portgroups: - searchnodes: - portgroups: - heavynodes: - portgroups: - searchnode: - chain: - DOCKER-USER: - hostgroups: - manager: - portgroups: - dockernet: - portgroups: - elasticsearch_rest: - portgroups: - searchnodes: - portgroups: - self: - portgroups: - INPUT: - hostgroups: - anywhere: - portgroups: - dockernet: - portgroups: - localhost: - portgroups: - sensor: - chain: - DOCKER-USER: - hostgroups: - self: - portgroups: - strelka_frontend: - portgroups: - INPUT: - hostgroups: - anywhere: - portgroups: - dockernet: - portgroups: - localhost: - portgroups: - heavynode: - chain: - DOCKER-USER: - hostgroups: - manager: - portgroups: - dockernet: - portgroups: - elasticsearch_rest: - portgroups: - self: - portgroups: - strelka_frontend: - portgroups: - INPUT: - hostgroups: - anywhere: - portgroups: - dockernet: - portgroups: - localhost: - portgroups: - import: - chain: - DOCKER-USER: - hostgroups: - manager: - portgroups: - sensors: - portgroups: - searchnodes: - portgroups: - beats_endpoint: - portgroups: - beats_endpoint_ssl: - portgroups: - elasticsearch_rest: - portgroups: - elastic_agent_endpoint: - portgroups: - analyst: - portgroups: - INPUT: - hostgroups: - anywhere: - portgroups: - dockernet: - portgroups: - localhost: - portgroups: - receiver: - chain: - DOCKER-USER: - hostgroups: - sensors: - portgroups: - searchnodes: - portgroups: - self: - portgroups: - syslog: - portgroups: - beats_endpoint: - portgroups: - beats_endpoint_ssl: - portgroups: - endgame: - portgroups: - INPUT: - hostgroups: - anywhere: - portgroups: - dockernet: - portgroups: - localhost: - portgroups: diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml new file mode 100644 index 000000000..a14b47e5c --- /dev/null +++ b/salt/logstash/defaults.yaml @@ -0,0 +1,28 @@ +logstash: + assigned_pipelines: + roles: + fleet: + - so/0012_input_elastic_agent.conf + - so/9806_output_lumberjack_fleet.conf.jinja + manager: + - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf + - so/0013_input_lumberjack_fleet.conf + - so/9999_output_redis.conf.jinja + receiver: + - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf + - so/9999_output_redis.conf.jinja + search: + - so/0900_input_redis.conf.jinja + - so/9805_output_elastic_agent.conf.jinja + - so/9900_output_endgame.conf.jinja + settings: + lsheap: 500m + config: + http_x_host: 0.0.0.0 + path_x_logs: /var/log/logstash + pipeline_x_workers: 1 + pipeline_x_batch_x_size: 125 + pipeline_x_ecs_compatibility: disabled + diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 7f3aef0aa..caabd10ea 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -11,7 +11,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} # Logstash Section - Decide which pillar to use -{% set lsheap = salt['pillar.get']('logstash_settings:lsheap') %} +{% set lsheap = salt['pillar.get']('logstash:settings:lsheap') %} {% if GLOBALS.role in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} {% set nodetype = GLOBALS.role %} {% endif %} diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml new file mode 100644 index 000000000..8e764b0c5 --- /dev/null +++ b/salt/logstash/soc_logstash.yaml @@ -0,0 +1,39 @@ +logstash: + assigned_pipelines: + roles: + reciever: &assigned_pipelines + description: List of pipelines assigned to this role. + advanced: True + helpLink: logstash.html + multiline: True + fleet: *assigned_pipelines + manager: *assigned_pipelines + nodes: *assigned_pipelines + search: *assigned_pipelines + settings: + lsheap: + description: Heap size to use for logstash + helpLink: logstash.html + global: False + config: + http_x_host: + description: Host interface to listen to connections. + helpLink: logstash.html + readonly: True + path_x_logs: + description: Path inside the container to wrote logs. + helpLink: logstash.html + readonly: True + pipeline_x_workers: + description: Number of worker threads to process events in logstash. + helpLink: logstash.html + global: False + pipeline_x_batch_x_size: + description: Logstash batch size. + helpLink: logstash.html + global: False + pipeline_x_ecs_compatibility: + description: Sets ECS compatibility. This is set per pipeline so you should never need to change this. + helpLink: logstash.html + readonly: True + From a38495ce39866a6a7ee1baf6ffd5209b76ea5db4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 1 May 2023 10:40:36 -0400 Subject: [PATCH 09/41] Fix pillar top for firewall --- pillar/logstash/fleet.sls | 6 - pillar/logstash/helix.sls | 42 ------ pillar/logstash/manager.sls | 8 -- pillar/logstash/receiver.sls | 8 -- pillar/logstash/search.sls | 7 - pillar/top.sls | 10 +- salt/logstash/defaults.yaml | 1 + salt/logstash/dmz_nodes.yaml | 15 --- salt/logstash/etc/logstash.yml | 221 -------------------------------- salt/logstash/soc_logstash.yaml | 7 +- 10 files changed, 9 insertions(+), 316 deletions(-) delete mode 100644 pillar/logstash/fleet.sls delete mode 100644 pillar/logstash/helix.sls delete mode 100644 pillar/logstash/manager.sls delete mode 100644 pillar/logstash/receiver.sls delete mode 100644 pillar/logstash/search.sls delete mode 100644 salt/logstash/dmz_nodes.yaml diff --git a/pillar/logstash/fleet.sls b/pillar/logstash/fleet.sls deleted file mode 100644 index fb70e7f0d..000000000 --- a/pillar/logstash/fleet.sls +++ /dev/null @@ -1,6 +0,0 @@ -logstash: - pipelines: - fleet: - config: - - so/0012_input_elastic_agent.conf - - so/9806_output_lumberjack_fleet.conf.jinja \ No newline at end of file diff --git a/pillar/logstash/helix.sls b/pillar/logstash/helix.sls deleted file mode 100644 index ddc1c745b..000000000 --- a/pillar/logstash/helix.sls +++ /dev/null @@ -1,42 +0,0 @@ -logstash: - pipelines: - helix: - config: - - so/0010_input_hhbeats.conf - - so/1033_preprocess_snort.conf - - so/1100_preprocess_bro_conn.conf - - so/1101_preprocess_bro_dhcp.conf - - so/1102_preprocess_bro_dns.conf - - so/1103_preprocess_bro_dpd.conf - - so/1104_preprocess_bro_files.conf - - so/1105_preprocess_bro_ftp.conf - - so/1106_preprocess_bro_http.conf - - so/1107_preprocess_bro_irc.conf - - so/1108_preprocess_bro_kerberos.conf - - so/1109_preprocess_bro_notice.conf - - so/1110_preprocess_bro_rdp.conf - - so/1111_preprocess_bro_signatures.conf - - so/1112_preprocess_bro_smtp.conf - - so/1113_preprocess_bro_snmp.conf - - so/1114_preprocess_bro_software.conf - - so/1115_preprocess_bro_ssh.conf - - so/1116_preprocess_bro_ssl.conf - - so/1117_preprocess_bro_syslog.conf - - so/1118_preprocess_bro_tunnel.conf - - so/1119_preprocess_bro_weird.conf - - so/1121_preprocess_bro_mysql.conf - - so/1122_preprocess_bro_socks.conf - - so/1123_preprocess_bro_x509.conf - - so/1124_preprocess_bro_intel.conf - - so/1125_preprocess_bro_modbus.conf - - so/1126_preprocess_bro_sip.conf - - so/1127_preprocess_bro_radius.conf - - so/1128_preprocess_bro_pe.conf - - so/1129_preprocess_bro_rfb.conf - - so/1130_preprocess_bro_dnp3.conf - - so/1131_preprocess_bro_smb_files.conf - - so/1132_preprocess_bro_smb_mapping.conf - - so/1133_preprocess_bro_ntlm.conf - - so/1134_preprocess_bro_dce_rpc.conf - - so/8001_postprocess_common_ip_augmentation.conf - - so/9997_output_helix.conf.jinja diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls deleted file mode 100644 index cee8eec02..000000000 --- a/pillar/logstash/manager.sls +++ /dev/null @@ -1,8 +0,0 @@ -logstash: - pipelines: - manager: - config: - - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/0013_input_lumberjack_fleet.conf - - so/9999_output_redis.conf.jinja \ No newline at end of file diff --git a/pillar/logstash/receiver.sls b/pillar/logstash/receiver.sls deleted file mode 100644 index 4d0637dde..000000000 --- a/pillar/logstash/receiver.sls +++ /dev/null @@ -1,8 +0,0 @@ -logstash: - pipelines: - receiver: - config: - - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/9999_output_redis.conf.jinja - diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls deleted file mode 100644 index 0b660b7ef..000000000 --- a/pillar/logstash/search.sls +++ /dev/null @@ -1,7 +0,0 @@ -logstash: - pipelines: - search: - config: - - so/0900_input_redis.conf.jinja - - so/9805_output_elastic_agent.conf.jinja - - so/9900_output_endgame.conf.jinja diff --git a/pillar/top.sls b/pillar/top.sls index 90b0a41b9..a0fbcb4c1 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -4,6 +4,8 @@ base: - global.adv_global - docker.soc_docker - docker.adv_docker + - firewall.soc_firewall + - firewall.adv_firewall - influxdb.token - logrotate.soc_logrotate - logrotate.adv_logrotate @@ -53,8 +55,6 @@ base: - elastalert.adv_elastalert - backup.soc_backup - backup.adv_backup - - firewall.soc_firewall - - firewall.adv_firewall - curator.soc_curator - curator.adv_curator - soctopus.soc_soctopus @@ -112,8 +112,6 @@ base: - influxdb.adv_influxdb - backup.soc_backup - backup.adv_backup - - firewall.soc_firewall - - firewall.adv_firewall - zeek.soc_zeek - zeek.adv_zeek - bpf.soc_bpf @@ -164,8 +162,6 @@ base: - curator.adv_curator - backup.soc_backup - backup.adv_backup - - firewall.soc_firewall - - firewall.adv_firewall - zeek.soc_zeek - zeek.adv_zeek - bpf.soc_bpf @@ -258,8 +254,6 @@ base: - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb - - firewall.soc_firewall - - firewall.adv_firewall - zeek.soc_zeek - zeek.adv_zeek - bpf.soc_bpf diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index a14b47e5c..dddab9ddf 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -25,4 +25,5 @@ logstash: pipeline_x_workers: 1 pipeline_x_batch_x_size: 125 pipeline_x_ecs_compatibility: disabled + dmz_nodes: {} diff --git a/salt/logstash/dmz_nodes.yaml b/salt/logstash/dmz_nodes.yaml deleted file mode 100644 index 460088a7d..000000000 --- a/salt/logstash/dmz_nodes.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -# Do not edit this file. Copy it to /opt/so/saltstack/local/salt/logstash/ and make changes there. It should be formatted as a list. -# logstash: -# dmz_nodes: -# - mydmznodehostname1 -# - mydmznodehostname2 -# - mydmznodehostname3 - -logstash: - dmz_nodes: \ No newline at end of file diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index 2a952c754..ca953975f 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -1,226 +1,5 @@ -# Settings file in YAML -# -# Settings can be specified either in hierarchical form, e.g.: -# -# pipeline: -# batch: -# size: 125 -# delay: 5 -# -# Or as flat keys: -# -# pipeline.batch.size: 125 -# pipeline.batch.delay: 5 -# -# ------------ Node identity ------------ -# -# Use a descriptive name for the node: -# -# node.name: test -# -# If omitted the node name will default to the machine's host name -# -# ------------ Data path ------------------ -# -# Which directory should be used by logstash and its plugins -# for any persistent needs. Defaults to LOGSTASH_HOME/data -# -# path.data: -# -# ------------ Pipeline Settings -------------- -# -# The ID of the pipeline. -# -# pipeline.id: main -# -# Set the number of workers that will, in parallel, execute the filters+outputs -# stage of the pipeline. -# -# This defaults to the number of the host's CPU cores. -# -# pipeline.workers: 2 -# -# How many events to retrieve from inputs before sending to filters+workers -# -# pipeline.batch.size: 125 -# -# How long to wait in milliseconds while polling for the next event -# before dispatching an undersized batch to filters+outputs -# -# pipeline.batch.delay: 50 -# -# Force Logstash to exit during shutdown even if there are still inflight -# events in memory. By default, logstash will refuse to quit until all -# received events have been pushed to the outputs. -# -# WARNING: enabling this can lead to data loss during shutdown -# -# pipeline.unsafe_shutdown: false -# -# ------------ Pipeline Configuration Settings -------------- -# -# Where to fetch the pipeline configuration for the main pipeline -# -# path.config: -# /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image -# Special Docker path -# path.config: /usr/share/logstash/pipeline - -# -# Pipeline configuration string for the main pipeline -# -# config.string: -# -# At startup, test if the configuration is valid and exit (dry run) -# -# config.test_and_exit: false -# -# Periodically check if the configuration has changed and reload the pipeline -# This can also be triggered manually through the SIGHUP signal -# -# config.reload.automatic: false -# -# How often to check if the pipeline configuration has changed (in seconds) -# -# config.reload.interval: 3s -# -# Show fully compiled configuration as debug log message -# NOTE: --log.level must be 'debug' -# -# config.debug: false -# -# When enabled, process escaped characters such as \n and \" in strings in the -# pipeline configuration files. -# -# config.support_escapes: false -# -# ------------ Module Settings --------------- -# Define modules here. Modules definitions must be defined as an array. -# The simple way to see this is to prepend each `name` with a `-`, and keep -# all associated variables under the `name` they are associated with, and -# above the next, like this: -# -# modules: -# - name: MODULE_NAME -# var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE -# var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE -# var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE -# var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE -# -# Module variable names must be in the format of -# -# var.PLUGIN_TYPE.PLUGIN_NAME.KEY -# -# modules: -# -# ------------ Cloud Settings --------------- -# Define Elastic Cloud settings here. -# Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy -# and it may have an label prefix e.g. staging:dXMtZ... -# This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host' -# cloud.id: -# -# Format of cloud.auth is: : -# This is optional -# If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password' -# If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password' -# cloud.auth: elastic: -# -# ------------ Queuing Settings -------------- -# -# Internal queuing model, "memory" for legacy in-memory based queuing and -# "persisted" for disk-based acked queueing. Defaults is memory -# -# queue.type: memory -# -# If using queue.type: persisted, the directory path where the data files will be stored. -# Default is path.data/queue -# -# path.queue: -# -# If using queue.type: persisted, the page data files size. The queue data consists of -# append-only data files separated into pages. Default is 64mb -# -# queue.page_capacity: 64mb -# -# If using queue.type: persisted, the maximum number of unread events in the queue. -# Default is 0 (unlimited) -# -# queue.max_events: 0 -# -# If using queue.type: persisted, the total capacity of the queue in number of bytes. -# If you would like more unacked events to be buffered in Logstash, you can increase the -# capacity using this setting. Please make sure your disk drive has capacity greater than -# the size specified here. If both max_bytes and max_events are specified, Logstash will pick -# whichever criteria is reached first -# Default is 1024mb or 1gb -# -# queue.max_bytes: 1024mb -# -# If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint -# Default is 1024, 0 for unlimited -# -# queue.checkpoint.acks: 1024 -# -# If using queue.type: persisted, the maximum number of written events before forcing a checkpoint -# Default is 1024, 0 for unlimited -# -# queue.checkpoint.writes: 1024 -# -# If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page -# Default is 1000, 0 for no periodic checkpoint. -# -# queue.checkpoint.interval: 1000 -# -# ------------ Dead-Letter Queue Settings -------------- -# Flag to turn on dead-letter queue. -# -# dead_letter_queue.enable: false - -# If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries -# will be dropped if they would increase the size of the dead letter queue beyond this setting. -# Default is 1024mb -# dead_letter_queue.max_bytes: 1024mb - -# If using dead_letter_queue.enable: true, the directory path where the data files will be stored. -# Default is path.data/dead_letter_queue -# -# path.dead_letter_queue: -# -# ------------ Metrics Settings -------------- -# -# Bind address for the metrics REST endpoint -# -# http.host: "127.0.0.1" http.host: 0.0.0.0 -# -# Bind port for the metrics REST endpoint, this option also accept a range -# (9600-9700) and logstash will pick up the first available ports. -# -# http.port: 9600-9700 -# -# ------------ Debugging Settings -------------- -# -# Options for log.level: -# * fatal -# * error -# * warn -# * info (default) -# * debug -# * trace -# -# log.level: info -# path.logs: path.logs: /var/log/logstash -# -# ------------ Other Settings -------------- -# -# Where to find custom plugins -# path.plugins: [] -{% set pipeline_workers = salt['pillar.get']('logstash_settings:ls_pipeline_workers', '1') %} -{% set pipeline_batch = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', '125') %} -{% set pipeline_ecs_compatibility = salt['pillar.get']('logstash_settings:ls_ecs_compatibility', 'disabled') %} - pipeline.workers: {{ pipeline_workers }} pipeline.batch.size: {{ pipeline_batch }} pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }} diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 8e764b0c5..a4d0b87bf 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -36,4 +36,9 @@ logstash: description: Sets ECS compatibility. This is set per pipeline so you should never need to change this. helpLink: logstash.html readonly: True - + dmz_nodes: + description: "List of receiver nodes in DMZs. Prevents sensors from sending to these receivers. Primarily used for external Elastic agents." + helpLink: logstash.html + multiline: True + advanced: True + forcedType: "[]string" From 102906f5ddd09b46f4e397b415bb8c7a95ef6fe0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 1 May 2023 13:39:24 -0400 Subject: [PATCH 10/41] Change to new repo layout --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index a3f1fe0d5..3b597770b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1994,8 +1994,8 @@ repo_sync_local() { info "Backing up old repos" mkdir -p /nsm/repo mkdir -p /opt/so/conf/reposync/cache - echo "https://repo.securityonion.net/file/so-repo/2.4/" > /opt/so/conf/reposync/mirror.txt - echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/2.4/" >> /opt/so/conf/reposync/mirror.txt + echo "https://repo.securityonion.net/file/so-repo/prod/2.4/rocky/9" > /opt/so/conf/reposync/mirror.txt + echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/prod/2.4/rocky/9" >> /opt/so/conf/reposync/mirror.txt echo "[main]" > /opt/so/conf/reposync/repodownload.conf echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf From 2d4f4791e0a61d78256e3f9705660d4c223a1e12 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 1 May 2023 15:21:31 -0400 Subject: [PATCH 11/41] Move files out of common --- salt/common/tools/sbin/so-nodered-restart | 12 -- salt/common/tools/sbin/so-nodered-start | 13 -- salt/common/tools/sbin/so-nodered-stop | 12 -- .../bin => tools/sbin}/so-curator-close | 0 .../sbin}/so-curator-cluster-close | 0 .../sbin}/so-curator-cluster-delete | 0 .../sbin}/so-curator-cluster-delete-delete | 0 .../bin => tools/sbin}/so-curator-delete | 0 .../bin => tools/sbin}/so-curator-restart | 0 .../bin => tools/sbin}/so-curator-start | 0 .../{files/bin => tools/sbin}/so-curator-stop | 0 .../{bin => tools/sbin}/so-elastalert-create | 0 .../{bin => tools/sbin}/so-elastalert-restart | 0 .../{bin => tools/sbin}/so-elastalert-start | 0 .../{bin => tools/sbin}/so-elastalert-stop | 0 .../{bin => tools/sbin}/so-elastalert-test | 0 .../sbin/so-elastic-fleet-agent-policy-delete | 0 .../sbin/so-elastic-fleet-agent-policy-list | 0 .../sbin/so-elastic-fleet-agent-policy-view | 0 .../sbin/so-elastic-fleet-data-streams-list | 0 ...astic-fleet-integration-policy-bulk-delete | 0 ...so-elastic-fleet-integration-policy-delete | 0 .../so-elastic-fleet-integration-policy-list | 0 .../so-elastic-fleet-integration-policy-load | 0 .../tools/sbin/so-elastic-fleet-restart | 0 .../tools/sbin/so-elastic-fleet-setup | 0 .../tools/sbin/so-elastic-fleet-start | 0 .../tools/sbin/so-elastic-fleet-stop | 0 .../elasticsearch/tools/sbin/so-elastic-clear | 154 ++++++++++++++++++ .../tools/sbin/so-elastic-diagnose | 25 +++ .../tools/sbin/so-elastic-restart | 31 ++++ .../elasticsearch/tools/sbin/so-elastic-start | 31 ++++ salt/elasticsearch/tools/sbin/so-elastic-stop | 31 ++++ .../{common => idh}/tools/sbin/so-idh-restart | 0 salt/{common => idh}/tools/sbin/so-idh-start | 0 salt/{common => idh}/tools/sbin/so-idh-stop | 0 .../{bin => tools/sbin}/so-idstools-restart | 0 .../{bin => tools/sbin}/so-idstools-start | 0 .../{bin => tools/sbin}/so-idstools-stop | 0 .../tools/sbin/so-influxdb-manage | 0 .../tools/sbin/so-influxdb-restart | 0 .../tools/sbin/so-influxdb-start | 0 .../tools/sbin/so-influxdb-stop | 0 .../tools/sbin/so-kibana-config-export | 0 .../{bin => tools/sbin}/so-kibana-config-load | 0 .../tools/sbin/so-kibana-restart | 0 .../sbin/so-kibana-savedobjects-defaults | 0 .../tools/sbin/so-kibana-space-defaults | 0 .../tools/sbin/so-kibana-start | 0 .../tools/sbin/so-kibana-stop | 0 .../tools/sbin/so-logstash-events | 0 .../tools/sbin/so-logstash-get-parsed | 0 .../tools/sbin/so-logstash-get-unparsed | 0 .../tools/sbin/so-logstash-pipeline-stats | 0 .../tools/sbin/so-logstash-restart | 0 .../tools/sbin/so-logstash-start | 0 .../tools/sbin/so-logstash-stop | 0 .../tools/sbin/so-mysql-restart | 0 .../tools/sbin/so-mysql-start | 0 .../tools/sbin/so-mysql-stop | 0 .../tools/sbin/so-playbook-import | 0 .../tools/sbin/so-playbook-reset | 0 .../tools/sbin/so-playbook-restart | 0 .../tools/sbin/so-playbook-ruleupdate | 0 .../tools/sbin/so-playbook-sigma-refresh | 0 .../tools/sbin/so-playbook-start | 0 .../tools/sbin/so-playbook-stop | 0 .../tools/sbin/so-playbook-sync | 0 .../tools/sbin/so-redis-count | 0 .../tools/sbin/so-redis-restart | 0 .../tools/sbin/so-redis-start | 0 .../tools/sbin/so-redis-stop | 0 .../{common => soc}/tools/sbin/so-soc-restart | 0 salt/{common => soc}/tools/sbin/so-soc-start | 0 salt/{common => soc}/tools/sbin/so-soc-stop | 0 .../tools/sbin/so-strelka-restart | 0 .../tools/sbin/so-strelka-start | 0 .../tools/sbin/so-strelka-stop | 0 .../tools/sbin/so-telegraf-restart | 0 .../tools/sbin/so-telegraf-start | 0 .../tools/sbin/so-telegraf-stop | 0 81 files changed, 272 insertions(+), 37 deletions(-) delete mode 100755 salt/common/tools/sbin/so-nodered-restart delete mode 100755 salt/common/tools/sbin/so-nodered-start delete mode 100755 salt/common/tools/sbin/so-nodered-stop rename salt/curator/{files/bin => tools/sbin}/so-curator-close (100%) rename salt/curator/{files/bin => tools/sbin}/so-curator-cluster-close (100%) rename salt/curator/{files/bin => tools/sbin}/so-curator-cluster-delete (100%) rename salt/curator/{files/bin => tools/sbin}/so-curator-cluster-delete-delete (100%) rename salt/curator/{files/bin => tools/sbin}/so-curator-delete (100%) rename salt/curator/{files/bin => tools/sbin}/so-curator-restart (100%) rename salt/curator/{files/bin => tools/sbin}/so-curator-start (100%) rename salt/curator/{files/bin => tools/sbin}/so-curator-stop (100%) rename salt/elastalert/{bin => tools/sbin}/so-elastalert-create (100%) rename salt/elastalert/{bin => tools/sbin}/so-elastalert-restart (100%) rename salt/elastalert/{bin => tools/sbin}/so-elastalert-start (100%) rename salt/elastalert/{bin => tools/sbin}/so-elastalert-stop (100%) rename salt/elastalert/{bin => tools/sbin}/so-elastalert-test (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-agent-policy-delete (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-agent-policy-list (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-agent-policy-view (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-data-streams-list (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-integration-policy-delete (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-integration-policy-list (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-integration-policy-load (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-restart (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-setup (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-start (100%) rename salt/{common => elasticfleet}/tools/sbin/so-elastic-fleet-stop (100%) create mode 100755 salt/elasticsearch/tools/sbin/so-elastic-clear create mode 100755 salt/elasticsearch/tools/sbin/so-elastic-diagnose create mode 100755 salt/elasticsearch/tools/sbin/so-elastic-restart create mode 100755 salt/elasticsearch/tools/sbin/so-elastic-start create mode 100755 salt/elasticsearch/tools/sbin/so-elastic-stop rename salt/{common => idh}/tools/sbin/so-idh-restart (100%) rename salt/{common => idh}/tools/sbin/so-idh-start (100%) rename salt/{common => idh}/tools/sbin/so-idh-stop (100%) rename salt/idstools/{bin => tools/sbin}/so-idstools-restart (100%) rename salt/idstools/{bin => tools/sbin}/so-idstools-start (100%) rename salt/idstools/{bin => tools/sbin}/so-idstools-stop (100%) rename salt/{common => influxdb}/tools/sbin/so-influxdb-manage (100%) rename salt/{common => influxdb}/tools/sbin/so-influxdb-restart (100%) rename salt/{common => influxdb}/tools/sbin/so-influxdb-start (100%) rename salt/{common => influxdb}/tools/sbin/so-influxdb-stop (100%) rename salt/{common => kibana}/tools/sbin/so-kibana-config-export (100%) rename salt/kibana/{bin => tools/sbin}/so-kibana-config-load (100%) rename salt/{common => kibana}/tools/sbin/so-kibana-restart (100%) rename salt/{common => kibana}/tools/sbin/so-kibana-savedobjects-defaults (100%) rename salt/{common => kibana}/tools/sbin/so-kibana-space-defaults (100%) rename salt/{common => kibana}/tools/sbin/so-kibana-start (100%) rename salt/{common => kibana}/tools/sbin/so-kibana-stop (100%) rename salt/{common => logstash}/tools/sbin/so-logstash-events (100%) rename salt/{common => logstash}/tools/sbin/so-logstash-get-parsed (100%) rename salt/{common => logstash}/tools/sbin/so-logstash-get-unparsed (100%) rename salt/{common => logstash}/tools/sbin/so-logstash-pipeline-stats (100%) rename salt/{common => logstash}/tools/sbin/so-logstash-restart (100%) rename salt/{common => logstash}/tools/sbin/so-logstash-start (100%) rename salt/{common => logstash}/tools/sbin/so-logstash-stop (100%) rename salt/{common => mysql}/tools/sbin/so-mysql-restart (100%) rename salt/{common => mysql}/tools/sbin/so-mysql-start (100%) rename salt/{common => mysql}/tools/sbin/so-mysql-stop (100%) rename salt/{common => playbook}/tools/sbin/so-playbook-import (100%) rename salt/{common => playbook}/tools/sbin/so-playbook-reset (100%) rename salt/{common => playbook}/tools/sbin/so-playbook-restart (100%) rename salt/{common => playbook}/tools/sbin/so-playbook-ruleupdate (100%) rename salt/{common => playbook}/tools/sbin/so-playbook-sigma-refresh (100%) rename salt/{common => playbook}/tools/sbin/so-playbook-start (100%) rename salt/{common => playbook}/tools/sbin/so-playbook-stop (100%) rename salt/{common => playbook}/tools/sbin/so-playbook-sync (100%) rename salt/{common => redis}/tools/sbin/so-redis-count (100%) rename salt/{common => redis}/tools/sbin/so-redis-restart (100%) rename salt/{common => redis}/tools/sbin/so-redis-start (100%) rename salt/{common => redis}/tools/sbin/so-redis-stop (100%) rename salt/{common => soc}/tools/sbin/so-soc-restart (100%) rename salt/{common => soc}/tools/sbin/so-soc-start (100%) rename salt/{common => soc}/tools/sbin/so-soc-stop (100%) rename salt/{common => strelka}/tools/sbin/so-strelka-restart (100%) rename salt/{common => strelka}/tools/sbin/so-strelka-start (100%) rename salt/{common => strelka}/tools/sbin/so-strelka-stop (100%) rename salt/{common => telegraf}/tools/sbin/so-telegraf-restart (100%) rename salt/{common => telegraf}/tools/sbin/so-telegraf-start (100%) rename salt/{common => telegraf}/tools/sbin/so-telegraf-stop (100%) diff --git a/salt/common/tools/sbin/so-nodered-restart b/salt/common/tools/sbin/so-nodered-restart deleted file mode 100755 index 06060b764..000000000 --- a/salt/common/tools/sbin/so-nodered-restart +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-restart nodered $1 diff --git a/salt/common/tools/sbin/so-nodered-start b/salt/common/tools/sbin/so-nodered-start deleted file mode 100755 index f5ab36c80..000000000 --- a/salt/common/tools/sbin/so-nodered-start +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-start nodered $1 - diff --git a/salt/common/tools/sbin/so-nodered-stop b/salt/common/tools/sbin/so-nodered-stop deleted file mode 100755 index 0286a175c..000000000 --- a/salt/common/tools/sbin/so-nodered-stop +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-stop nodered $1 diff --git a/salt/curator/files/bin/so-curator-close b/salt/curator/tools/sbin/so-curator-close similarity index 100% rename from salt/curator/files/bin/so-curator-close rename to salt/curator/tools/sbin/so-curator-close diff --git a/salt/curator/files/bin/so-curator-cluster-close b/salt/curator/tools/sbin/so-curator-cluster-close similarity index 100% rename from salt/curator/files/bin/so-curator-cluster-close rename to salt/curator/tools/sbin/so-curator-cluster-close diff --git a/salt/curator/files/bin/so-curator-cluster-delete b/salt/curator/tools/sbin/so-curator-cluster-delete similarity index 100% rename from salt/curator/files/bin/so-curator-cluster-delete rename to salt/curator/tools/sbin/so-curator-cluster-delete diff --git a/salt/curator/files/bin/so-curator-cluster-delete-delete b/salt/curator/tools/sbin/so-curator-cluster-delete-delete similarity index 100% rename from salt/curator/files/bin/so-curator-cluster-delete-delete rename to salt/curator/tools/sbin/so-curator-cluster-delete-delete diff --git a/salt/curator/files/bin/so-curator-delete b/salt/curator/tools/sbin/so-curator-delete similarity index 100% rename from salt/curator/files/bin/so-curator-delete rename to salt/curator/tools/sbin/so-curator-delete diff --git a/salt/curator/files/bin/so-curator-restart b/salt/curator/tools/sbin/so-curator-restart similarity index 100% rename from salt/curator/files/bin/so-curator-restart rename to salt/curator/tools/sbin/so-curator-restart diff --git a/salt/curator/files/bin/so-curator-start b/salt/curator/tools/sbin/so-curator-start similarity index 100% rename from salt/curator/files/bin/so-curator-start rename to salt/curator/tools/sbin/so-curator-start diff --git a/salt/curator/files/bin/so-curator-stop b/salt/curator/tools/sbin/so-curator-stop similarity index 100% rename from salt/curator/files/bin/so-curator-stop rename to salt/curator/tools/sbin/so-curator-stop diff --git a/salt/elastalert/bin/so-elastalert-create b/salt/elastalert/tools/sbin/so-elastalert-create similarity index 100% rename from salt/elastalert/bin/so-elastalert-create rename to salt/elastalert/tools/sbin/so-elastalert-create diff --git a/salt/elastalert/bin/so-elastalert-restart b/salt/elastalert/tools/sbin/so-elastalert-restart similarity index 100% rename from salt/elastalert/bin/so-elastalert-restart rename to salt/elastalert/tools/sbin/so-elastalert-restart diff --git a/salt/elastalert/bin/so-elastalert-start b/salt/elastalert/tools/sbin/so-elastalert-start similarity index 100% rename from salt/elastalert/bin/so-elastalert-start rename to salt/elastalert/tools/sbin/so-elastalert-start diff --git a/salt/elastalert/bin/so-elastalert-stop b/salt/elastalert/tools/sbin/so-elastalert-stop similarity index 100% rename from salt/elastalert/bin/so-elastalert-stop rename to salt/elastalert/tools/sbin/so-elastalert-stop diff --git a/salt/elastalert/bin/so-elastalert-test b/salt/elastalert/tools/sbin/so-elastalert-test similarity index 100% rename from salt/elastalert/bin/so-elastalert-test rename to salt/elastalert/tools/sbin/so-elastalert-test diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete b/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-delete similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-agent-policy-delete rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-delete diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-list b/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-list similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-agent-policy-list rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-list diff --git a/salt/common/tools/sbin/so-elastic-fleet-agent-policy-view b/salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-view similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-agent-policy-view rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-agent-policy-view diff --git a/salt/common/tools/sbin/so-elastic-fleet-data-streams-list b/salt/elasticfleet/tools/sbin/so-elastic-fleet-data-streams-list similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-data-streams-list rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-data-streams-list diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-bulk-delete diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-delete similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-integration-policy-delete rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-delete diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-list b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-list similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-integration-policy-list rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-list diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-integration-policy-load rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load diff --git a/salt/common/tools/sbin/so-elastic-fleet-restart b/salt/elasticfleet/tools/sbin/so-elastic-fleet-restart similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-restart rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-restart diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin/so-elastic-fleet-setup similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-setup rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-setup diff --git a/salt/common/tools/sbin/so-elastic-fleet-start b/salt/elasticfleet/tools/sbin/so-elastic-fleet-start similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-start rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-start diff --git a/salt/common/tools/sbin/so-elastic-fleet-stop b/salt/elasticfleet/tools/sbin/so-elastic-fleet-stop similarity index 100% rename from salt/common/tools/sbin/so-elastic-fleet-stop rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-stop diff --git a/salt/elasticsearch/tools/sbin/so-elastic-clear b/salt/elasticsearch/tools/sbin/so-elastic-clear new file mode 100755 index 000000000..f491fb62f --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elastic-clear @@ -0,0 +1,154 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} +. /usr/sbin/so-common + +SKIP=0 +######################################### +# Options +######################################### +usage() +{ +cat < /dev/null 2>&1 + done +fi + +# Delete Elastalert data +if [ ! -z "$DELETE_ELASTALERT_DATA" ]; then + # Delete Elastalert data + echo "Deleting Elastalert data..." + INDXS=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index | grep "elastalert") + for INDX in ${INDXS} + do + echo "Deleting $INDX" + /usr/sbin/so-elasticsearch-query ${INDX} -XDELETE > /dev/null 2>&1 + done +fi + +# Delete log data +if [ ! -z "$DELETE_LOG_DATA" ]; then + echo "Deleting log data ..." + DATASTREAMS=$(/usr/sbin/so-elasticsearch-query _data_stream | jq -r '.[] |.[].name') + for DATASTREAM in ${DATASTREAMS} + do + # Delete the data stream + echo "Deleting $DATASTREAM..." + /usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} -XDELETE > /dev/null 2>&1 + done +fi + +if [ -z "$DONT_STOP_SERVICES" ]; then + #Start Logstash + if [ ! -z "$LS_ENABLED" ]; then + /usr/sbin/so-logstash-start + + fi + + #Start Elastic Fleet + #if [ ! -z "$EF_ENABLED" ]; then + # /usr/sbin/so-elastic-fleet-start + #fi + + #Start Elastalert + if [ ! -z "$EA_ENABLED" ]; then + /usr/sbin/so-elastalert-start + fi + + # Start Elastic Agent + /usr/bin/elastic-agent restart +fi diff --git a/salt/elasticsearch/tools/sbin/so-elastic-diagnose b/salt/elasticsearch/tools/sbin/so-elastic-diagnose new file mode 100755 index 000000000..a94384fe8 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elastic-diagnose @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +# Source common settings +. /usr/sbin/so-common + +# Check for log files +for FILE in /opt/so/log/elasticsearch/*.log /opt/so/log/logstash/*.log /opt/so/log/kibana/*.log /opt/so/log/elastalert/*.log /opt/so/log/curator/*.log /opt/so/log/freqserver/*.log /opt/so/log/nginx/*.log; do + +# If file exists, then look for errors or warnings +if [ -f $FILE ]; then + MESSAGE=`grep -i 'ERROR\|FAIL\|WARN' $FILE` + if [ ! -z "$MESSAGE" ]; then + header $FILE + echo $MESSAGE | sed 's/WARN/\nWARN/g' | sed 's/WARNING/\nWARNING/g' | sed 's/ERROR/\nERROR/g' | sort | uniq -c | sort -nr + echo + fi +fi +done diff --git a/salt/elasticsearch/tools/sbin/so-elastic-restart b/salt/elasticsearch/tools/sbin/so-elastic-restart new file mode 100755 index 000000000..67988193f --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elastic-restart @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + + +{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} +/usr/sbin/so-restart elasticsearch $1 +{%- endif %} + +{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} +/usr/sbin/so-restart kibana $1 +{%- endif %} + +{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-restart logstash $1 +{%- endif %} + +{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-restart curator $1 +{%- endif %} + +{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} +/usr/sbin/so-restart elastalert $1 +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin/so-elastic-start b/salt/elasticsearch/tools/sbin/so-elastic-start new file mode 100755 index 000000000..fd78d1859 --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elastic-start @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + + +{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} +/usr/sbin/so-start elasticsearch $1 +{%- endif %} + +{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} +/usr/sbin/so-start kibana $1 +{%- endif %} + +{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-start logstash $1 +{%- endif %} + +{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-start curator $1 +{%- endif %} + +{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} +/usr/sbin/so-start elastalert $1 +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin/so-elastic-stop b/salt/elasticsearch/tools/sbin/so-elastic-stop new file mode 100755 index 000000000..88350a8fe --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elastic-stop @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + + +{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} +/usr/sbin/so-stop elasticsearch $1 +{%- endif %} + +{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} +/usr/sbin/so-stop kibana $1 +{%- endif %} + +{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-stop logstash $1 +{%- endif %} + +{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-stop curator $1 +{%- endif %} + +{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} +/usr/sbin/so-stop elastalert $1 +{%- endif %} diff --git a/salt/common/tools/sbin/so-idh-restart b/salt/idh/tools/sbin/so-idh-restart similarity index 100% rename from salt/common/tools/sbin/so-idh-restart rename to salt/idh/tools/sbin/so-idh-restart diff --git a/salt/common/tools/sbin/so-idh-start b/salt/idh/tools/sbin/so-idh-start similarity index 100% rename from salt/common/tools/sbin/so-idh-start rename to salt/idh/tools/sbin/so-idh-start diff --git a/salt/common/tools/sbin/so-idh-stop b/salt/idh/tools/sbin/so-idh-stop similarity index 100% rename from salt/common/tools/sbin/so-idh-stop rename to salt/idh/tools/sbin/so-idh-stop diff --git a/salt/idstools/bin/so-idstools-restart b/salt/idstools/tools/sbin/so-idstools-restart similarity index 100% rename from salt/idstools/bin/so-idstools-restart rename to salt/idstools/tools/sbin/so-idstools-restart diff --git a/salt/idstools/bin/so-idstools-start b/salt/idstools/tools/sbin/so-idstools-start similarity index 100% rename from salt/idstools/bin/so-idstools-start rename to salt/idstools/tools/sbin/so-idstools-start diff --git a/salt/idstools/bin/so-idstools-stop b/salt/idstools/tools/sbin/so-idstools-stop similarity index 100% rename from salt/idstools/bin/so-idstools-stop rename to salt/idstools/tools/sbin/so-idstools-stop diff --git a/salt/common/tools/sbin/so-influxdb-manage b/salt/influxdb/tools/sbin/so-influxdb-manage similarity index 100% rename from salt/common/tools/sbin/so-influxdb-manage rename to salt/influxdb/tools/sbin/so-influxdb-manage diff --git a/salt/common/tools/sbin/so-influxdb-restart b/salt/influxdb/tools/sbin/so-influxdb-restart similarity index 100% rename from salt/common/tools/sbin/so-influxdb-restart rename to salt/influxdb/tools/sbin/so-influxdb-restart diff --git a/salt/common/tools/sbin/so-influxdb-start b/salt/influxdb/tools/sbin/so-influxdb-start similarity index 100% rename from salt/common/tools/sbin/so-influxdb-start rename to salt/influxdb/tools/sbin/so-influxdb-start diff --git a/salt/common/tools/sbin/so-influxdb-stop b/salt/influxdb/tools/sbin/so-influxdb-stop similarity index 100% rename from salt/common/tools/sbin/so-influxdb-stop rename to salt/influxdb/tools/sbin/so-influxdb-stop diff --git a/salt/common/tools/sbin/so-kibana-config-export b/salt/kibana/tools/sbin/so-kibana-config-export similarity index 100% rename from salt/common/tools/sbin/so-kibana-config-export rename to salt/kibana/tools/sbin/so-kibana-config-export diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/tools/sbin/so-kibana-config-load similarity index 100% rename from salt/kibana/bin/so-kibana-config-load rename to salt/kibana/tools/sbin/so-kibana-config-load diff --git a/salt/common/tools/sbin/so-kibana-restart b/salt/kibana/tools/sbin/so-kibana-restart similarity index 100% rename from salt/common/tools/sbin/so-kibana-restart rename to salt/kibana/tools/sbin/so-kibana-restart diff --git a/salt/common/tools/sbin/so-kibana-savedobjects-defaults b/salt/kibana/tools/sbin/so-kibana-savedobjects-defaults similarity index 100% rename from salt/common/tools/sbin/so-kibana-savedobjects-defaults rename to salt/kibana/tools/sbin/so-kibana-savedobjects-defaults diff --git a/salt/common/tools/sbin/so-kibana-space-defaults b/salt/kibana/tools/sbin/so-kibana-space-defaults similarity index 100% rename from salt/common/tools/sbin/so-kibana-space-defaults rename to salt/kibana/tools/sbin/so-kibana-space-defaults diff --git a/salt/common/tools/sbin/so-kibana-start b/salt/kibana/tools/sbin/so-kibana-start similarity index 100% rename from salt/common/tools/sbin/so-kibana-start rename to salt/kibana/tools/sbin/so-kibana-start diff --git a/salt/common/tools/sbin/so-kibana-stop b/salt/kibana/tools/sbin/so-kibana-stop similarity index 100% rename from salt/common/tools/sbin/so-kibana-stop rename to salt/kibana/tools/sbin/so-kibana-stop diff --git a/salt/common/tools/sbin/so-logstash-events b/salt/logstash/tools/sbin/so-logstash-events similarity index 100% rename from salt/common/tools/sbin/so-logstash-events rename to salt/logstash/tools/sbin/so-logstash-events diff --git a/salt/common/tools/sbin/so-logstash-get-parsed b/salt/logstash/tools/sbin/so-logstash-get-parsed similarity index 100% rename from salt/common/tools/sbin/so-logstash-get-parsed rename to salt/logstash/tools/sbin/so-logstash-get-parsed diff --git a/salt/common/tools/sbin/so-logstash-get-unparsed b/salt/logstash/tools/sbin/so-logstash-get-unparsed similarity index 100% rename from salt/common/tools/sbin/so-logstash-get-unparsed rename to salt/logstash/tools/sbin/so-logstash-get-unparsed diff --git a/salt/common/tools/sbin/so-logstash-pipeline-stats b/salt/logstash/tools/sbin/so-logstash-pipeline-stats similarity index 100% rename from salt/common/tools/sbin/so-logstash-pipeline-stats rename to salt/logstash/tools/sbin/so-logstash-pipeline-stats diff --git a/salt/common/tools/sbin/so-logstash-restart b/salt/logstash/tools/sbin/so-logstash-restart similarity index 100% rename from salt/common/tools/sbin/so-logstash-restart rename to salt/logstash/tools/sbin/so-logstash-restart diff --git a/salt/common/tools/sbin/so-logstash-start b/salt/logstash/tools/sbin/so-logstash-start similarity index 100% rename from salt/common/tools/sbin/so-logstash-start rename to salt/logstash/tools/sbin/so-logstash-start diff --git a/salt/common/tools/sbin/so-logstash-stop b/salt/logstash/tools/sbin/so-logstash-stop similarity index 100% rename from salt/common/tools/sbin/so-logstash-stop rename to salt/logstash/tools/sbin/so-logstash-stop diff --git a/salt/common/tools/sbin/so-mysql-restart b/salt/mysql/tools/sbin/so-mysql-restart similarity index 100% rename from salt/common/tools/sbin/so-mysql-restart rename to salt/mysql/tools/sbin/so-mysql-restart diff --git a/salt/common/tools/sbin/so-mysql-start b/salt/mysql/tools/sbin/so-mysql-start similarity index 100% rename from salt/common/tools/sbin/so-mysql-start rename to salt/mysql/tools/sbin/so-mysql-start diff --git a/salt/common/tools/sbin/so-mysql-stop b/salt/mysql/tools/sbin/so-mysql-stop similarity index 100% rename from salt/common/tools/sbin/so-mysql-stop rename to salt/mysql/tools/sbin/so-mysql-stop diff --git a/salt/common/tools/sbin/so-playbook-import b/salt/playbook/tools/sbin/so-playbook-import similarity index 100% rename from salt/common/tools/sbin/so-playbook-import rename to salt/playbook/tools/sbin/so-playbook-import diff --git a/salt/common/tools/sbin/so-playbook-reset b/salt/playbook/tools/sbin/so-playbook-reset similarity index 100% rename from salt/common/tools/sbin/so-playbook-reset rename to salt/playbook/tools/sbin/so-playbook-reset diff --git a/salt/common/tools/sbin/so-playbook-restart b/salt/playbook/tools/sbin/so-playbook-restart similarity index 100% rename from salt/common/tools/sbin/so-playbook-restart rename to salt/playbook/tools/sbin/so-playbook-restart diff --git a/salt/common/tools/sbin/so-playbook-ruleupdate b/salt/playbook/tools/sbin/so-playbook-ruleupdate similarity index 100% rename from salt/common/tools/sbin/so-playbook-ruleupdate rename to salt/playbook/tools/sbin/so-playbook-ruleupdate diff --git a/salt/common/tools/sbin/so-playbook-sigma-refresh b/salt/playbook/tools/sbin/so-playbook-sigma-refresh similarity index 100% rename from salt/common/tools/sbin/so-playbook-sigma-refresh rename to salt/playbook/tools/sbin/so-playbook-sigma-refresh diff --git a/salt/common/tools/sbin/so-playbook-start b/salt/playbook/tools/sbin/so-playbook-start similarity index 100% rename from salt/common/tools/sbin/so-playbook-start rename to salt/playbook/tools/sbin/so-playbook-start diff --git a/salt/common/tools/sbin/so-playbook-stop b/salt/playbook/tools/sbin/so-playbook-stop similarity index 100% rename from salt/common/tools/sbin/so-playbook-stop rename to salt/playbook/tools/sbin/so-playbook-stop diff --git a/salt/common/tools/sbin/so-playbook-sync b/salt/playbook/tools/sbin/so-playbook-sync similarity index 100% rename from salt/common/tools/sbin/so-playbook-sync rename to salt/playbook/tools/sbin/so-playbook-sync diff --git a/salt/common/tools/sbin/so-redis-count b/salt/redis/tools/sbin/so-redis-count similarity index 100% rename from salt/common/tools/sbin/so-redis-count rename to salt/redis/tools/sbin/so-redis-count diff --git a/salt/common/tools/sbin/so-redis-restart b/salt/redis/tools/sbin/so-redis-restart similarity index 100% rename from salt/common/tools/sbin/so-redis-restart rename to salt/redis/tools/sbin/so-redis-restart diff --git a/salt/common/tools/sbin/so-redis-start b/salt/redis/tools/sbin/so-redis-start similarity index 100% rename from salt/common/tools/sbin/so-redis-start rename to salt/redis/tools/sbin/so-redis-start diff --git a/salt/common/tools/sbin/so-redis-stop b/salt/redis/tools/sbin/so-redis-stop similarity index 100% rename from salt/common/tools/sbin/so-redis-stop rename to salt/redis/tools/sbin/so-redis-stop diff --git a/salt/common/tools/sbin/so-soc-restart b/salt/soc/tools/sbin/so-soc-restart similarity index 100% rename from salt/common/tools/sbin/so-soc-restart rename to salt/soc/tools/sbin/so-soc-restart diff --git a/salt/common/tools/sbin/so-soc-start b/salt/soc/tools/sbin/so-soc-start similarity index 100% rename from salt/common/tools/sbin/so-soc-start rename to salt/soc/tools/sbin/so-soc-start diff --git a/salt/common/tools/sbin/so-soc-stop b/salt/soc/tools/sbin/so-soc-stop similarity index 100% rename from salt/common/tools/sbin/so-soc-stop rename to salt/soc/tools/sbin/so-soc-stop diff --git a/salt/common/tools/sbin/so-strelka-restart b/salt/strelka/tools/sbin/so-strelka-restart similarity index 100% rename from salt/common/tools/sbin/so-strelka-restart rename to salt/strelka/tools/sbin/so-strelka-restart diff --git a/salt/common/tools/sbin/so-strelka-start b/salt/strelka/tools/sbin/so-strelka-start similarity index 100% rename from salt/common/tools/sbin/so-strelka-start rename to salt/strelka/tools/sbin/so-strelka-start diff --git a/salt/common/tools/sbin/so-strelka-stop b/salt/strelka/tools/sbin/so-strelka-stop similarity index 100% rename from salt/common/tools/sbin/so-strelka-stop rename to salt/strelka/tools/sbin/so-strelka-stop diff --git a/salt/common/tools/sbin/so-telegraf-restart b/salt/telegraf/tools/sbin/so-telegraf-restart similarity index 100% rename from salt/common/tools/sbin/so-telegraf-restart rename to salt/telegraf/tools/sbin/so-telegraf-restart diff --git a/salt/common/tools/sbin/so-telegraf-start b/salt/telegraf/tools/sbin/so-telegraf-start similarity index 100% rename from salt/common/tools/sbin/so-telegraf-start rename to salt/telegraf/tools/sbin/so-telegraf-start diff --git a/salt/common/tools/sbin/so-telegraf-stop b/salt/telegraf/tools/sbin/so-telegraf-stop similarity index 100% rename from salt/common/tools/sbin/so-telegraf-stop rename to salt/telegraf/tools/sbin/so-telegraf-stop From e60e21d9ffbc7af0841cda2c2f709ee3ba23d769 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 May 2023 09:40:02 -0400 Subject: [PATCH 12/41] Move files out of common --- salt/common/tools/sbin/so-elastic-clear | 154 ------------------ salt/common/tools/sbin/so-elastic-diagnose | 25 --- salt/common/tools/sbin/so-elastic-restart | 31 ---- salt/common/tools/sbin/so-elastic-start | 31 ---- salt/common/tools/sbin/so-elastic-stop | 31 ---- .../so-elastic-agent-gen-installers.jinja} | 0 ...eet-setup => so-elastic-fleet-setup.jinja} | 0 .../tools/sbin/so-index-list | 0 salt/{common => idstools}/tools/sbin/so-rule | 0 .../tools/sbin/so-rule-update | 0 ...g-export => so-kibana-config-export.jinja} | 0 ...onfig-load => so-kibana-config-load.jinja} | 0 ...efaults => so-kibana-space-defaults.jinja} | 0 salt/manager/sbin/so-saltstack-update | 53 ------ salt/{common => manager}/tools/sbin/so-allow | 0 .../tools/sbin/so-allow-view | 0 salt/{common => manager}/tools/sbin/so-deny | 0 .../tools/sbin/so-docker-refresh | 0 .../tools/sbin/so-elastic-auth-password-reset | 0 .../tools/sbin/so-firewall | 0 .../tools/sbin/so-firewall-minion | 0 salt/{common => manager}/tools/sbin/so-minion | 0 salt/manager/{ => tools}/sbin/so-repo-sync | 0 .../tools/sbin/so-saltstack-update | 0 salt/{common => manager}/tools/sbin/so-user | 0 .../tools/sbin/so-user-add | 0 .../tools/sbin/so-user-disable | 0 .../tools/sbin/so-user-enable | 0 .../tools/sbin/so-user-list | 0 salt/{common => manager}/tools/sbin/soup | 0 .../toos}/sbin/so-nginx-restart | 0 .../tools => nginx/toos}/sbin/so-nginx-start | 0 .../tools => nginx/toos}/sbin/so-nginx-stop | 0 .../tools/sbin/so-pcap-export | 0 .../tools/sbin/so-pcap-restart | 0 .../{common => pcap}/tools/sbin/so-pcap-start | 0 salt/{common => pcap}/tools/sbin/so-pcap-stop | 0 .../tools/sbin/so-sensoroni-restart | 0 .../tools/sbin/so-sensoroni-start | 0 .../tools/sbin/so-sensoroni-stop | 0 .../tools/sbin/so-soctopus-restart | 0 .../tools/sbin/so-soctopus-start | 0 .../tools/sbin/so-soctopus-stop | 0 .../tools/sbin/so-suricata-restart | 0 .../tools/sbin/so-suricata-start | 0 .../tools/sbin/so-suricata-stop | 0 .../tools/sbin/so-suricata-testrule | 0 47 files changed, 325 deletions(-) delete mode 100755 salt/common/tools/sbin/so-elastic-clear delete mode 100755 salt/common/tools/sbin/so-elastic-diagnose delete mode 100755 salt/common/tools/sbin/so-elastic-restart delete mode 100755 salt/common/tools/sbin/so-elastic-start delete mode 100755 salt/common/tools/sbin/so-elastic-stop rename salt/{common/tools/sbin/so-elastic-agent-gen-installers => elasticfleet/tools/sbin/so-elastic-agent-gen-installers.jinja} (100%) rename salt/elasticfleet/tools/sbin/{so-elastic-fleet-setup => so-elastic-fleet-setup.jinja} (100%) rename salt/{common => elasticsearch}/tools/sbin/so-index-list (100%) rename salt/{common => idstools}/tools/sbin/so-rule (100%) rename salt/{common => idstools}/tools/sbin/so-rule-update (100%) rename salt/kibana/tools/sbin/{so-kibana-config-export => so-kibana-config-export.jinja} (100%) rename salt/kibana/tools/sbin/{so-kibana-config-load => so-kibana-config-load.jinja} (100%) rename salt/kibana/tools/sbin/{so-kibana-space-defaults => so-kibana-space-defaults.jinja} (100%) delete mode 100755 salt/manager/sbin/so-saltstack-update rename salt/{common => manager}/tools/sbin/so-allow (100%) rename salt/{common => manager}/tools/sbin/so-allow-view (100%) rename salt/{common => manager}/tools/sbin/so-deny (100%) rename salt/{common => manager}/tools/sbin/so-docker-refresh (100%) rename salt/{common => manager}/tools/sbin/so-elastic-auth-password-reset (100%) rename salt/{common => manager}/tools/sbin/so-firewall (100%) rename salt/{common => manager}/tools/sbin/so-firewall-minion (100%) rename salt/{common => manager}/tools/sbin/so-minion (100%) rename salt/manager/{ => tools}/sbin/so-repo-sync (100%) rename salt/{common => manager}/tools/sbin/so-saltstack-update (100%) rename salt/{common => manager}/tools/sbin/so-user (100%) rename salt/{common => manager}/tools/sbin/so-user-add (100%) rename salt/{common => manager}/tools/sbin/so-user-disable (100%) rename salt/{common => manager}/tools/sbin/so-user-enable (100%) rename salt/{common => manager}/tools/sbin/so-user-list (100%) rename salt/{common => manager}/tools/sbin/soup (100%) rename salt/{common/tools => nginx/toos}/sbin/so-nginx-restart (100%) rename salt/{common/tools => nginx/toos}/sbin/so-nginx-start (100%) rename salt/{common/tools => nginx/toos}/sbin/so-nginx-stop (100%) rename salt/{common => pcap}/tools/sbin/so-pcap-export (100%) rename salt/{common => pcap}/tools/sbin/so-pcap-restart (100%) rename salt/{common => pcap}/tools/sbin/so-pcap-start (100%) rename salt/{common => pcap}/tools/sbin/so-pcap-stop (100%) rename salt/{common => sensoroni}/tools/sbin/so-sensoroni-restart (100%) rename salt/{common => sensoroni}/tools/sbin/so-sensoroni-start (100%) rename salt/{common => sensoroni}/tools/sbin/so-sensoroni-stop (100%) rename salt/{common => soctopus}/tools/sbin/so-soctopus-restart (100%) rename salt/{common => soctopus}/tools/sbin/so-soctopus-start (100%) rename salt/{common => soctopus}/tools/sbin/so-soctopus-stop (100%) rename salt/{common => suricata}/tools/sbin/so-suricata-restart (100%) rename salt/{common => suricata}/tools/sbin/so-suricata-start (100%) rename salt/{common => suricata}/tools/sbin/so-suricata-stop (100%) rename salt/{common => suricata}/tools/sbin/so-suricata-testrule (100%) diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear deleted file mode 100755 index f491fb62f..000000000 --- a/salt/common/tools/sbin/so-elastic-clear +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash -# -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} -. /usr/sbin/so-common - -SKIP=0 -######################################### -# Options -######################################### -usage() -{ -cat < /dev/null 2>&1 - done -fi - -# Delete Elastalert data -if [ ! -z "$DELETE_ELASTALERT_DATA" ]; then - # Delete Elastalert data - echo "Deleting Elastalert data..." - INDXS=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index | grep "elastalert") - for INDX in ${INDXS} - do - echo "Deleting $INDX" - /usr/sbin/so-elasticsearch-query ${INDX} -XDELETE > /dev/null 2>&1 - done -fi - -# Delete log data -if [ ! -z "$DELETE_LOG_DATA" ]; then - echo "Deleting log data ..." - DATASTREAMS=$(/usr/sbin/so-elasticsearch-query _data_stream | jq -r '.[] |.[].name') - for DATASTREAM in ${DATASTREAMS} - do - # Delete the data stream - echo "Deleting $DATASTREAM..." - /usr/sbin/so-elasticsearch-query _data_stream/${DATASTREAM} -XDELETE > /dev/null 2>&1 - done -fi - -if [ -z "$DONT_STOP_SERVICES" ]; then - #Start Logstash - if [ ! -z "$LS_ENABLED" ]; then - /usr/sbin/so-logstash-start - - fi - - #Start Elastic Fleet - #if [ ! -z "$EF_ENABLED" ]; then - # /usr/sbin/so-elastic-fleet-start - #fi - - #Start Elastalert - if [ ! -z "$EA_ENABLED" ]; then - /usr/sbin/so-elastalert-start - fi - - # Start Elastic Agent - /usr/bin/elastic-agent restart -fi diff --git a/salt/common/tools/sbin/so-elastic-diagnose b/salt/common/tools/sbin/so-elastic-diagnose deleted file mode 100755 index a94384fe8..000000000 --- a/salt/common/tools/sbin/so-elastic-diagnose +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -# Source common settings -. /usr/sbin/so-common - -# Check for log files -for FILE in /opt/so/log/elasticsearch/*.log /opt/so/log/logstash/*.log /opt/so/log/kibana/*.log /opt/so/log/elastalert/*.log /opt/so/log/curator/*.log /opt/so/log/freqserver/*.log /opt/so/log/nginx/*.log; do - -# If file exists, then look for errors or warnings -if [ -f $FILE ]; then - MESSAGE=`grep -i 'ERROR\|FAIL\|WARN' $FILE` - if [ ! -z "$MESSAGE" ]; then - header $FILE - echo $MESSAGE | sed 's/WARN/\nWARN/g' | sed 's/WARNING/\nWARNING/g' | sed 's/ERROR/\nERROR/g' | sort | uniq -c | sort -nr - echo - fi -fi -done diff --git a/salt/common/tools/sbin/so-elastic-restart b/salt/common/tools/sbin/so-elastic-restart deleted file mode 100755 index 67988193f..000000000 --- a/salt/common/tools/sbin/so-elastic-restart +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} -/usr/sbin/so-restart elasticsearch $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} -/usr/sbin/so-restart kibana $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-restart logstash $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-restart curator $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} -/usr/sbin/so-restart elastalert $1 -{%- endif %} diff --git a/salt/common/tools/sbin/so-elastic-start b/salt/common/tools/sbin/so-elastic-start deleted file mode 100755 index fd78d1859..000000000 --- a/salt/common/tools/sbin/so-elastic-start +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} -/usr/sbin/so-start elasticsearch $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} -/usr/sbin/so-start kibana $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-start logstash $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-start curator $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} -/usr/sbin/so-start elastalert $1 -{%- endif %} diff --git a/salt/common/tools/sbin/so-elastic-stop b/salt/common/tools/sbin/so-elastic-stop deleted file mode 100755 index 88350a8fe..000000000 --- a/salt/common/tools/sbin/so-elastic-stop +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} -/usr/sbin/so-stop elasticsearch $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} -/usr/sbin/so-stop kibana $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-stop logstash $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-stop curator $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} -/usr/sbin/so-stop elastalert $1 -{%- endif %} diff --git a/salt/common/tools/sbin/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin/so-elastic-agent-gen-installers.jinja similarity index 100% rename from salt/common/tools/sbin/so-elastic-agent-gen-installers rename to salt/elasticfleet/tools/sbin/so-elastic-agent-gen-installers.jinja diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin/so-elastic-fleet-setup.jinja similarity index 100% rename from salt/elasticfleet/tools/sbin/so-elastic-fleet-setup rename to salt/elasticfleet/tools/sbin/so-elastic-fleet-setup.jinja diff --git a/salt/common/tools/sbin/so-index-list b/salt/elasticsearch/tools/sbin/so-index-list similarity index 100% rename from salt/common/tools/sbin/so-index-list rename to salt/elasticsearch/tools/sbin/so-index-list diff --git a/salt/common/tools/sbin/so-rule b/salt/idstools/tools/sbin/so-rule similarity index 100% rename from salt/common/tools/sbin/so-rule rename to salt/idstools/tools/sbin/so-rule diff --git a/salt/common/tools/sbin/so-rule-update b/salt/idstools/tools/sbin/so-rule-update similarity index 100% rename from salt/common/tools/sbin/so-rule-update rename to salt/idstools/tools/sbin/so-rule-update diff --git a/salt/kibana/tools/sbin/so-kibana-config-export b/salt/kibana/tools/sbin/so-kibana-config-export.jinja similarity index 100% rename from salt/kibana/tools/sbin/so-kibana-config-export rename to salt/kibana/tools/sbin/so-kibana-config-export.jinja diff --git a/salt/kibana/tools/sbin/so-kibana-config-load b/salt/kibana/tools/sbin/so-kibana-config-load.jinja similarity index 100% rename from salt/kibana/tools/sbin/so-kibana-config-load rename to salt/kibana/tools/sbin/so-kibana-config-load.jinja diff --git a/salt/kibana/tools/sbin/so-kibana-space-defaults b/salt/kibana/tools/sbin/so-kibana-space-defaults.jinja similarity index 100% rename from salt/kibana/tools/sbin/so-kibana-space-defaults rename to salt/kibana/tools/sbin/so-kibana-space-defaults.jinja diff --git a/salt/manager/sbin/so-saltstack-update b/salt/manager/sbin/so-saltstack-update deleted file mode 100755 index 73c9c7791..000000000 --- a/salt/manager/sbin/so-saltstack-update +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -default_salt_dir=/opt/so/saltstack/default -clone_to_tmp() { - - # Make a temp location for the files - mkdir /tmp/sogh - cd /tmp/sogh - git clone https://github.com/Security-Onion-Solutions/securityonion.git - cd /tmp - -} - -copy_new_files() { - - # Copy new files over to the salt dir - cd /tmp/sogh/securityonion - git checkout $BRANCH - VERSION=$(cat VERSION) - # We need to overwrite if there is a repo file - if [ -d /opt/so/repo ]; then - tar -czf /opt/so/repo/"$VERSION".tar.gz -C "$(pwd)/.." . - fi - rsync -a salt $default_salt_dir/ - rsync -a pillar $default_salt_dir/ - chown -R socore:socore $default_salt_dir/salt - chown -R socore:socore $default_salt_dir/pillar - chmod 755 $default_salt_dir/pillar/firewall/addfirewall.sh - - rm -rf /tmp/sogh -} - -got_root(){ - if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" - exit 1 - fi -} - -got_root -if [ $# -ne 1 ] ; then - BRANCH=2.4/main -else - BRANCH=$1 -fi -clone_to_tmp -copy_new_files diff --git a/salt/common/tools/sbin/so-allow b/salt/manager/tools/sbin/so-allow similarity index 100% rename from salt/common/tools/sbin/so-allow rename to salt/manager/tools/sbin/so-allow diff --git a/salt/common/tools/sbin/so-allow-view b/salt/manager/tools/sbin/so-allow-view similarity index 100% rename from salt/common/tools/sbin/so-allow-view rename to salt/manager/tools/sbin/so-allow-view diff --git a/salt/common/tools/sbin/so-deny b/salt/manager/tools/sbin/so-deny similarity index 100% rename from salt/common/tools/sbin/so-deny rename to salt/manager/tools/sbin/so-deny diff --git a/salt/common/tools/sbin/so-docker-refresh b/salt/manager/tools/sbin/so-docker-refresh similarity index 100% rename from salt/common/tools/sbin/so-docker-refresh rename to salt/manager/tools/sbin/so-docker-refresh diff --git a/salt/common/tools/sbin/so-elastic-auth-password-reset b/salt/manager/tools/sbin/so-elastic-auth-password-reset similarity index 100% rename from salt/common/tools/sbin/so-elastic-auth-password-reset rename to salt/manager/tools/sbin/so-elastic-auth-password-reset diff --git a/salt/common/tools/sbin/so-firewall b/salt/manager/tools/sbin/so-firewall similarity index 100% rename from salt/common/tools/sbin/so-firewall rename to salt/manager/tools/sbin/so-firewall diff --git a/salt/common/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion similarity index 100% rename from salt/common/tools/sbin/so-firewall-minion rename to salt/manager/tools/sbin/so-firewall-minion diff --git a/salt/common/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion similarity index 100% rename from salt/common/tools/sbin/so-minion rename to salt/manager/tools/sbin/so-minion diff --git a/salt/manager/sbin/so-repo-sync b/salt/manager/tools/sbin/so-repo-sync similarity index 100% rename from salt/manager/sbin/so-repo-sync rename to salt/manager/tools/sbin/so-repo-sync diff --git a/salt/common/tools/sbin/so-saltstack-update b/salt/manager/tools/sbin/so-saltstack-update similarity index 100% rename from salt/common/tools/sbin/so-saltstack-update rename to salt/manager/tools/sbin/so-saltstack-update diff --git a/salt/common/tools/sbin/so-user b/salt/manager/tools/sbin/so-user similarity index 100% rename from salt/common/tools/sbin/so-user rename to salt/manager/tools/sbin/so-user diff --git a/salt/common/tools/sbin/so-user-add b/salt/manager/tools/sbin/so-user-add similarity index 100% rename from salt/common/tools/sbin/so-user-add rename to salt/manager/tools/sbin/so-user-add diff --git a/salt/common/tools/sbin/so-user-disable b/salt/manager/tools/sbin/so-user-disable similarity index 100% rename from salt/common/tools/sbin/so-user-disable rename to salt/manager/tools/sbin/so-user-disable diff --git a/salt/common/tools/sbin/so-user-enable b/salt/manager/tools/sbin/so-user-enable similarity index 100% rename from salt/common/tools/sbin/so-user-enable rename to salt/manager/tools/sbin/so-user-enable diff --git a/salt/common/tools/sbin/so-user-list b/salt/manager/tools/sbin/so-user-list similarity index 100% rename from salt/common/tools/sbin/so-user-list rename to salt/manager/tools/sbin/so-user-list diff --git a/salt/common/tools/sbin/soup b/salt/manager/tools/sbin/soup similarity index 100% rename from salt/common/tools/sbin/soup rename to salt/manager/tools/sbin/soup diff --git a/salt/common/tools/sbin/so-nginx-restart b/salt/nginx/toos/sbin/so-nginx-restart similarity index 100% rename from salt/common/tools/sbin/so-nginx-restart rename to salt/nginx/toos/sbin/so-nginx-restart diff --git a/salt/common/tools/sbin/so-nginx-start b/salt/nginx/toos/sbin/so-nginx-start similarity index 100% rename from salt/common/tools/sbin/so-nginx-start rename to salt/nginx/toos/sbin/so-nginx-start diff --git a/salt/common/tools/sbin/so-nginx-stop b/salt/nginx/toos/sbin/so-nginx-stop similarity index 100% rename from salt/common/tools/sbin/so-nginx-stop rename to salt/nginx/toos/sbin/so-nginx-stop diff --git a/salt/common/tools/sbin/so-pcap-export b/salt/pcap/tools/sbin/so-pcap-export similarity index 100% rename from salt/common/tools/sbin/so-pcap-export rename to salt/pcap/tools/sbin/so-pcap-export diff --git a/salt/common/tools/sbin/so-pcap-restart b/salt/pcap/tools/sbin/so-pcap-restart similarity index 100% rename from salt/common/tools/sbin/so-pcap-restart rename to salt/pcap/tools/sbin/so-pcap-restart diff --git a/salt/common/tools/sbin/so-pcap-start b/salt/pcap/tools/sbin/so-pcap-start similarity index 100% rename from salt/common/tools/sbin/so-pcap-start rename to salt/pcap/tools/sbin/so-pcap-start diff --git a/salt/common/tools/sbin/so-pcap-stop b/salt/pcap/tools/sbin/so-pcap-stop similarity index 100% rename from salt/common/tools/sbin/so-pcap-stop rename to salt/pcap/tools/sbin/so-pcap-stop diff --git a/salt/common/tools/sbin/so-sensoroni-restart b/salt/sensoroni/tools/sbin/so-sensoroni-restart similarity index 100% rename from salt/common/tools/sbin/so-sensoroni-restart rename to salt/sensoroni/tools/sbin/so-sensoroni-restart diff --git a/salt/common/tools/sbin/so-sensoroni-start b/salt/sensoroni/tools/sbin/so-sensoroni-start similarity index 100% rename from salt/common/tools/sbin/so-sensoroni-start rename to salt/sensoroni/tools/sbin/so-sensoroni-start diff --git a/salt/common/tools/sbin/so-sensoroni-stop b/salt/sensoroni/tools/sbin/so-sensoroni-stop similarity index 100% rename from salt/common/tools/sbin/so-sensoroni-stop rename to salt/sensoroni/tools/sbin/so-sensoroni-stop diff --git a/salt/common/tools/sbin/so-soctopus-restart b/salt/soctopus/tools/sbin/so-soctopus-restart similarity index 100% rename from salt/common/tools/sbin/so-soctopus-restart rename to salt/soctopus/tools/sbin/so-soctopus-restart diff --git a/salt/common/tools/sbin/so-soctopus-start b/salt/soctopus/tools/sbin/so-soctopus-start similarity index 100% rename from salt/common/tools/sbin/so-soctopus-start rename to salt/soctopus/tools/sbin/so-soctopus-start diff --git a/salt/common/tools/sbin/so-soctopus-stop b/salt/soctopus/tools/sbin/so-soctopus-stop similarity index 100% rename from salt/common/tools/sbin/so-soctopus-stop rename to salt/soctopus/tools/sbin/so-soctopus-stop diff --git a/salt/common/tools/sbin/so-suricata-restart b/salt/suricata/tools/sbin/so-suricata-restart similarity index 100% rename from salt/common/tools/sbin/so-suricata-restart rename to salt/suricata/tools/sbin/so-suricata-restart diff --git a/salt/common/tools/sbin/so-suricata-start b/salt/suricata/tools/sbin/so-suricata-start similarity index 100% rename from salt/common/tools/sbin/so-suricata-start rename to salt/suricata/tools/sbin/so-suricata-start diff --git a/salt/common/tools/sbin/so-suricata-stop b/salt/suricata/tools/sbin/so-suricata-stop similarity index 100% rename from salt/common/tools/sbin/so-suricata-stop rename to salt/suricata/tools/sbin/so-suricata-stop diff --git a/salt/common/tools/sbin/so-suricata-testrule b/salt/suricata/tools/sbin/so-suricata-testrule similarity index 100% rename from salt/common/tools/sbin/so-suricata-testrule rename to salt/suricata/tools/sbin/so-suricata-testrule From 7595072e859e57a8ea041f9ea5c9edbb260cb62d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 May 2023 12:15:05 -0400 Subject: [PATCH 13/41] Fix some files --- salt/common/init.sls | 24 +++++++------- .../{sbin => sbin_jinja}/so-analyst-install | 0 .../tools/{sbin => sbin_jinja}/so-import-evtx | 0 .../tools/{sbin => sbin_jinja}/so-import-pcap | 0 .../tools/{sbin => sbin_jinja}/so-raid-status | 0 salt/curator/init.sls | 31 +++++++----------- .../so-curator-cluster-delete-delete | 0 salt/elastalert/init.sls | 17 ++++++++++ salt/elasticfleet/init.sls | 15 +++++++++ .../so-elastic-agent-gen-installers} | 0 .../so-elastic-fleet-setup} | 0 .../elasticsearch/tools/sbin/so-elastic-clear | 3 +- .../tools/sbin/so-elastic-restart | 31 ------------------ .../elasticsearch/tools/sbin/so-elastic-start | 31 ------------------ salt/elasticsearch/tools/sbin/so-elastic-stop | 31 ------------------ .../so-elasticsearch-component-templates-list | 5 ++- .../so-elasticsearch-ilm-lifecycle-status | 6 ++-- .../sbin/so-elasticsearch-ilm-policy-delete | 4 +-- .../so-elasticsearch-ilm-policy-load copy | 21 ------------ .../sbin/so-elasticsearch-ilm-policy-view | 5 ++- .../tools/sbin/so-elasticsearch-ilm-start | 3 +- .../tools/sbin/so-elasticsearch-ilm-status | 6 ++-- .../tools/sbin/so-elasticsearch-ilm-stop | 4 +-- .../so-elasticsearch-index-templates-list | 5 ++- .../tools/sbin/so-elasticsearch-indices-list | 4 +-- .../tools/sbin/so-elasticsearch-indices-rw | 6 ++-- .../sbin/so-elasticsearch-pipeline-stats | 6 ++-- .../tools/sbin/so-elasticsearch-pipeline-view | 6 ++-- .../sbin/so-elasticsearch-pipelines-list | 5 ++- .../tools/sbin/so-elasticsearch-roles-load | 4 +-- .../tools/sbin/so-elasticsearch-shards-list | 4 +-- .../sbin/so-elasticsearch-template-remove | 4 +-- .../tools/sbin/so-elasticsearch-template-view | 6 ++-- .../sbin/so-elasticsearch-templates-list | 6 ++-- .../sbin/so-elasticsearch-templates-load | 3 -- .../tools/sbin_jinja/so-elastic-restart | 32 +++++++++++++++++++ .../tools/sbin_jinja/so-elastic-start | 31 ++++++++++++++++++ .../tools/sbin_jinja/so-elastic-stop | 31 ++++++++++++++++++ .../so-elasticsearch-cluster-space-used | 0 .../so-elasticsearch-ilm-policy-load | 3 +- 40 files changed, 182 insertions(+), 211 deletions(-) rename salt/common/tools/{sbin => sbin_jinja}/so-analyst-install (100%) rename salt/common/tools/{sbin => sbin_jinja}/so-import-evtx (100%) rename salt/common/tools/{sbin => sbin_jinja}/so-import-pcap (100%) rename salt/common/tools/{sbin => sbin_jinja}/so-raid-status (100%) rename salt/curator/tools/{sbin => sbin_jinja}/so-curator-cluster-delete-delete (100%) rename salt/elasticfleet/tools/{sbin/so-elastic-agent-gen-installers.jinja => sbin_jinja/so-elastic-agent-gen-installers} (100%) rename salt/elasticfleet/tools/{sbin/so-elastic-fleet-setup.jinja => sbin_jinja/so-elastic-fleet-setup} (100%) delete mode 100755 salt/elasticsearch/tools/sbin/so-elastic-restart delete mode 100755 salt/elasticsearch/tools/sbin/so-elastic-start delete mode 100755 salt/elasticsearch/tools/sbin/so-elastic-stop delete mode 100755 salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load copy create mode 100755 salt/elasticsearch/tools/sbin_jinja/so-elastic-restart create mode 100755 salt/elasticsearch/tools/sbin_jinja/so-elastic-start create mode 100755 salt/elasticsearch/tools/sbin_jinja/so-elastic-stop rename salt/elasticsearch/tools/{sbin => sbin_jinja}/so-elasticsearch-cluster-space-used (100%) rename salt/elasticsearch/tools/{sbin => sbin_jinja}/so-elasticsearch-ilm-policy-load (77%) diff --git a/salt/common/init.sls b/salt/common/init.sls index 2feee941c..8723cc3c5 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -111,21 +111,23 @@ elastic_curl_config: {% endif %} {% endif %} -# Sync some Utilities -utilsyncscripts: + +common_sbin: file.recurse: - name: /usr/sbin - - user: root - - group: root + - source: salt://common/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +common_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://common/tools/sbin_jinja + - user: 939 + - group: 939 - file_mode: 755 - template: jinja - - source: salt://common/tools/sbin - - exclude_pat: - - so-common - - so-firewall - - so-image-common - - soup - - so-status so-status_script: file.managed: diff --git a/salt/common/tools/sbin/so-analyst-install b/salt/common/tools/sbin_jinja/so-analyst-install similarity index 100% rename from salt/common/tools/sbin/so-analyst-install rename to salt/common/tools/sbin_jinja/so-analyst-install diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx similarity index 100% rename from salt/common/tools/sbin/so-import-evtx rename to salt/common/tools/sbin_jinja/so-import-evtx diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin_jinja/so-import-pcap similarity index 100% rename from salt/common/tools/sbin/so-import-pcap rename to salt/common/tools/sbin_jinja/so-import-pcap diff --git a/salt/common/tools/sbin/so-raid-status b/salt/common/tools/sbin_jinja/so-raid-status similarity index 100% rename from salt/common/tools/sbin/so-raid-status rename to salt/common/tools/sbin_jinja/so-raid-status diff --git a/salt/curator/init.sls b/salt/curator/init.sls index d1e4276e1..eaa5639ff 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -60,30 +60,21 @@ curconf: - template: jinja - show_changes: False -curclusterclose: - file.managed: - - name: /usr/sbin/so-curator-cluster-close - - source: salt://curator/files/bin/so-curator-cluster-close +curator_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://curator/tools/sbin - user: 934 - group: 939 - - mode: 755 - - template: jinja + - file_mode: 755 -curclusterdelete: - file.managed: - - name: /usr/sbin/so-curator-cluster-delete - - source: salt://curator/files/bin/so-curator-cluster-delete +curator_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://curator/tools/sbin_jinja - user: 934 - - group: 939 - - mode: 755 - -curclusterdeletedelete: - file.managed: - - name: /usr/sbin/so-curator-cluster-delete-delete - - source: salt://curator/files/bin/so-curator-cluster-delete-delete - - user: 934 - - group: 939 - - mode: 755 + - group: 939 + - file_mode: 755 - template: jinja so-curator: diff --git a/salt/curator/tools/sbin/so-curator-cluster-delete-delete b/salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete similarity index 100% rename from salt/curator/tools/sbin/so-curator-cluster-delete-delete rename to salt/curator/tools/sbin_jinja/so-curator-cluster-delete-delete diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index b04fe1147..148fe7e1b 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -29,6 +29,23 @@ elastalogdir: - group: 933 - makedirs: True +elastalert_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://elastalert/tools/sbin + - user: 933 + - group: 939 + - file_mode: 755 + +#elastalert_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://elastalert/tools/sbin_jinja +# - user: 933 +# - group: 939 +# - file_mode: 755 +# - template: jinja + elastarules: file.directory: - name: /opt/so/rules/elastalert diff --git a/salt/elasticfleet/init.sls b/salt/elasticfleet/init.sls index da735ffac..924d2cb3f 100644 --- a/salt/elasticfleet/init.sls +++ b/salt/elasticfleet/init.sls @@ -25,6 +25,21 @@ elastic-agent: - home: /opt/so/conf/elastic-fleet - createhome: False +elasticfleet_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://elasticfleet/tools/sbin + - user: 947 + - group: 939 + +elasticfleet_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://elasticfleet/tools/sbin_jinja + - user: 947 + - group: 939 + - template: jinja + eaconfdir: file.directory: - name: /opt/so/conf/elastic-fleet diff --git a/salt/elasticfleet/tools/sbin/so-elastic-agent-gen-installers.jinja b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers similarity index 100% rename from salt/elasticfleet/tools/sbin/so-elastic-agent-gen-installers.jinja rename to salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-setup.jinja b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup similarity index 100% rename from salt/elasticfleet/tools/sbin/so-elastic-fleet-setup.jinja rename to salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup diff --git a/salt/elasticsearch/tools/sbin/so-elastic-clear b/salt/elasticsearch/tools/sbin/so-elastic-clear index f491fb62f..3b4f5fc62 100755 --- a/salt/elasticsearch/tools/sbin/so-elastic-clear +++ b/salt/elasticsearch/tools/sbin/so-elastic-clear @@ -5,7 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common SKIP=0 @@ -59,7 +58,7 @@ done if [ $SKIP -ne 1 ]; then # List indices echo - curl -K /opt/so/conf/elasticsearch/curl.config -k -L https://{{ NODEIP }}:9200/_cat/indices?v + curl -K /opt/so/conf/elasticsearch/curl.config -k -L https://localhost:9200/_cat/indices?v echo # Inform user we are about to delete all data echo diff --git a/salt/elasticsearch/tools/sbin/so-elastic-restart b/salt/elasticsearch/tools/sbin/so-elastic-restart deleted file mode 100755 index 67988193f..000000000 --- a/salt/elasticsearch/tools/sbin/so-elastic-restart +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} -/usr/sbin/so-restart elasticsearch $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} -/usr/sbin/so-restart kibana $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-restart logstash $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-restart curator $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} -/usr/sbin/so-restart elastalert $1 -{%- endif %} diff --git a/salt/elasticsearch/tools/sbin/so-elastic-start b/salt/elasticsearch/tools/sbin/so-elastic-start deleted file mode 100755 index fd78d1859..000000000 --- a/salt/elasticsearch/tools/sbin/so-elastic-start +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} -/usr/sbin/so-start elasticsearch $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} -/usr/sbin/so-start kibana $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-start logstash $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-start curator $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} -/usr/sbin/so-start elastalert $1 -{%- endif %} diff --git a/salt/elasticsearch/tools/sbin/so-elastic-stop b/salt/elasticsearch/tools/sbin/so-elastic-stop deleted file mode 100755 index 88350a8fe..000000000 --- a/salt/elasticsearch/tools/sbin/so-elastic-stop +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} -/usr/sbin/so-stop elasticsearch $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} -/usr/sbin/so-stop kibana $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-stop logstash $1 -{%- endif %} - -{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} -/usr/sbin/so-stop curator $1 -{%- endif %} - -{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} -/usr/sbin/so-stop elastalert $1 -{%- endif %} diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list index 976499574..2fccce9cb 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-component-templates-list @@ -5,10 +5,9 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_component_template | jq '.component_templates[] |.name'| sort + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_component_template/$1 | jq + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-lifecycle-status b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-lifecycle-status index 130a7cf16..db72f8078 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-lifecycle-status +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-lifecycle-status @@ -6,10 +6,8 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_all/_ilm/explain | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_all/_ilm/explain | jq . else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/$1/_ilm/explain | jq .[] + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/$1/_ilm/explain | jq .[] fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-delete b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-delete index 2be9dabb2..ef936b742 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-delete +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-delete @@ -6,6 +6,4 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X DELETE https://{{ NODEIP }}:9200/_ilm/policy/$1 +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X DELETE https://localhost:9200/_ilm/policy/$1 diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load copy b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load copy deleted file mode 100755 index 26ce487a7..000000000 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load copy +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -. /usr/sbin/so-common - -{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - -{%- for index, settings in ES_INDEX_SETTINGS.items() %} - {%- if settings.policy is defined %} -echo -echo "Setting up {{ index }}-logs policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' -echo - {%- endif %} -{%- endfor %} -echo diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-view b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-view index 426b6938d..f488bab87 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-view +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-view @@ -6,10 +6,9 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/policy | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ilm/policy | jq . else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/policy/$1 | jq .[] + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ilm/policy/$1 | jq .[] fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start index 98dd38e9e..d9c63f8ea 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-start @@ -6,7 +6,6 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} echo "Starting ILM..." -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://{{ NODEIP }}:9200/_ilm/start +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/start diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-status b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-status index 8d78adc5b..7ba0201a4 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-status +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-status @@ -1,4 +1,4 @@ -/bin/bash +#!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the @@ -6,6 +6,4 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/status | jq . +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ilm/status | jq . diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop index 4868fd86d..034082699 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-stop @@ -6,7 +6,5 @@ . /usr/sbin/so-common -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - echo "Stopping ILM..." -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://{{ NODEIP }}:9200/_ilm/stop +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/stop diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-templates-list index bea975c93..6df836c1d 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-index-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-index-templates-list @@ -5,10 +5,9 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_index_template | jq '.index_templates[] |.name'| sort + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_index_template | jq '.index_templates[] |.name'| sort else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_index_template/$1 | jq + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_index_template/$1 | jq fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-list index da8ea4cca..57cc5e799 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-list @@ -5,8 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://{{ NODEIP }}:9200/_cat/indices?pretty&v&s=index" +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://localhost:9200/_cat/indices?pretty&v&s=index" diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-rw b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-rw index 724dd9dcf..5e6bf71a5 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-rw +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-indices-rw @@ -6,10 +6,8 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - -IP={{ salt['grains.get']('ip_interfaces').get(salt['pillar.get']('sensor:mainint', salt['pillar.get']('manager:mainint', salt['pillar.get']('elasticsearch:mainint', salt['pillar.get']('host:mainint')))))[0] }} -ESPORT=9200 +. /usr/sbin/so-common echo "Removing read only attributes for indices..." echo -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -H "Content-Type: application/json" -L https://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi; diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-stats b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-stats index 952773cda..fd06eeb78 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-stats +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-stats @@ -5,12 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines" else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\"" fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-view b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-view index 32a26b948..8de82f901 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-view +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipeline-view @@ -5,12 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/* | jq . else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .[] + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/$1 | jq .[] fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines-list index b690d5846..feeecb68b 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-pipelines-list @@ -5,10 +5,9 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys' + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/* | jq 'keys' else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/$1 | jq fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load index 17265a7c4..b6b593320 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load @@ -7,8 +7,6 @@ . /usr/sbin/so-common default_conf_dir=/opt/so/conf -ELASTICSEARCH_HOST="{{ GLOBALS.node_ip }}" -ELASTICSEARCH_PORT=9200 # Define a default directory to load roles from ELASTICSEARCH_ROLES="$default_conf_dir/elasticsearch/roles/" @@ -18,7 +16,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://localhost:9200" if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-shards-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-shards-list index 8865e05ac..cd6410b99 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-shards-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-shards-list @@ -5,8 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_cat/shards?pretty diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-template-remove b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-remove index f69495152..7d5ae5b3e 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-template-remove +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-remove @@ -5,8 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common -curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1 +curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -XDELETE https://localhost:9200/_template/$1 diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-template-view b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-view index c56127703..cc2678582 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-template-view +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-template-view @@ -5,12 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} - . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/* | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/* | jq . else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq . + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/$1 | jq . fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-list b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-list index 7db4fdeff..28f23c6e1 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-list +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-list @@ -5,10 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} . /usr/sbin/so-common + if [ "$1" == "" ]; then - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys' + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/* | jq 'keys' else - curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq + curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/$1 | jq fi diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load index 386026f0c..bce8af1ff 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-templates-load @@ -7,9 +7,6 @@ . /usr/sbin/so-common default_conf_dir=/opt/so/conf -ELASTICSEARCH_HOST="{{ GLOBALS.node_ip }}" -ELASTICSEARCH_PORT=9200 -#ELASTICSEARCH_AUTH="" # Define a default directory to load pipelines from ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elastic-restart b/salt/elasticsearch/tools/sbin_jinja/so-elastic-restart new file mode 100755 index 000000000..1b5e9bf03 --- /dev/null +++ b/salt/elasticsearch/tools/sbin_jinja/so-elastic-restart @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} +/usr/sbin/so-restart elasticsearch $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} +/usr/sbin/so-restart kibana $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-restart logstash $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-restart curator $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} +/usr/sbin/so-restart elastalert $1 +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elastic-start b/salt/elasticsearch/tools/sbin_jinja/so-elastic-start new file mode 100755 index 000000000..6be969e9d --- /dev/null +++ b/salt/elasticsearch/tools/sbin_jinja/so-elastic-start @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} +/usr/sbin/so-start elasticsearch $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} +/usr/sbin/so-start kibana $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-start logstash $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-start curator $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} +/usr/sbin/so-start elastalert $1 +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elastic-stop b/salt/elasticsearch/tools/sbin_jinja/so-elastic-stop new file mode 100755 index 000000000..b6ea04964 --- /dev/null +++ b/salt/elasticsearch/tools/sbin_jinja/so-elastic-stop @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'vars/globals.map.jinja' import GLOBALS %} + +. /usr/sbin/so-common + + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%} +/usr/sbin/so-stop elasticsearch $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%} +/usr/sbin/so-stop kibana $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-stop logstash $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%} +/usr/sbin/so-stop curator $1 +{%- endif %} + +{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%} +/usr/sbin/so-stop elastalert $1 +{%- endif %} diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-space-used b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used similarity index 100% rename from salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-space-used rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-used diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load similarity index 77% rename from salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index 26ce487a7..afeddfa01 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -8,13 +8,12 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %} {%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %} -{%- set NODEIP = salt['pillar.get']('host:mainip', '') %} {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} echo echo "Setting up {{ index }}-logs policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }' echo {%- endif %} {%- endfor %} From 2419cf86eefa13c661368b9b491ec10895e02ebd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 May 2023 12:41:49 -0400 Subject: [PATCH 14/41] Fix some files --- salt/elasticsearch/init.sls | 70 ++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index bfe288b8e..54d427e6d 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -21,6 +21,27 @@ vm.max_map_count: sysctl.present: - value: 262144 +# Add ES Group +elasticsearchgroup: + group.present: + - name: elasticsearch + - gid: 930 + +esconfdir: + file.directory: + - name: /opt/so/conf/elasticsearch + - user: 930 + - group: 939 + - makedirs: True + +# Add ES user +elasticsearch: + user.present: + - uid: 930 + - gid: 930 + - home: /opt/so/conf/elasticsearch + - createhome: False + {% if GLOBALS.is_manager %} # We have to add the Manager CA to the CA list cascriptsync: @@ -42,20 +63,26 @@ cascriptfun: - file: cascriptsync {% endif %} -# Sync some es scripts -es_sync_scripts: +elasticsearch_sbin: file.recurse: - name: /usr/sbin - - user: root - - group: root + - source: salt://elasticsearch/tools/sbin + - user: 930 + - group: 939 + - file_mode: 755 + - exclude_pat: + - so-catrust + - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state + - so-elasticsearch-ilm-policy-load + +elasticsearch_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://elasticsearch/tools/sbin_jinja + - user: 939 + - group: 939 - file_mode: 755 - template: jinja - - source: salt://elasticsearch/tools/sbin - - exclude_pat: - - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state - - so-elasticsearch-ilm-policy-load - - defaults: - GLOBALS: {{ GLOBALS }} so-elasticsearch-ilm-policy-load-script: file.managed: @@ -96,29 +123,6 @@ capemz: - user: 939 - group: 939 - - -# Add ES Group -elasticsearchgroup: - group.present: - - name: elasticsearch - - gid: 930 - -# Add ES user -elasticsearch: - user.present: - - uid: 930 - - gid: 930 - - home: /opt/so/conf/elasticsearch - - createhome: False - -esconfdir: - file.directory: - - name: /opt/so/conf/elasticsearch - - user: 930 - - group: 939 - - makedirs: True - esingestdir: file.directory: - name: /opt/so/conf/elasticsearch/ingest From dd034edad6b3ece6e4967f5e212969bda86a82d4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 May 2023 13:12:14 -0400 Subject: [PATCH 15/41] Fix some files --- salt/logstash/soc_logstash.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index a4d0b87bf..a780c1ebe 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -1,14 +1,13 @@ logstash: assigned_pipelines: roles: - reciever: &assigned_pipelines + receiver: &assigned_pipelines description: List of pipelines assigned to this role. advanced: True helpLink: logstash.html multiline: True fleet: *assigned_pipelines manager: *assigned_pipelines - nodes: *assigned_pipelines search: *assigned_pipelines settings: lsheap: @@ -20,10 +19,12 @@ logstash: description: Host interface to listen to connections. helpLink: logstash.html readonly: True + advanced: True path_x_logs: description: Path inside the container to wrote logs. helpLink: logstash.html readonly: True + advanced: True pipeline_x_workers: description: Number of worker threads to process events in logstash. helpLink: logstash.html @@ -36,6 +37,7 @@ logstash: description: Sets ECS compatibility. This is set per pipeline so you should never need to change this. helpLink: logstash.html readonly: True + advanced: True dmz_nodes: description: "List of receiver nodes in DMZs. Prevents sensors from sending to these receivers. Primarily used for external Elastic agents." helpLink: logstash.html From c7604e893e3451ab27fbf737ba274ebfc6b8b2c4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 May 2023 09:17:37 -0400 Subject: [PATCH 16/41] Fix annotations and defaults for logstash --- salt/elasticfleet/init.sls | 2 ++ salt/logstash/defaults.yaml | 64 +++++++++++++++++++++++++-------- salt/logstash/soc_logstash.yaml | 35 +++++++++++++++--- 3 files changed, 81 insertions(+), 20 deletions(-) diff --git a/salt/elasticfleet/init.sls b/salt/elasticfleet/init.sls index 924d2cb3f..9476c3b94 100644 --- a/salt/elasticfleet/init.sls +++ b/salt/elasticfleet/init.sls @@ -31,6 +31,7 @@ elasticfleet_sbin: - source: salt://elasticfleet/tools/sbin - user: 947 - group: 939 + - file_mode: 755 elasticfleet_sbin_jinja: file.recurse: @@ -38,6 +39,7 @@ elasticfleet_sbin_jinja: - source: salt://elasticfleet/tools/sbin_jinja - user: 947 - group: 939 + - file_mode: 755 - template: jinja eaconfdir: diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index dddab9ddf..21667ece8 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -1,22 +1,56 @@ logstash: assigned_pipelines: roles: - fleet: - - so/0012_input_elastic_agent.conf - - so/9806_output_lumberjack_fleet.conf.jinja - manager: - - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/0013_input_lumberjack_fleet.conf - - so/9999_output_redis.conf.jinja + standalone: + - manager + - search receiver: - - so/0011_input_endgame.conf - - so/0012_input_elastic_agent.conf - - so/9999_output_redis.conf.jinja - search: - - so/0900_input_redis.conf.jinja - - so/9805_output_elastic_agent.conf.jinja - - so/9900_output_endgame.conf.jinja + - receiver + heavynode: + - search + searchnode: + - search + manager: + - manager + managersearch: + - manager + - search + fleet: + - fleet + defined_pipelines: + fleet: + - so/0012_input_elastic_agent.conf + - so/9806_output_lumberjack_fleet.conf.jinja + manager: + - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf + - so/0013_input_lumberjack_fleet.conf + - so/9999_output_redis.conf.jinja + receiver: + - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf + - so/9999_output_redis.conf.jinja + searchnode: + - so/0900_input_redis.conf.jinja + - so/9805_output_elastic_agent.conf.jinja + - so/9900_output_endgame.conf.jinja + custom0: [] + custom1: [] + custom2: [] + custom3: [] + custom4: [] + docker_options: + port_bindings: + - 0.0.0.0:3765:3765 + - 0.0.0.0:5044:5044 + - 0.0.0.0:5055:5055 + - 0.0.0.0:5056:5056 + - 0.0.0.0:5644:5644 + - 0.0.0.0:6050:6050 + - 0.0.0.0:6051:6051 + - 0.0.0.0:6052:6052 + - 0.0.0.0:6053:6053 + - 0.0.0.0:9600:9600 settings: lsheap: 500m config: diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index a780c1ebe..0539a9243 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -1,14 +1,33 @@ logstash: assigned_pipelines: + standalone: &assigned_pipelines + description: List of defined pipelines to add to this role. + advanced: True + helpLink: logstash.html + multiline: True + forcedType: "[]string" + receiver: *assigned_pipelines + heavynode: *assigned_pipelines + searchnode: *assigned_pipelines + manager: *assigned_pipelines + managersearch: *assigned_pipelines + fleet: *assigned_pipelines + defined_pipelines: roles: - receiver: &assigned_pipelines - description: List of pipelines assigned to this role. + receiver: &defined_pipelines + description: List of pipeline configurations assign to this group. advanced: True helpLink: logstash.html multiline: True - fleet: *assigned_pipelines - manager: *assigned_pipelines - search: *assigned_pipelines + forcedType: "[]string" + fleet: *defined_pipelines + manager: *defined_pipelines + search: *defined_pipelines + custom0: *defined_pipelines + custom1: *defined_pipelines + custom2: *defined_pipelines + custom3: *defined_pipelines + custom4: *defined_pipelines settings: lsheap: description: Heap size to use for logstash @@ -38,6 +57,12 @@ logstash: helpLink: logstash.html readonly: True advanced: True + docker_options: + port_bindings: + description: List of ports to open to the logstash docker container. Firewall ports will still need to be added to the firewall configuration. + helpLink: logstash.html + advanced: True + multiline: True dmz_nodes: description: "List of receiver nodes in DMZs. Prevents sensors from sending to these receivers. Primarily used for external Elastic agents." helpLink: logstash.html From 220c534ad4003ad79b03ce9a8900611dd52e1c73 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 May 2023 09:32:03 -0400 Subject: [PATCH 17/41] Fix annotations and defaults for logstash --- salt/logstash/soc_logstash.yaml | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index 0539a9243..e41ff000f 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -1,33 +1,33 @@ logstash: assigned_pipelines: - standalone: &assigned_pipelines - description: List of defined pipelines to add to this role. - advanced: True - helpLink: logstash.html - multiline: True - forcedType: "[]string" - receiver: *assigned_pipelines - heavynode: *assigned_pipelines - searchnode: *assigned_pipelines - manager: *assigned_pipelines - managersearch: *assigned_pipelines - fleet: *assigned_pipelines - defined_pipelines: roles: - receiver: &defined_pipelines - description: List of pipeline configurations assign to this group. + standalone: &assigned_pipelines + description: List of defined pipelines to add to this role. advanced: True helpLink: logstash.html multiline: True - forcedType: "[]string" - fleet: *defined_pipelines - manager: *defined_pipelines - search: *defined_pipelines - custom0: *defined_pipelines - custom1: *defined_pipelines - custom2: *defined_pipelines - custom3: *defined_pipelines - custom4: *defined_pipelines + forcedType: "[]string" + receiver: *assigned_pipelines + heavynode: *assigned_pipelines + searchnode: *assigned_pipelines + manager: *assigned_pipelines + managersearch: *assigned_pipelines + fleet: *assigned_pipelines + defined_pipelines: + receiver: &defined_pipelines + description: List of pipeline configurations assign to this group. + advanced: True + helpLink: logstash.html + multiline: True + forcedType: "[]string" + fleet: *defined_pipelines + manager: *defined_pipelines + search: *defined_pipelines + custom0: *defined_pipelines + custom1: *defined_pipelines + custom2: *defined_pipelines + custom3: *defined_pipelines + custom4: *defined_pipelines settings: lsheap: description: Heap size to use for logstash From 3d10a60502fe104d7ab4b07ebe4eed84c9663f67 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 May 2023 10:01:44 -0400 Subject: [PATCH 18/41] Fix annotations and defaults for logstash --- salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja | 4 ++-- .../logstash/pipelines/config/so/9999_output_redis.conf.jinja | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index b4251b81a..e0999e490 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,5 +1,5 @@ -{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} -{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} +{%- set THREADS = salt['pillar.get']('logstash:settings:pipeline_x_workers') %} +{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size', 125) %} {%- from 'logstash/map.jinja' import REDIS_NODES with context %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 6b8b8503f..7c4dacf12 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -3,7 +3,7 @@ {%- else %} {%- set HOST = GLOBALS.manager %} {%- endif %} -{%- set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %} +{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size') %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} output { From a5b1660778e7a5307f1bb6b1b68e16249504f777 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 3 May 2023 14:12:32 -0400 Subject: [PATCH 19/41] Fix firewall changes --- salt/idh/init.sls | 17 +++++++++++++++++ salt/idstools/init.sls | 17 +++++++++++++++++ salt/influxdb/init.sls | 17 +++++++++++++++++ salt/kibana/init.sls | 17 +++++++++++++++++ .../so-kibana-config-export} | 0 .../so-kibana-config-load} | 0 .../so-kibana-space-defaults} | 0 salt/logstash/init.sls | 17 +++++++++++++++++ salt/logstash/tools/sbin/so-logstash-events | 7 ++----- salt/logstash/tools/sbin/so-logstash-get-parsed | 12 ------------ .../tools/sbin/so-logstash-pipeline-stats | 6 ++---- salt/manager/tools/sbin/so-firewall | 2 +- salt/manager/tools/sbin/so-firewall-minion | 2 +- 13 files changed, 91 insertions(+), 23 deletions(-) rename salt/kibana/tools/{sbin/so-kibana-config-export.jinja => sbin_jinja/so-kibana-config-export} (100%) rename salt/kibana/tools/{sbin/so-kibana-config-load.jinja => sbin_jinja/so-kibana-config-load} (100%) rename salt/kibana/tools/{sbin/so-kibana-space-defaults.jinja => sbin_jinja/so-kibana-space-defaults} (100%) delete mode 100755 salt/logstash/tools/sbin/so-logstash-get-parsed diff --git a/salt/idh/init.sls b/salt/idh/init.sls index 20a6412ce..895cd61ac 100644 --- a/salt/idh/init.sls +++ b/salt/idh/init.sls @@ -60,6 +60,23 @@ opencanary_config: - defaults: OPENCANARYCONFIG: {{ OPENCANARYCONFIG }} +idh_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://idh/tools/sbin + - user: 934 + - group: 939 + - file_mode: 755 + +#idh_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://idh/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + so-idh: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idh:{{ GLOBALS.so_version }} diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls index 5ec9464cc..7ad22e58b 100644 --- a/salt/idstools/init.sls +++ b/salt/idstools/init.sls @@ -20,6 +20,23 @@ idstoolslogdir: - group: 939 - makedirs: True +idstools_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://idstools/tools/sbin + - user: 934 + - group: 939 + - file_mode: 755 + +#idstools_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://idstools/tools/sbin_jinja +# - user: 934 +# - group: 939 +# - file_mode: 755 +# - template: jinja + so-rule-update: cron.present: - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1 diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index b4824825b..7e10a6798 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -31,6 +31,23 @@ influxdbdir: - name: /nsm/influxdb - makedirs: True +influxdb_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://influxdb/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#influxdb_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://influxdb/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + influxdbconf: file.managed: - name: /opt/so/conf/influxdb/config.yaml diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index a974dcf48..015aa4396 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -34,6 +34,23 @@ kibanaconfdir: - group: 939 - makedirs: True +kibana_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://kibana/tools/sbin + - user: 932 + - group: 939 + - file_mode: 755 + +curator_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://kibana/tools/sbin_jinja + - user: 932 + - group: 939 + - file_mode: 755 + - template: jinja + kibanaconfig: file.managed: - name: /opt/so/conf/kibana/etc/kibana.yml diff --git a/salt/kibana/tools/sbin/so-kibana-config-export.jinja b/salt/kibana/tools/sbin_jinja/so-kibana-config-export similarity index 100% rename from salt/kibana/tools/sbin/so-kibana-config-export.jinja rename to salt/kibana/tools/sbin_jinja/so-kibana-config-export diff --git a/salt/kibana/tools/sbin/so-kibana-config-load.jinja b/salt/kibana/tools/sbin_jinja/so-kibana-config-load similarity index 100% rename from salt/kibana/tools/sbin/so-kibana-config-load.jinja rename to salt/kibana/tools/sbin_jinja/so-kibana-config-load diff --git a/salt/kibana/tools/sbin/so-kibana-space-defaults.jinja b/salt/kibana/tools/sbin_jinja/so-kibana-space-defaults similarity index 100% rename from salt/kibana/tools/sbin/so-kibana-space-defaults.jinja rename to salt/kibana/tools/sbin_jinja/so-kibana-space-defaults diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index caabd10ea..c80df1f5c 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -43,6 +43,23 @@ lslibdir: file.absent: - name: /opt/so/conf/logstash/lib +logstash_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://logstash/tools/sbin + - user: 931 + - group: 939 + - file_mode: 755 + +#logstash_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://logstash/tools/sbin_jinja +# - user: 931 +# - group: 939 +# - file_mode: 755 +# - template: jinja + lsetcdir: file.directory: - name: /opt/so/conf/logstash/etc diff --git a/salt/logstash/tools/sbin/so-logstash-events b/salt/logstash/tools/sbin/so-logstash-events index 5ea34ad80..60d02e8d9 100755 --- a/salt/logstash/tools/sbin/so-logstash-events +++ b/salt/logstash/tools/sbin/so-logstash-events @@ -5,13 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{% set MAININT = salt['pillar.get']('host:mainint') -%} -{% set NODEIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] -%} - . /usr/sbin/so-common if [ "$1" == "" ]; then - for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done + for i in $(curl -s -L http://localhost:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done else - curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events + curl -s -L http://localhost:9600/_node/stats | jq .pipelines.$1.events fi diff --git a/salt/logstash/tools/sbin/so-logstash-get-parsed b/salt/logstash/tools/sbin/so-logstash-get-parsed deleted file mode 100755 index 1575010ac..000000000 --- a/salt/logstash/tools/sbin/so-logstash-get-parsed +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -docker exec -it so-redis redis-cli llen logstash:unparsed diff --git a/salt/logstash/tools/sbin/so-logstash-pipeline-stats b/salt/logstash/tools/sbin/so-logstash-pipeline-stats index 4ad58e5b3..badcddf72 100755 --- a/salt/logstash/tools/sbin/so-logstash-pipeline-stats +++ b/salt/logstash/tools/sbin/so-logstash-pipeline-stats @@ -5,13 +5,11 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{% set MAININT = salt['pillar.get']('host:mainint') -%} -{% set NODEIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] -%} . /usr/sbin/so-common if [ "$1" == "" ]; then - curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines + curl -s -L http://localhost:9600/_node/stats | jq .pipelines else - curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1 + curl -s -L http://localhost:9600/_node/stats | jq .pipelines.$1 fi diff --git a/salt/manager/tools/sbin/so-firewall b/salt/manager/tools/sbin/so-firewall index 94302b5b2..6c47a3719 100755 --- a/salt/manager/tools/sbin/so-firewall +++ b/salt/manager/tools/sbin/so-firewall @@ -144,4 +144,4 @@ def main(): sys.exit(code) if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/salt/manager/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion index 610d0fc3a..4834f0e41 100755 --- a/salt/manager/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -79,4 +79,4 @@ fi 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; - esac + esac \ No newline at end of file From b14d33ced8f422808d07242146146f0b41cee9b1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 3 May 2023 15:22:03 -0400 Subject: [PATCH 20/41] add logstash jinja for ui changes --- salt/logstash/etc/pipelines.yml.jinja | 6 ++-- salt/logstash/init.sls | 41 ++++++++++++++------------- salt/logstash/map.jinja | 3 ++ 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/salt/logstash/etc/pipelines.yml.jinja b/salt/logstash/etc/pipelines.yml.jinja index 3ee7a0d3b..07eedce25 100644 --- a/salt/logstash/etc/pipelines.yml.jinja +++ b/salt/logstash/etc/pipelines.yml.jinja @@ -1,4 +1,4 @@ -{%- for pl in pipelines %} -- pipeline.id: {{ pl }} - path.config: "/usr/share/logstash/pipelines/{{ pl }}/" +{%- for ap in assigned_pipelines %} +- pipeline.id: {{ ap }} + path.config: "/usr/share/logstash/pipelines/{{ ap }}/" {% endfor -%} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index caabd10ea..54a038668 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -6,19 +6,19 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% from 'docker/docker.map.jinja' import DOCKER %} -{% from 'logstash/map.jinja' import REDIS_NODES with context %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'logstash/map.jinja' import REDIS_NODES %} +{% from 'logstash/map.jinja' import LOGSTASH_MERGED %} # Logstash Section - Decide which pillar to use -{% set lsheap = salt['pillar.get']('logstash:settings:lsheap') %} +{% set lsheap = LOGSTASH_MERGED.settings.lsheap %} {% if GLOBALS.role in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} {% set nodetype = GLOBALS.role %} {% endif %} -{% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %} -{% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} +{% set ASSIGNED_PIPELINES = LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} +{% set DOCKER_OPTIONS = LOGSTASH_MERGED.docker_options %} include: - ssl @@ -56,20 +56,20 @@ lspipelinedir: - user: 931 - group: 939 - {% for PL in PIPELINES %} - {% for CONFIGFILE in PIPELINES[PL].config %} -ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: +{% for assigned_pipeline in ASSIGNED_PIPELINES %} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} +ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: file.managed: - source: salt://logstash/pipelines/config/{{CONFIGFILE}} {% if 'jinja' in CONFIGFILE.split('.')[-1] %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} - template: jinja - defaults: GLOBALS: {{ GLOBALS }} ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}" ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}" {% else %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}} {% endif %} - user: 931 - group: 939 @@ -78,18 +78,19 @@ ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: - show_changes: False {% endfor %} -ls_pipeline_{{PL}}: +ls_pipeline_{{assigned_pipeline}}: file.directory: - - name: /opt/so/conf/logstash/pipelines/{{PL}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}} - user: 931 - group: 939 - require: - {% for CONFIGFILE in PIPELINES[PL].config %} - - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} + - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} - clean: True {% endfor %} +{% endfor %} lspipelinesyml: file.managed: @@ -97,7 +98,7 @@ lspipelinesyml: - source: salt://logstash/etc/pipelines.yml.jinja - template: jinja - defaults: - pipelines: {{ PIPELINES }} + assigned_pipelines: {{ ASSIGNED_PIPELINES }} # Copy down all the configs lsetcsync: @@ -185,10 +186,10 @@ so-logstash: {%- endif %} - watch: - file: lsetcsync - {% for PL in PIPELINES %} - - file: ls_pipeline_{{PL}} - {% for CONFIGFILE in PIPELINES[PL].config %} - - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} + {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} + - file: ls_pipeline_{{assigned_pipeline}} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[ap] %} + - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} {% endfor %} - require: diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index e23f944a2..c4ad5d96a 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -1,4 +1,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %} +{% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} + {% set REDIS_NODES = [] %} {% set LOGSTASH_NODES = [] %} {% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} From cbd1c0592906433294fc47655ea68770097aa488 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 10:36:03 -0400 Subject: [PATCH 21/41] Sbin Changes --- salt/manager/init.sls | 17 +++++++++++++++++ salt/mysql/init.sls | 17 +++++++++++++++++ salt/nginx/init.sls | 17 +++++++++++++++++ .../nginx/{toos => tools}/sbin/so-nginx-restart | 0 salt/nginx/{toos => tools}/sbin/so-nginx-start | 0 salt/nginx/{toos => tools}/sbin/so-nginx-stop | 0 salt/pcap/init.sls | 17 +++++++++++++++++ salt/playbook/init.sls | 17 +++++++++++++++++ salt/redis/init.sls | 17 +++++++++++++++++ .../tools/{sbin => sbin_jinja}/so-redis-count | 0 salt/sensoroni/init.sls | 17 +++++++++++++++++ salt/soc/init.sls | 17 +++++++++++++++++ salt/soctopus/init.sls | 17 +++++++++++++++++ salt/strelka/init.sls | 17 +++++++++++++++++ salt/suricata/init.sls | 17 +++++++++++++++++ .../{sbin => sbin_jinja}/so-suricata-testrule | 0 salt/telegraf/init.sls | 17 +++++++++++++++++ salt/zeek/init.sls | 17 +++++++++++++++++ salt/zeek/{bin => tools/sbin}/so-zeek-restart | 0 salt/zeek/{bin => tools/sbin}/so-zeek-start | 0 salt/zeek/{bin => tools/sbin}/so-zeek-stats | 0 salt/zeek/{bin => tools/sbin}/so-zeek-stop | 0 22 files changed, 221 insertions(+) rename salt/nginx/{toos => tools}/sbin/so-nginx-restart (100%) rename salt/nginx/{toos => tools}/sbin/so-nginx-start (100%) rename salt/nginx/{toos => tools}/sbin/so-nginx-stop (100%) rename salt/redis/tools/{sbin => sbin_jinja}/so-redis-count (100%) rename salt/suricata/tools/{sbin => sbin_jinja}/so-suricata-testrule (100%) rename salt/zeek/{bin => tools/sbin}/so-zeek-restart (100%) rename salt/zeek/{bin => tools/sbin}/so-zeek-start (100%) rename salt/zeek/{bin => tools/sbin}/so-zeek-stats (100%) rename salt/zeek/{bin => tools/sbin}/so-zeek-stop (100%) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 9973dcb41..eea0f9568 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -43,6 +43,23 @@ repo_dir: - user - group +manager_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://manager/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#manager_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://manager/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + repo_sync_script: file.managed: - name: /usr/sbin/so-repo-sync diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 44e6789af..1c0ca70c0 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -69,6 +69,23 @@ mysqldatadir: - group: 939 - makedirs: True +mysql_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://mysql/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#mysql_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://mysql/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + {% if MYSQLPASS == None %} mysql_password_none: diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index 833bda98a..c66af0837 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -81,6 +81,23 @@ navigatorenterpriseattack: - makedirs: True - replace: False +nginx_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://nginx/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#nginx_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://nginx/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + so-nginx: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }} diff --git a/salt/nginx/toos/sbin/so-nginx-restart b/salt/nginx/tools/sbin/so-nginx-restart similarity index 100% rename from salt/nginx/toos/sbin/so-nginx-restart rename to salt/nginx/tools/sbin/so-nginx-restart diff --git a/salt/nginx/toos/sbin/so-nginx-start b/salt/nginx/tools/sbin/so-nginx-start similarity index 100% rename from salt/nginx/toos/sbin/so-nginx-start rename to salt/nginx/tools/sbin/so-nginx-start diff --git a/salt/nginx/toos/sbin/so-nginx-stop b/salt/nginx/tools/sbin/so-nginx-stop similarity index 100% rename from salt/nginx/toos/sbin/so-nginx-stop rename to salt/nginx/tools/sbin/so-nginx-stop diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 73b384a53..d71a9b1dd 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -33,6 +33,23 @@ stenoconfdir: - group: 939 - makedirs: True +pcap_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://pcap/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#pcap_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://pcap/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + {% if PCAPBPF %} {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %} {% if BPF_CALC['stderr'] == "" %} diff --git a/salt/playbook/init.sls b/salt/playbook/init.sls index f76292333..930c3b9ec 100644 --- a/salt/playbook/init.sls +++ b/salt/playbook/init.sls @@ -66,6 +66,23 @@ query_updatepluginurls: - connection_user: root - connection_pass: {{ MYSQLPASS }} +playbook_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://playbook/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#playbook_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://playbook/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + playbooklogdir: file.directory: - name: /opt/so/log/playbook diff --git a/salt/redis/init.sls b/salt/redis/init.sls index ebaad842b..c01b4e547 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -41,6 +41,23 @@ redisconf: - group: 939 - template: jinja +redis_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://redis/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +redis_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://redis/tools/sbin_jinja + - user: 939 + - group: 939 + - file_mode: 755 + - template: jinja + so-redis: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-redis:{{ GLOBALS.so_version }} diff --git a/salt/redis/tools/sbin/so-redis-count b/salt/redis/tools/sbin_jinja/so-redis-count similarity index 100% rename from salt/redis/tools/sbin/so-redis-count rename to salt/redis/tools/sbin_jinja/so-redis-count diff --git a/salt/sensoroni/init.sls b/salt/sensoroni/init.sls index c410a6fd9..3540fe40a 100644 --- a/salt/sensoroni/init.sls +++ b/salt/sensoroni/init.sls @@ -39,6 +39,23 @@ analyzerscripts: - template: jinja - source: salt://sensoroni/files/analyzers +sensoroni_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://sensoroni/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +sensoroni_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://sensoroni/tools/sbin_jinja + - user: 939 + - group: 939 + - file_mode: 755 + - template: jinja + so-sensoroni: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-soc:{{ GLOBALS.so_version }} diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 9460eeac2..8c3ed5104 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -64,6 +64,23 @@ socbanner: - mode: 600 - template: jinja +soc_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://soc/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#soc_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://soc/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + soccustom: file.managed: - name: /opt/so/conf/soc/custom.js diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index 203950bb4..6470d1163 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -58,6 +58,23 @@ playbookrulessync: - defaults: GLOBALS: {{ GLOBALS }} +soctopus_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://soctopus/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#soctopus_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://soctopus/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + so-soctopus: docker_container.running: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-soctopus:{{ GLOBALS.so_version }} diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 081f2ebd1..6b7a2bbd2 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -57,6 +57,23 @@ backend_passwords: - defaults: PASSWORDS: {{ STRELKAMERGED.config.backend.passwords }} +strelka_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://strelka/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#strelka_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://strelka/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + backend_taste: file.managed: - name: /opt/so/conf/strelka/backend/taste/taste.yara diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index 159e59f4f..7788fa94a 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -38,6 +38,23 @@ socoregroupwithsuricata: - addusers: - suricata +suricata_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://suricata/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +suricata_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://suricata/tools/sbin_jinja + - user: 939 + - group: 939 + - file_mode: 755 + - template: jinja + suridir: file.directory: - name: /opt/so/conf/suricata diff --git a/salt/suricata/tools/sbin/so-suricata-testrule b/salt/suricata/tools/sbin_jinja/so-suricata-testrule similarity index 100% rename from salt/suricata/tools/sbin/so-suricata-testrule rename to salt/suricata/tools/sbin_jinja/so-suricata-testrule diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls index e5574e7d1..f14ef14e4 100644 --- a/salt/telegraf/init.sls +++ b/salt/telegraf/init.sls @@ -39,6 +39,23 @@ tgrafsyncscripts: - exclude_pat: zeekcaptureloss.sh {% endif %} +telegraf_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://telegraf/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#telegraf_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://telegraf/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + tgrafconf: file.managed: - name: /opt/so/conf/telegraf/etc/telegraf.conf diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index 3b8390a77..ce5996888 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -92,6 +92,23 @@ zeekstatedbownership: - replace: False - create: False +zeek_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://zeek/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +#zeek_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://zeek/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja + # Sync Intel zeekintelloadsync: file.managed: diff --git a/salt/zeek/bin/so-zeek-restart b/salt/zeek/tools/sbin/so-zeek-restart similarity index 100% rename from salt/zeek/bin/so-zeek-restart rename to salt/zeek/tools/sbin/so-zeek-restart diff --git a/salt/zeek/bin/so-zeek-start b/salt/zeek/tools/sbin/so-zeek-start similarity index 100% rename from salt/zeek/bin/so-zeek-start rename to salt/zeek/tools/sbin/so-zeek-start diff --git a/salt/zeek/bin/so-zeek-stats b/salt/zeek/tools/sbin/so-zeek-stats similarity index 100% rename from salt/zeek/bin/so-zeek-stats rename to salt/zeek/tools/sbin/so-zeek-stats diff --git a/salt/zeek/bin/so-zeek-stop b/salt/zeek/tools/sbin/so-zeek-stop similarity index 100% rename from salt/zeek/bin/so-zeek-stop rename to salt/zeek/tools/sbin/so-zeek-stop From c5c2600799d1046c589b50cdca46d39a3036d499 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 12:56:04 -0400 Subject: [PATCH 22/41] Fix some errors --- salt/kibana/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 015aa4396..10f410e70 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -42,7 +42,7 @@ kibana_sbin: - group: 939 - file_mode: 755 -curator_sbin_jinja: +kibana_sbin_jinja: file.recurse: - name: /usr/sbin - source: salt://kibana/tools/sbin_jinja From 7e71c6033457f3b66c3c36d0fa94b42a521b7cac Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 12:57:35 -0400 Subject: [PATCH 23/41] Fix some errors --- salt/redis/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/redis/init.sls b/salt/redis/init.sls index c01b4e547..5806d99f3 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -54,7 +54,7 @@ redis_sbin_jinja: - name: /usr/sbin - source: salt://redis/tools/sbin_jinja - user: 939 - - group: 939 + - group: 939 - file_mode: 755 - template: jinja From 71b6311edcc6fa21acf782dfa68bc33c474fbe5d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 4 May 2023 13:05:16 -0400 Subject: [PATCH 24/41] add logstash.nodes to pillar top --- pillar/top.sls | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index a0fbcb4c1..259e87c96 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -35,6 +35,7 @@ base: - manager.adv_manager - idstools.soc_idstools - idstools.adv_idstools + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - soc.soc_soc @@ -124,9 +125,7 @@ base: - minions.adv_{{ grains.id }} '*_standalone': - - logstash - - logstash.manager - - logstash.search + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.index_templates @@ -175,6 +174,7 @@ base: '*_heavynode': - elasticsearch.auth + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.soc_elasticsearch @@ -203,6 +203,7 @@ base: - minions.adv_{{ grains.id }} '*_searchnode': + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.soc_elasticsearch @@ -214,6 +215,7 @@ base: - minions.adv_{{ grains.id }} '*_receiver': + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} @@ -270,6 +272,7 @@ base: '*_fleet': - backup.soc_backup - backup.adv_backup + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - minions.{{ grains.id }} From 082704ce1f51c26345d10d341bd900dc7793f6b4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 4 May 2023 13:07:07 -0400 Subject: [PATCH 25/41] logstash jinja for ui --- salt/logstash/defaults.yaml | 4 ++-- salt/logstash/etc/logstash.yml | 6 +----- salt/logstash/etc/pipelines.yml.jinja | 6 +++--- salt/logstash/init.sls | 12 +++++++----- .../pipelines/config/so/0900_input_redis.conf.jinja | 2 -- .../pipelines/config/so/9999_output_redis.conf.jinja | 1 - 6 files changed, 13 insertions(+), 18 deletions(-) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 21667ece8..d253a6b51 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -30,7 +30,7 @@ logstash: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - searchnode: + search: - so/0900_input_redis.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja @@ -59,5 +59,5 @@ logstash: pipeline_x_workers: 1 pipeline_x_batch_x_size: 125 pipeline_x_ecs_compatibility: disabled - dmz_nodes: {} + dmz_nodes: [] diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index ca953975f..973b2ab10 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -1,5 +1 @@ -http.host: 0.0.0.0 -path.logs: /var/log/logstash -pipeline.workers: {{ pipeline_workers }} -pipeline.batch.size: {{ pipeline_batch }} -pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }} +{{ LOGSTASH_MERGED.config | yaml(False) | replace("_x_", ".") }} diff --git a/salt/logstash/etc/pipelines.yml.jinja b/salt/logstash/etc/pipelines.yml.jinja index 07eedce25..427cc9f14 100644 --- a/salt/logstash/etc/pipelines.yml.jinja +++ b/salt/logstash/etc/pipelines.yml.jinja @@ -1,4 +1,4 @@ -{%- for ap in assigned_pipelines %} -- pipeline.id: {{ ap }} - path.config: "/usr/share/logstash/pipelines/{{ ap }}/" +{%- for assigned_pipeline in ASSIGNED_PIPELINES %} +- pipeline.id: {{ assigned_pipeline }} + path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/" {% endfor -%} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 54a038668..8d00d059f 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -68,6 +68,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") GLOBALS: {{ GLOBALS }} ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}" ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}" + THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }} + BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }} {% else %} - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}} {% endif %} @@ -88,19 +90,17 @@ ls_pipeline_{{assigned_pipeline}}: - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} - clean: True - - {% endfor %} {% endfor %} +# Copy down all the configs lspipelinesyml: file.managed: - name: /opt/so/conf/logstash/etc/pipelines.yml - source: salt://logstash/etc/pipelines.yml.jinja - template: jinja - defaults: - assigned_pipelines: {{ ASSIGNED_PIPELINES }} + ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }} -# Copy down all the configs lsetcsync: file.recurse: - name: /opt/so/conf/logstash/etc @@ -110,6 +110,8 @@ lsetcsync: - template: jinja - clean: True - exclude_pat: pipelines* + - defaults: + LOGSTASH_MERGED: {{ LOGSTASH_MERGED }} # Create the import directory importdir: @@ -188,7 +190,7 @@ so-logstash: - file: lsetcsync {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} - file: ls_pipeline_{{assigned_pipeline}} - {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[ap] %} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} {% endfor %} diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index e0999e490..661bc0b73 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,5 +1,3 @@ -{%- set THREADS = salt['pillar.get']('logstash:settings:pipeline_x_workers') %} -{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size', 125) %} {%- from 'logstash/map.jinja' import REDIS_NODES with context %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 7c4dacf12..0d3b3324b 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set HOST = GLOBALS.manager %} {%- endif %} -{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size') %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} output { From fbacfce0e4fbec882fff1e5c7b7f49af6b9c9bd6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 13:18:08 -0400 Subject: [PATCH 26/41] Fix some errors --- salt/sensoroni/init.sls | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/sensoroni/init.sls b/salt/sensoroni/init.sls index 3540fe40a..df6b99948 100644 --- a/salt/sensoroni/init.sls +++ b/salt/sensoroni/init.sls @@ -47,14 +47,14 @@ sensoroni_sbin: - group: 939 - file_mode: 755 -sensoroni_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://sensoroni/tools/sbin_jinja - - user: 939 - - group: 939 - - file_mode: 755 - - template: jinja +#sensoroni_sbin_jinja: +# file.recurse: +# - name: /usr/sbin +# - source: salt://sensoroni/tools/sbin_jinja +# - user: 939 +# - group: 939 +# - file_mode: 755 +# - template: jinja so-sensoroni: docker_container.running: From d0cfaaeb26526b45f188768bfe39f4148f837dcf Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 13:28:11 -0400 Subject: [PATCH 27/41] Fix some errors --- salt/elasticsearch/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 54d427e6d..8a1d4a346 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -378,7 +378,7 @@ so-es-cluster-settings: - template: jinja - require: - docker_container: so-elasticsearch - - file: es_sync_scripts + - file: elasticsearch_sbin_jinja so-elasticsearch-ilm-policy-load: cmd.run: @@ -397,7 +397,7 @@ so-elasticsearch-templates: - template: jinja - require: - docker_container: so-elasticsearch - - file: es_sync_scripts + - file: elasticsearch_sbin_jinja so-elasticsearch-pipelines: cmd.run: @@ -413,7 +413,7 @@ so-elasticsearch-roles-load: - template: jinja - require: - docker_container: so-elasticsearch - - file: es_sync_scripts + - file: elasticsearch_sbin_jinja {% endif %} {% else %} From 8055088d25f6fca23017971744f38866eada6ae6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 13:35:44 -0400 Subject: [PATCH 28/41] Fix some errors --- salt/kibana/init.sls | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 10f410e70..a9d3c6da9 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -50,6 +50,8 @@ kibana_sbin_jinja: - group: 939 - file_mode: 755 - template: jinja + - defaults: + GLOBALS: {{ GLOBALS }} kibanaconfig: file.managed: @@ -84,15 +86,6 @@ synckibanacustom: - user: 932 - group: 939 -kibanabin: - file.managed: - - name: /usr/sbin/so-kibana-config-load - - source: salt://kibana/bin/so-kibana-config-load - - mode: 755 - - template: jinja - - defaults: - GLOBALS: {{ GLOBALS }} - # Start the kibana docker so-kibana: docker_container.running: From dc77b2072303ae569b02013ae457bbe4e4ee76e3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 4 May 2023 14:54:37 -0400 Subject: [PATCH 29/41] remove extra " from so-elasticsearch-roles-load --- salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load index b6b593320..90b262989 100755 --- a/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-roles-load @@ -16,7 +16,7 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do - curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://localhost:9200" + curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://localhost:9200 if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" From f34627f709dba8125b3a8e654acdcdf6a3afc9ad Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 4 May 2023 15:13:42 -0400 Subject: [PATCH 30/41] source from sbin_jinja and exlude pat --- salt/elasticsearch/init.sls | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 8a1d4a346..0507a8c2c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -73,7 +73,6 @@ elasticsearch_sbin: - exclude_pat: - so-catrust - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state - - so-elasticsearch-ilm-policy-load elasticsearch_sbin_jinja: file.recurse: @@ -83,11 +82,13 @@ elasticsearch_sbin_jinja: - group: 939 - file_mode: 755 - template: jinja + - exclude_pat: + - so-elasticsearch-ilm-policy-load so-elasticsearch-ilm-policy-load-script: file.managed: - name: /usr/sbin/so-elasticsearch-ilm-policy-load - - source: salt://elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load + - source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load - user: 930 - group: 939 - mode: 754 From a64eb0ba978916e194b53c8e75b04cb5dc0abf97 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 15:30:44 -0400 Subject: [PATCH 31/41] Update so-setup --- setup/so-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 72549d79d..d3dd76202 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -239,7 +239,8 @@ esac # Allow execution of SO tools during setup local_sbin="$(pwd)/../salt/common/tools/sbin" -export PATH=$PATH:$local_sbin +manager_sbin="$(pwd)/../salt/manager/tools/sbin" +export PATH=$PATH:$local_sbin:$manager_sbin # Ubuntu whiptail pallete to make it look the same as CentOS and Rocky. set_palette >> $setup_log 2>&1 From 80cbe5f6e8701de752ed0216318a43dcaee5c9d3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 15:45:05 -0400 Subject: [PATCH 32/41] Update so-functions --- setup/so-functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index b7e473ca6..1cd81e644 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -908,6 +908,8 @@ create_manager_pillars() { soc_pillar idh_pillar influxdb_pillar + logrotate_pillar + patch_pillar } From 4fec2a18a5e87ba465a96ecb64e7638e094ddf38 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 16:26:40 -0400 Subject: [PATCH 33/41] Update so-functions --- setup/so-functions | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 1cd81e644..cf6bb83ab 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1675,51 +1675,6 @@ parse_install_username() { INSTALLUSERNAME=${SUDO_USER:-${USER}} } -patch_pillar() { - title "Create the patch pillar file" - local pillar_file=$local_salt_dir/pillar/minions/$MINION_ID.sls - - - if [[ $MANAGERUPDATES == 1 ]]; then - local source="manager" - else - local source="direct" - fi - - printf '%s\n'\ - "patch:"\ - " os:"\ - " source: '$source'"\ - " schedule_name: '$PATCHSCHEDULENAME'"\ - " enabled: True"\ - " splay: 300"\ - "" > "$pillar_file" - -} - -patch_schedule_os_new() { - title "Create the patch schedule" - local OSPATCHSCHEDULEDIR="$temp_install_dir/salt/patch/os/schedules" - local OSPATCHSCHEDULE="$OSPATCHSCHEDULEDIR/$PATCHSCHEDULENAME.yml" - - logCmd "mkdir -p $OSPATCHSCHEDULEDIR" - - printf '%s\n'\ - "patch:"\ - " os:"\ - " schedule:"> "$OSPATCHSCHEDULE" - for psd in "${PATCHSCHEDULEDAYS[@]}";do - psd="${psd//\"/}" - echo " - $psd:" >> "$OSPATCHSCHEDULE" - for psh in "${PATCHSCHEDULEHOURS[@]}" - do - psh="${psh//\"/}" - echo " - '$psh'" >> "$OSPATCHSCHEDULE" - done - done - -} - print_salt_state_apply() { local state=$1 From 469258ee5e8b8cde2211fe9413b4d5b6186f582e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 4 May 2023 16:46:54 -0400 Subject: [PATCH 34/41] Update init.sls --- salt/docker/init.sls | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/docker/init.sls b/salt/docker/init.sls index f2a4c80a9..36530c9c3 100644 --- a/salt/docker/init.sls +++ b/salt/docker/init.sls @@ -26,10 +26,10 @@ dockerheldpackages: dockerheldpackages: pkg.installed: - pkgs: - - containerd.io: 1.6.18-3.1.el9 - - docker-ce: 23.0.1-1.el9 - - docker-ce-cli: 23.0.1-1.el9 - - docker-ce-rootless-extras: 23.0.1-1.el9 + - containerd.io: 1.6.20-3.1.el9 + - docker-ce: 23.0.5-1.el9 + - docker-ce-cli: 23.0.5-1.el9 + - docker-ce-rootless-extras: 23.0.5-1.el9 - hold: True - update_holds: True {% endif %} From ddb776c80e9b06f401005ca8c208e85a8ee7ee9c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 4 May 2023 17:26:18 -0400 Subject: [PATCH 35/41] add redis pillars to searchnode. move redis scripts with jinja to sbin_jinja --- pillar/top.sls | 2 ++ salt/elasticsearch/init.sls | 6 ++++-- salt/elasticsearch/tools/{sbin => sbin_jinja}/so-catrust | 0 .../{sbin => sbin_jinja}/so-elasticsearch-cluster-settings | 0 .../so-elasticsearch-cluster-space-total | 0 5 files changed, 6 insertions(+), 2 deletions(-) rename salt/elasticsearch/tools/{sbin => sbin_jinja}/so-catrust (100%) rename salt/elasticsearch/tools/{sbin => sbin_jinja}/so-elasticsearch-cluster-settings (100%) rename salt/elasticsearch/tools/{sbin => sbin_jinja}/so-elasticsearch-cluster-space-total (100%) diff --git a/pillar/top.sls b/pillar/top.sls index 259e87c96..ac46bfc12 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -211,6 +211,8 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + - redis.soc_redis + - redis.adv_redis - minions.{{ grains.id }} - minions.adv_{{ grains.id }} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 0507a8c2c..5e2ffae9d 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -47,7 +47,7 @@ elasticsearch: cascriptsync: file.managed: - name: /usr/sbin/so-catrust - - source: salt://elasticsearch/tools/sbin/so-catrust + - source: salt://elasticsearch/tools/sbin_jinja/so-catrust - user: 939 - group: 939 - mode: 750 @@ -83,7 +83,9 @@ elasticsearch_sbin_jinja: - file_mode: 755 - template: jinja - exclude_pat: - - so-elasticsearch-ilm-policy-load + - so-elasticsearch-ilm-policy-load # exclude this because we need to watch it for changes, we sync it in another state + - defaults: + GLOBALS: {{ GLOBALS }} so-elasticsearch-ilm-policy-load-script: file.managed: diff --git a/salt/elasticsearch/tools/sbin/so-catrust b/salt/elasticsearch/tools/sbin_jinja/so-catrust similarity index 100% rename from salt/elasticsearch/tools/sbin/so-catrust rename to salt/elasticsearch/tools/sbin_jinja/so-catrust diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings similarity index 100% rename from salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-settings rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-space-total b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total similarity index 100% rename from salt/elasticsearch/tools/sbin/so-elasticsearch-cluster-space-total rename to salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-space-total From d5a1406095ffd70e6849c1495dcada469940c7d5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 5 May 2023 09:15:52 -0400 Subject: [PATCH 36/41] Update so-user --- salt/manager/tools/sbin/so-user | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-user b/salt/manager/tools/sbin/so-user index 3c712491a..989b8d554 100755 --- a/salt/manager/tools/sbin/so-user +++ b/salt/manager/tools/sbin/so-user @@ -7,7 +7,8 @@ -source $(dirname $0)/so-common +#source $(dirname $0)/so-common +source /usr/sbin/so-common DEFAULT_ROLE=analyst From f6dcefe0f87d3b66bcd098874483bb662f1f9cce Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 5 May 2023 09:34:33 -0400 Subject: [PATCH 37/41] Update so-user --- salt/manager/tools/sbin/so-user | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/so-user b/salt/manager/tools/sbin/so-user index 989b8d554..8234f7ae5 100755 --- a/salt/manager/tools/sbin/so-user +++ b/salt/manager/tools/sbin/so-user @@ -5,11 +5,13 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +if [[ -f /usr/sbin/so-common ]]; then + source /usr/sbin/so-common +else + source $(dirname $0)/../../../common/tools/sbin/so-common +fi -#source $(dirname $0)/so-common -source /usr/sbin/so-common - DEFAULT_ROLE=analyst function usage() { From d7972032e455d590a948389fc8188ae593534345 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 5 May 2023 10:33:11 -0400 Subject: [PATCH 38/41] Update init.sls --- salt/manager/init.sls | 8 -------- 1 file changed, 8 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index eea0f9568..47867edaf 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -60,14 +60,6 @@ manager_sbin: # - file_mode: 755 # - template: jinja -repo_sync_script: - file.managed: - - name: /usr/sbin/so-repo-sync - - source: salt://manager/files/so-repo-sync - - user: root - - group: root - - mode: 755 - so-repo-sync: {% if MANAGERMERGED.reposync.enabled %} cron.present: From 77cbf35625071f1f89d7d904ea128f1a811bef54 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 5 May 2023 11:26:10 -0400 Subject: [PATCH 39/41] Update so-functions --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index cf6bb83ab..8b361597b 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2289,11 +2289,11 @@ set_initial_firewall_policy() { set_initial_firewall_access() { if [[ ! -z "$ALLOW_CIDR" ]]; then - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost analyst $ALLOW_CIDR --apply + so-firewall includehost analyst $ALLOW_CIDR --apply fi if [[ ! -z "$MINION_CIDR" ]]; then - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost sensors $MINION_CIDR - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost searchnodes $MINION_CIDR --apply + so-firewall includehost sensors $MINION_CIDR + so-firewall includehost searchnodes $MINION_CIDR --apply fi } From 563c0631ba2b1db811e007eaef78bdf15a294c59 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 5 May 2023 13:01:40 -0400 Subject: [PATCH 40/41] Update so-functions --- setup/so-functions | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 8b361597b..1a96d4bd0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1240,11 +1240,6 @@ firewall_generate_templates() { logCmd "cp -r ../files/firewall/* /opt/so/saltstack/local/salt/firewall/" - # i think this can be commented out for 2.4 - #for i in analyst beats_endpoint endgame sensors manager managersearch elastic_agent_endpoint searchnodes; do - # $default_salt_dir/salt/common/tools/sbin/so-firewall --role="$i" --ip=127.0.0.1 - #done - } generate_ca() { @@ -2277,12 +2272,9 @@ set_hostname() { } set_initial_firewall_policy() { - title "Setting Initial Firewall Policy" - if [ -f $default_salt_dir/salt/common/tools/sbin/so-firewall ]; then chmod +x $default_salt_dir/salt/common/tools/sbin/so-firewall; fi - case "$install_type" in 'EVAL' | 'MANAGER' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT') - $default_salt_dir/salt/common/tools/sbin/so-firewall includehost $minion_type $MAINIP --apply + so-firewall includehost $minion_type $MAINIP --apply ;; esac } @@ -2369,19 +2361,6 @@ update_sudoers_for_testing() { fi } -update_sudoers() { - - if ! grep -qE '^soremote\ ALL=\(ALL\)\ NOPASSWD:(\/usr\/bin\/salt\-key|\/opt\/so\/saltstack)' /etc/sudoers; then - # Update Sudoers so that soremote can accept keys without a password - echo "soremote ALL=(ALL) NOPASSWD:/usr/bin/salt-key" | tee -a /etc/sudoers - echo "soremote ALL=(ALL) NOPASSWD:$default_salt_dir/salt/common/tools/sbin/so-firewall" | tee -a /etc/sudoers - echo "soremote ALL=(ALL) NOPASSWD:$default_salt_dir/pillar/data/addtotab.sh" | tee -a /etc/sudoers - echo "soremote ALL=(ALL) NOPASSWD:$default_salt_dir/salt/manager/files/add_minion.sh" | tee -a /etc/sudoers - else - info "User soremote already granted sudo privileges" - fi -} - update_packages() { if [[ $is_rocky ]]; then logCmd "dnf repolist" From 21ffcbf2fd051f53a8960bfe66f38fd2abcb720c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 5 May 2023 13:16:45 -0400 Subject: [PATCH 41/41] Update so-setup --- setup/so-setup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index d3dd76202..b1c4ce42b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -608,9 +608,6 @@ if ! [[ -f $install_opt_file ]]; then securityonion_repo # Update existing packages update_packages - # Download Elastic Agent Artifacts - title "Downloading Elastic Agent Artifacts" - download_elastic_agent_artifacts # Install salt saltify # Start the master service @@ -627,6 +624,9 @@ if ! [[ -f $install_opt_file ]]; then logCmd "salt-call state.apply docker" firewall_generate_templates set_initial_firewall_policy + # Download Elastic Agent Artifacts + title "Downloading Elastic Agent Artifacts" + download_elastic_agent_artifacts generate_ca generate_ssl