Merge branch '2.4/dev' into kilo

2026-04-26 22:47:49 +02:00 · 2023-05-08 12:19:00 -04:00
parent bd23d1ab7b a1e0041b14
commit 40c3f9a156
207 changed files with 944 additions and 869 deletions
@@ -49,13 +49,12 @@ so-status.conf:
    - name: /opt/so/conf/so-status/so-status.conf
    - unless: ls /opt/so/conf/so-status/so-status.conf

-sosaltstackperms:
+socore_opso_perms:
  file.directory:
-    - name: /opt/so/saltstack
+    - name: /opt/so
    - user: 939
    - group: 939
-    - dir_mode: 770
-
+    
 so_log_perms:
  file.directory:
    - name: /opt/so/log
@@ -112,21 +111,23 @@ elastic_curl_config:
  {% endif %}
 {% endif %}

-# Sync some Utilities
-utilsyncscripts:
+
+common_sbin:
  file.recurse:
    - name: /usr/sbin
-    - user: root
-    - group: root
+    - source: salt://common/tools/sbin
+    - user: 939
+    - group: 939
+    - file_mode: 755
+
+common_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://common/tools/sbin_jinja
+    - user: 939
+    - group: 939 
    - file_mode: 755
    - template: jinja
-    - source: salt://common/tools/sbin
-    - exclude_pat:
-        - so-common
-        - so-firewall
-        - so-image-common
-        - soup
-        - so-status

 so-status_script:
  file.managed:
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
-/usr/sbin/so-restart elasticsearch $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
-/usr/sbin/so-restart kibana $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-restart logstash $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-restart curator $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
-/usr/sbin/so-restart elastalert $1
-{%- endif %}
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
-/usr/sbin/so-start elasticsearch $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
-/usr/sbin/so-start kibana $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-start logstash $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-start curator $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
-/usr/sbin/so-start elastalert $1
-{%- endif %}
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
-/usr/sbin/so-stop elasticsearch $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
-/usr/sbin/so-stop kibana $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-stop logstash $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-stop curator $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
-/usr/sbin/so-stop elastalert $1
-{%- endif %}
@@ -1,15 +0,0 @@
-#!/bin/bash
-#
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-IP={{ salt['grains.get']('ip_interfaces').get(salt['pillar.get']('sensor:mainint', salt['pillar.get']('manager:mainint', salt['pillar.get']('elasticsearch:mainint', salt['pillar.get']('host:mainint')))))[0] }}
-ESPORT=9200
-
-echo "Removing read only attributes for indices..."
-echo
-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-local_salt_dir=/opt/so/saltstack/local
-
-got_root() {
-
-  # Make sure you are root
-  if [ "$(id -u)" -ne 0 ]; then
-          echo "This script must be run using sudo!"
-          exit 1
-  fi
-
-}
-
-got_root
-if [ ! -f $local_salt_dir/pillar/fireeye/init.sls ]; then
-  echo "This is nto configured for Helix Mode. Please re-install."
-  exit
-else
-  echo "Enter your Helix API Key: "
-  read APIKEY
-  sed -i "s/^    api_key.*/    api_key: $APIKEY/g" $local_salt_dir/pillar/fireeye/init.sls
-  docker stop so-logstash
-  docker rm so-logstash
-  echo "Restarting Logstash for updated key"
-  salt-call state.apply logstash queue=True
-fi
@@ -1,17 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% set MAININT = salt['pillar.get']('host:mainint') -%}
-{% set NODEIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] -%}
-
-. /usr/sbin/so-common
-
-if [ "$1" == "" ]; then
-        for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done
-else
-        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events
-fi
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-docker exec -it so-redis redis-cli llen logstash:unparsed
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-restart nodered $1
@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start nodered $1
-
@@ -60,30 +60,21 @@ curconf:
    - template: jinja
    - show_changes: False

-curclusterclose: 
-  file.managed:
-    - name: /usr/sbin/so-curator-cluster-close
-    - source: salt://curator/files/bin/so-curator-cluster-close
+curator_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://curator/tools/sbin
    - user: 934
    - group: 939
-    - mode: 755
-    - template: jinja
+    - file_mode: 755

-curclusterdelete:
-  file.managed:
-    - name: /usr/sbin/so-curator-cluster-delete
-    - source: salt://curator/files/bin/so-curator-cluster-delete
+curator_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://curator/tools/sbin_jinja
    - user: 934
-    - group: 939
-    - mode: 755
-
-curclusterdeletedelete:
-  file.managed:
-    - name: /usr/sbin/so-curator-cluster-delete-delete
-    - source: salt://curator/files/bin/so-curator-cluster-delete-delete
-    - user: 934
-    - group: 939
-    - mode: 755
+    - group: 939 
+    - file_mode: 755
    - template: jinja

 so-curator:
@@ -8,30 +8,44 @@ docker:
      final_octet: 20
      port_bindings:
        - 0.0.0.0:5000:5000
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-elastic-fleet':
      final_octet: 21
      port_bindings:
        - 0.0.0.0:8220:8220/tcp
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-elasticsearch':
      final_octet: 22
      port_bindings:
        - 0.0.0.0:9200:9200/tcp
        - 0.0.0.0:9300:9300/tcp
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-idstools':
      final_octet: 25
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-influxdb':
      final_octet: 26
      port_bindings:
        - 0.0.0.0:8086:8086
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-kibana':
      final_octet: 27
      port_bindings:
        - 0.0.0.0:5601:5601
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-kratos':
      final_octet: 28
      port_bindings:
        - 0.0.0.0:4433:4433
        - 0.0.0.0:4434:4434
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-logstash':
      final_octet: 29
      port_bindings:
@@ -45,58 +59,92 @@ docker:
        - 0.0.0.0:6052:6052
        - 0.0.0.0:6053:6053
        - 0.0.0.0:9600:9600
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-mysql':
      final_octet: 30
      port_bindings:
        - 0.0.0.0:3306:3306
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-nginx':
      final_octet: 31
      port_bindings:
        - 80:80
        - 443:443
        - 8443:8443
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-playbook':
      final_octet: 32
      port_bindings:
        - 0.0.0.0:3000:3000
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-redis':
      final_octet: 33
      port_bindings:
        - 0.0.0.0:6379:6379
        - 0.0.0.0:9696:9696
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-soc':
      final_octet: 34
      port_bindings:
        - 0.0.0.0:9822:9822
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-soctopus':
      final_octet: 35
      port_bindings:
        - 0.0.0.0:7000:7000
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-strelka-backend':
      final_octet: 36
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-strelka-filestream':
      final_octet: 37
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-strelka-frontend':
      final_octet: 38
      port_bindings:
        - 0.0.0.0:57314:57314
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-strelka-manager':
      final_octet: 39
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-strelka-gatekeeper':
      final_octet: 40
      port_bindings:
        - 0.0.0.0:6381:6379
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-strelka-coordinator':
      final_octet: 41
      port_bindings:
        - 0.0.0.0:6380:6379
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-elastalert':
      final_octet: 42
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-curator':
      final_octet: 43
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-elastic-fleet-package-registry':
      final_octet: 44
      port_bindings:
        - 0.0.0.0:8080:8080/tcp
+      custom_bind_mounts: []
+      extra_hosts: []
    'so-idh':
-      final_octet: 45
+      final_octet: 45
+      custom_bind_mounts: []
+      extra_hosts: []
@@ -26,10 +26,10 @@ dockerheldpackages:
 dockerheldpackages:
  pkg.installed:
    - pkgs:
-      - containerd.io: 1.6.18-3.1.el9
-      - docker-ce: 23.0.1-1.el9
-      - docker-ce-cli: 23.0.1-1.el9
-      - docker-ce-rootless-extras: 23.0.1-1.el9
+      - containerd.io: 1.6.20-3.1.el9
+      - docker-ce: 23.0.5-1.el9
+      - docker-ce-cli: 23.0.5-1.el9
+      - docker-ce-rootless-extras: 23.0.5-1.el9
    - hold: True
    - update_holds: True
 {% endif %}
@@ -28,6 +28,18 @@ docker:
        helpLink: docker.html
        advanced: True
        multiline: True
+      custom_bind_mounts:
+        description: List of custom local volume bindings.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
+      extra_hosts:
+        description: List of additional host entries for the container.
+        advanced: True
+        helpLink: docker.html
+        multiline: True
+        forcedType: "[]string"
    so-dockerregistry: *dockerOptions
    so-elastalert: *dockerOptions
    so-elastic-fleet-package-registry: *dockerOptions
@@ -29,6 +29,23 @@ elastalogdir:
    - group: 933
    - makedirs: True

+elastalert_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://elastalert/tools/sbin
+    - user: 933
+    - group: 939
+    - file_mode: 755
+
+#elastalert_sbin_jinja:
+#  file.recurse:
+#    - name: /usr/sbin
+#    - source: salt://elastalert/tools/sbin_jinja
+#    - user: 933
+#    - group: 939 
+#    - file_mode: 755
+#    - template: jinja
+
 elastarules:
  file.directory:
    - name: /opt/so/rules/elastalert
@@ -25,6 +25,23 @@ elastic-agent:
    - home: /opt/so/conf/elastic-fleet
    - createhome: False

+elasticfleet_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://elasticfleet/tools/sbin
+    - user: 947
+    - group: 939
+    - file_mode: 755
+
+elasticfleet_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://elasticfleet/tools/sbin_jinja
+    - user: 947
+    - group: 939 
+    - file_mode: 755
+    - template: jinja
+
 eaconfdir:
  file.directory:
    - name: /opt/so/conf/elastic-fleet
@@ -21,12 +21,33 @@ vm.max_map_count:
  sysctl.present:
    - value: 262144

+# Add ES Group
+elasticsearchgroup:
+  group.present:
+    - name: elasticsearch
+    - gid: 930
+
+esconfdir:
+  file.directory:
+    - name: /opt/so/conf/elasticsearch
+    - user: 930
+    - group: 939
+    - makedirs: True
+
+# Add ES user
+elasticsearch:
+  user.present:
+    - uid: 930
+    - gid: 930
+    - home: /opt/so/conf/elasticsearch
+    - createhome: False
+
 {% if GLOBALS.is_manager %}
 # We have to add the Manager CA to the CA list
 cascriptsync:
  file.managed:
    - name: /usr/sbin/so-catrust
-    - source: salt://elasticsearch/tools/sbin/so-catrust
+    - source: salt://elasticsearch/tools/sbin_jinja/so-catrust
    - user: 939
    - group: 939
    - mode: 750
@@ -42,25 +63,34 @@ cascriptfun:
        - file: cascriptsync
 {% endif %}

-# Sync some es scripts
-es_sync_scripts:
+elasticsearch_sbin:
  file.recurse:
    - name: /usr/sbin
-    - user: root
-    - group: root
+    - source: salt://elasticsearch/tools/sbin
+    - user: 930
+    - group: 939
+    - file_mode: 755
+    - exclude_pat:
+      - so-catrust
+      - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+
+elasticsearch_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://elasticsearch/tools/sbin_jinja
+    - user: 939
+    - group: 939 
    - file_mode: 755
    - template: jinja
-    - source: salt://elasticsearch/tools/sbin
    - exclude_pat:
-        - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
-        - so-elasticsearch-ilm-policy-load 
+      - so-elasticsearch-ilm-policy-load # exclude this because we need to watch it for changes, we sync it in another state
    - defaults:
        GLOBALS: {{ GLOBALS }}

 so-elasticsearch-ilm-policy-load-script:
  file.managed:
    - name: /usr/sbin/so-elasticsearch-ilm-policy-load
-    - source: salt://elasticsearch/tools/sbin/so-elasticsearch-ilm-policy-load
+    - source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
    - user: 930
    - group: 939
    - mode: 754
@@ -96,29 +126,6 @@ capemz:
    - user: 939
    - group: 939

-
-
-# Add ES Group
-elasticsearchgroup:
-  group.present:
-    - name: elasticsearch
-    - gid: 930
-
-# Add ES user
-elasticsearch:
-  user.present:
-    - uid: 930
-    - gid: 930
-    - home: /opt/so/conf/elasticsearch
-    - createhome: False
-
-esconfdir:
-  file.directory:
-    - name: /opt/so/conf/elasticsearch
-    - user: 930
-    - group: 939
-    - makedirs: True
-
 esingestdir:
  file.directory:
    - name: /opt/so/conf/elasticsearch/ingest
@@ -374,7 +381,7 @@ so-es-cluster-settings:
    - template: jinja
    - require:
      - docker_container: so-elasticsearch
-      - file: es_sync_scripts
+      - file: elasticsearch_sbin_jinja

 so-elasticsearch-ilm-policy-load:
  cmd.run:
@@ -393,7 +400,7 @@ so-elasticsearch-templates:
    - template: jinja
    - require:
      - docker_container: so-elasticsearch
-      - file: es_sync_scripts
+      - file: elasticsearch_sbin_jinja

 so-elasticsearch-pipelines:
  cmd.run:
@@ -409,7 +416,7 @@ so-elasticsearch-roles-load:
    - template: jinja
    - require:
      - docker_container: so-elasticsearch
-      - file: es_sync_scripts
+      - file: elasticsearch_sbin_jinja
 {% endif %}
 {% else %}

@@ -5,7 +5,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
 . /usr/sbin/so-common

 SKIP=0
@@ -59,7 +58,7 @@ done
 if [ $SKIP -ne 1 ]; then
        # List indices
        echo 
-        curl -K /opt/so/conf/elasticsearch/curl.config -k -L https://{{ NODEIP }}:9200/_cat/indices?v
+        curl -K /opt/so/conf/elasticsearch/curl.config -k -L https://localhost:9200/_cat/indices?v
        echo
        # Inform user we are about to delete all data
        echo
@@ -5,10 +5,9 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_component_template | jq '.component_templates[] |.name'| sort
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template | jq '.component_templates[] |.name'| sort
 else
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_component_template/$1 | jq
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_component_template/$1 | jq
 fi
@@ -6,10 +6,8 @@

 . /usr/sbin/so-common

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
 if [ "$1" == "" ]; then
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_all/_ilm/explain | jq .
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_all/_ilm/explain | jq .
 else
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/$1/_ilm/explain | jq .[]
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/$1/_ilm/explain | jq .[]
 fi
@@ -6,6 +6,4 @@

 . /usr/sbin/so-common

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X DELETE https://{{ NODEIP }}:9200/_ilm/policy/$1
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X DELETE https://localhost:9200/_ilm/policy/$1
@@ -1,21 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
-{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
-{%- for index, settings in ES_INDEX_SETTINGS.items() %}
-  {%- if settings.policy is defined %}
-echo
-echo "Setting up {{ index }}-logs policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-echo
-  {%- endif %}
-{%- endfor %}
-echo
@@ -6,10 +6,9 @@

 . /usr/sbin/so-common

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}

 if [ "$1" == "" ]; then
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/policy | jq .
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ilm/policy | jq .
 else
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/policy/$1 | jq .[]
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ilm/policy/$1 | jq .[]
 fi
@@ -6,7 +6,6 @@

 . /usr/sbin/so-common

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}

 echo "Starting ILM..."
-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://{{ NODEIP }}:9200/_ilm/start
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/start
@@ -1,4 +1,4 @@
-/bin/bash
+#!/bin/bash
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
 # https://securityonion.net/license; you may not use this file except in compliance with the
@@ -6,6 +6,4 @@

 . /usr/sbin/so-common

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ilm/status | jq .
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ilm/status | jq .
@@ -6,7 +6,5 @@

 . /usr/sbin/so-common

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
 echo "Stopping ILM..."
-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://{{ NODEIP }}:9200/_ilm/stop
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -X POST https://localhost:9200/_ilm/stop
@@ -5,10 +5,9 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_index_template | jq '.index_templates[] |.name'| sort
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_index_template | jq '.index_templates[] |.name'| sort
 else
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_index_template/$1 | jq
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_index_template/$1 | jq
 fi
@@ -5,8 +5,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
 . /usr/sbin/so-common

-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://{{ NODEIP }}:9200/_cat/indices?pretty&v&s=index"
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L "https://localhost:9200/_cat/indices?pretty&v&s=index"
@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+echo "Removing read only attributes for indices..."
+echo
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -H "Content-Type: application/json" -L https://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;
@@ -5,12 +5,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
 . /usr/sbin/so-common

 if [ "$1" == "" ]; then
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
 else
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
 fi
@@ -5,12 +5,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
 . /usr/sbin/so-common

 if [ "$1" == "" ]; then
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq .
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/* | jq .
 else
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq .[]
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/$1 | jq .[]
 fi
@@ -5,10 +5,9 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/* | jq 'keys'
 else
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_ingest/pipeline/$1 | jq
 fi
@@ -7,8 +7,6 @@
 . /usr/sbin/so-common

 default_conf_dir=/opt/so/conf
-ELASTICSEARCH_HOST="{{ GLOBALS.node_ip }}"
-ELASTICSEARCH_PORT=9200

 # Define a default directory to load roles from
 ELASTICSEARCH_ROLES="$default_conf_dir/elasticsearch/roles/"
@@ -18,7 +16,7 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
-    curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+    curl -K /opt/so/conf/elasticsearch/curl.config -k --output /dev/null --silent --head --fail -L https://localhost:9200
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -5,8 +5,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
 . /usr/sbin/so-common

-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_cat/shards?pretty
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_cat/shards?pretty
@@ -5,8 +5,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
 . /usr/sbin/so-common

-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -XDELETE https://{{ NODEIP }}:9200/_template/$1
+curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -XDELETE https://localhost:9200/_template/$1
@@ -5,12 +5,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
 . /usr/sbin/so-common

 if [ "$1" == "" ]; then
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/* | jq .
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/* | jq .
 else
-        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq .
+        curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/$1 | jq .
 fi
@@ -5,10 +5,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
 . /usr/sbin/so-common
+
 if [ "$1" == "" ]; then
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys'
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/* | jq 'keys'
 else
-    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq
+    curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L https://localhost:9200/_template/$1 | jq
 fi
@@ -7,9 +7,6 @@
 . /usr/sbin/so-common

 default_conf_dir=/opt/so/conf
-ELASTICSEARCH_HOST="{{ GLOBALS.node_ip }}"
-ELASTICSEARCH_PORT=9200
-#ELASTICSEARCH_AUTH=""

 # Define a default directory to load pipelines from
 ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/"
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+. /usr/sbin/so-common
+
+
+{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
+/usr/sbin/so-restart elasticsearch $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
+/usr/sbin/so-restart kibana $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
+/usr/sbin/so-restart logstash $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
+/usr/sbin/so-restart curator $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
+/usr/sbin/so-restart elastalert $1
+{%- endif %}
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+. /usr/sbin/so-common
+
+
+{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
+/usr/sbin/so-start elasticsearch $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
+/usr/sbin/so-start kibana $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
+/usr/sbin/so-start logstash $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
+/usr/sbin/so-start curator $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
+/usr/sbin/so-start elastalert $1
+{%- endif %}
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+. /usr/sbin/so-common
+
+
+{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
+/usr/sbin/so-stop elasticsearch $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
+/usr/sbin/so-stop kibana $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
+/usr/sbin/so-stop logstash $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
+/usr/sbin/so-stop curator $1
+{%- endif %}
+
+{%- if GLOBALS.role in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
+/usr/sbin/so-stop elastalert $1
+{%- endif %}
@@ -8,13 +8,12 @@

 {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
 {%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}

 {%- for index, settings in ES_INDEX_SETTINGS.items() %}
  {%- if settings.policy is defined %}
 echo
 echo "Setting up {{ index }}-logs policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
+curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://localhost:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
 echo
  {%- endif %}
 {%- endfor %}
@@ -60,6 +60,23 @@ opencanary_config:
    - defaults:
        OPENCANARYCONFIG: {{ OPENCANARYCONFIG }}

+idh_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://idh/tools/sbin
+    - user: 934
+    - group: 939
+    - file_mode: 755
+
+#idh_sbin_jinja:
+#  file.recurse:
+#    - name: /usr/sbin
+#    - source: salt://idh/tools/sbin_jinja
+#    - user: 939
+#    - group: 939 
+#    - file_mode: 755
+#    - template: jinja
+
 so-idh:
  docker_container.running:
    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idh:{{ GLOBALS.so_version }}
@@ -20,6 +20,23 @@ idstoolslogdir:
    - group: 939
    - makedirs: True

+idstools_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://idstools/tools/sbin
+    - user: 934
+    - group: 939
+    - file_mode: 755
+
+#idstools_sbin_jinja:
+#  file.recurse:
+#    - name: /usr/sbin
+#    - source: salt://idstools/tools/sbin_jinja
+#    - user: 934
+#    - group: 939 
+#    - file_mode: 755
+#    - template: jinja
+
 so-rule-update:
  cron.present:
    - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1
@@ -31,6 +31,23 @@ influxdbdir:
    - name: /nsm/influxdb
    - makedirs: True

+influxdb_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://influxdb/tools/sbin
+    - user: 939
+    - group: 939
+    - file_mode: 755
+
+#influxdb_sbin_jinja:
+#  file.recurse:
+#    - name: /usr/sbin
+#    - source: salt://influxdb/tools/sbin_jinja
+#    - user: 939
+#    - group: 939 
+#    - file_mode: 755
+#    - template: jinja
+
 influxdbconf:
  file.managed:
    - name: /opt/so/conf/influxdb/config.yaml
--- a/Show More
+++ b/Show More