Merge branch 'dev' of https://github.com/Security-Onion-Solutions/securityonion into dev

salt/common/init.sls

@@ -186,6 +186,8 @@ utilsyncscripts:
     - template: jinja
     - source: salt://common/tools/sbin
     - defaults:
+        ELASTICCURL: 'curl'
+    - context:
         ELASTICCURL: {{ ELASTICAUTH.elasticcurl }}
 
 {% if role in ['eval', 'standalone', 'sensor', 'heavynode'] %}

salt/common/tools/sbin/so-user

@@ -45,6 +45,13 @@ elasticRolesFile=${ELASTIC_ROLES_FILE:-/opt/so/saltstack/local/salt/elasticsearc
 esUID=${ELASTIC_UID:-930}
 esGID=${ELASTIC_GID:-930}
 
+function lock() {
+  # Obtain file descriptor lock
+  exec 99>/var/tmp/so-user.lock || fail "Unable to create lock descriptor; if the system was not shutdown gracefully you may need to remove /var/tmp/so-user.lock manually."
+  flock -w 10 99 || fail "Another process is using so-user; if the system was not shutdown gracefully you may need to remove /var/tmp/so-user.lock manually."
+  trap 'rm -f /var/tmp/so-user.lock' EXIT
+}
+
 function fail() {
   msg=$1
   echo "$1"
@@ -219,12 +226,9 @@ function syncElastic() {
     mv "${rolesTmpFile}" "${elasticRolesFile}"
 
     if [[ -z "$SKIP_STATE_APPLY" ]]; then
-      echo "Applying elastic state locally; This can take a few minutes..."
-      echo "Applying elastic state locally at $(date)" >> /opt/so/log/soc/sync.log 2>&1
-      salt-call state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1
-      echo "Applying elastic state to elastic minions; This can take a few minutes..."
+      echo "Elastic state will be re-applied to affected minions in the background."
       echo "Applying elastic state to elastic minions at $(date)" >> /opt/so/log/soc/sync.log 2>&1
-      salt -C 'G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1
+      salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 &
     fi
   else
     echo "Newly generated users/roles files are incomplete; aborting."
@@ -331,6 +335,7 @@ case "${operation}" in
     verifyEnvironment
     [[ "$email" == "" ]] && fail "Email address must be provided"
 
+    lock
     validateEmail "$email"
     updatePassword
     createUser "$email"
@@ -349,6 +354,7 @@ case "${operation}" in
     verifyEnvironment
     [[ "$email" == "" ]] && fail "Email address must be provided"
 
+    lock
     updateUser "$email"
     syncAll
     echo "Successfully updated user"
@@ -358,6 +364,7 @@ case "${operation}" in
     verifyEnvironment
     [[ "$email" == "" ]] && fail "Email address must be provided"
 
+    lock
     updateStatus "$email" 'active'
     syncAll
     echo "Successfully enabled user"
@@ -369,6 +376,7 @@ case "${operation}" in
     verifyEnvironment
     [[ "$email" == "" ]] && fail "Email address must be provided"
 
+    lock
     updateStatus "$email" 'locked'
     syncAll
     echo "Successfully disabled user"
@@ -380,6 +388,7 @@ case "${operation}" in
     verifyEnvironment
     [[ "$email" == "" ]] && fail "Email address must be provided"
 
+    lock
     deleteUser "$email"
     syncAll
     echo "Successfully deleted user"
@@ -388,7 +397,8 @@ case "${operation}" in
     ;;
 
   "sync")
-    syncAll && echo "Synchronization completed at $(date)"
+    lock
+    syncAll
     ;;
 
   "validate")

salt/common/tools/sbin/soup

@@ -392,7 +392,7 @@ rc1_to_rc2() {
   local NAME=$(echo $p | awk '{print $1}')
   local IP=$(echo $p | awk '{print $2}')
   echo "Removing the old cross cluster config for $NAME"
-  {{ ELASTICCURL }} -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}'
+  curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}'
   done </tmp/nodes.txt
   # Add the nodes back using hostname
   while read p; do
@@ -400,7 +400,7 @@ rc1_to_rc2() {
     local EHOSTNAME=$(echo $p | awk -F"_" '{print $1}')
     local IP=$(echo $p | awk '{print $2}')
     echo "Adding the new cross cluster config for $NAME"
-    {{ ELASTICCURL }} -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["'$EHOSTNAME':9300"]}}}}}'
+    curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["'$EHOSTNAME':9300"]}}}}}'
   done </tmp/nodes.txt
 
   INSTALLEDVERSION=rc.2

salt/soctopus/files/templates/es-generic.template

@@ -1,13 +1,7 @@
 {% set ES = salt['pillar.get']('global:managerip', '') %}
-{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
-{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
 alert: modules.so.playbook-es.PlaybookESAlerter
 elasticsearch_host: "{{ ES }}:9200"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
-elasticsearch_user: "{{ ES_USER }}"
-elasticsearch_pass: "{{ ES_PASS }}"
-{% endif %}
 play_title: ""
 play_url: "https://{{ ES }}/playbook/issues/6000"
 sigma_level: ""

salt/soctopus/files/templates/generic.template

@@ -1,15 +1,9 @@
 {% set es = salt['pillar.get']('global:url_base', '') %}
-{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
-{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
 alert:
 - "modules.so.playbook-es.PlaybookESAlerter"
 
 elasticsearch_host: "{{ es }}:9200"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
-elasticsearch_user: "{{ ES_USER }}"
-elasticsearch_pass: "{{ ES_PASS }}"
-{% endif %}
 play_title: ""
 play_id: ""
 event.module: "playbook"

salt/soctopus/files/templates/osquery.template

@@ -1,15 +1,9 @@
 {% set es = salt['pillar.get']('global:url_base', '') %}
-{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
-{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
 alert:
 - "modules.so.playbook-es.PlaybookESAlerter"
  
 elasticsearch_host: "{{ es }}:9200"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
-elasticsearch_user: "{{ ES_USER }}"
-elasticsearch_pass: "{{ ES_PASS }}"
-{% endif %}
 play_title: ""
 event.module: "playbook"
 event.dataset: "alert"

salt/soctopus/init.sls

@@ -21,7 +21,6 @@ soctopus-sync:
     - source: salt://soctopus/files/templates
     - user: 939
     - group: 939
-    - file_mode: 600
     - template: jinja
 
 soctopusconf: