remove use of pipe

salt/soc/config.sls

@@ -32,9 +32,10 @@ soclogdir:
 
 socsaltdir:
   file.directory:
-    - name: /opt/so/conf/soc/salt
+    - name: /opt/so/conf/soc/queue
     - user: 939
     - group: 939
+    - mode: 770
     - makedirs: True
 
 socconfig:

salt/soc/defaults.yaml

@@ -1039,7 +1039,7 @@ soc:
           bucket: telegraf/so_short_term
           verifyCert: false
         salt:
-          saltPipe: /opt/sensoroni/salt/pipe
+          queueDir: /opt/sensoroni/queue
         sostatus:
           refreshIntervalMs: 30000
           offlineThresholdMs: 900000

salt/soc/enabled.sls

@@ -31,7 +31,7 @@ so-soc:
       - /opt/so/conf/soc/custom.js:/opt/sensoroni/html/js/custom.js:ro
       - /opt/so/conf/soc/custom_roles:/opt/sensoroni/rbac/custom_roles:ro
       - /opt/so/conf/soc/soc_users_roles:/opt/sensoroni/rbac/users_roles:rw
-      - /opt/so/conf/soc/salt:/opt/sensoroni/salt:rw
+      - /opt/so/conf/soc/queue:/opt/sensoroni/queue:rw
       - /opt/so/saltstack:/opt/so/saltstack:rw
       {% if DOCKER.containers['so-soc'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-soc'].custom_bind_mounts %}
@@ -73,7 +73,7 @@ delete_so-soc_so-status.disabled:
 
 salt-relay:
   cron.present:
-    - name: 'ps -ef | grep salt-relay.sh | grep -v grep > /dev/null 2>&1 || /opt/so/saltstack/default/salt/soc/files/bin/salt-relay.sh >> /opt/so/log/soc/salt-relay.log 2>&1 &'
+    - name: '/opt/so/saltstack/default/salt/soc/files/bin/salt-relay.sh &'
     - identifier: salt-relay
 
 {% else %}

salt/soc/files/bin/salt-relay.sh

@@ -6,58 +6,78 @@
 
 . /usr/sbin/so-common
 
-PIPE_OWNER=${PIPE_OWNER:-socore}
+QUEUE_OWNER=${QUEUE_OWNER:-socore}
-PIPE_GROUP=${PIPE_GROUP:-socore}
+QUEUE_GROUP=${QUEUE_GROUP:-socore}
-SOC_PIPE=${SOC_PIPE:-/opt/so/conf/soc/salt/pipe}
+MIN_POLL_INTERVAL=${MIN_POLL_INTERVAL:-1}
-CMD_PREFIX=${CMD_PREFIX:-""}
+LOG_FILE=${LOG_FILE:-/opt/so/log/soc/salt-relay.log}
 PATH=${PATH}:/usr/sbin
 
+# USE CAUTION when changing this value as all files in this dir will be deleted
+QUEUE_DIR=/opt/so/conf/soc/queue
+
 function log() {
-  echo "$(date) | $1"
+  echo "$(date) | $1" >> $LOG_FILE
 }
 
-function make_pipe() {
+function poll() {
-  path=$1
+  # Purge any expired files older than 1 minute. SOC will have already errored out to the user
+  # if a response hasn't been detected by this time.
+  find "$QUEUE_DIR" -type f -mmin +1 -delete
 
-  log "Creating pipe: $path"
+  file=$(ls -1trI "*.response" "$QUEUE_DIR" | head -1)
-  rm -f "${path}"
+  if [[ "$file" != "" ]]; then
-  mkfifo "${path}"
+    contents=$(cat "$QUEUE_DIR/$file")
-  chmod 0660 "${path}"
+    # Delete immediately to prevent a crash from potentially causing the same
-  chown ${PIPE_OWNER}:${PIPE_GROUP} "${path}"
+    # command to be executed multiple times -> Safer to not run at all than to
+    # potentially execute multiple times (Ex: user management)
+    rm -f "$QUEUE_DIR/$file"
+    echo "$contents"
+  fi
 }
 
-make_pipe "${SOC_PIPE}"
+function respond() {
+  file="$QUEUE_DIR/$1.response"
+  response=$2
+
+  touch "$file"
+  chmod 660 "$file"
+  chown "$QUEUE_OWNER:$QUEUE_GROUP" "$file"
+  echo "$response" > "$file"
+}
 
 function list_minions() {
-  response=$($CMD_PREFIX so-minion -o=list)
+  id=$1
+  response=$(so-minion -o=list)
   exit_code=$?
   if [[ $exit_code -eq 0 ]]; then
     log "Successful command execution"
-    $(echo "$response" > "${SOC_PIPE}")
+    respond "$id" "$response"
   else
     log "Unsuccessful command execution: $exit_code"
-    $(echo "false" > "${SOC_PIPE}")
+    respond "$id" "false"
   fi
 }
 
 function manage_minion() {
-  request=$1
+  id=$1
+  request=$2
   op=$(echo "$request" | jq -r .operation)
-  id=$(echo "$request" | jq -r .id)
+  minion_id=$(echo "$request" | jq -r .id)
 
-  response=$($CMD_PREFIX so-minion "-o=$op" "-m=$id")
+  response=$(so-minion "-o=$op" "-m=$minion_id")
   exit_code=$?
   if [[ exit_code -eq 0 ]]; then
     log "Successful command execution"
-    $(echo "true" > "${SOC_PIPE}")
+    respond "$id" "true"
   else
     log "Unsuccessful command execution: $response ($exit_code)"
-    $(echo "false" > "${SOC_PIPE}")
+    respond "$id" "false"
   fi
 }
 
 function manage_user() {
-  request=$1
+  id=$1
+  request=$2
   op=$(echo "$request" | jq -r .operation)
 
   max_tries=10
@@ -72,20 +92,20 @@ function manage_user() {
         lastName=$(echo "$request" | jq -r .lastName)
         note=$(echo "$request" | jq -r .note)
         log "Performing user '$op' for user '$email' with firstname '$firstName', lastname '$lastName', note '$note' and role '$role'"
-        response=$(echo "$password" | $CMD_PREFIX so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note" --role "$role" --skip-sync)
+        response=$(echo "$password" | so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note" --role "$role" --skip-sync)
         exit_code=$?
         ;;
       add|enable|disable|delete)
         email=$(echo "$request" | jq -r .email)
         log "Performing user '$op' for user '$email'"
-        response=$($CMD_PREFIX so-user "$op" --email "$email" --skip-sync)
+        response=$(so-user "$op" --email "$email" --skip-sync)
         exit_code=$?
         ;;
       addrole|delrole)
         email=$(echo "$request" | jq -r .email)
         role=$(echo "$request" | jq -r .role)
         log "Performing '$op' for user '$email' with role '$role'"
-        response=$($CMD_PREFIX so-user "$op" --email "$email" --role "$role" --skip-sync)
+        response=$(so-user "$op" --email "$email" --role "$role" --skip-sync)
         exit_code=$?
         ;;
       password)
@@ -101,12 +121,12 @@ function manage_user() {
         lastName=$(echo "$request" | jq -r .lastName)
         note=$(echo "$request" | jq -r .note)
         log "Performing '$op' update for user '$email' with firstname '$firstName', lastname '$lastName', and note '$note'"
-        response=$($CMD_PREFIX so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note")
+        response=$(so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note")
         exit_code=$?
         ;;
       sync)
         log "Performing '$op'"
-        response=$($CMD_PREFIX so-user "$op")
+        response=$(so-user "$op")
         exit_code=$?
         ;;
       *)
@@ -126,15 +146,16 @@ function manage_user() {
 
   if [[ exit_code -eq 0 ]]; then
     log "Successful command execution: $response"
-    $(echo "true" > "${SOC_PIPE}")
+    respond "$id" "true"
   else
     log "Unsuccessful command execution: $response ($exit_code)"
-    $(echo "false" > "${SOC_PIPE}")
+    respond "$id" "false"
   fi
 }
 
 function manage_salt() {
-  request=$1
+  id=$1
+  request=$2
   op=$(echo "$request" | jq -r .operation)
   minion=$(echo "$request" | jq -r .minion)
   if [[ -s $minion || "$minion" == "null" ]]; then
@@ -145,18 +166,18 @@ function manage_salt() {
     state)
       log "Performing '$op' for '$state' on minion '$minion'"
       state=$(echo "$request" | jq -r .state)
-      response=$($CMD_PREFIX salt --async "$minion" state.apply "$state" queue=2)
+      response=$(salt --async "$minion" state.apply "$state" queue=2)
       exit_code=$?
       ;;
     highstate)
       log "Performing '$op' on minion $minion"
-      response=$($CMD_PREFIX salt --async "$minion" state.highstate queue=2)
+      response=$(salt --async "$minion" state.highstate queue=2)
       exit_code=$?
       ;;
     activejobs)
-      response=$($CMD_PREFIX salt-run jobs.active -out json -l quiet)
+      response=$(salt-run jobs.active -out json -l quiet)
       log "Querying active salt jobs"
-      $(echo "$response" > "${SOC_PIPE}")
+      respond "$id" "$response"
       return
       ;;
     *)
@@ -167,15 +188,16 @@ function manage_salt() {
 
   if [[ exit_code -eq 0 ]]; then
     log "Successful command execution: $response"
-    $(echo "true" > "${SOC_PIPE}")
+    respond "$id" "true"
   else
     log "Unsuccessful command execution: $response ($exit_code)"
-    $(echo "false" > "${SOC_PIPE}")
+    respond "$id" "false"
   fi
 }
 
 function send_file() {
-  request=$1
+  id=$1
+  request=$2
   from=$(echo "$request" | jq -r .from)
   to=$(echo "$request" | jq -r .to)
   node=$(echo "$request" | jq -r .node)
@@ -195,7 +217,7 @@ function send_file() {
   filename=$(basename "$fromgpg")
 
   log "sending..."
-  response=$($CMD_PREFIX salt-cp -C "$node" "$fromgpg" "$to")
+  response=$(salt-cp -C "$node" "$fromgpg" "$to")
   # salt-cp returns 0 even if the file transfer fails, so we need to check the response.
   # Remove the node and filename from the response on the off-chance they contain
   # the word "True" in them
@@ -213,14 +235,15 @@ function send_file() {
   fi
 
   if [[ exit_code -eq 0 ]]; then
-    $(echo "true" > "${SOC_PIPE}")
+    respond "$id" "true"
   else
-    $(echo "false" > "${SOC_PIPE}")
+    respond "$id" "false"
   fi
 }
 
 function import_file() {
-  request=$1
+  id=$1
+  request=$2
   node=$(echo "$request" | jq -r .node)
   file=$(echo "$request" | jq -r .file)
   importer=$(echo "$request" | jq -r .importer)
@@ -234,7 +257,7 @@ function import_file() {
   log "decrypting..."
   password=$(lookup_pillar_secret import_pass)
   decrypt_cmd="gpg --passphrase $password -o $file.tmp --batch --decrypt $filegpg"
-  $CMD_PREFIX salt "$node" cmd.run "\"$decrypt_cmd\""
+  salt "$node" cmd.run "\"$decrypt_cmd\""
   decrypt_code=$?
 
   if [[ $decrypt_code -eq 0 ]]; then
@@ -243,12 +266,12 @@ function import_file() {
     case $importer in
       pcap)
         import_cmd="so-import-pcap $file --json"
-        response=$($CMD_PREFIX salt "$node" cmd.run "\"$import_cmd\"")
+        response=$(salt "$node" cmd.run "\"$import_cmd\"")
         exit_code=$?
         ;;
       evtx)
         import_cmd="so-import-evtx $file --json"
-        response=$($CMD_PREFIX salt "$node" cmd.run "\"$import_cmd\"")
+        response=$(salt "$node" cmd.run "\"$import_cmd\"")
         exit_code=$?
         ;;
       *)
@@ -269,45 +292,51 @@ function import_file() {
   if [[ exit_code -eq 0 ]]; then
     # trim off the node header ("manager_standalone:\n") and parse out the URL
     url=$(echo "$response" | tail -n +2 | jq -r .url)
-    $(echo "$url" > "${SOC_PIPE}")
+    respond "$id" "$url"
   else
     log "false"
-    $(echo "false" > "${SOC_PIPE}")
+    respond "$id" "false"
   fi
 }
 
+# Ensure there are not multiple salt-relay.sh programs running.
+num_relays_running=$(pgrep salt-relay.sh -c)
+if [[ $num_relays_running -gt 1 ]]; then
+  exit;
+fi
+
+# loop indefinitely
+log "Polling for requests: ${QUEUE_DIR}"
 while true; do
-  log "Listening for request"
+  request=$(poll)
-  request=$(cat ${SOC_PIPE})
   if [[ "$request" != "" ]]; then
     command=$(echo "$request" | jq -r .command)
-    log "Received request; command=${command}"
+    id=$(echo "$request" | jq -r .command_id)
+    log "Received request; command=${command}; id=${id}"
     case "$command" in
       list-minions)
-        list_minions
+        list_minions "$id" 
         ;;
       manage-minion)
-        manage_minion "${request}"
+        manage_minion "$id" "${request}"
         ;;
       manage-user)
-        manage_user "${request}"
+        manage_user "$id" "${request}"
         ;;
       manage-salt)
-        manage_salt "${request}"
+        manage_salt "$id" "${request}"
         ;;
       send-file)
-        send_file "${request}"
+        send_file "$id" "${request}"
         ;;
       import-file)
-        import_file "${request}"
+        import_file "$id" "${request}"
         ;;
       *)
         log "Unsupported command: $command"
-        $(echo "false" > "${SOC_PIPE}")
+        respond "$id" "false"
         ;;
     esac
-
-    # allow remote reader to get a clean reader before we try to read again on next loop
-    sleep 1
   fi
+  sleep $MIN_POLL_INTERVAL
 done