Merge branch 'dev' into 23100soup_jpp

pillar/elasticsearch/eval.sls

@@ -1,6 +1,7 @@
 elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
+    - so/so-case-template.json.jinja
     - so/so-common-template.json.jinja
     - so/so-firewall-template.json.jinja
     - so/so-flow-template.json.jinja

pillar/elasticsearch/manager.sls

@@ -1,6 +1,7 @@
 elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
+    - so/so-case-template.json.jinja
     - so/so-common-template.json.jinja
     - so/so-endgame-template.json.jinja
     - so/so-firewall-template.json.jinja

pillar/elasticsearch/search.sls

@@ -1,6 +1,7 @@
 elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
+    - so/so-case-template.json.jinja
     - so/so-common-template.json.jinja
     - so/so-endgame-template.json.jinja
     - so/so-firewall-template.json.jinja

salt/common/tools/sbin/so-common

@@ -298,6 +298,7 @@ retry() {
         sleepDelay=$2
         cmd=$3
         expectedOutput=$4
+        failedOutput=$5
         attempt=0
         local exitcode=0
         while [[ $attempt -lt $maxAttempts ]]; do
@@ -308,12 +309,28 @@ retry() {
                 echo "Results: $output ($exitcode)"
                 if [ -n "$expectedOutput" ]; then
                         if [[ "$output" =~ "$expectedOutput" ]]; then
-				return $exitCode
+                                return $exitcode
                         else
-				echo "Expected '$expectedOutput' but got '$output'"
+                                echo "Did not find expectedOutput: '$expectedOutput' in the output below from running the command: '$cmd'"
+								echo "<Start of output>"
+								echo "$output"
+								echo "<End of output>"
+                        fi
+                elif [ -n "$failedOutput" ]; then
+                        if [[ "$output" =~ "$failedOutput" ]]; then
+                                echo "Found failedOutput: '$failedOutput' in the output below from running the command: '$cmd'"
+								echo "<Start of output>"
+								echo "$output"
+								echo "<End of output>"
+								if [[ $exitcode -eq 0 ]]; then
+									echo "The exitcode was 0, but we are setting to 1 since we found $failedOutput in the output."
+									exitcode=1
+								fi
+                        else
+                                return $exitcode
                         fi
                 elif [[ $exitcode -eq 0 ]]; then
-			return $exitCode
+                        return $exitcode
                 fi
                 echo "Command failed with exit code $exitcode; will retry in $sleepDelay seconds ($attempt / $maxAttempts)..."
                 sleep $sleepDelay

salt/common/tools/sbin/soup

@@ -1221,8 +1221,16 @@ Please review the following for more information about the update process and re
 https://docs.securityonion.net/soup
 https://blog.securityonion.net
 
-Press Enter to continue or Ctrl-C to cancel.
+EOF
 
+  if [ -n "$BRANCH" ]; then
+    cat << EOF
+SOUP will use the $BRANCH branch.
+
+EOF
+  fi
+  cat << EOF
+Press Enter to continue or Ctrl-C to cancel.
 EOF
 
   read -r input

salt/elasticsearch/defaults.yaml

@@ -56,6 +56,12 @@ elasticsearch:
       query:
         bool:
           max_clause_count: 1500
+      id_field_data:
+        enabled: false
+    logger:
+      org:
+        elasticsearch:
+          deprecation: ERROR
 
 
 

salt/elasticsearch/files/ingest/ossec

@@ -39,6 +39,12 @@
     { "rename":         { "field": "decoder.name",                   "target_field": "event.dataset",         "ignore_missing": true  } },
     { "rename":         { "field": "rule.description",                   "target_field": "rule.name",         "ignore_missing": true  } },
     { "rename":         { "field": "rule.id",                            "target_field": "rule.uuid",         "ignore_missing": true  } },
+    { "script": {
+        "if": "ctx.winlog?.event_data != null", 
+        "lang": "painless", 
+        "source": "Map eventData = ctx['winlog']['event_data']; Map updatedEventData = new HashMap(); for (String key: eventData.keySet()) { updatedEventData[key.substring(0,1).toUpperCase() + key.substring(1)] = eventData[key] } ctx['winlog']['event_data'] = updatedEventData"
+            }
+          },
     { "set":            { "if": "ctx.rule != null && ctx.rule.level == 1",          "field": "rule.category", "value": "None"                               } },
     { "set":            { "if": "ctx.rule != null && ctx.rule.level == 2",          "field": "rule.category", "value": "System low priority notification"   } },
     { "set":            { "if": "ctx.rule != null && ctx.rule.level == 3",          "field": "rule.category", "value": "Successful/authorized event"        } },

salt/elasticsearch/init.sls

@@ -24,6 +24,7 @@ include:
 {% set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip') %}
+{% set ESMOUNT = salt['pillar.get']('elasticsearch:extramount', False) %}
 
 {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %}
   {% set esclustername = salt['pillar.get']('manager:esclustername') %}
@@ -288,6 +289,9 @@ so-elasticsearch:
       - /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro
       - /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro
       {% endif %}
+      {% if ESMOUNT %}
+      - {{ ESMOUNT }}:/snapshots:rw
+      {% endif %}
     - watch:
       - file: cacertz
       - file: esyml
@@ -337,6 +341,8 @@ so-elasticsearch-pipelines:
       - file: esingestdynamicconf
       - file: esyml
       - file: so-elasticsearch-pipelines-file
+    - require:
+      - docker_container: so-elasticsearch
 
 {% if TEMPLATES %}
 so-elasticsearch-templates:
@@ -344,6 +350,8 @@ so-elasticsearch-templates:
     - name: /usr/sbin/so-elasticsearch-templates-load
     - cwd: /opt/so
     - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
 {% endif %}
 
 so-elasticsearch-roles-load:
@@ -351,6 +359,8 @@ so-elasticsearch-roles-load:
     - name: /usr/sbin/so-elasticsearch-roles-load
     - cwd: /opt/so
     - template: jinja
+    - require:
+      - docker_container: so-elasticsearch
 
 {% endif %} {# if grains['role'] != 'so-helix' #}
 

salt/elasticsearch/templates/so/so-case-template.json.jinja

@@ -0,0 +1,261 @@
+{%- set INDEX_SORTING = salt['pillar.get']('elasticsearch:index_sorting', True) %}
+{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
+{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-common:refresh', '30s') %}
+{
+  "index_patterns": ["so-case*"],
+  "version":50001,
+  "order":10,
+  "settings":{
+    "number_of_replicas":{{ REPLICAS }},
+    "number_of_shards":1,
+    "index.refresh_interval":"{{ REFRESH }}",
+    "index.routing.allocation.require.box_type":"hot",
+{%- if INDEX_SORTING is sameas true %}
+    "index.sort.field": "@timestamp",
+    "index.sort.order": "desc",
+{%- endif %}
+    "index.mapping.total_fields.limit": "1500"
+  },
+  "mappings": {
+    "_meta": {
+      "version": "1.5.0"
+    },
+    "date_detection": false,
+    "dynamic_templates": [
+      {
+        "strings_as_keyword": {
+          "mapping": {
+            "ignore_above": 1024,
+            "type": "keyword"
+          },
+          "match_mapping_type": "string"
+        }
+      }
+    ],
+    "properties": {
+      "@timestamp": {
+        "type": "date"
+      },
+      "kind": {
+        "type": "keyword",
+        "ignore_above": 1024
+      },
+      "operation": {
+        "type": "keyword",
+        "ignore_above": 1024
+      },      
+      "so_audit_doc_id": {
+        "type": "keyword",
+        "ignore_above": 1024
+      },
+      "artifact": {
+        "properties": {
+          "artifactType": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "caseId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "createTime": {
+            "type": "date"
+          },
+          "description": {
+            "type": "text"
+          },
+          "groupId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "groupType": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "ioc": {
+            "type": "boolean"
+          },
+          "md5": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "mimeType": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "sha1": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "sha256": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "streamId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "streamLength": {
+            "type": "long"
+          },
+          "tags": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "tlp": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "userId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "value": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 1024
+              }
+            }
+          }
+        }
+      },
+      "artifactstream": {
+        "properties": {
+          "content": {
+            "type": "text"
+          },
+          "createTime": {
+            "type": "date"
+          },
+          "userId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          }
+        }
+      },
+      "case": {
+        "properties": {
+          "assigneeId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "category": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "completeTime": {
+            "type": "date"
+          },
+          "createTime": {
+            "type": "date"
+          },
+          "description": {
+            "type": "text"
+          },
+          "pap": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "priority": {
+            "type": "long"
+          },
+          "severity": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "startTime": {
+            "type": "date"
+          },
+          "status": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "tags": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "template": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "title": {
+            "type": "text"
+          },
+          "tlp": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "userId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          }
+        }
+      },
+      "comment": {
+        "properties": {
+          "caseId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "createTime": {
+            "type": "date"
+          },
+          "description": {
+            "type": "text"
+          },
+          "userId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          }
+        }
+      },
+      "related": {
+        "properties": {
+          "caseId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          },
+          "createTime": {
+            "type": "date"
+          },
+          "fields": {
+            "properties": {
+              "@timestamp": {
+                "type": "date"
+              },
+              "event": {
+                "properties": {
+                  "dataset": {
+                    "type": "keyword",
+                    "ignore_above": 1024
+                  },
+                  "module": {
+                    "type": "keyword",
+                    "ignore_above": 1024
+                  },
+                  "category": {
+                    "type": "keyword",
+                    "ignore_above": 1024
+                  }                  
+                }
+              },
+              "message": {
+                "type": "text"
+              },
+              "tags": {
+                "type": "keyword",
+                "ignore_above": 1024
+              }
+            }
+          },
+          "userId": {
+            "type": "keyword",
+            "ignore_above": 1024
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/so/so-common-template.json.jinja

@@ -165,9 +165,56 @@
             }
           }
         },
-        "agent":{
+        "agent": {
           "type":"object",
-          "dynamic": true
+          "dynamic": true,
+          "properties": {
+            "ephemeral_id": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
         },     
         "as":{
           "type":"object",
@@ -225,17 +272,180 @@
           "type":"object",
           "dynamic": true
         },
-        "ecs":{
+        "ecs": {
-          "type":"object",
+          "properties": {
-          "dynamic": true
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
         },
         "error":{
           "type":"object",
           "dynamic": true
         },
-        "event":{
+        "event": {
-          "type":"object",
+          "properties": {
-          "dynamic": true
+	    "acknowledged": {
+              "type": "boolean",
+	      "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "action": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "category": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "code": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "created": {
+              "type": "date",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "dataset": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            }, 
+            "duration": {
+              "type": "long"
+            },
+            "end": {
+              "type": "date"
+            },
+	    "escalated": {
+              "type": "boolean",
+	      "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "hash": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "ingested": {
+              "type": "date",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "kind": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "module": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "original": {
+              "doc_values": false,
+              "ignore_above": 1024,
+              "index": false,
+              "type": "keyword"
+            },
+            "outcome": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "provider": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "reference": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "risk_score": {
+              "type": "float"
+            },
+            "risk_score_norm": {
+              "type": "float"
+            },
+            "sequence": {
+              "type": "long"
+            },
+            "severity": {
+              "type": "long"
+            },
+	    "severity_label": {
+              "ignore_above": 1024,
+              "type": "keyword",
+	      "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "start": {
+              "type": "date"
+            },
+            "timezone": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "url": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
         },
         "event_data":{
           "type":"object",
@@ -267,11 +477,97 @@
         },
         "host":{
           "type":"object",
-          "dynamic": true
+          "dynamic": true,
+          "properties": {
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
         },
         "http":{
           "type":"object",
-          "dynamic": true
+          "dynamic": true,
+          "properties": {
+            "request": {
+              "properties": {
+                "body": {
+                  "properties": {
+                    "bytes": {
+                      "type": "long"
+                    },
+                    "content": {
+                      "fields": {
+                        "text": {
+                          "norms": false,
+                          "type": "text"
+                        }
+                      },
+                      "ignore_above": 1024,
+                      "type": "keyword"
+                    }
+                  }
+                },
+                "bytes": {
+                  "type": "long"
+                },
+                "method": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword"
+                    }
+                  }
+                },
+                "referrer": {
+                  "ignore_above": 1024,
+                  "type": "keyword",
+                  "fields": {
+                    "keyword": {
+                      "type": "keyword"
+                    }
+                  }
+                }
+              }
+            },
+            "response": {
+              "properties": {
+                "body": {
+                  "properties": {
+                    "bytes": {
+                      "type": "long"
+                    },
+                    "content": {
+                      "fields": {
+                        "text": {
+                          "norms": false,
+                          "type": "text"
+                        }
+                      },
+                      "ignore_above": 1024,
+                      "type": "keyword"
+                    }
+                  }
+                },
+                "bytes": {
+                  "type": "long"
+                },
+                "status_code": {
+                  "type": "long"
+                }
+              }
+            },
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
         },
         "import":{
           "type":"object",
@@ -318,7 +614,18 @@
         },
         "log":{
           "type":"object",
-          "dynamic": true
+          "dynamic": true,
+          "properties": {
+            "level": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
         },
         "logscan": {
           "type": "object",
@@ -436,7 +743,27 @@
         },
         "service":{
           "type":"object",
-          "dynamic": true
+          "dynamic": true,
+          "properties": {
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "name": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
         },
         "sip":{
           "type":"object",
@@ -464,7 +791,18 @@
         },
 	"source":{
           "type":"object",
-          "dynamic": true
+          "dynamic": true,
+          "properties" : {
+            "address": {
+              "ignore_above": 1024,
+              "type": "keyword",
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
         },
         "ssh":{
           "type":"object",
@@ -478,11 +816,12 @@
           "type":"object",
           "dynamic": true
         },
-        "tags":{
+        "tags": {
-          "type":"text",
+          "ignore_above": 1024,
-          "fields":{
+          "type": "keyword",
-            "keyword":{
+          "fields": {
-              "type":"keyword"
+            "keyword": {
+              "type": "keyword"
             }
           }
         },
@@ -508,7 +847,22 @@
         },
         "user_agent":{
           "type":"object",
-          "dynamic": true
+          "dynamic": true,
+          "properties": {
+            "original": {
+              "fields": {
+                "keyword": {
+                  "type": "keyword"
+                },
+                "text": {
+                  "norms": false,
+                  "type": "text"
+                }
+              },
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
         },
         "version":{
           "type":"object",

salt/logstash/etc/logstash.yml

@@ -219,6 +219,8 @@ path.logs: /var/log/logstash
 # path.plugins: []
 {% set pipeline_workers = salt['pillar.get']('logstash_settings:ls_pipeline_workers', '1') %}
 {% set pipeline_batch = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', '125') %}
+{% set pipeline_ecs_compatibility = salt['pillar.get']('logstash_settings:ls_ecs_compatibility', 'disabled') %}
 
 pipeline.workers: {{ pipeline_workers }}
 pipeline.batch.size: {{ pipeline_batch }}
+pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }}

salt/soc/files/soc/cases.queries.json

@@ -1,5 +1,5 @@
 [
-  { "name": "Open Cases", "query": "!case.status:Closed AND !case.category:Template" },
+  { "name": "Open Cases", "query": "NOT case.status:closed AND NOT case.category:template" },
-  { "name": "Closed Cases", "query": "case.status:Closed AND !case.category:Template" },
+  { "name": "Closed Cases", "query": "case.status:closed AND NOT case.category:template" },
-  { "name": "Templates", "query": "case.category:Template" }
+  { "name": "Templates", "query": "case.category:template" }
 ]

salt/soc/files/soc/menu.actions.json

@@ -18,7 +18,8 @@
             "links": [
               "/joblookup?esid={:soc_id}&time={:@timestamp}", 
               "/joblookup?ncid={:network.community_id}&time={:@timestamp}"
-            ]},
+            ],
+            "categories": ["hunt", "alerts"]},
   { "name": "actionCyberChef", "description": "actionCyberChefHelp", "icon": "fas fa-bread-slice", "target": "_blank", 
             "links": [
               "/cyberchef/#input={value|base64}"

salt/soc/files/soc/presets.category.json

@@ -1,7 +1,7 @@
 {
   "labels": [
-    "General",
+    "general",
-    "Template"
+    "template"
   ],
   "customEnabled": true
 }

salt/soc/files/soc/presets.pap.json

@@ -1,9 +1,9 @@
 {
   "labels": [
-    "White",
+    "white",
-    "Green",
+    "green",
-    "Amber",
+    "amber",
-    "Red"
+    "red"
   ],
   "customEnabled": false
 }

salt/soc/files/soc/presets.severity.json

@@ -1,9 +1,9 @@
 {
   "labels": [
-    "Low",
+    "low",
-    "Medium",
+    "medium",
-    "High",
+    "high",
-    "Critical"
+    "critical"
   ],
   "customEnabled": false
 }

salt/soc/files/soc/presets.status.json

@@ -1,8 +1,8 @@
 {
   "labels": [
-    "New",
+    "new",
-    "In Progress",
+    "in progress",
-    "Closed"
+    "closed"
   ],
   "customEnabled": false
 }

salt/soc/files/soc/presets.tlp.json

@@ -1,9 +1,9 @@
 {
   "labels": [
-    "White",
+    "white",
-    "Green",
+    "green",
-    "Amber",
+    "amber",
-    "Red"
+    "red"
   ],
   "customEnabled": false
 }

salt/soc/files/soc/soc.json

@@ -3,8 +3,8 @@
 {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
 {%- set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
 {%- set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
-{% set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) %}
+{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) %}
-{% set FLEETNODE = salt['pillar.get']('global:fleet_node', False) %}
+{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) %}
 {%- set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
 {%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
 {%- set API_TIMEOUT = salt['pillar.get']('sensoroni:api_timeout_ms', 0) %}
@@ -89,23 +89,23 @@
         "refreshIntervalMs": 30000,
         "offlineThresholdMs": 900000
       },
-{% if CASE_MODULE == 'thehive' and THEHIVEKEY != '' %}
+{%- if CASE_MODULE == 'thehive' and THEHIVEKEY != '' %}
       "thehive": {
         "hostUrl": "http://{{ MANAGERIP }}:9000/thehive",
         "key": "{{ THEHIVEKEY }}",
         "verifyCert": false
       },
-{% elif CASE_MODULE == 'elasticcases' %}
+{%- elif CASE_MODULE == 'elasticcases' %}
       "elasticcases": {
         "hostUrl": "https://{{ MANAGERIP }}:5601",
         "username": "{{ ES_USER }}",
         "password": "{{ ES_PASS }}",
       },
-{% elif CASE_MODULE == 'generichttp' %}
+{%- elif CASE_MODULE == 'generichttp' %}
       "generichttp": {
         {{ GENERIC_CASE_CONFIG }}
       },
-{% endif %}
+{%- endif %}
       "statickeyauth": {
         "anonymousCidr": "{{ DNET }}/24",
         "apiKey": "{{ SENSORONIKEY }}"
@@ -166,7 +166,9 @@
         "escalateRelatedEventsEnabled": {{ 'true' if CASE_MODULE == 'soc' else 'false' }},
         "eventFields": {{ hunt_eventfields | json }},
         "queryBaseFilter": "",
-        "queryToggleFilters": [],
+        "queryToggleFilters": [
+          { "name": "caseExcludeToggle", "filter": "NOT _index:\"*:so-case*\"", "enabled": true }
+        ],
         "queries": {{ hunt_queries | json }},
         "actions": {{ menu_actions | json }}           
       },
@@ -207,8 +209,9 @@
         "escalateEnabled": false,
         "escalateRelatedEventsEnabled": false,
         "viewEnabled": true,
+        "createLink": "/case/create",
         "eventFields": {{ cases_eventfields | json }},
-        "queryBaseFilter": "_index:so-case AND kind:case",
+        "queryBaseFilter": "_index:\"*:so-case\" AND kind:case",
         "queryToggleFilters": [
         ],
         "queries": {{ cases_queries | json }},

salt/wazuh/files/agent/wazuh-register-agent

@@ -60,8 +60,14 @@ register_agent() {
   GOT_KEY=$?
 
   if [[ -z "$AGENT_ID" || -z "$AGENT_KEY" || $GOT_ID -ne 0 || $GOT_KEY -ne 0 ]]; then
+    if echo "$API_RESULT" | jq -er .message | grep -q "There is an agent with the same name"; then
+      echo "Agent $AGENT_NAME already registered!"
+      touch /opt/so/conf/wazuh/initial_agent_registration.log
+      exit 0
+    else
       echo "Failed Result: $API_RESULT"
       return 1
+    fi
   else
     echo "Agent '$AGENT_NAME' with ID '$AGENT_ID' added."
     echo "Key for agent '$AGENT_ID' received."

salt/wazuh/init.sls

@@ -127,7 +127,7 @@ registertheagent:
   cmd.run:
     - name: /usr/sbin/wazuh-register-agent
     - cwd: /
-    #- stateful: True
+    - unless: ls /opt/so/conf/wazuh/initial_agent_registration.log
 
 # Whitelist manager IP
 whitelistmanager:

setup/so-functions

@@ -1033,6 +1033,8 @@ copy_minion_tmp_files() {
 			} >> "$setup_log" 2>&1
 			;;
 	esac
+	echo "Syncing all salt modules." >> "$setup_log" 2>&1
+	salt-call saltutil.sync_modules >> "$setup_log" 2>&1
 }
 
 copy_ssh_key() {
@@ -1145,7 +1147,7 @@ installer_prereq_packages() {
 		logCmd "systemctl start NetworkManager"
 	elif [ "$OS" == ubuntu ]; then
 		# Print message to stdout so the user knows setup is doing something
-		retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+		retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 		# Install network manager so we can do interface stuff
 		if ! command -v nmcli > /dev/null 2>&1; then
 			retry 50 10 "apt-get -y install network-manager" >> "$setup_log" 2>&1 || exit 1
@@ -1202,18 +1204,24 @@ docker_install() {
 	else
 		case "$install_type" in
 			'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT')
-				retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+				retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 				;;
 			*)
 				retry 50 10 "apt-key add $temp_install_dir/gpg/docker.pub" >> "$setup_log" 2>&1 || exit 1
 				add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" >> "$setup_log" 2>&1
-				retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+				retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 				;;
 		esac
 		if [ $OSVER == "bionic" ]; then
+		        service docker stop
+			apt -y purge docker-ce docker-ce-cli docker-ce-rootless-extras
 			retry 50 10 "apt-get -y install docker-ce=5:20.10.5~3-0~ubuntu-bionic docker-ce-cli=5:20.10.5~3-0~ubuntu-bionic docker-ce-rootless-extras=5:20.10.5~3-0~ubuntu-bionic python3-docker" >> "$setup_log" 2>&1 || exit 1
+	        apt-mark hold docker-ce docker-ce-cli docker-ce-rootless-extras
 		elif [ $OSVER == "focal" ]; then
+		        service docker stop
+			apt -y purge docker-ce docker-ce-cli docker-ce-rootless-extras
 			retry 50 10 "apt-get -y install docker-ce=5:20.10.8~3-0~ubuntu-focal docker-ce-cli=5:20.10.8~3-0~ubuntu-focal docker-ce-rootless-extras=5:20.10.8~3-0~ubuntu-focal python3-docker" >> "$setup_log" 2>&1 || exit 1
+                        apt-mark hold docker-ce docker-ce-cli docker-ce-rootless-extras		
 		fi
 	fi
 	docker_registry
@@ -1436,16 +1444,20 @@ generate_ca() {
 		echo "Building Certificate Authority";
 		salt-call state.apply ca;
 
-		echo " Confirming existence of the CA certificate"
+		echo "Confirming existence of the CA certificate"
 		openssl x509 -in /etc/pki/ca.crt -noout -subject -issuer -dates
 	} >> "$setup_log" 2>&1
 }
 
 generate_ssl() {
 	{
+		# if the install type is a manager then we need to wait for the minion to be ready before trying
+		# to run the ssl state since we need the minion to sign the certs
+		if [[ "$install_type" =~ ^(EVAL|MANAGER|MANAGERSEARCH|STANDALONE|IMPORT|HELIXSENSOR)$ ]]; then
+			wait_for_salt_minion
+		fi
 		echo "Applying SSL state";
 		salt-call state.apply ssl;
-		salt-call saltutil.sync_modules;
 	} >> "$setup_log" 2>&1
 }
 
@@ -2030,6 +2042,9 @@ reserve_ports() {
 reinstall_init() {
 	info "Putting system in state to run setup again"
 	
+	# remove all of root's cronjobs
+	crontab -r -u root
+
 	if [[ $install_type =~ ^(MANAGER|EVAL|HELIXSENSOR|MANAGERSEARCH|STANDALONE|FLEET|IMPORT)$ ]]; then
 		local salt_services=( "salt-master" "salt-minion" )
 	else
@@ -2069,7 +2084,7 @@ reinstall_init() {
 		done
 
 		# Remove all salt configs
-		rm -rf /etc/salt/grains /etc/salt/minion /etc/salt/pki/*
+		rm -rf /etc/salt/engines/* /etc/salt/grains /etc/salt/master /etc/salt/master.d/* /etc/salt/minion /etc/salt/minion.d/* /etc/salt/pki/* /etc/salt/proxy /etc/salt/proxy.d/* /var/cache/salt/
 
 		if command -v docker &> /dev/null; then
 			# Stop and remove all so-* containers so files can be changed with more safety
@@ -2084,6 +2099,12 @@ reinstall_init() {
 
 		# Backup /opt/so since we'll be rebuilding this directory during setup
 		backup_dir /opt/so "$date_string"
+		# We need to restore these files during a reinstall so python3-influxdb state doesn't try to patch again
+		restore_file "/opt/so_old_$date_string/state/influxdb_continuous_query.py.patched" "/opt/so/state/"
+		restore_file "/opt/so_old_$date_string/state/influxdb_retention_policy.py.patched" "/opt/so/state/"
+		restore_file "/opt/so_old_$date_string/state/influxdbmod.py.patched" "/opt/so/state/"
+		# If the elastic license has been accepted restore the state file
+		restore_file "/opt/so_old_$date_string/state/yeselastic.txt" "/opt/so/state/"
 
 		# Backup directories in /nsm to prevent app errors
 		backup_dir /nsm/mysql "$date_string"
@@ -2117,6 +2138,16 @@ reset_proxy() {
 	fi
 }
 
+restore_file() {
+	src=$1
+	dst=$2
+	if [ -f "$src" ]; then
+		[ ! -d "$dst" ] && mkdir -v -p "$dst"
+		echo "Restoring $src to $dst." >> "$setup_log" 2>&1
+		cp -v "$src" "$dst"
+	fi
+}
+
 backup_dir() {
 	dir=$1
 	backup_suffix=$2
@@ -2253,7 +2284,7 @@ saltify() {
 				# Add repo
 				echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
 				
-				retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+				retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 				set_progress_str 6 'Installing various dependencies'
 				retry 50 10 "apt-get -y install sqlite3 libssl-dev" >> "$setup_log" 2>&1 || exit 1
 				set_progress_str 7 'Installing salt-master'
@@ -2273,7 +2304,7 @@ saltify() {
 				;;
 		esac
 
-		retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+		retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 		set_progress_str 8 'Installing salt-minion & python modules'
 		retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1
 		retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1
@@ -2787,6 +2818,10 @@ wait_for_file() {
 	return 1
 }
 
+wait_for_salt_minion() {
+	retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$setup_log" 2>&1 || exit 1
+}
+
 # Enable Zeek Logs
 zeek_logs_enabled() {
 	echo "Enabling Zeek Logs" >> "$setup_log" 2>&1

setup/so-setup

@@ -766,10 +766,8 @@ echo "1" > /root/accept_changes
 		generate_ca >> $setup_log 2>&1
 	fi
 
-	if [[ $is_minion ]]; then
 	set_progress_str 24 'Generating SSL'
 	generate_ssl >> $setup_log 2>&1
-	fi
 
 	if [[ $is_manager || $is_helix || $is_import ]]; then
 		set_progress_str 25 'Configuring firewall'

setup/so-whiptail

@@ -213,7 +213,7 @@ whiptail_create_web_user() {
 	[ -n "$TESTING" ] && return
 
 	WEBUSER=$(whiptail --title "$whiptail_title" --inputbox \
-	"Please enter an email address to create an administrator account for the web interface.\n\nThis will also be used for Elasticsearch, Kibana, TheHive, Cortex, and Fleet." 12 60 "$1" 3>&1 1>&2 2>&3)
+	"Please enter an email address to create an administrator account for the web interface.\n\nThis will also be used for Elasticsearch, Kibana, and Fleet." 12 60 "$1" 3>&1 1>&2 2>&3)
 
 	local exitstatus=$?
 	whiptail_check_exitstatus $exitstatus
@@ -396,7 +396,6 @@ if [[ $is_eval ]]; then
 	GRAFANA "Enable Grafana for system monitoring" ON \
 	OSQUERY "Enable Fleet with osquery" ON \
 	WAZUH "Enable Wazuh" ON \
-	THEHIVE "Enable TheHive" ON \
 	PLAYBOOK "Enable Playbook" ON \
 	STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3)
 else
@@ -404,7 +403,6 @@ else
 	"$description" 20 75 7 \
 	OSQUERY "Enable Fleet with osquery" ON \
 	WAZUH "Enable Wazuh" ON \
-	THEHIVE "Enable TheHive" ON \
 	PLAYBOOK "Enable Playbook" ON \
 	STRELKA "Enable Strelka" ON 3>&1 1>&2 2>&3)
 	export "GRAFANA=1"