Merge pull request #16366 from Security-Onion-Solutions/3/dev

3.4.0

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -14,7 +14,6 @@ body:
         - 3.2.0
         - 3.3.0
         - 3.4.0
-        - 3.5.0
         - Other (please provide detail below)
     validations:
       required: true

VERSION

@@ -1 +1 @@
-3.5.0
+3.4.0

salt/desktop/packages.sls

@@ -438,10 +438,6 @@ install_networkminer:
   pkg.latest:
     - name: securityonion-networkminer
 
-install_so_crates:
-  pkg.latest:
-    - name: securityonion-so-crates
-
 {% else %}
 
 desktop_packages_os_fail:

salt/docker/defaults.yaml

@@ -103,6 +103,7 @@ docker:
         - 80:80
         - 443:443
         - 8443:8443
+        - 7788:7788
         - 7789:7789
       custom_bind_mounts: []
       extra_hosts: []

salt/elasticsearch/cluster.sls

@@ -107,12 +107,6 @@ so-elasticsearch-system-indices-patch:
       - file: so-elasticsearch-system-indices-patch-script
 {%  endif %}
 
-so-elasticsearch-ilm-policy-reload:
-  file.absent:
-    - name: /opt/so/state/esilm.txt
-    - onchanges:
-      - file: so-elasticsearch-ilm-policy-load-script
-
 # heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
 so-elasticsearch-ilm-policy-load:
   cmd.run:
@@ -121,9 +115,8 @@ so-elasticsearch-ilm-policy-load:
     - require:
       - docker_container: so-elasticsearch
       - file: so-elasticsearch-ilm-policy-load-script
-      - http: wait_for_so-elasticsearch
-      - file: so-elasticsearch-ilm-policy-reload
-    - unless: test -f /opt/so/state/esilm.txt
+    - onchanges:
+      - file: so-elasticsearch-ilm-policy-load-script
 
 so-elasticsearch-templates-reload:
   file.absent:

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load

@@ -7,17 +7,11 @@
 . /usr/sbin/so-common
 
 MAX_JOBS=${MAX_ILM_JOBS:-10}
-ILM_STATEFILE=/opt/so/state/esilm.txt
-rm -f "$ILM_STATEFILE"
 
-check_elasticsearch_responsive
-
-ILM_TMP_DIR=$(mktemp -d)
-trap 'rm -rf "$ILM_TMP_DIR"' EXIT
 # Lock used to serialize block writes so concurrent jobs never interleave their output.
-ILM_OUTPUT_LOCK="$ILM_TMP_DIR/output.lock"
-ILM_FAIL_FILE="$ILM_TMP_DIR/failed"
-touch "$ILM_FAIL_FILE"
+ILM_OUTPUT_LOCK=$(mktemp)
+ILM_FAIL_FILE=$(mktemp)
+trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT
 
 # Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
 # printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
@@ -26,30 +20,13 @@ echo "Loading ILM policies concurrently; output below appears in completion orde
 echo
 
 put_policy() {
-  local desc="$1" policyname="$2" data="$3" rc=1 attempt out http_code body result reason tmpdata
-  tmpdata=$(mktemp "$ILM_TMP_DIR/${policyname}.XXXXXX.json")
-  printf '%s' "$data" >"$tmpdata"
-
-  for attempt in 1 2 3; do
-    out=$(so-elasticsearch-query "_ilm/policy/${policyname}" -d@"${tmpdata}" -XPUT -w '\n%{http_code}' 2>/dev/null)
-    http_code=${out##*$'\n'} body=${out%$'\n'*}
-
-    if [[ "$http_code" == 200 ]] && jq -e '.acknowledged == true' <<<"$body" >/dev/null 2>&1; then
-      rc=0
-      break
-    fi
-    # 429 = ES rejected/throttled (may succeed later); other 4xx will not
-    [[ "$http_code" == 4?? && "$http_code" != 429 ]] && break
-    (( attempt < 3 )) && sleep 3
-  done
-
-  if (( rc == 0 )); then
-    result='{"acknowledged":true}'
-  else
-    result=${body:-"HTTP ${http_code:-000}"}
-    # flatten tabs/newlines so the TAB fail-file stays one record
-    reason=$(jq -r '(if (.error|type) == "object" then (.error.reason // .error.type) elif (.error|type) == "string" then .error else empty end) // empty | tostring | gsub("[\t\r\n]+"; " ")' <<<"$body" 2>/dev/null)
-    reason="HTTP ${http_code:-000}${reason:+: ${reason}}"
+  local desc="$1" policyname="$2" data="$3" result rc=0
+  if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \
+    -X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
+    -H 'Content-Type: application/json' -d"${data}" 2>&1); then
+    rc=1
+  elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then
+    rc=1
   fi
 
   # curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
@@ -57,7 +34,7 @@ put_policy() {
     flock 200
     printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
     if (( rc != 0 )); then
-      printf '%s\t%s\n' "${policyname}" "${reason}" >>"$ILM_FAIL_FILE"
+      printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
     fi
   } 200>>"${ILM_OUTPUT_LOCK}"
 
@@ -105,15 +82,10 @@ wait || true
 
 if [[ -s "$ILM_FAIL_FILE" ]]; then
   echo "ERROR: Failed to load ILM policy(s):"
-  while IFS=$'\t' read -r POLICY REASON; do
-    if [[ -n "$REASON" ]]; then
-      echo " - ${POLICY}: ${REASON}"
-    else
-      echo " - ${POLICY}"
-    fi
+  while read -r POLICY; do
+    echo " - $POLICY"
   done < "$ILM_FAIL_FILE"
   exit 1
 else
   echo "Successfully loaded all ILM policies."
-  touch "$ILM_STATEFILE"
 fi

salt/firewall/defaults.yaml

@@ -118,6 +118,10 @@ firewall:
       tcp:
         - 5601
       udp: []
+    localrules:
+      tcp:
+        - 7788
+      udp: []
     nginx:
       tcp:
         - 80
@@ -196,6 +200,7 @@ firewall:
                 - postgres
                 - elasticsearch_rest
                 - elasticsearch_node
+                - localrules
             sensor:
               portgroups:
                 - beats_5044
@@ -386,6 +391,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - localrules
                 - sensoroni
             fleet:
               portgroups:
@@ -605,6 +611,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - localrules
                 - sensoroni
             fleet:
               portgroups:
@@ -822,6 +829,7 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+                - localrules
                 - sensoroni
             fleet:
               portgroups:
@@ -1050,6 +1058,7 @@ firewall:
                 - elastic_agent_update
                 - endgame
                 - strelka_frontend
+                - localrules
             fleet:
               portgroups:
                 - elasticsearch_rest
@@ -1509,6 +1518,7 @@ firewall:
                 - elasticsearch_rest
                 - elasticsearch_node
                 - elastic_agent_control
+                - localrules
             sensor:
               portgroups:
                 - beats_5044

salt/firewall/soc_firewall.yaml

@@ -138,6 +138,9 @@ firewall:
     kibana:
       tcp: *tcpsettings
       udp: *udpsettings
+    localrules:
+      tcp: *tcpsettings
+      udp: *udpsettings
     nginx:
       tcp: *tcpsettings
       udp: *udpsettings

salt/manager/tools/sbin/so-yaml.py

@@ -20,7 +20,6 @@ def showUsage(args):
     print('    append           - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr)
     print('    removelistitem   - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
-    print('    purgelistitem    - Remove every occurrence of a list item from all lists nested under a yaml key, if it exists. Requires KEY and LISTITEM args.', file=sys.stderr)
     print('    replacelistobject  - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
     print('    add              - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
     print('    get [-r]         - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
@@ -200,33 +199,6 @@ def removelistitem(args):
     return 0
 
 
-def purgeListItem(content, listItem):
-    if isinstance(content, dict):
-        for value in content.values():
-            purgeListItem(value, listItem)
-    elif isinstance(content, list):
-        content[:] = [item for item in content if item != listItem]
-        for item in content:
-            purgeListItem(item, listItem)
-
-
-def purgelistitem(args):
-    if len(args) != 3:
-        print('Missing filename, key arg, or list item to purge', file=sys.stderr)
-        showUsage(None)
-        return 1
-
-    filename = args[0]
-    key = args[1]
-    listItem = args[2]
-
-    content = loadYaml(filename)
-    purgeListItem(getKeyValue(content, key), convertType(listItem))
-    writeYaml(filename, content)
-
-    return 0
-
-
 def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject):
     pieces = key.split(".", 1)
     if len(pieces) > 1:
@@ -413,7 +385,6 @@ def main():
         "append": append,
         "appendlistobject": appendlistobject,
         "removelistitem": removelistitem,
-        "purgelistitem": purgelistitem,
         "replacelistobject": replacelistobject,
         "get": get,
         "remove": remove,

salt/manager/tools/sbin/so-yaml_test.py

@@ -709,223 +709,6 @@ class TestRemoveListItem(unittest.TestCase):
                 self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
 
 
-class TestPurgeListItem(unittest.TestCase):
-
-    def test_purgelistitem_missing_arg(self):
-        with patch('sys.exit', new=MagicMock()) as sysmock:
-            with patch('sys.stderr', new=StringIO()) as mock_stderr:
-                sys.argv = ["cmd", "help"]
-                soyaml.purgelistitem(["file", "key"])
-                sysmock.assert_called()
-                self.assertIn("Missing filename, key arg, or list item to purge", mock_stderr.getvalue())
-
-    def test_purgelistitem(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [a,b,c], key2: false}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- a\n- c\nkey2: false\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_nested_lists(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: [a,b,c], child2: { deep1: [b,d], deep2: 45 } }, key2: [b,e]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1:\n  - a\n  - c\n  child2:\n    deep1:\n    - d\n    deep2: 45\nkey2:\n- b\n- e\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_duplicates(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [b,a,b,c,b]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- a\n- c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_list_of_objects(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [{name: item1, values: [a,b]}, b, {name: item2, values: [b,c]}]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- name: item1\n  values:\n  - a\n- name: item2\n  values:\n  - c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_typed(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [1, '1', 2]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "1"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- '1'\n- 2\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_item_not_in_list(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: [a,b,c] }}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "d"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1:\n  - a\n  - b\n  - c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_key_noexist(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [a,b,c]}")
-        file.close()
-
-        result = soyaml.purgelistitem([filename, "key2.child1", "b"])
-        self.assertEqual(result, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- a\n- b\n- c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_key_noexist_deep(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: [a,b,c] }}")
-        file.close()
-
-        result = soyaml.purgelistitem([filename, "key1.child2.deep1", "b"])
-        self.assertEqual(result, 0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1:\n  - a\n  - b\n  - c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_scalar_and_null_values(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: b, child2: null, child3: [b] }}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1: b\n  child2: null\n  child3: []\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_lists_of_lists(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [[a,b], [b,[b,c]], b]}")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "b"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- - a\n- - - c\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_file_object(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: [{name: x}, a, {name: y}, {name: x}]}")
-        file.close()
-
-        itemfile = "/tmp/so-yaml_test-purgelistitem-item.yaml"
-        file = open(itemfile, "w")
-        file.write("name: x\n")
-        file.close()
-
-        soyaml.purgelistitem([filename, "key1", "file:" + itemfile])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n- a\n- name: y\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_main(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: [a,b,c] }, key2: [b]}")
-        file.close()
-
-        with patch('sys.exit', new=MagicMock()) as sysmock:
-            sys.argv = ["cmd", "purgelistitem", filename, "key1", "b"]
-            soyaml.main()
-            sysmock.assert_called_once_with(0)
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child1:\n  - a\n  - c\nkey2:\n- b\n"
-        self.assertEqual(actual, expected)
-
-    def test_purgelistitem_with_remove(self):
-        filename = "/tmp/so-yaml_test-purgelistitem.yaml"
-        file = open(filename, "w")
-        file.write("{key1: { child1: { deep1: [a] }, child2: [b] }, key2: { group1: { items: [x,child1,y] }, group2: { items: [child1] } }}")
-        file.close()
-
-        soyaml.remove([filename, "key1.child1"])
-        soyaml.purgelistitem([filename, "key2", "child1"])
-
-        file = open(filename, "r")
-        actual = file.read()
-        file.close()
-
-        expected = "key1:\n  child2:\n  - b\nkey2:\n  group1:\n    items:\n    - x\n    - y\n  group2:\n    items: []\n"
-        self.assertEqual(actual, expected)
-
-
 class TestAppendListObject(unittest.TestCase):
 
     def test_appendlistobject_missing_arg(self):

salt/manager/tools/sbin/soup

@@ -121,6 +121,9 @@ check_err() {
       161)
         echo 'Required intermediate Elasticsearch upgrade not complete'
       ;;
+      162)
+        echo 'One or more Elastic Agent nodes do not support the x86-64-v3 CPU instruction set'
+      ;;
       170)
         echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
         echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
@@ -348,6 +351,83 @@ check_cluster_health() {
   exit 0
 }
 
+no_soup_for_you() {
+  echo ""
+  echo "No soup for you!"
+  exit 162
+}
+
+check_cpu_compatibility() {
+  # Roles running a container built from the so-elastic-agent image; mirrors the
+  # elasticagent and elasticfleet entries in salt/reactor/pillar_push_map.yaml.
+  local cpu_target='G@role:so-heavynode or G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
+  local expected_nodes cpu_results node result confirm
+  local -a unsupported=() offline=()
+
+  echo "Checking that Elastic Agent nodes support the x86-64-v3 CPU instruction set now required by Elastic."
+
+  if [[ "$SKIP_CPU_CHECK" == "true" ]]; then
+    printf "\nSkipping the x86-64-v3 CPU check because --skip-cpu-check was specified.\n\n"
+    return
+  fi
+
+  # Nodes that never answer are absent from the results, so diff against who should have.
+  expected_nodes=$(salt -C "$cpu_target" --preview-target --out=json 2>/dev/null | jq -r '.[]?') || true
+  if [[ -z "$expected_nodes" ]]; then
+    printf "\nCould not determine which nodes run the Elastic Agent, so the x86-64-v3 CPU check cannot run.\n"
+    no_soup_for_you
+  fi
+
+  cpu_results=$(salt -t 30 -C "$cpu_target" cmd.run "/lib64/ld-linux-x86-64.so.2 --help | grep x86-64-v3" --out=json 2>/dev/null) || true
+
+  while IFS= read -r node; do
+    [[ -z "$node" ]] && continue
+    result=$(jq -r --arg node "$node" '.[$node] // empty' <<< "$cpu_results" 2>/dev/null)
+    if [[ -z "$result" || "$result" == *"did not return"* ]]; then
+      offline+=("$node")
+    elif [[ "$result" != *"x86-64-v3 (supported"* ]]; then
+      # glibc appends "(supported, searched)" only when supported; the open paren keeps
+      # this from matching a future "(unsupported".
+      unsupported+=("$node")
+    fi
+  done <<< "$expected_nodes"
+
+  if [[ ${#unsupported[@]} -eq 0 && ${#offline[@]} -eq 0 ]]; then
+    printf "\nAll Elastic Agent nodes support x86-64-v3. We can proceed with SOUP.\n\n"
+    return
+  fi
+
+  echo ""
+  if [[ ${#unsupported[@]} -gt 0 ]]; then
+    echo "The following node(s) do NOT support the x86-64-v3 CPU instruction set:"
+    printf '  %s\n' "${unsupported[@]}"
+    echo ""
+    echo "Upstream Elastic now builds its binaries for x86-64-v3, so these nodes can no"
+    echo "longer run Elastic. Upgrading them WILL BREAK them."
+    echo ""
+  fi
+  if [[ ${#offline[@]} -gt 0 ]]; then
+    echo "The following node(s) did not respond and could not be checked:"
+    printf '  %s\n' "${offline[@]}"
+    echo ""
+    echo "These nodes are offline, so we cannot confirm they support x86-64-v3, which"
+    echo "upstream Elastic now requires."
+    echo ""
+  fi
+
+  if [[ -n $UNATTENDED ]]; then
+    echo "Unattended mode cannot prompt for an override. Re-run soup interactively, or pass --skip-cpu-check to bypass this check."
+    no_soup_for_you
+  fi
+
+  read -rp "Type 'override' to continue anyway, or press Enter to exit: " confirm
+  if [[ "${confirm,,}" == "override" ]]; then
+    printf "\nOverride accepted. Continuing at your own risk.\n\n"
+  else
+    no_soup_for_you
+  fi
+}
+
 check_fleet_server() {
   echo "Checking that Elastic Fleet Server is responding."
   # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
@@ -527,7 +607,6 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
     [[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
     [[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
-    [[ "$INSTALLEDVERSION" == "3.4.0" ]] && up_to_3.5.0
     true
 }
 
@@ -1203,25 +1282,6 @@ add_detection_runtime_fields() {
 }
 ### 3.4.0 End ###
 
-### 3.5.0 Scripts ###
-up_to_3.5.0() {
-  remove_localrules_portgroup
-
-  INSTALLEDVERSION=3.5.0
-}
-
-remove_localrules_portgroup() {
-  local pillar_file
-
-  for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do
-    grep -qs localrules "$pillar_file" || continue
-    echo "Removing the localrules portgroup from $pillar_file."
-    so-yaml.py remove "$pillar_file" firewall.portgroups.localrules
-    so-yaml.py purgelistitem "$pillar_file" firewall.role localrules
-  done
-}
-### 3.5.0 End ###
-
 
 repo_sync() {
   echo "Sync the local repo."
@@ -2108,6 +2168,8 @@ main() {
   echo "Let's see if we need to update Security Onion."
   upgrade_check
 
+  check_cpu_compatibility
+
   upgrade_space
 
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
@@ -2386,6 +2448,17 @@ fi
   echo "### soup has been served at $(date) ###"
 }
 
+SKIP_CPU_CHECK=false
+declare -a SOUP_ARGS=()
+for arg in "$@"; do
+  if [[ "$arg" == "--skip-cpu-check" ]]; then
+    SKIP_CPU_CHECK=true
+  else
+    SOUP_ARGS+=("$arg")
+  fi
+done
+set -- "${SOUP_ARGS[@]}"
+
 while getopts ":b:f:y" opt; do
   case ${opt} in
     b )
@@ -2409,7 +2482,7 @@ while getopts ":b:f:y" opt; do
       ISOLOC="$OPTARG"
     ;;
     \? )
-      echo "Usage: soup [-b] [-y] [-f <iso location>]"
+      echo "Usage: soup [-b] [-y] [-f <iso location>] [--skip-cpu-check]"
       exit 1
     ;;
     : )

salt/nginx/enabled.sls

@@ -54,9 +54,9 @@ so-nginx:
       - /opt/so/log/nginx/:/var/log/nginx:rw
       - /opt/so/tmp/nginx/:/var/lib/nginx:rw
       - /opt/so/tmp/nginx/:/run:rw
-      - /nsm/elastic-fleet/artifacts/:/opt/socore/elastic-artifacts/artifacts:ro
-      {% if GLOBALS.is_manager %}
       - /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
+      - /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts 
+      {% if GLOBALS.is_manager %}
       - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
       - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
       # ATT&CK Navigator binds
@@ -64,6 +64,7 @@ so-nginx:
       - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
       - /nsm/repo:/opt/socore/html/repo:ro
       - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
+      - /nsm/rules:/nsm/rules:ro
       {%   if NGINXMERGED.external_suricata %}
       - /opt/so/rules/nids/suri:/surirules:ro
       {%   endif %}

salt/nginx/etc/nginx.conf

@@ -4,8 +4,6 @@
 {%- set role = grains.id.split('_') | last %}
 {%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
 {%- set influxauth = ('so:' + influxpass) | base64_encode %}
-{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
-{%- set full_server_name = all_names | unique | join(' ') %}
 
 worker_processes auto;
 error_log /var/log/nginx/error.log;
@@ -41,31 +39,52 @@ http {
 
 	include /etc/nginx/conf.d/*.conf;
 
+	{%- if role in ['fleet'] %}
+
+	server {
+		listen 8443;
+		server_name {{ GLOBALS.hostname }};
+		root /opt/socore/html;
+		location /artifacts/ {
+			try_files $uri =206;
+			proxy_read_timeout    90;
+			proxy_connect_timeout 90;
+			proxy_set_header      Host $host;
+			proxy_set_header      X-Real-IP $remote_addr;
+			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+			proxy_set_header      Proxy "";
+			proxy_set_header      X-Forwarded-Proto $scheme;
+		}
+	}
+
+    {%- endif %}
+
 	{%- if GLOBALS.is_manager %}
+	{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
+	{%- set full_server_name = all_names | unique | join(' ') %}
+
 	server {
 		listen 80 default_server;
 		server_name _;
 		return 307 https://{{ GLOBALS.url_base }}$request_uri;
 	}
-  {%- endif %}
-  {%- if GLOBALS.is_manager or role == 'fleet' %}
+
 	server {
 		listen 8443;
-    {%- if GLOBALS.is_manager %}
 		server_name {{ full_server_name }};
-    {%- else %}
-    server_name {{ GLOBALS.hostname }};
-    {%- endif %}
-		root /opt/socore/elastic-artifacts;
+		root /opt/socore/html;
 		location /artifacts/ {
-			try_files $uri =404;
-		}
-		location / {
-			return 404;
+			try_files $uri =206;
+			proxy_read_timeout    90;
+			proxy_connect_timeout 90;
+			proxy_set_header      Host $host;
+			proxy_set_header      X-Real-IP $remote_addr;
+			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+			proxy_set_header      Proxy "";
+			proxy_set_header      X-Forwarded-Proto $scheme;
 		}
 	}
-  {%- endif %}
-  {%- if GLOBALS.is_manager %}
+
 	server {
 		listen 443 ssl default_server;
 		http2 on;
@@ -92,6 +111,21 @@ http {
 	{%- endif %}
 
 	{%- if GLOBALS.is_manager %}
+
+	server {
+		listen 7788;
+		server_name {{ full_server_name }};
+		root /nsm/rules;
+		location / {
+			allow all;
+			sendfile on;
+			sendfile_max_chunk 1m;
+			autoindex on;
+			autoindex_exact_size off;
+			autoindex_format html;
+			autoindex_localtime on;
+		}
+	}
 	{%-   if NGINXMERGED.external_suricata %}
     server {
         listen 7789 ssl;

setup/so-functions

@@ -976,50 +976,27 @@ docker_seed_update() {
 	set_progress_str "$docker_seed_update_percent" "Downloading $name"
 }
 
-registry_tar_has_soversion() {
-	local tarfile=$1
-	local version=$2
-
-	[ -n "$version" ] || return 1
-	[ -f "$tarfile" ] || return 1
-	# grep -q in a pipe SIGPIPEs tar; under pipefail a match would look like failure.
-	grep -qF "/_manifests/tags/${version}/" < <(tar -tf "$tarfile" 2>/dev/null)
-}
-
 docker_seed_registry() {
 	local VERSION="$SOVERSION"
-	local REGISTRY_TAR=/nsm/docker-registry/docker/registry.tar
 
-	if [ -f "$REGISTRY_TAR" ] && registry_tar_has_soversion "$REGISTRY_TAR" "$VERSION"; then
-		info "Extracting $REGISTRY_TAR"
-		if ! tar xvf "$REGISTRY_TAR" -C /nsm/docker-registry/docker >> "$setup_log" 2>&1; then
-			error "Failed to extract $REGISTRY_TAR"
-			return 1
-		fi
-		logCmd "rm -f $REGISTRY_TAR"
-		return 0
-	elif [ -f "$REGISTRY_TAR" ]; then
-		info "Not using $REGISTRY_TAR; it has no images tagged '${VERSION}'"
-		if [[ "$is_airgap" == true ]]; then
-			error "Airgap install: $REGISTRY_TAR does not match Security Onion version '${VERSION}'. Use the ISO for this version."
-			return 1
-		fi
+	if [ -f /nsm/docker-registry/docker/registry.tar ]; then
+		logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
+		logCmd "rm /nsm/docker-registry/docker/registry.tar"
 	elif [[ -d /nsm/docker-registry/docker/registry && ( -f /etc/SOCLOUD || "$is_airgap" == true ) ]]; then
 		echo "Using existing docker registry content"
-		return 0
-	fi
-
-	if [ "$install_type" == 'IMPORT' ]; then
-		container_list 'so-import'	
 	else
-		container_list
-	fi
+		if [ "$install_type" == 'IMPORT' ]; then
+			container_list 'so-import'	
+		else
+			container_list
+		fi
 
-	docker_seed_update_percent=25
+		docker_seed_update_percent=25
 
-	update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
+		update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
         # Use pipe exit status of 'update_docker_containers' for return code
-	return ${PIPESTATUS[0]}
+		return ${PIPESTATUS[0]}
+	fi
 }
 
 elasticfleet_pillar() {