merge

unmount current agupdate dir, before final upgrade on airgap

VERSION

@@ -1 +1 @@
-3.0.0
+3.0.0-foxtrot

pillar/top.sls

@@ -87,6 +87,8 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - minions.{{ grains.id }}
@@ -132,6 +134,8 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - minions.{{ grains.id }}
@@ -181,6 +185,8 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - minions.{{ grains.id }}
@@ -203,6 +209,8 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - strelka.soc_strelka
@@ -289,6 +297,8 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - strelka.soc_strelka

salt/allowed_states.map.jinja

@@ -38,6 +38,7 @@
 ] %}
 
 {% set sensor_states = [
+    'pcap',
     'suricata',
     'healthcheck',
     'tcpreplay',

salt/bpf/pcap.map.jinja

@@ -1,15 +1,21 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% set PCAP_BPF_STATUS = 0  %}
+{% set STENO_BPF_COMPILED = "" %}
 
+{% if GLOBALS.pcap_engine == "TRANSITION" %}
+{%   set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
+{% else %}
 {%   import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {%   set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
 {%   import 'bpf/macros.jinja' as MACROS %}
 {{   MACROS.remove_comments(BPFMERGED, 'pcap') }}
 {%   set PCAPBPF = BPFMERGED.pcap %}
+{% endif %}
 
 {% if PCAPBPF %}
    {% set PCAP_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %}
    {% if PCAP_BPF_CALC['retcode'] == 0 %}
       {% set PCAP_BPF_STATUS = 1 %}
+      {% set STENO_BPF_COMPILED =  ",\\\"--filter=" + PCAP_BPF_CALC['stdout'] + "\\\""  %}
    {% endif %}
 {% endif %}

salt/common/tools/sbin/so-bpf-compile

@@ -16,7 +16,7 @@
 
 if [ "$#" -lt 2 ]; then
   cat 1>&2 <<EOF
-$0 compiles a BPF expression to be passed to PCAP to apply a socket filter.
+$0 compiles a BPF expression to be passed to stenotype to apply a socket filter.
 Its first argument is the interface (link type is required) and all other arguments
 are passed to TCPDump.
 

salt/common/tools/sbin/so-image-common

@@ -32,6 +32,7 @@ container_list() {
       "so-nginx"
       "so-pcaptools"
       "so-soc"
+      "so-steno"
       "so-suricata"
       "so-telegraf"
       "so-zeek"
@@ -57,6 +58,7 @@ container_list() {
       "so-pcaptools"
       "so-redis"
       "so-soc"
+      "so-steno"
       "so-strelka-backend"
       "so-strelka-manager"
       "so-suricata"
@@ -69,6 +71,7 @@ container_list() {
       "so-logstash"
       "so-nginx"
       "so-redis"
+      "so-steno"
       "so-suricata"
       "so-soc"
       "so-telegraf"

salt/common/tools/sbin/so-log-check

@@ -179,6 +179,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check"            # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics"           # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf"            # known issue with reposync on pre-2.4.20 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record"      # stenographer corrupt index
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field."                   # known ingest type collisions issue with earlier versions of SO
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature"      # Malformed Suricata rule, from upstream provider
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error

salt/common/tools/sbin/so-nsm-clear

@@ -55,22 +55,19 @@ if [ $SKIP -ne 1 ]; then
 fi
 
 delete_pcap() {
-  PCAP_DATA="/nsm/suripcap/"
-  [ -d $PCAP_DATA ] && rm -rf $PCAP_DATA/*
+  PCAP_DATA="/nsm/pcap/"
+  [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
 }
 delete_suricata() {
   SURI_LOG="/nsm/suricata/"
-  [ -d $SURI_LOG ] && rm -rf $SURI_LOG/*
+  [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
 }
 delete_zeek() {
   ZEEK_LOG="/nsm/zeek/logs/"
   [ -d $ZEEK_LOG ] && so-zeek-stop && rm -rf $ZEEK_LOG/* && so-zeek-start
 }
 
-so-suricata-stop
 delete_pcap
 delete_suricata
 delete_zeek
-so-suricata-start
-
 

salt/common/tools/sbin/so-restart

@@ -23,6 +23,7 @@ if [ $# -ge 1 ]; then
         fi
 
         case $1 in
+                "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
                 "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
                 *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
         esac

salt/common/tools/sbin/so-sensor-clean

@@ -72,7 +72,7 @@ clean() {
 		done
 	fi
 
-	## Clean up extracted pcaps
+	## Clean up extracted pcaps from Steno
 	PCAPS='/nsm/pcapout'
 	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
 	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then

salt/common/tools/sbin/so-start

@@ -23,6 +23,7 @@ if [ $# -ge 1 ]; then
 
 	case $1 in
    		"all") salt-call state.highstate queue=True;;
+   		"steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;; 
    		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
    		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
 	esac

salt/docker/defaults.yaml

@@ -174,6 +174,11 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+    'so-steno':
+      final_octet: 99
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []
     'so-suricata':
       final_octet: 99
       custom_bind_mounts: []

salt/docker/soc_docker.yaml

@@ -62,6 +62,7 @@ docker:
     so-idh: *dockerOptions
     so-elastic-agent: *dockerOptions
     so-telegraf: *dockerOptions
+    so-steno: *dockerOptions
     so-suricata:
       final_octet:
         description: Last octet of the container IP address.

salt/global/defaults.yaml

@@ -1,3 +1,3 @@
 global:
-  pcapengine: SURICATA
+  pcapengine: STENO
   pipeline: REDIS

salt/global/soc_global.yaml

@@ -18,11 +18,13 @@ global:
     regexFailureMessage: You must enter either ZEEK or SURICATA.
     global: True
   pcapengine:
-    description: Which engine to use for generating pcap. Currently only SURICATA is supported.
-    regex: ^(SURICATA)$
+    description: Which engine to use for generating pcap. Options are STENO, SURICATA or TRANSITION.
+    regex: ^(STENO|SURICATA|TRANSITION)$
     options:
+      - STENO
       - SURICATA
-    regexFailureMessage: You must enter either SURICATA.
+      - TRANSITION
+    regexFailureMessage: You must enter either STENO, SURICATA or TRANSITION.
     global: True
   ids:
     description: Which IDS engine to use. Currently only Suricata is supported.

salt/influxdb/templates/alarm_steno_packet_loss.json

@@ -0,0 +1,27 @@
+[{
+	"apiVersion": "influxdata.com/v2alpha1",
+	"kind": "CheckThreshold",
+	"metadata": {
+		"name": "steno-packet-loss"
+	},
+	"spec": {
+		"description": "Triggers when the average percent of packet loss is above the defined threshold. To tune this alert, modify the value for the appropriate alert level.",
+		"every": "1m",
+		"name": "Stenographer Packet Loss",
+		"query": "from(bucket: \"telegraf/so_short_term\")\n  |\u003e range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |\u003e filter(fn: (r) =\u003e r[\"_measurement\"] == \"stenodrop\")\n  |\u003e filter(fn: (r) =\u003e r[\"_field\"] == \"drop\")\n  |\u003e aggregateWindow(every: 1m, fn: mean, createEmpty: false)\n  |\u003e yield(name: \"mean\")",
+		"status": "active",
+		"statusMessageTemplate": "Stenographer Packet Loss on node ${r.host} has reached the ${ r._level } threshold. The current packet loss is ${ r.drop }%.",
+		"thresholds": [
+			{
+				"level": "CRIT",
+				"type": "greater", 
+				"value": 5
+			},
+			{
+				"level": "WARN",
+				"type": "greater",
+				"value": 3
+			}
+		]
+	}
+}]

salt/logrotate/defaults.yaml

@@ -180,6 +180,16 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/stenographer/*_x_log:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /opt/so/log/salt/so-salt-minion-check:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -112,6 +112,13 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/stenographer/*_x_log":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/stenographer/*.log
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/opt/so/log/salt/so-salt-minion-check":
       description: List of logrotate options for this file.
       title: /opt/so/log/salt/so-salt-minion-check

salt/pcap/ca.sls

@@ -0,0 +1,22 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states or sls in allowed_states%}
+
+stenoca:
+  file.directory:
+    - name: /opt/so/conf/steno/certs
+    - user: 941
+    - group: 939
+    - makedirs: True
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/pcap/cleanup.sls

@@ -1,59 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-{% if GLOBALS.is_sensor %}
-
-delete_so-steno_so-status.conf:
-  file.line:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - mode: delete
-    - match: so-steno
-
-remove_stenographer_user:
-  user.absent:
-    - name: stenographer
-    - force: True
-
-remove_stenographer_log_dir:
-  file.absent:
-    - name: /opt/so/log/stenographer
-
-remove_stenoloss_script:
-  file.absent:
-    - name: /opt/so/conf/telegraf/scripts/stenoloss.sh
-
-remove_steno_conf_dir:
-  file.absent:
-    - name: /opt/so/conf/steno
-
-remove_so_pcap_export:
-  file.absent:
-    - name: /usr/sbin/so-pcap-export
-
-remove_so_pcap_restart:
-  file.absent:
-    - name: /usr/sbin/so-pcap-restart
-
-remove_so_pcap_start:
-  file.absent:
-    - name: /usr/sbin/so-pcap-start
-
-remove_so_pcap_stop:
-  file.absent:
-    - name: /usr/sbin/so-pcap-stop
-
-so-steno:
-  docker_container.absent:
-    - force: True
-
-{% else %}
-
-{{sls}}.non_sensor_node:
-  test.show_notification:
-    - text: "Stenographer cleanup not applicable on non-sensor nodes."
-
-{% endif %}

salt/pcap/config.map.jinja

@@ -0,0 +1,13 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% import_yaml 'pcap/defaults.yaml' as PCAPDEFAULTS %}
+{% set PCAPMERGED = salt['pillar.get']('pcap', PCAPDEFAULTS.pcap, merge=True) %}
+
+{# disable stenographer if the pcap engine is set to SURICATA #}
+{% if GLOBALS.pcap_engine == "SURICATA" %}
+{%   do PCAPMERGED.update({'enabled': False}) %}
+{% endif %}

salt/pcap/config.sls

@@ -0,0 +1,87 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% from "pcap/config.map.jinja" import PCAPMERGED %}
+{% from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS, PCAP_BPF_CALC, STENO_BPF_COMPILED %}
+
+# PCAP Section
+stenographergroup:
+  group.present:
+    - name: stenographer
+    - gid: 941
+
+stenographer:
+  user.present:
+    - uid: 941
+    - gid: 941
+    - home: /opt/so/conf/steno
+
+stenoconfdir:
+  file.directory:
+    - name: /opt/so/conf/steno
+    - user: 941
+    - group: 939
+    - makedirs: True
+
+pcap_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://pcap/tools/sbin
+    - user: 939
+    - group: 939
+    - file_mode: 755
+
+{% if PCAPBPF and not PCAP_BPF_STATUS %}
+stenoPCAPbpfcompilationfailure:
+  test.configurable_test_state:
+   - changes: False
+   - result: False
+  - comment: "BPF Syntax Error - Discarding Specified BPF. Error: {{ PCAP_BPF_CALC['stderr'] }}"
+{% endif %}
+
+stenoconf:
+  file.managed:
+    - name: /opt/so/conf/steno/config
+    - source: salt://pcap/files/config.jinja
+    - user: stenographer
+    - group: stenographer
+    - mode: 644
+    - template: jinja
+    - defaults:
+        PCAPMERGED: {{ PCAPMERGED }}
+        STENO_BPF_COMPILED: "{{ STENO_BPF_COMPILED }}"
+
+pcaptmpdir:
+  file.directory:
+    - name: /nsm/pcaptmp
+    - user: 941
+    - group: 941
+    - makedirs: True
+
+pcapindexdir:
+  file.directory:
+    - name: /nsm/pcapindex
+    - user: 941
+    - group: 941
+    - makedirs: True
+
+stenolog:
+  file.directory:
+    - name: /opt/so/log/stenographer
+    - user: 941
+    - group: 941
+    - makedirs: True
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/pcap/defaults.yaml

@@ -0,0 +1,11 @@
+pcap:
+  enabled: False
+  config:
+    maxdirectoryfiles: 30000
+    diskfreepercentage: 10
+    blocks: 2048
+    preallocate_file_mb: 4096
+    aiops: 128
+    pin_to_cpu: False
+    cpus_to_pin_to: []
+    disks: []

salt/pcap/disabled.sls

@@ -0,0 +1,27 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+include:
+  - pcap.sostatus
+  
+so-steno:
+  docker_container.absent:
+    - force: True
+
+so-steno_so-status.disabled:
+  file.comment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-steno$
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/pcap/enabled.sls

@@ -0,0 +1,63 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}
+
+
+include:
+  - pcap.ca
+  - pcap.config
+  - pcap.sostatus
+
+so-steno:
+  docker_container.running:
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-steno:{{ GLOBALS.so_version }}
+    - start: True
+    - network_mode: host
+    - privileged: True
+    - binds:
+      - /opt/so/conf/steno/certs:/etc/stenographer/certs:rw
+      - /opt/so/conf/steno/config:/etc/stenographer/config:rw
+      - /nsm/pcap:/nsm/pcap:rw
+      - /nsm/pcapindex:/nsm/pcapindex:rw
+      - /nsm/pcaptmp:/tmp:rw
+      - /opt/so/log/stenographer:/var/log/stenographer:rw
+    {% if DOCKER.containers['so-steno'].custom_bind_mounts %}
+        {% for BIND in DOCKER.containers['so-steno'].custom_bind_mounts %}
+      - {{ BIND }}
+        {% endfor %}
+      {% endif %}
+    {% if DOCKER.containers['so-steno'].extra_hosts %}
+    - extra_hosts:
+      {% for XTRAHOST in DOCKER.containers['so-steno'].extra_hosts %}
+      - {{ XTRAHOST }}
+      {% endfor %}
+    {% endif %}
+    {% if DOCKER.containers['so-steno'].extra_env %}
+    - environment:
+      {% for XTRAENV in DOCKER.containers['so-steno'].extra_env %}
+      - {{ XTRAENV }}
+      {% endfor %}
+    {% endif %}
+    - watch:
+      - file: stenoconf
+    - require:
+      - file: stenoconf
+
+delete_so-steno_so-status.disabled:
+  file.uncomment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-steno$
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/pcap/files/config.jinja

@@ -0,0 +1,11 @@
+{
+  "Threads": [
+    { "PacketsDirectory": "/nsm/pcap", "IndexDirectory": "/nsm/pcapindex", "MaxDirectoryFiles": {{ PCAPMERGED.config.maxdirectoryfiles }}, "DiskFreePercentage": {{ PCAPMERGED.config.diskfreepercentage }} }
+  ]
+  , "StenotypePath": "/usr/bin/stenotype"
+  , "Interface": "{{ pillar.sensor.interface }}"
+  , "Port": 1234
+  , "Host": "127.0.0.1"
+  , "Flags": ["-v", "--blocks={{ PCAPMERGED.config.blocks }}", "--preallocate_file_mb={{ PCAPMERGED.config.preallocate_file_mb }}", "--aiops={{ PCAPMERGED.config.aiops }}", "--uid=stenographer", "--gid=stenographer"{{ STENO_BPF_COMPILED }}]
+  , "CertPath": "/etc/stenographer/certs"
+}

salt/pcap/init.sls

@@ -0,0 +1,41 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'pcap/config.map.jinja' import PCAPMERGED %}
+
+include:
+{% if PCAPMERGED.enabled and GLOBALS.role != 'so-import'%}
+  - pcap.enabled
+{% elif GLOBALS.role == 'so-import' %}
+  - pcap.config
+  - pcap.disabled
+{% else %}
+  - pcap.disabled
+{% endif %}
+
+# This directory needs to exist regardless of whether STENO is enabled or not, in order for
+# Sensoroni to be able to look at old steno PCAP data
+
+# if stenographer has never run as the pcap engine no 941 user is created, so we use socore as a placeholder.
+# /nsm/pcap is empty until stenographer is used as pcap engine
+{% set pcap_id = 941 %}
+{% set user_list = salt['user.list_users']() %}
+{% if GLOBALS.pcap_engine == "SURICATA" and 'stenographer' not in user_list %}
+{%   set pcap_id = 939 %}
+{% endif %}
+pcapdir:
+  file.directory:
+    - name: /nsm/pcap
+    - user: {{ pcap_id }}
+    - group: {{ pcap_id }}
+    - makedirs: True
+
+pcapoutdir:
+  file.directory:
+    - name: /nsm/pcapout
+    - user: 939
+    - group: 939
+    - makedirs: True

salt/pcap/soc_pcap.yaml

@@ -0,0 +1,35 @@
+pcap:
+  enabled: 
+    description: Enables or disables the Stenographer packet recording process. This process may already be disabled if Suricata is being used as the packet capture process.
+    helpLink: stenographer.html
+  config:
+    maxdirectoryfiles:
+      description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting.
+      helpLink: stenographer.html
+    diskfreepercentage:
+      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated Sensor nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
+      helpLink: stenographer.html
+    blocks:
+      description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. 
+      advanced: True
+      helpLink: stenographer.html
+    preallocate_file_mb:
+      description: File size to pre-allocate for individual Stenographer PCAP files. You shouldn't need to change this. 
+      advanced: True
+      helpLink: stenographer.html
+    aiops:
+      description: The max number of async writes to allow for Stenographer at once.
+      advanced: True
+      helpLink: stenographer.html
+    pin_to_cpu:
+      description: Enable CPU pinning for Stenographer PCAP.
+      advanced: True
+      helpLink: stenographer.html
+    cpus_to_pin_to:
+      description: CPU to pin Stenographer PCAP to. Currently only a single CPU is supported.
+      advanced: True
+      helpLink: stenographer.html
+    disks:
+      description: List of disks to use for Stenographer PCAP. This is currently not used.
+      advanced: True
+      helpLink: stenographer.html

salt/pcap/sostatus.sls

@@ -0,0 +1,21 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+append_so-steno_so-status.conf:
+  file.append:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - text: so-steno
+    - unless: grep -q so-steno /opt/so/conf/so-status/so-status.conf
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/pcap/tools/sbin/so-pcap-export

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+if [ $# -lt 2 ]; then
+  echo "Usage: $0 <steno-query> Output-Filename"
+  exit 1
+fi
+
+docker exec -t so-sensoroni scripts/stenoquery.sh "$1" -w /nsm/pcapout/$2.pcap
+
+echo ""
+echo "If successful, the output was written to: /nsm/pcapout/$2.pcap"

salt/pcap/tools/sbin/so-pcap-restart

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-restart steno $1

salt/pcap/tools/sbin/so-pcap-start

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-start steno $1

salt/pcap/tools/sbin/so-pcap-stop

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+. /usr/sbin/so-common
+
+/usr/sbin/so-stop steno $1

salt/salt/cloud/cloud.profiles.d/socloud.conf.jinja

@@ -29,11 +29,7 @@ sool9_{{host}}:
     hypervisor_host: {{host ~ "_" ~ role}}
   preflight_cmds:
     - |
-      {%- set hostnames = [MANAGERHOSTNAME] %}
-      {%- if not (URL_BASE | ipaddr) and URL_BASE != MANAGERHOSTNAME %}
-      {%-   do hostnames.append(URL_BASE) %}
-      {%- endif %}
-      tee -a /etc/hosts <<< "{{ MANAGERIP }}    {{ hostnames | join('  ') }}"
+      tee -a /etc/hosts <<< "{{ MANAGERIP }}    {{ MANAGERHOSTNAME }}"
     - |
       timeout 600 bash -c 'trap "echo \"Preflight Check: Failed to establish repo connectivity\"; exit 1" TERM; \
         while ! dnf makecache --repoid=securityonion >/dev/null 2>&1; do echo "Preflight Check: Waiting for repo connectivity..."; \

salt/salt/cloud/config.sls

@@ -14,7 +14,6 @@
 {%   if 'vrt' in salt['pillar.get']('features', []) %}
 {%     set HYPERVISORS = salt['pillar.get']('hypervisor:nodes', {} ) %}
 {%     from 'salt/map.jinja' import SALTVERSION %}
-{%     from 'vars/globals.map.jinja' import GLOBALS %}
 
 {%     if HYPERVISORS %}
 cloud_providers:
@@ -35,7 +34,6 @@ cloud_profiles:
         MANAGERHOSTNAME: {{ grains.host }}
         MANAGERIP: {{ pillar.host.mainip }}
         SALTVERSION: {{ SALTVERSION }}
-        URL_BASE: {{ GLOBALS.url_base }}
     - template: jinja
     - makedirs: True
 {%     else %}

salt/salt/engines/master/virtual_node_manager.py

@@ -805,6 +805,11 @@ def process_vm_creation(hypervisor_path: str, vm_config: dict) -> None:
                     mark_invalid_hardware(hypervisor_path, vm_name, vm_config,
                                         {'nsm_size': 'Invalid nsm_size: must be positive integer'})
                     return
+                if size > 10000:  # 10TB reasonable maximum
+                    log.error("VM: %s - nsm_size %dGB exceeds reasonable maximum (10000GB)", vm_name, size)
+                    mark_invalid_hardware(hypervisor_path, vm_name, vm_config,
+                                        {'nsm_size': f'Invalid nsm_size: {size}GB exceeds maximum (10000GB)'})
+                    return
                 log.debug("VM: %s - nsm_size validated: %dGB", vm_name, size)
             except (ValueError, TypeError) as e:
                 log.error("VM: %s - nsm_size must be a valid integer, got: %s", vm_name, vm_config.get('nsm_size'))

salt/sensoroni/enabled.sls

@@ -8,6 +8,9 @@
 
 
 include:
+{% if GLOBALS.is_sensor or GLOBALS.role == 'so-import' %}
+  - pcap.ca
+{% endif %}
   - sensoroni.config
   - sensoroni.sostatus
 
@@ -16,6 +19,10 @@ so-sensoroni:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-soc:{{ GLOBALS.so_version }}
     - network_mode: host
     - binds:
+      {% if GLOBALS.is_sensor or GLOBALS.role == 'so-import' %}
+      - /opt/so/conf/steno/certs:/etc/stenographer/certs:rw
+      {% endif %}
+      - /nsm/pcap:/nsm/pcap:rw
       - /nsm/import:/nsm/import:rw
       - /nsm/pcapout:/nsm/pcapout:rw
       - /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro

salt/sensoroni/files/sensoroni.json

@@ -32,6 +32,11 @@
         "apiKey": "{{ GLOBALS.sensoroni_key }}"
 {%  if GLOBALS.is_sensor %}
       },     
+      "stenoquery": {
+        "executablePath": "/opt/sensoroni/scripts/stenoquery.sh",
+        "pcapInputPath": "/nsm/pcap",
+        "pcapOutputPath": "/nsm/pcapout"
+      },     
       "suriquery": {
         "pcapInputPath": "/nsm/suripcap",
         "pcapOutputPath": "/nsm/pcapout",

salt/suricata/config.sls

@@ -10,7 +10,7 @@
 {%   from 'suricata/map.jinja' import SURICATAMERGED %}
 {%   from 'bpf/suricata.map.jinja' import SURICATABPF, SURICATA_BPF_STATUS, SURICATA_BPF_CALC %}
 
-{%   if GLOBALS.pcap_engine in ["SURICATA"] %}
+{%   if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %}
 {%     from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS, PCAP_BPF_CALC %}
 # BPF compilation and configuration
 {%     if PCAPBPF and not PCAP_BPF_STATUS %}

salt/suricata/map.jinja

@@ -9,7 +9,7 @@
 {% set surimeta_filestore_index = [] %}
 
 {# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
-{% if GLOBALS.pcap_engine in ["SURICATA"] %}
+{% if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %}
 
 {%   from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS %}
 {%   if PCAPBPF and PCAP_BPF_STATUS %}

salt/suricata/pcap.sls

@@ -2,7 +2,7 @@
 {% from 'suricata/map.jinja' import SURICATAMERGED %}
 
 # This directory needs to exist regardless of whether SURIPCAP is enabled or not, in order for
-# Sensoroni to mount it
+# Sensoroni to be able to look at old Suricata PCAP data
 suripcapdir:
   file.directory:
     - name: /nsm/suripcap
@@ -11,14 +11,7 @@ suripcapdir:
     - mode: 775
     - makedirs: True
 
-pcapoutdir:
-  file.directory:
-    - name: /nsm/pcapout
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-{%   if GLOBALS.pcap_engine in ["SURICATA"] %}
+{%   if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %}
 
 {# there should only be 1 interface in af-packet so we can just reference the first list item #}
 {% for i in range(1, SURICATAMERGED.config['af-packet'][0].threads + 1) %}

salt/suricata/tools/sbin_jinja/so-suricata-testrule

@@ -31,9 +31,8 @@ mkdir -p /tmp/nids-testing/output
 chown suricata:socore /tmp/nids-testing/output
 mkdir -p /tmp/nids-testing/rules
 
-cp /opt/so/rules/suricata/all-rulesets.rules /tmp/nids-testing/rules/all-rulesets.rules
-cat $TESTRULE >> /tmp/nids-testing/rules/all-rulesets.rules
-
+cp /opt/so/conf/suricata/rules/all.rules /tmp/nids-testing/rules/all.rules
+cat $TESTRULE >> /tmp/nids-testing/rules/all.rules
 
 echo "==== Begin Suricata Output ==="
 

salt/telegraf/defaults.yaml

@@ -19,6 +19,7 @@ telegraf:
       - os.sh
       - raid.sh
       - sostatus.sh
+      - stenoloss.sh
       - suriloss.sh
       - surirules.sh
       - zeekcaptureloss.sh
@@ -34,6 +35,7 @@ telegraf:
       - raid.sh
       - redis.sh
       - sostatus.sh
+      - stenoloss.sh
       - suriloss.sh
       - surirules.sh
       - zeekcaptureloss.sh
@@ -79,6 +81,7 @@ telegraf:
       - os.sh
       - raid.sh
       - sostatus.sh
+      - stenoloss.sh
       - suriloss.sh
       - surirules.sh
       - zeekcaptureloss.sh
@@ -93,6 +96,7 @@ telegraf:
       - raid.sh
       - redis.sh
       - sostatus.sh
+      - stenoloss.sh
       - suriloss.sh
       - surirules.sh
       - zeekcaptureloss.sh

salt/telegraf/enabled.sls

@@ -47,6 +47,7 @@ so-telegraf:
       - /etc/pki/telegraf.crt:/etc/telegraf/telegraf.crt:ro
       - /etc/pki/telegraf.key:/etc/telegraf/telegraf.key:ro
       - /opt/so/conf/telegraf/scripts:/scripts:ro
+      - /opt/so/log/stenographer:/var/log/stenographer:ro
       - /opt/so/log/suricata:/var/log/suricata:ro
       - /opt/so/log/raid:/var/log/raid:ro
       - /opt/so/log/sostatus:/var/log/sostatus:ro

salt/telegraf/map.jinja

@@ -14,6 +14,13 @@
 {%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %}
 {%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %}
 {%   endif %}
+
+{%   from 'pcap/config.map.jinja' import PCAPMERGED %}
+{#   PCAPMERGED.enabled is set false in soc ui or if suricata is the pcap engine #}
+{%   if not PCAPMERGED.enabled %}
+{%     do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('stenoloss.sh') %}
+{%   endif %}
+
 {% endif %}
 
 {% if GLOBALS.pipeline != 'REDIS' %}

salt/telegraf/scripts/oldpcap.sh

@@ -5,7 +5,11 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+{%- if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %}
 PCAPLOC=/host/nsm/suripcap
+{%- else %}
+PCAPLOC=/host/nsm/pcap
+{%- endif %}
 
 # if this script isn't already running
 if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then

salt/telegraf/scripts/stenoloss.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+# if this script isn't already running
+if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
+
+    CHECKIT=$(grep "Thread 0 stats" /var/log/stenographer/stenographer.log |tac |head -2|wc -l)
+    STENOGREP=$(grep "Thread 0 stats" /var/log/stenographer/stenographer.log |tac |head -2)
+
+    declare RESULT=($STENOGREP)
+
+    CURRENT_PACKETS=$(echo ${RESULT[9]} | awk -F'=' '{print $2 }')
+    CURRENT_DROPS=$(echo ${RESULT[12]} | awk -F'=' '{print $2 }')
+    PREVIOUS_PACKETS=$(echo ${RESULT[23]} | awk -F'=' '{print $2 }')
+    PREVIOUS_DROPS=$(echo ${RESULT[26]} | awk -F'=' '{print $2 }')
+
+    DROPPED=$((CURRENT_DROPS - PREVIOUS_DROPS))
+    TOTAL_CURRENT=$((CURRENT_PACKETS + CURRENT_DROPS))
+    TOTAL_PAST=$((PREVIOUS_PACKETS + PREVIOUS_DROPS))
+    TOTAL=$((TOTAL_CURRENT - TOTAL_PAST))
+
+    if [ $CHECKIT == 2 ]; then
+      if [ $DROPPED == 0 ]; then
+        echo "stenodrop drop=$DROPPED"
+      else
+        LOSS=$(echo "4 k $DROPPED $TOTAL / 100 * p" | dc)
+        echo "stenodrop drop=$LOSS"
+      fi
+    fi
+
+fi
+
+exit 0

salt/top.sls

@@ -78,6 +78,7 @@ base:
     - elasticsearch
     - elastic-fleet-package-registry
     - kibana
+    - pcap
     - suricata
     - zeek
     - strelka
@@ -85,7 +86,6 @@ base:
     - elastalert
     - utility
     - elasticfleet
-    - pcap.cleanup
 
   '*_standalone and G@saltversion:{{saltversion}} and not I@node_data:False':
     - match: compound
@@ -108,6 +108,7 @@ base:
     - redis
     - elastic-fleet-package-registry
     - kibana
+    - pcap
     - suricata
     - zeek
     - strelka
@@ -117,7 +118,6 @@ base:
     - elasticfleet
     - stig
     - kafka
-    - pcap.cleanup
 
   '*_manager or *_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False':
     - match: compound
@@ -192,6 +192,7 @@ base:
     - sensoroni
     - telegraf
     - firewall
+    - pcap
     - elasticsearch
     - elastic-fleet-package-registry
     - kibana
@@ -199,7 +200,6 @@ base:
     - suricata
     - zeek
     - elasticfleet
-    - pcap.cleanup
 
   '*_searchnode and G@saltversion:{{saltversion}}':
     - match: compound
@@ -220,13 +220,13 @@ base:
     - telegraf
     - firewall
     - nginx
+    - pcap
     - suricata
     - healthcheck
     - zeek
     - strelka
     - elasticfleet.install_agent_grid
     - stig
-    - pcap.cleanup
 
   '*_heavynode and G@saltversion:{{saltversion}}':
     - match: compound
@@ -240,11 +240,11 @@ base:
     - redis
     - curator.disabled
     - strelka
+    - pcap
     - suricata
     - zeek
     - elasticfleet.install_agent_grid
     - elasticagent
-    - pcap.cleanup
 
   '*_receiver and G@saltversion:{{saltversion}}':
     - match: compound

setup/so-functions

@@ -1382,7 +1382,9 @@ create_global() {
 	echo "  registry_host: '$HOSTNAME'" >> $global_pillar_file
 	echo "  endgamehost: '$ENDGAMEHOST'" >> $global_pillar_file
 
-   	echo "  pcapengine: SURICATA" >> $global_pillar_file
+	if [[ $is_standalone || $is_eval ]]; then
+        	echo "  pcapengine: SURICATA" >> $global_pillar_file
+	fi
 }
 
 create_sensoroni_pillar() {
@@ -1444,7 +1446,7 @@ make_some_dirs() {
 	mkdir -p $local_salt_dir/salt/firewall/portgroups
 	mkdir -p $local_salt_dir/salt/firewall/ports
 
-	for THEDIR in bpf elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do
+	for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do
 	mkdir -p $local_salt_dir/pillar/$THEDIR
 	touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls
 	touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls
@@ -1598,6 +1600,7 @@ reserve_group_ids() {
 	logCmd "groupadd -g 940 suricata"
 	logCmd "groupadd -g 948 elastic-agent-pr"
 	logCmd "groupadd -g 949 elastic-agent"
+	logCmd "groupadd -g 941 stenographer"
 	logCmd "groupadd -g 947 elastic-fleet"
 	logCmd "groupadd -g 960 kafka"
 }