Merge pull request #12513 from Security-Onion-Solutions/newsuripcap

Change Factoring for so-minion pcap disk space
2025-12-06 17:22:49 +01:00 · 2024-03-07 10:14:34 -05:00
parent 8f36a8a4b6 70f3ce0536
commit b5f1733e97
5 changed files with 95 additions and 75 deletions
--- a/salt/manager/tools/sbin/so-minion
+++ b/salt/manager/tools/sbin/so-minion
@@ -81,6 +81,9 @@ function getinstallinfo() {

 function pcapspace() {
 	if [[ "$OPERATION" == "setup" ]]; then
+		# Use 25% for PCAP
+		PCAP_PERCENTAGE=1
+		DFREEPERCENT=75
 		local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n)
 	else

@@ -96,10 +99,9 @@ function pcapspace() {
 	fi

 	local s=$(( $SPACESIZE / 1000000 ))
-	local s1=$(( $s / 4 ))
-	local s2=$(( $s1 / $CORECOUNT ))
+	local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))

-	MAXPCAPFILES=$s2
+	MAX_PCAP_SPACE=$s1

 }

@@ -268,20 +270,18 @@ function add_sensor_to_minion() {
 	echo "      lb_procs: '$CORECOUNT'" >> $PILLARFILE
 	echo "suricata:" >> $PILLARFILE
 	echo "  enabled: True " >> $PILLARFILE
+	if [[ $is_pcaplimit ]]; then
+	echo "  pcap:" >> $PILLARFILE
+	echo "    maxsize: $MAX_PCAP_SPACE" >> $PILLARFILE
+	fi
 	echo "  config:" >> $PILLARFILE
 	echo "    af-packet:" >> $PILLARFILE
 	echo "      threads: '$CORECOUNT'" >> $PILLARFILE
-	if [[ $is_pcaplimit ]]; then
-		echo "    output:" >> $PILLARFILE
-		echo "      pcap-log:" >> $PILLARFILE
-		echo "        max-files: '$MAXPCAPFILES'" >> $PILLARFILE
-	fi
 	echo "pcap:" >> $PILLARFILE
 	echo "  enabled: True" >> $PILLARFILE
 	if [[ $is_pcaplimit ]]; then
 		echo "  config:" >> $PILLARFILE
-		echo "    diskfreepercentage: 75" >> $PILLARFILE
-		pcapspace
+		echo "    diskfreepercentage: $DFREEPERCENT" >> $PILLARFILE
 	fi
 	echo "  " >> $PILLARFILE
 }
@@ -563,6 +563,8 @@ function createIDH() {

 function createHEAVYNODE() {
 	is_pcaplimit=true
+	PCAP_PERCENTAGE=1
+	DFREEPERCENT=75
 	pcapspace
 	add_elasticsearch_to_minion
 	add_elastic_agent_to_minion
@@ -575,6 +577,8 @@ function createHEAVYNODE() {

 function createSENSOR() {
 	is_pcaplimit=true
+	DFREEPERCENT=10
+	PCAP_PERCENTAGE=3
 	pcapspace
 	add_sensor_to_minion
 	add_strelka_to_minion
--- a/salt/suricata/defaults.yaml
+++ b/salt/suricata/defaults.yaml
@@ -1,5 +1,16 @@
 suricata:
  enabled: False
+  pcap:
+    filesize: 1000mb
+    maxsize: 25
+    compression: "none"
+    lz4-checksum: "no"
+    lz4-level: 8
+    filename: "%n/so-pcap.%t"
+    mode: "multi"
+    use-stream-depth: "no"
+    conditional: "all"
+    dir: "/nsm/suripcap"
  config:
    threading:
      set-cpu-affinity: "no"
@@ -128,16 +139,6 @@ suricata:
        enabled: "no"
      pcap-log:
        enabled: "no"
-        compression: "none"
-        lz4-checksum: "no"
-        lz4-level: 8
-        filename: "%n/so-pcap.%t"
-        limit: "1000mb"
-        mode: "multi"
-        max-files: 10
-        use-stream-depth: "no"
-        conditional: "all"
-        dir: "/nsm/suripcap"
      alert-debug:
        enabled: "no"
      alert-prelude:
--- a/salt/suricata/map.jinja
+++ b/salt/suricata/map.jinja
@@ -8,6 +8,24 @@
 {% set surimeta_evelog_index = [] %}
 {% set surimeta_filestore_index = [] %}

+{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
+{% if GLOBALS.pcap_engine == "SURICATA" %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
+{#   move the items in suricata.pcap into suricata.config.outputs.pcap-log. these items were placed under suricata.config for ease of access in SOC #}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'compression': SURICATAMERGED.pcap.compression}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-checksum': SURICATAMERGED.pcap['lz4-checksum']}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'lz4-level': SURICATAMERGED.pcap['lz4-level']}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'filename': SURICATAMERGED.pcap.filename}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'limit': SURICATAMERGED.pcap.filesize}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'mode': SURICATAMERGED.pcap.mode}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'use-stream-depth': SURICATAMERGED.pcap['use-stream-depth']}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'conditional': SURICATAMERGED.pcap.conditional}) %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'dir': SURICATAMERGED.pcap.dir}) %}
+{#   multiply maxsize by 1000 since it is saved in GB, i.e. 52 = 52000MB. filesize is also saved in MB and we strip the MB and convert to int #}
+{%   set maxfiles = (SURICATAMERGED.pcap.maxsize * 1000 / (SURICATAMERGED.pcap.filesize[:-2] | int) / SURICATAMERGED.config['af-packet'].threads | int) | round | int %}
+{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'max-files': maxfiles}) %}
+{% endif %}
+
 {# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #}
 {# we are limited to only one iterface #}
 {% load_yaml as afpacket %}
@@ -60,11 +78,6 @@
 {%   do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
 {% endif %}

-{# before we change outputs back to list, enable pcap-log if suricata is the pcapengine #}
-{% if GLOBALS.pcap_engine == "SURICATA" %}
-{%   do SURICATAMERGED.config.outputs['pcap-log'].update({'enabled': 'yes'}) %}
-{% endif %}
-
 {# outputs is a list but we convert to dict in defaults to work with ui #}
 {# below they are converted back to lists #}
 {% load_yaml as outputs %}
--- a/salt/suricata/soc_suricata.yaml
+++ b/salt/suricata/soc_suricata.yaml
@@ -19,6 +19,55 @@ suricata:
      multiline: True
      title: Classifications
      helpLink: suricata.html
+  pcap:
+    filesize: 
+      description: Max file size for individual PCAP files written by Suricata. Increasing this number could improve write performance at the expense of pcap retrieval times.
+      advanced: True
+      helplink: suricata.html 
+    maxsize:
+      description: Size in GB for total usage size of PCAP on disk. 
+      helplink: suricata.html
+    compression: 
+      description: Enable compression of Suricata PCAP. Currently unsupported
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    lz4-checksum: 
+      description: Enable PCAP lz4 checksum. Currently unsupported
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    lz4-level: 
+      description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    filename:
+      description: Filename output for Suricata PCAP.
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    mode: 
+      description: Suricata PCAP mode. Currently only multi is supported.
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
+    use-stream-depth: 
+      description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. 
+      advanced: True
+      regex: ^(yes|no)$
+      regexFailureMessage: You must enter either yes or no.
+      helpLink: suricata.html
+    conditional: 
+      description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules.
+      regex: ^(all|alert|tag)$
+      regexFailureMessage: You must enter either all, alert or tag.
+      helpLink: suricata.html
+    dir:
+      description: Parent directory to store PCAP.
+      advanced: True
+      readonly: True
+      helpLink: suricata.html
  config:
    af-packet:
      interface:
@@ -168,53 +217,6 @@ suricata:
          readonly: True
          helpLink: suricata.html
          advanced: True        
-        compression: 
-          description: Enable compression of Suricata PCAP. Currently unsupported
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        lz4-checksum: 
-          description: Enable PCAP lz4 checksum. Currently unsupported
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        lz4-level: 
-          description: lz4 compression level of PCAP. 0 for no compression 16 for max compression. Currently unsupported
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        filename:
-          description: Filename output for Suricata PCAP.
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        limit:
-          description: File size limit per thread. To determine max PCAP size multiple threads x max-files x limit.
-          helpLink: suricata.html
-        mode: 
-          description: Suricata PCAP mode. Currently only multi is supported.
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
-        max-files: 
-          description: Max PCAP files per thread. To determine max PCAP size multiple threads x max-files x limit.
-          helpLink: suricata.html
-        use-stream-depth: 
-          description: Set to "no" to ignore the stream depth and capture the entire flow. Set this to "yes" to truncate the flow based on the stream depth. 
-          advanced: True
-          regex: ^(yes|no)$
-          regexFailureMessage: You must enter either yes or no.
-          helpLink: suricata.html
-        conditional: 
-          description: Set to "all" to capture PCAP for all flows. Set to "alert" to capture PCAP just for alerts or set to "tag" to capture PCAP for just tagged rules.
-          regex: ^(all|alert|tag)$
-          regexFailureMessage: You must enter either all, alert or tag.
-          helpLink: suricata.html
-        dir:
-          description: Parent directory to store PCAP.
-          advanced: True
-          readonly: True
-          helpLink: suricata.html
    asn1-max-frames:
      description: Maximum nuber of asn1 frames to decode.
      helpLink: suricata.html
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1811,8 +1811,8 @@ repo_sync_local() {
 		info "Adding Repo Download Configuration"
 		mkdir -p /nsm/repo
 		mkdir -p /opt/so/conf/reposync/cache
-		echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9.3" > /opt/so/conf/reposync/mirror.txt
-		echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9.3" >> /opt/so/conf/reposync/mirror.txt
+		echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /opt/so/conf/reposync/mirror.txt
+		echo "https://repo-alt.securityonion.net/prod/2.4/oracle/9" >> /opt/so/conf/reposync/mirror.txt
 		echo "[main]" > /opt/so/conf/reposync/repodownload.conf
 		echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
 		echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf