Merge branch '2.4/dev' of https://github.com/Security-Onion-Solutions/securityonion into airgaps

2026-04-26 22:47:49 +02:00 · 2023-05-26 15:16:38 -04:00
parent b2d2a9f0ed 32021cf272
commit 38881231ac
9 changed files with 275 additions and 141 deletions
@@ -10,12 +10,12 @@

 . /usr/sbin/so-common

-FLEETHOST="https://{{ GLOBALS.manager_ip }}:8220"
+#FLEETHOST="https://{{ GLOBALS.manager_ip }}:8220"

 for i in {1..30}
 do
-   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints")) | .api_key')
-   #FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts' | jq -r  '.items[].host_urls[]' | paste -sd ',')
+   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+   FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r  '.item.host_urls[]' | paste -sd ',')
 if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
 done
 if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then printf "\nFleet Host URL or Enrollment Token empty - exiting..." && exit; fi
@@ -35,9 +35,16 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
 printf "\n\n"
 {%- endif %}

+# Add Manager IP & URL Base to Fleet Host URLs
 printf "\nAdd SO-Manager Fleet URL\n"
+if [ "{{ GLOBALS.manager_ip }}" = "{{ GLOBALS.url_base }}" ]; then
+    JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220"]}')
+else
+    JSON_STRING=$( jq -n '{"id":"grid-default","name":"grid-default","is_default":true,"host_urls":["https://{{ GLOBALS.url_base }}:8220", "https://{{ GLOBALS.manager_ip }}:8220"]}')
+fi
+
 ## This array replaces whatever URLs are currently configured
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220", "https://{{ GLOBALS.manager }}:8220"]}'
+curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/fleet_server_hosts" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  "$JSON_STRING"
 printf "\n\n"


@@ -74,7 +81,7 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
 ### Finalization ###

 # Query for Enrollment Tokens for default policies
-ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-default")) | .api_key')
+ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
 GRIDNODESENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes")) | .api_key')

 # Store needed data in minion pillar
@@ -6,8 +6,13 @@ global:
  managerip:
    description: The IP address of the grid manager.
    global: True
+    advanced: True
+    regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$
+    regexFailureMessage: You must enter a valid IP address or CIDR.
  mdengine:
    description: What engine to use for meta data generation. Options are ZEEK and SURICATA.
+    regex: ^(ZEEK|SURICATA)$
+    regexFailureMessage: You must enter either ZEEK or SURICATA.
    global: True
  ids:
    description: Which IDS engine to use. Currently only Suricata is supported.
@@ -2,56 +2,83 @@ suricata:
  enabled: False
  config:
    threading:
-        set-cpu-affinity: 'no'
-        detect-thread-ratio: 1.0
-        cpu-affinity:
-            - management-cpu-set:
-                cpu: []
-            - receive-cpu-set:
-                cpu: []
-            - worker-cpu-set:
-                cpu: []
-                mode: exclusive
-                threads: 1
-                prio:
-                    default: high
+      set-cpu-affinity: "no"
+      cpu-affinity:
+        management-cpu-set:
+          cpu:
+            - 1
+        worker-cpu-set:
+          cpu:
+            - 2-3
+          mode: exclusive
+          prio:
+            default: high
    af-packet:
-        interface: bond0
-        cluster-id: 59
-        cluster-type: cluster_flow
-        defrag: true
-        use-mmap: true
-        threads: 1
-        tpacket-v3: true
-        ring-size: 5000
+      interface: bond0
+      cluster-id: 59
+      cluster-type: cluster_flow
+      defrag: "yes"
+      use-mmap: "yes"
+      threads: 1
+      tpacket-v3: "yes"
+      ring-size: 5000
    vars:
      address-groups:
-        HOME_NET: "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
-        EXTERNAL_NET: "any"
-        HTTP_SERVERS: "$HOME_NET"
-        SMTP_SERVERS: "$HOME_NET"
-        SQL_SERVERS: "$HOME_NET"
-        DNS_SERVERS: "$HOME_NET"
-        TELNET_SERVERS: "$HOME_NET"
-        AIM_SERVERS: "$EXTERNAL_NET"
-        DC_SERVERS: "$HOME_NET"
-        DNP3_SERVER: "$HOME_NET"
-        DNP3_CLIENT: "$HOME_NET"
-        MODBUS_CLIENT: "$HOME_NET"
-        MODBUS_SERVER: "$HOME_NET"
-        ENIP_CLIENT: "$HOME_NET"
-        ENIP_SERVER: "$HOME_NET"  
+        HOME_NET:
+          - 192.168.0.0/16
+          - 10.0.0.0/8
+          - 172.16.0.0/12
+        EXTERNAL_NET:
+          - any
+        HTTP_SERVERS:
+          - $HOME_NET
+        SMTP_SERVERS:
+          - $HOME_NET
+        SQL_SERVERS:
+          - $HOME_NET
+        DNS_SERVERS:
+          - $HOME_NET
+        TELNET_SERVERS:
+          - $HOME_NET
+        AIM_SERVERS:
+          - $EXTERNAL_NET
+        DC_SERVERS:
+          - $HOME_NET
+        DNP3_SERVER:
+          - $HOME_NET
+        DNP3_CLIENT:
+          - $HOME_NET
+        MODBUS_CLIENT:
+          - $HOME_NET
+        MODBUS_SERVER:
+          - $HOME_NET
+        ENIP_CLIENT:
+          - $HOME_NET
+        ENIP_SERVER:
+          - $HOME_NET
      port-groups:
-        HTTP_PORTS: "80"
-        SHELLCODE_PORTS: "!80"
-        ORACLE_PORTS: "1521"
-        SSH_PORTS: "22"
-        DNP3_PORTS: "20000"
-        MODBUS_PORTS: "502"
-        FILE_DATA_PORTS: "[$HTTP_PORTS,110,143]"
-        FTP_PORTS: "21"
-        VXLAN_PORTS: "4789"
-        TEREDO_PORTS: "3544"
+        HTTP_PORTS:
+          - 80
+        SHELLCODE_PORTS:
+          - "!80"
+        ORACLE_PORTS:
+          - 1521
+        SSH_PORTS:
+          - 22
+        DNP3_PORTS:
+          - 20000
+        MODBUS_PORTS:
+          - 502
+        FILE_DATA_PORTS:
+          - $HTTP_PORTS
+          - 110
+          - 143
+        FTP_PORTS:
+          - 21
+        VXLAN_PORTS:
+          - 4789
+        TEREDO_PORTS:
+          - 3544
    default-log-dir: /var/log/suricata/
    stats:
      enabled: "yes"
@@ -69,24 +96,24 @@ suricata:
        pcap-file: false
        community-id: true
        community-id-seed: 0
-        xff:
-          enabled: "no"
-          mode: extra-data
-          deployment: reverse
-          header: X-Forwarded-For
        types:
-          - alert:
-              payload: "no"
-              payload-buffer-size: 4kb
-              payload-printable: "yes"
-              packet: "yes"
-              metadata: 
-                app-layer: false
-                flow: false
-                rule:
-                  metadata: true
-                  raw: true
-              tagged-packets: "no"
+          alert:
+            payload: "no"
+            payload-buffer-size: 4kb
+            payload-printable: "yes"
+            packet: "yes"
+            metadata:
+              app-layer: false
+              flow: false
+              rule:
+                metadata: true
+                raw: true
+            tagged-packets: "no"
+            xff:
+              enabled: "no"
+              mode: extra-data
+              deployment: reverse
+              header: X-Forwarded-For
      unified2-alert:
        enabled: "no"
      http-log:
@@ -1,4 +1,4 @@
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'suricata/defaults.yaml' as SURICATADEFAULTS %}
 {% set SURICATAMERGED = salt['pillar.get']('suricata', SURICATADEFAULTS.suricata, merge=True) %}
 {% import_yaml 'suricata/suricata_mdengine.yaml' as suricata_mdengine %}
@@ -23,6 +23,45 @@
 {% do SURICATAMERGED.config.pop('af-packet') %}
 {% do SURICATAMERGED.config.update({'af-packet': afpacket}) %}

+{# eve-log.types is a list but we convert to dict in defaults to work with ui #}
+{# below they are converted back to lists #}
+{% load_yaml as evelogtypes %}
+{% for le, ld in SURICATAMERGED.config.outputs['eve-log'].types.items() %}
+  - {{ le }}: {{ ld }}
+{% endfor %}
+{% endload %}
+{% do SURICATAMERGED.config.outputs['eve-log'].pop('types') %}
+{% do SURICATAMERGED.config.outputs['eve-log'].update({'types': evelogtypes}) %}
+
+{# threading.cpu-affinity is a list but we convert to dict in defaults to work with ui #}
+{# below they are converted back to lists #}
+{% load_yaml as cpuaffinity %}
+{% for le, ld in SURICATAMERGED.config.threading['cpu-affinity'].items() %}
+  - {{ le }}: {{ ld }}
+{% endfor %}
+{% endload %}
+{% do SURICATAMERGED.config.threading.pop('cpu-affinity') %}
+{% do SURICATAMERGED.config.threading.update({'cpu-affinity': cpuaffinity}) %}
+
+{# Find the index of eve-log and file-store in suricata_mdengine.suricata.config.outputs #}
+{# update outputs eve-log.types and filestore with config for Suricata metadata engine #}
+{% if GLOBALS.md_engine == 'SURICATA' %}
+{%   for li in suricata_mdengine.suricata.config.outputs %}
+{%     if 'eve-log' in li.keys() %}
+{%       do surimeta_evelog_index.append(loop.index0) %}
+{%     endif %}
+{%     if 'file-store' in li.keys() %}
+{%       do surimeta_filestore_index.append(loop.index0) %}
+{%     endif %}
+{%   endfor %}
+{%   set surimeta_evelog_index = surimeta_evelog_index[0] %}
+{%   set surimeta_filestore_index = surimeta_filestore_index[0] %}
+{%   do SURICATAMERGED.config.outputs['eve-log'].types.extend(suricata_mdengine.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %}
+{%   do SURICATAMERGED.config.outputs['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
+{% endif %}
+
+{# outputs is a list but we convert to dict in defaults to work with ui #}
+{# below they are converted back to lists #}
 {% load_yaml as outputs %}
 {% for le, ld in SURICATAMERGED.config.outputs.items() %}
  - {{ le }}: {{ ld }}
@@ -31,31 +70,22 @@
 {% do SURICATAMERGED.config.pop('outputs') %}
 {% do SURICATAMERGED.config.update({'outputs': outputs}) %}

-{# Find the index of eve-log so it can be updated later #}
-{% for li in SURICATAMERGED.config.outputs %}
-  {% if 'eve-log' in li.keys() %}
-    {% do default_evelog_index.append(loop.index0) %}
-  {% endif %}
-  {% if 'file-store' in li.keys() %}
-    {% do default_filestore_index.append(loop.index0) %}
-  {% endif %}
+{# change address-groups vars from list to comma seperated string #}
+{% for k, v in SURICATAMERGED.config.vars['address-groups'].items() %}
+{#   if address-group value is a list #}
+{%   if v is iterable and (v is not string and v is not mapping and v | length > 1) %}
+{%     do SURICATAMERGED.config.vars['address-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %}
+{%   else %}
+{%     do SURICATAMERGED.config.vars['address-groups'].update({k: v[0]}) %}
+{%   endif %}
 {% endfor %}
-{% set default_evelog_index = default_evelog_index[0]  %}
-{% set default_filestore_index = default_filestore_index[0]  %}

-{# Find the index of eve-log so it can be grabbed later #}
-{% for li in suricata_mdengine.suricata.config.outputs %}
-  {% if 'eve-log' in li.keys() %}
-    {% do surimeta_evelog_index.append(loop.index0) %}
-  {% endif %}
-  {% if 'file-store' in li.keys() %}
-    {% do surimeta_filestore_index.append(loop.index0) %}
-  {% endif %}
+{# change port-groups vars from list to comma seperated string #}
+{% for k, v in SURICATAMERGED.config.vars['port-groups'].items() %}
+{#   if address-group value is a list #}
+{%   if v is iterable and (v is not string and v is not mapping and v | length > 1) %}
+{%     do SURICATAMERGED.config.vars['port-groups'].update({k: '[' ~ v | join(',') ~ ']'}) %}
+{%   else %}
+{%     do SURICATAMERGED.config.vars['port-groups'].update({k: v[0]}) %}
+{%   endif %}
 {% endfor %}
-{% set surimeta_evelog_index = surimeta_evelog_index[0] %}
-{% set surimeta_filestore_index = surimeta_filestore_index[0] %}
-
-{% if GLOBALS.md_engine == 'SURICATA' %}
-  {% do SURICATAMERGED.config.outputs[default_evelog_index]['eve-log'].types.extend(suricata_mdengine.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %}
-  {% do SURICATAMERGED.config.outputs[default_filestore_index]['file-store'].update({'enabled':suricata_mdengine.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
-{% endif %}
@@ -12,10 +12,54 @@ suricata:
      title: SIDS
      helpLink: suricata.html
  config:
+    af-packet:
+      interface:
+        description: The network interface that Suricata will monitor.
+        helpLink: suricata.html
+      cluster-id:
+        advanced: True
+      cluster-type:
+        advanced: True
+        regex: ^(cluster_flow|cluster_qm)$
+      defrag:
+        advanced: True
+        regex: ^(yes|no)$
+      use-mmap:
+        advanced: True
+        readonly: True
+      threads:
+        description: The amount of worker threads.
+        helpLink: suricata.html
+        forcedType: int
+      tpacket-v3:
+        advanced: True
+        readonly: True
+      ring-size:
+        description: Buffer size for packets per thread.
+        forcedType: int
+        helpLink: suricata.html
+    threading:
+      set-cpu-affinity:
+        description: Bind(yes) or unbind(no) management and worker threads to a core or range of cores.
+        regex: ^(yes|no)$
+        helpLink: suricata.html
+      cpu-affinity:
+        management-cpu-set:
+          cpu:
+            description: Bind management threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to 'yes' for this to be used.
+            forcedType: "[]string"
+            helpLink: suricata.html
+        worker-cpu-set:
+          cpu:
+            description: Bind worker threads to a core or range of cores. This can be a sigle core, list of cores, or list of range of cores. set-cpu-affinity must be set to 'yes' for this to be used.
+            forcedType: "[]string"
+            helpLink: suricata.html
    vars:
      address-groups:
        HOME_NET:
          description: List of hosts or networks.
+          regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$
+          regexFailureMessage: You must enter a valid IP address or CIDR.
          helpLink: suricata.html
        EXTERNAL_NET: 
          description: List of hosts or networks.
@@ -92,19 +136,21 @@ suricata:
          helpLink: suricata.html
    outputs:
      eve-log:
-        xff:
-          enabled:
-            description: Enable X-Forward-For support. 
-            helpLink: suricata.html
-          mode: 
-            description: Operation mode. This should always be extra-data if you use PCAP.
-            helpLink: suricata.html
-          deployment: 
-            description: forward would use the first IP address and reverse would use the last.
-            helpLink: suricata.html
-          header: 
-            description: Header name where the actual IP address will be reported.
-            helpLink: suricata.html
+        types:
+          alert:
+            xff:
+              enabled:
+                description: Enable X-Forward-For support.
+                helpLink: suricata.html
+              mode:
+                description: Operation mode. This should always be extra-data if you use PCAP.
+                helpLink: suricata.html
+              deployment:
+                description: forward would use the first IP address and reverse would use the last.
+                helpLink: suricata.html
+              header:
+                description: Header name where the actual IP address will be reported.
+                helpLink: suricata.html
    asn1-max-frames:
      description: Maximum nuber of asn1 frames to decode.
      helpLink: suricata.html