Merge branch 'dev' into feature/nginx-update

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.0
+## Security Onion 2.3.1
 
-Security Onion 2.3.0 is here!
+Security Onion 2.3.1 is here!
 
 
 ### Release Notes

VERIFY_ISO.md

@@ -1,16 +1,16 @@
-### 2.3.0 ISO image built on 2020/10/15
+### 2.3.1 ISO image built on 2020/10/22
 
 ### Download and Verify
 
-2.3.0 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.0.iso
+2.3.1 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.1.iso
 
-MD5: E05B220E4FD7C054DF5C50906EE1375B
-SHA1: 55E93C6EAB140AB4A0F07873CC871EBFDC699CD6  
-SHA256: 57B96A6E0951143E123BFC0CD0404F7466776E69F3C115F5A0444C0C6D5A6E32 
+MD5: EF2DEBCCBAE0B0BCCC906552B5FF918A
+SHA1: 16AFCACB102BD217A038044D64E7A86DA351640E  
+SHA256: 7125F90B6323179D0D29F5745681BE995BD2615E64FA1E0046D94888A72C539E 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.0.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.1.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -24,22 +24,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.0.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.1.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.0.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.1.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.0.iso.sig securityonion-2.3.0.iso
+gpg --verify securityonion-2.3.1.iso.sig securityonion-2.3.1.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Thu 15 Oct 2020 08:06:28 PM EDT using RSA key ID FE507013
+gpg: Signature made Thu 22 Oct 2020 10:34:27 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.1
+2.3.10

salt/common/maps/manager.map.jinja

@@ -5,7 +5,6 @@
     'so-telegraf',
     'so-soc',
     'so-kratos',
-    'so-aptcacherng',
     'so-idstools',
     'so-redis',
     'so-elasticsearch',
@@ -15,4 +14,8 @@
     'so-filebeat',
     'so-soctopus'
   ]
-} %}
+} %}
+
+{% if salt['pillar.get']('global:managerupdate') == 1 %}
+   {% do docker.containers.append('so-aptcacherng') %}
+{% endif %}

salt/common/maps/managersearch.map.jinja

@@ -4,7 +4,6 @@
     'so-telegraf',
     'so-soc',
     'so-kratos',
-    'so-aptcacherng',
     'so-idstools',
     'so-redis',
     'so-logstash',
@@ -15,4 +14,8 @@
     'so-filebeat',
     'so-soctopus'
   ]
-} %}
+} %}
+
+{% if salt['pillar.get']('global:managerupdate') == 1 %}
+   {% do docker.containers.append('so-aptcacherng') %}
+{% endif %}

salt/common/maps/standalone.map.jinja

@@ -4,7 +4,6 @@
     'so-telegraf',
     'so-soc',
     'so-kratos',
-    'so-aptcacherng',
     'so-idstools',
     'so-redis',
     'so-logstash',
@@ -19,4 +18,8 @@
     'so-soctopus',
     'so-sensoroni'
   ]
-} %}
+} %}
+
+{% if salt['pillar.get']('global:managerupdate') == 1 %}
+   {% do docker.containers.append('so-aptcacherng') %}
+{% endif %}

salt/common/tools/sbin/so-features-enable

@@ -51,7 +51,7 @@ manager_check() {
 }
 
 manager_check
-VERSION=$(grep soversion $local_salt_dir/pillar/global.sls | cut -d':' -f2|sed 's/ //g')
+VERSION=$(lookup_pillar soversion) 
 # Modify global.sls to enable Features
 sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls
 SUFFIX="-features"

salt/common/tools/sbin/so-firewall

@@ -116,7 +116,7 @@ def addhostgroup(args):
     print('Missing host group name argument', file=sys.stderr)
     showUsage(args)
 
-  name = args[1]
+  name = args[0]
   content = loadYaml(hostgroupsFilename)
   if name in content['firewall']['hostgroups']:
     print('Already exists', file=sys.stderr)

salt/common/tools/sbin/so-status

@@ -27,10 +27,15 @@ ERROR_STRING="ERROR"
 SUCCESS_STRING="OK"
 PENDING_STRING="PENDING"
 MISSING_STRING='MISSING'
+DISABLED_STRING='DISABLED'
 CALLER=$(ps -o comm= $PPID)
 declare -a BAD_STATUSES=("removing" "paused" "exited" "dead")
 declare -a PENDING_STATUSES=("paused" "created" "restarting")
 declare -a GOOD_STATUSES=("running")
+declare -a DISABLED_CONTAINERS=()
+{%- if salt['pillar.get']('steno:enabled', 'True') is sameas false %}
+DISABLED_CONTAINERS+=("so-steno")
+{%- endif %}
 
 declare -a temp_container_name_list=()
 declare -a temp_container_state_list=()
@@ -104,6 +109,7 @@ populate_container_lists() {
 
 parse_status() {
     local container_state=${1}
+    local service_name=${2}
 
     [[ $container_state = "missing" ]] && printf $MISSING_STRING && return 1
 
@@ -117,7 +123,13 @@ parse_status() {
 
     # This is technically not needed since the default is error state
     for state in "${BAD_STATUSES[@]}"; do
-        [[ $container_state = "$state" ]] && printf $ERROR_STRING && return 1
+        if [[ " ${DISABLED_CONTAINERS[@]} " =~ " ${service_name} " ]]; then
+            printf $DISABLED_STRING
+            return 0
+        elif [[ $container_state = "$state" ]]; then
+            printf $ERROR_STRING
+            return 1
+        fi
     done
 
     printf $ERROR_STRING && return 1
@@ -127,7 +139,7 @@ parse_status() {
 
 print_line() {
     local service_name=${1}
-    local service_state="$( parse_status ${2} )"
+    local service_state="$( parse_status ${2} ${1} )"
     local columns=$(tput cols)
     local state_color="\e[0m"
 
@@ -137,7 +149,7 @@ print_line() {
         state_color="\e[1;31m"
     elif [[ $service_state = "$SUCCESS_STRING" ]]; then
         state_color="\e[1;32m"
-    elif [[ $service_state = "$PENDING_STRING" ]]; then
+    elif [[ $service_state = "$PENDING_STRING" ]] || [[ $service_state = "$DISABLED_STRING" ]]; then
         state_color="\e[1;33m"
     fi
 

salt/common/tools/sbin/soup

@@ -47,7 +47,7 @@ airgap_mounted() {
     echo "If you just copied the .iso file over you can specify the path."
     echo "If you burned the ISO to a disk the standard way you can specify the device."
     echo "Example: /home/user/securityonion-2.X.0.iso"
-    echo "Example: /dev/cdrom"
+    echo "Example: /dev/sdx1"
     echo ""
     read -p 'Enter the location of the iso: ' ISOLOC
     if [ -f $ISOLOC ]; then

salt/docker_clean/init.sls

@@ -1,6 +1,6 @@
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
-{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3']%}
+{% set OLDVERSIONS = ['2.0.0-rc.1','2.0.1-rc.1','2.0.2-rc.1','2.0.3-rc.1','2.1.0-rc.2','2.2.0-rc.3','2.3.0']%}
 
 {% for VERSION in OLDVERSIONS %}
 remove_images_{{ VERSION }}:
@@ -42,4 +42,4 @@ remove_images_{{ VERSION }}:
       - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-thehive-es:{{ VERSION }}'
       - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-wazuh:{{ VERSION }}'
       - '{{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}'
-{% endfor %}
+{% endfor %}

salt/elasticsearch/files/ingest/common.nids

@@ -6,7 +6,7 @@
     { "set":            { "if": "ctx.rule?.uuid > 1999999",             "field": "rule.reference", "value": "https://doc.emergingthreats.net/{{rule.uuid}}"                     } },
     { "convert":        { "if": "ctx.rule.uuid != null",                        "field": "rule.uuid",                   "type": "string"                                                                               } },
     { "dissect":        { "if": "ctx.rule.name != null",                        "field": "rule.name",                   "pattern" : "%{rule_type} %{rest_of_rulename} ",                                "ignore_failure": true  } },
-    { "set":            { "if": "ctx.rule_type == 'GPL'",       "field": "rule_ruleset", "value": "Snort GPL"                                                   } },
+    { "set":            { "if": "ctx.rule_type == 'GPL'",       "field": "rule.ruleset", "value": "Snort GPL"                                                   } },
     { "set":            { "if": "ctx.rule_type == 'ET'",        "field": "rule.ruleset", "value": "Emerging Threats"                                            } },
     { "set":            { "if": "ctx.rule.severity == 3",   "field": "event.severity", "value": 1, "override": true }  },
     { "set":            { "if": "ctx.rule.severity == 2",   "field": "event.severity", "value": 2, "override": true }  },

salt/filebeat/etc/filebeat.yml

@@ -74,7 +74,6 @@ filebeat.modules:
 # List of prospectors to fetch data.
 filebeat.inputs:
 #------------------------------ Log prospector --------------------------------
-{%- if grains['role'] in ['so-sensor', "so-eval", "so-helix", "so-heavynode", "so-standalone", "so-import"] %}
 - type: udp
   enabled: true
   host: "0.0.0.0:514"
@@ -100,6 +99,8 @@ filebeat.inputs:
   - drop_fields:
       fields: ["source", "prospector", "input", "offset", "beat"]
   fields_under_root: true
+
+{%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor',  'so-helix', 'so-heavynode', 'so-import'] %}
   {%- if ZEEKVER != 'SURICATA' %}
     {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '')  %}
 - type: log

salt/filebeat/init.sls

@@ -82,6 +82,7 @@ so-filebeat:
       - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro
     - port_bindings:
         - 0.0.0.0:514:514/udp
+        - 0.0.0.0:514:514/tcp
     - watch:
       - file: /opt/so/conf/filebeat/etc/filebeat.yml
 

salt/firewall/assigned_hostgroups.map.yaml

@@ -134,6 +134,7 @@ role:
               - {{ portgroups.redis }}
               - {{ portgroups.minio }}
               - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.beats_5644 }}
           self:
             portgroups:
               - {{ portgroups.syslog}}
@@ -424,6 +425,9 @@ role:
           elasticsearch_rest:
             portgroups:
               - {{ portgroups.elasticsearch_rest }}
+          self:
+            portgroups:
+              - {{ portgroups.syslog}}
       INPUT:
         hostgroups:
           anywhere:
@@ -437,6 +441,11 @@ role:
               - {{ portgroups.all }}
   sensor:
     chain:
+      DOCKER-USER:
+        hostgroups:
+          self:
+            portgroups:
+              - {{ portgroups.syslog}}
       INPUT:
         hostgroups:
           anywhere:
@@ -463,6 +472,9 @@ role:
           elasticsearch_rest:
             portgroups:
               - {{ portgroups.elasticsearch_rest }}
+          self:
+            portgroups:
+              - {{ portgroups.syslog}}
       INPUT:
         hostgroups:
           anywhere:
@@ -530,9 +542,6 @@ role:
             portgroups:
               - {{ portgroups.redis }}
               - {{ portgroups.elasticsearch_node }}
-          self:
-            portgroups:
-              - {{ portgroups.syslog}}
           beats_endpoint:
             portgroups:
               - {{ portgroups.beats_5044 }}

salt/pcap/init.sls

@@ -23,7 +23,7 @@
 {% set INTERFACE = salt['pillar.get']('sensor:interface', 'bond0') %}
 {% set BPF_STENO = salt['pillar.get']('steno:bpf', None) %}
 {% set BPF_COMPILED = "" %}
-{% from "pcap/map.jinja" import START with context %}
+{% from "pcap/map.jinja" import STENOOPTIONS with context %}
 
 # PCAP Section
 
@@ -135,9 +135,9 @@ sensoronilog:
     - makedirs: True
 
 so-steno:
-  docker_container.running:
+  docker_container.{{ STENOOPTIONS.status }}:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-steno:{{ VERSION }}
-    - start: {{ START }}
+    - start: {{ STENOOPTIONS.start }}
     - network_mode: host
     - privileged: True
     - port_bindings:

salt/pcap/map.jinja

@@ -1,6 +1,15 @@
-# don't start the docker container if it is an import node
-{% if grains.id.split('_')|last == 'import' %}
-  {% set START = False %}
+{% set STENOOPTIONS = {} %}
+{% set ENABLED = salt['pillar.get']('steno:enabled', 'True') %}
+
+# don't start the docker container if it is an import node or disabled via pillar
+{% if grains.id.split('_')|last == 'import' or ENABLED is sameas false %}
+  {% do STENOOPTIONS.update({'start': False}) %}
 {% else %}
-  {% set START = True %}
+  {% do STENOOPTIONS.update({'start': True}) %}
+{% endif %}
+
+{% if ENABLED is sameas false %}
+  {% do STENOOPTIONS.update({'status': 'stopped'}) %}
+{% else %}
+  {% do STENOOPTIONS.update({'status': 'running'}) %}
 {% endif %}

salt/soc/files/soc/alerts.actions.json

@@ -1,6 +1,6 @@
 [
-          { "name": "", "description": "actionHuntHelp", "icon": "fa-search", "link": "/#/hunt?q=\"{value}\" | groupby event.module event.dataset", "target": "" },
-          { "name": "", "description": "actionPcapHelp", "icon": "fa-stream", "link": "/joblookup?esid={eventId}", "target": "" },
-          { "name": "", "description": "actionGoogleHelp", "icon": "fab fa-google", "link": "https://www.google.com/search?q={value}", "target": "_blank" },
-          { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "", "link": "https://www.virustotal.com/gui/search/{value}", "target": "_blank" }
-        ]
+  { "name": "", "description": "actionHuntHelp", "icon": "fa-crosshairs", "link": "/#/hunt?q=\"{value}\" | groupby event.module event.dataset", "target": "" },
+  { "name": "", "description": "actionPcapHelp", "icon": "fa-stream", "link": "/joblookup?esid={eventId}", "target": "" },
+  { "name": "", "description": "actionGoogleHelp", "icon": "fab fa-google", "link": "https://www.google.com/search?q={value}", "target": "_blank" },
+  { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "", "link": "https://www.virustotal.com/gui/search/{value}", "target": "_blank" }
+]

salt/soc/files/soc/changes.json

@@ -1,25 +1,13 @@
 {
-  "title": "Security Onion 2.3.0 is here!",
+  "title": "Security Onion 2.3.1 is here!",
   "changes": [
-    { "summary": "We have a new Alerts interface for reviewing alerts and acknowledging or escalating them. Escalating creates a new case in TheHive. Please note that TheHive no longer receives alerts directly." },
-    { "summary": "Kibana no longer presents the option to create alerts from events, but instead allows creation of cases from events." },
-    { "summary": "Our Security Onion ISO now works for UEFI as well as Secure Boot." },
-    { "summary": "Airgap deployments can now be updated using the latest ISO. Please read this documentation carefully." },
-    { "summary": "Suricata has been updated to version 5.0.4." },
-    { "summary": "Zeek has been updated to version 3.0.11." },
-    { "summary": "Stenographer has been updated to the latest version." },
-    { "summary": "soup will now attempt to clean up old docker images to free up space." },
-    { "summary": "Hunt actions can be customized via hunt.actions.json." },
-    { "summary": "Hunt queries can be customized via hunt.queries.json." },
-    { "summary": "Hunt event fields can be customized via hunt.eventfields.json." },
-    { "summary": "Alerts actions can be customized via alerts.actions.json." },
-    { "summary": "Alerts queries can be customized via alerts.queries.json." },
-    { "summary": "Alerts event fields can be customized via alerts.eventfields.json." },
-    { "summary": "The help documentation is now viewable offline for airgap installations." },
-    { "summary": "The script so-user-add will now validate the password is acceptable before attempting to create the user." },
-    { "summary": "Playbook and Grafana no longer use static passwords for their admin accounts." },
-    { "summary": "Analyst VM now comes with NetworkMiner 2.6 installed." },
-    { "summary": "Strelka YARA matches now generate alerts that can be viewed through the Alerts interface." },
+    { "summary": "Fixed a SOC issue in airgap mode that was preventing people from logging in." },
+    { "summary": "Downloading Elastic features images will now download the correct images." },
+    { "summary": "Winlogbeat download no longer requires Internet access." },
+    { "summary": "Adjusted Alerts quick action bar to allow searching for a specific value while remaining in Alerts view." },
+    { "summary": "/nsm will properly display disk usage on the standalone Grafana dashboard." },
+    { "summary": "The manager node now has syslog listener enabled by default (you'll still need to allow syslog traffic through the firewall of course)." },
+    { "summary": "Fixed an issue when creating host groups with so-firewall." },
     { "summary": "Known Issues <ul><li>It is still possible to update your grid from any release candidate to 2.3. However, if you have a true production deployment, then we recommend a fresh image and install for best results.</li><li>In 2.3.0 we made some changes to data types in the elastic index templates. This will cause some errors in Kibana around field conflicts. You can address this in 2 ways:<ol><li>Delete all the data on the ES nodes preserving all of your other settings suchs as BPFs by running sudo so-elastic-clear on all the search nodes</li><li>Re-Index the data. This is not a quick process but you can find more information at <a href='https://docs.securityonion.net/en/2.3/elasticsearch.html#re-indexing' target='so-help'>https://docs.securityonion.net/en/2.3/elasticsearch.html#re-indexing</a></li></ol><li>Please be patient as we update our documentation. We have made a concerted effort to update as much as possible but some things still may be incorrect or ommited. If you have questions or feedback, please start a discussion at <a href='https://securityonion.net/discuss' target='so-discuss'>https://securityonion.net/discuss</a>.</li><li>Once you update your grid to 2.3.0, any new nodes that join the grid must be 2.3.0. For example, if you try to join a new RC1 node it will fail. For best results, use the latest ISO (or 2.3.0 installer from github) when joining to an 2.3.0 grid.</li><li>Shipping Windows Eventlogs with Osquery will fail intermittently with utf8 errors logged in the Application log. This is scheduled to be fixed in Osquery 4.5.</li><li>When running soup to upgrade from RC1/RC2/RC3 to 2.3.0, there is a Salt error that occurs during the final highstate. This error is related to the patch_os_schedule and can be ignored as it will not occur again in subsequent highstates.</li><li>When Search Nodes are upgraded from RC1 to 2.3.0, there is a chance of a race condition where certificates are missing. This will show errors in the manager log to the remote node. To fix this run the following on the search node that is having the issue:<ol><li>Stop elasticsearch - <i>sudo so-elasticsearch-stop</i></li><li>Run the SSL state - <i>sudo salt-call state.apply ssl</i></li><li>Restart elasticsearch - <i>sudo so-elasticsearch-restart</i></li></ol></li><li>If you are upgrading from RC1 you might see errors around registry:2 missing. This error does not break the actual upgrade. To fix, run the following on the manager:</li><ol><li>Stop the Docker registry - sudo docker stop so-dockerregistry</li><li>Remove the container - sudo docker rm so-dockerregistry</li><li>Run the registry state - sudo salt-call state.apply registry</li></ol></ul>" }
   ]
 }

salt/soc/files/soc/hunt.actions.json

@@ -1,6 +1,5 @@
 [
-          { "name": "", "description": "actionHuntHelp", "icon": "fa-search", "link": "/#/hunt?q=\"{value}\" | groupby event.module event.dataset", "target": "" },
-          { "name": "", "description": "actionPcapHelp", "icon": "fa-stream", "link": "/joblookup?esid={eventId}", "target": "" },
-          { "name": "", "description": "actionGoogleHelp", "icon": "fab fa-google", "link": "https://www.google.com/search?q={value}", "target": "_blank" },
-          { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "", "link": "https://www.virustotal.com/gui/search/{value}", "target": "_blank" }
-        ]
+  { "name": "", "description": "actionPcapHelp", "icon": "fa-stream", "link": "/joblookup?esid={eventId}", "target": "" },
+  { "name": "", "description": "actionGoogleHelp", "icon": "fab fa-google", "link": "https://www.google.com/search?q={value}", "target": "_blank" },
+  { "name": "actionVirusTotal", "description": "actionVirusTotalHelp", "icon": "", "link": "https://www.virustotal.com/gui/search/{value}", "target": "_blank" }
+]

salt/soc/files/soc/soc.json

@@ -16,7 +16,7 @@
     "baseUrl": "/",
     "maxPacketCount": 5000,
     "htmlDir": "html",
-    {%- if ISAIRGAP is sameas true -%}
+    {%- if ISAIRGAP is sameas true %}
     "airgapEnabled": true,
     {%- else %}
     "airgapEnabled": false,
@@ -54,8 +54,8 @@
       }
     },
     "client": {
-      {%- if ISAIRGAP is sameas true -%}
-      "docsUrl": "/docs/,
+      {%- if ISAIRGAP is sameas true %}
+      "docsUrl": "/docs/",
       {%- else %}
       "docsUrl": "https://docs.securityonion.net/en/2.3/",
       {%- endif %}