Merge remote-tracking branch 'remotes/origin/dev' into issue/6469

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.90-20211206
+## Security Onion 2.3.90-20211213
 
-Security Onion 2.3.90-20211206 is here!
+Security Onion 2.3.90-20211213 is here!
 
 ## Screenshots
 

VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.3.90-20211206 ISO image built on 2021/12/06
+### 2.3.90-20211213 ISO image built on 2021/12/13
 
 
 
 ### Download and Verify
 
-2.3.90-20211206 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.90-20211206.iso
+2.3.90-20211213 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.3.90-20211213.iso
 
-MD5: 8A5FDF731D548E27D123E5B711890AEC  
-SHA1: B4AF33FE1D64592D46C780AF0C5E7FBD21A22BDE  
-SHA256: 091DA2D06C82447639D324EE32DBC385AE407078B3A55F4E0704B22DB6B29A7E 
+MD5: D7E90433B416627347DD54B7C3C07F18  
+SHA1: 11E212B2237162749F5E3BD959C84D6C4720D213  
+SHA256: 01DD0AF3CF5BBFD4AF7463F8897935A885E3D9CC8B9B3B5E9A01E0A2EF37ED95 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.90-20211206.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.90-20211213.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.90-20211206.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.90-20211213.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.90-20211206.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.90-20211213.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.90-20211206.iso.sig securityonion-2.3.90-20211206.iso
+gpg --verify securityonion-2.3.90-20211213.iso.sig securityonion-2.3.90-20211213.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 06 Dec 2021 10:14:29 AM EST using RSA key ID FE507013
+gpg: Signature made Mon 13 Dec 2021 11:46:27 AM EST using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

salt/common/tools/sbin/so-image-common

@@ -132,7 +132,7 @@ update_docker_containers() {
   # Let's make sure we have the public key
   run_check_net_err \
   "curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
-  "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \
+  "Could not pull signature key file, please ensure connectivity to https://raw.githubusercontent.com" \
   noretry >> "$LOG_FILE" 2>&1
   result=$?
   if [[ $result -eq 0 ]]; then

salt/elasticsearch/init.sls

@@ -131,6 +131,10 @@ esrolesdir:
     - group: 939
     - makedirs: True
 
+eslibdir:
+  file.absent:
+    - name: /opt/so/conf/elasticsearch/lib
+   
 esingestdynamicconf:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/ingest
@@ -258,7 +262,7 @@ so-elasticsearch:
       {% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %}
       - discovery.type=single-node
       {% endif %}
-      - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true
+      - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
       ulimits:
       - memlock=-1:-1
       - nofile=65536:65536

salt/logstash/etc/jvm.options

@@ -0,0 +1 @@
+-Dlog4j2.formatMsgNoLookups=true

salt/logstash/init.sls

@@ -57,6 +57,10 @@ logstash:
     - gid: 931
     - home: /opt/so/conf/logstash
 
+lslibdir:
+  file.absent:
+    - name: /opt/so/conf/logstash/lib
+
 lsetcdir:
   file.directory:
     - name: /opt/so/conf/logstash/etc

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -7,13 +7,30 @@
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
   if "beat-ext" in [tags] and "import" not in [tags] and "filebeat" not in [metadata][pipeline] {
+    if [metadata][_id] {
       elasticsearch {
         pipeline => "beats.common" 
         hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
+  {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
         user => "{{ ES_USER }}"
         password => "{{ ES_PASS }}"
-{% endif %}
+  {% endif %}
+        index => "so-beats"
+        template_name => "so-beats"
+        template => "/templates/so-beats-template.json"
+        template_overwrite => true
+        ssl => true
+        ssl_certificate_verification => false
+        document_id => "%{[metadata][_id]}" 
+      }
+      } else {
+      elasticsearch {
+        pipeline => "beats.common" 
+        hosts => "{{ ES }}"
+  {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
+        user => "{{ ES_USER }}"
+        password => "{{ ES_PASS }}"
+  {% endif %}
         index => "so-beats"
         template_name => "so-beats"
         template => "/templates/so-beats-template.json"
@@ -22,4 +39,5 @@ output {
         ssl_certificate_verification => false
       }
   }
+ }
 }

salt/thehive/init.sls

@@ -95,7 +95,7 @@ so-thehive-es:
       - /opt/so/conf/thehive/etc/es/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro
       - /opt/so/log/thehive:/var/log/elasticsearch:rw
     - environment:
-      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
     - port_bindings:
       - 0.0.0.0:9400:9400
       - 0.0.0.0:9500:9500