Merge branch 'dev' into kilo

salt/common/tools/sbin/so-image-common

@@ -132,7 +132,7 @@ update_docker_containers() {
   # Let's make sure we have the public key
   run_check_net_err \
   "curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
-  "Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \
+  "Could not pull signature key file, please ensure connectivity to https://raw.githubusercontent.com" \
   noretry >> "$LOG_FILE" 2>&1
   result=$?
   if [[ $result -eq 0 ]]; then

salt/elasticsearch/files/scripts/so-catrust

@@ -24,9 +24,9 @@ set -e
 
 # Check to see if we have extracted the ca cert.
 if [ ! -f /opt/so/saltstack/local/salt/common/cacerts ]; then
-    docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }} -keystore /etc/pki/ca-trust/extracted/java/cacerts -alias SOSCA -import -file /etc/pki/ca.crt -storepass changeit -noprompt
-    docker cp so-elasticsearchca:/etc/pki/ca-trust/extracted/java/cacerts /opt/so/saltstack/local/salt/common/cacerts
-    docker cp so-elasticsearchca:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /opt/so/saltstack/local/salt/common/tls-ca-bundle.pem
+    docker run -v /etc/pki/ca.crt:/etc/ssl/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }} -keystore /etc/ssl/certs/java/cacerts -alias SOSCA -import -file /etc/ssl/ca.crt -storepass changeit -noprompt
+    docker cp so-elasticsearchca:/etc/ssl/certs/java/cacerts /opt/so/saltstack/local/salt/common/cacerts
+    docker cp so-elasticsearchca:/etc/ssl/certs/ca-certificates.crt /opt/so/saltstack/local/salt/common/tls-ca-bundle.pem
     docker rm so-elasticsearchca
     echo "" >> /opt/so/saltstack/local/salt/common/tls-ca-bundle.pem
     echo "sosca" >> /opt/so/saltstack/local/salt/common/tls-ca-bundle.pem

salt/elasticsearch/init.sls

@@ -131,6 +131,10 @@ esrolesdir:
     - group: 939
     - makedirs: True
 
+eslibdir:
+  file.absent:
+    - name: /opt/so/conf/elasticsearch/lib
+   
 esingestdynamicconf:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/ingest
@@ -258,7 +262,7 @@ so-elasticsearch:
       {% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %}
       - discovery.type=single-node
       {% endif %}
-      - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true
+      - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
       ulimits:
       - memlock=-1:-1
       - nofile=65536:65536
@@ -271,7 +275,7 @@ so-elasticsearch:
       - /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro
       - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
       - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
-      - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
+      - /opt/so/conf/ca/cacerts:/etc/ssl/certs/java/cacerts:ro
       {% if ismanager %}
       - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro
       {% else %}

salt/logstash/etc/jvm.options

@@ -0,0 +1 @@
+-Dlog4j2.formatMsgNoLookups=true

salt/logstash/init.sls

@@ -61,6 +61,10 @@ logstash:
     - gid: 931
     - home: /opt/so/conf/logstash
 
+lslibdir:
+  file.absent:
+    - name: /opt/so/conf/logstash/lib
+
 lsetcdir:
   file.directory:
     - name: /opt/so/conf/logstash/etc

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -7,19 +7,37 @@
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 output {
   if "beat-ext" in [tags] and "import" not in [tags] and "filebeat" not in [metadata][pipeline] {
-    elasticsearch {
-      pipeline => "beats.common" 
-      hosts => "{{ ES }}"
-{% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
-      user => "{{ ES_USER }}"
-      password => "{{ ES_PASS }}"
-{% endif %}
-      index => "so-beats"
-      template_name => "so-beats"
-      template => "/templates/so-beats-template.json"
-      template_overwrite => true
-      ssl => true
-      ssl_certificate_verification => false
-    }
+    if [metadata][_id] {
+      elasticsearch {
+        pipeline => "beats.common" 
+        hosts => "{{ ES }}"
+  {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
+        user => "{{ ES_USER }}"
+        password => "{{ ES_PASS }}"
+  {% endif %}
+        index => "so-beats"
+        template_name => "so-beats"
+        template => "/templates/so-beats-template.json"
+        template_overwrite => true
+        ssl => true
+        ssl_certificate_verification => false
+        document_id => "%{[metadata][_id]}" 
+      }
+      } else {
+      elasticsearch {
+        pipeline => "beats.common" 
+        hosts => "{{ ES }}"
+  {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
+        user => "{{ ES_USER }}"
+        password => "{{ ES_PASS }}"
+  {% endif %}
+        index => "so-beats"
+        template_name => "so-beats"
+        template => "/templates/so-beats-template.json"
+        template_overwrite => true
+        ssl => true
+        ssl_certificate_verification => false
+      }
   }
+ }
 }

salt/thehive/init.sls

@@ -95,7 +95,7 @@ so-thehive-es:
       - /opt/so/conf/thehive/etc/es/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro
       - /opt/so/log/thehive:/var/log/elasticsearch:rw
     - environment:
-      - ES_JAVA_OPTS=-Xms512m -Xmx512m
+      - ES_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true
     - port_bindings:
       - 0.0.0.0:9400:9400
       - 0.0.0.0:9500:9500