Merge remote-tracking branch 'remotes/origin/dev' into issue/1049

2026-02-20 14:05:26 +01:00 · 2020-08-10 16:36:49 -04:00
parent 30e0abf326 6bb84f8513
commit 11433b87e6
21 changed files with 853 additions and 215 deletions
--- a/salt/ca/init.sls
+++ b/salt/ca/init.sls
@@ -36,6 +36,7 @@ pki_private_key:
    - days_valid: 3650
    - days_remaining: 0
    - backup: True
+    - replace: False
    - require:
      - file: /etc/pki

--- a/salt/common/tools/sbin/so-playbook-sync
+++ b/salt/common/tools/sbin/so-playbook-sync
@@ -17,4 +17,4 @@

 . /usr/sbin/so-common

-docker exec so-soctopus python3 playbook_play-sync.py >> /opt/so/log/soctopus/so-playbook-sync.log 2>&1
+docker exec so-soctopus python3 playbook_play-sync.py
--- a/salt/elasticsearch/files/ingest/sysmon
+++ b/salt/elasticsearch/files/ingest/sysmon
@@ -7,6 +7,9 @@
    { "set":           { "if": "ctx.winlog?.computer_name != null",   "field": "observer.name", "value": "{{winlog.computer_name}}", "override": true }  },
    { "set":           { "if": "ctx.event?.code == '3'",   "field": "event.category", "value": "host,process,network", "override": true }  },
    { "set":           { "if": "ctx.event?.code == '1'",   "field": "event.category", "value": "host,process", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '5'",   "field": "event.category", "value": "host,process", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '6'",   "field": "event.category", "value": "host,driver", "override": true }  },
+    { "set":           { "if": "ctx.event?.code == '22'",   "field": "event.category", "value": "network", "override": true }  },
    { "set":          { "if": "ctx.event?.code == '1'",   "field": "event.dataset", "value": "process_creation", "override": true }  },
    { "set":          { "if": "ctx.event?.code == '2'",   "field": "event.dataset", "value": "process_changed_file", "override": true }  },
    { "set":          { "if": "ctx.event?.code == '3'",   "field": "event.dataset", "value": "network_connection", "override": true }  },
@@ -34,6 +37,7 @@
    { "rename": 	   { "field": "winlog.event_data.CurrentDirectory", 			"target_field": "process.working_directory",		"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.Description", 			      "target_field": "process.pe.description",		"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.Product", 			          "target_field": "process.pe.product",		"ignore_missing": true 	} },
+    { "rename": 	   { "field": "winlog.event_data.Company", 			          "target_field": "process.pe.company",		"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.OriginalFileName", 			"target_field": "process.pe.original_file_name",		"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.FileVersion", 			      "target_field": "process.pe.file_version",		"ignore_missing": true 	} },
    { "rename": 	   { "field": "winlog.event_data.ParentCommandLine", 			"target_field": "process.parent.command_line",		"ignore_missing": true 	} },
--- a/salt/elasticsearch/files/scripts/so-catrust
+++ b/salt/elasticsearch/files/scripts/so-catrust
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+{%- set VERSION = salt['pillar.get']('global:soversion', '') %}
+{%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
+{%- set MANAGER = salt['grains.get']('master') %}
+. /usr/sbin/so-common
+# Check to see if we have extracted the ca cert.
+if [ ! -f /opt/so/saltstack/local/salt/common/cacerts ]; then
+    docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-elasticsearchca --user root --entrypoint keytool {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }} -keystore /etc/pki/ca-trust/extracted/java/cacerts -alias SOSCA -import -file /etc/pki/ca.crt -storepass changeit -noprompt
+    docker cp so-elasticsearchca:/etc/pki/ca-trust/extracted/java/cacerts /opt/so/saltstack/local/salt/common/cacerts
+    docker cp so-elasticsearchca:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /opt/so/saltstack/local/salt/common/tls-ca-bundle.pem
+    docker rm so-elasticsearchca
+    echo "" >> /opt/so/saltstack/local/salt/common/tls-ca-bundle.pem
+    echo "sosca" >> /opt/so/saltstack/local/salt/common/tls-ca-bundle.pem
+    cat /etc/pki/ca.crt >> /opt/so/saltstack/local/salt/common/tls-ca-bundle.pem
+else
+    exit 0
+fi
--- a/salt/elasticsearch/init.sls
+++ b/salt/elasticsearch/init.sls
@@ -26,9 +26,11 @@
 {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-importpcap'] %}
  {% set esclustername = salt['pillar.get']('manager:esclustername', '') %}
  {% set esheap = salt['pillar.get']('manager:esheap', '') %}
+  {% set ismanager = True %}
 {% elif grains['role'] in ['so-node','so-heavynode'] %}
  {% set esclustername = salt['pillar.get']('elasticsearch:esclustername', '') %}
  {% set esheap = salt['pillar.get']('elasticsearch:esheap', '') %}
+  {% set ismanager = False %}
 {% endif %}

 {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
@@ -37,6 +39,46 @@ vm.max_map_count:
  sysctl.present:
    - value: 262144

+{% if ismanager %}
+# We have to add the Manager CA to the CA list
+cascriptsync:
+  file.managed:
+    - name: /usr/sbin/so-catrust
+    - source: salt://elasticsearch/files/scripts/so-catrust
+    - user: 939
+    - group: 939
+    - mode: 750
+    - template: jinja
+
+# Run the CA magic
+cascriptfun:
+  cmd.run:
+    - name: /usr/sbin/so-catrust
+
+{% endif %}
+
+# Move our new CA over so Elastic and Logstash can use SSL with the internal CA
+catrustdir:
+  file.directory:
+    - name: /opt/so/conf/ca
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+cacertz:
+  file.managed:
+    - name: /opt/so/conf/ca/cacerts
+    - source: salt://common/cacerts
+    - user: 939
+    - group: 939
+
+capemz:
+  file.managed:
+    - name: /opt/so/conf/ca/tls-ca-bundle.pem
+    - source: salt://common/tls-ca-bundle.pem
+    - user: 939
+    - group: 939
+
 # Add ES Group
 elasticsearchgroup:
  group.present:
@@ -149,6 +191,9 @@ so-elasticsearch:
      - /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro
      - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
      - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
+      - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
+    - watch:
+      - file: cacertz

 so-elasticsearch-pipelines-file:
  file.managed:
--- a/salt/firewall/portgroups.yaml
+++ b/salt/firewall/portgroups.yaml
@@ -64,6 +64,7 @@ firewall:
      redis:
        tcp:
          - 6379
+          - 9696
      salt_manager:
        tcp:
          - 4505
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -148,7 +148,6 @@ so-logstash:
    - user: logstash
    - environment:
      - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }}
-      - SSL_CERT_FILE=/etc/ssl/certs/ca.crt
    - port_bindings:
 {% for BINDING in DOCKER_OPTIONS.port_bindings %}
      - {{ BINDING }}
@@ -167,7 +166,8 @@ so-logstash:
      - /etc/pki/filebeat.crt:/usr/share/logstash/filebeat.crt:ro
      - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
      - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro
-      - /etc/ssl/certs/intca.crt:/etc/ssl/certs/ca.crt:ro
+      - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
+      - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
      {%- if grains['role'] == 'so-eval' %}
      - /nsm/zeek:/nsm/zeek:ro
      - /nsm/suricata:/suricata:ro
--- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
+++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
@@ -1,13 +1,11 @@
-{%- if grains.role == 'so-heavynode' %}
-{%- set MANAGER = salt['pillar.get']('elasticsearch:mainip', '') %}
-{%- else %}
-{%- set MANAGER = salt['pillar.get']('global:managerip', '') %}
-{% endif -%}
+{%- set MANAGER = salt['grains.get']('master') %}
 {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}

 input {
 	redis {
 		host => '{{ MANAGER }}'
+		port => 9696
+		ssl => true
 		data_type => 'list'
 		key => 'logstash:unparsed'
 		type => 'redis-input'
--- a/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9998_output_minio.conf.jinja
@@ -17,6 +17,7 @@ output {
        encoding => {{ ENCODING }}
        upload_queue_size => {{ UPLOAD_QUEUE_SIZE }}
        temporary_directory => "/usr/share/logstash/data/tmp"
+        validate_credentials_on_root_bucket => false
        additional_settings => {
            "force_path_style" => true
            }
--- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja
@@ -1,8 +1,9 @@
-{% set MANAGER = salt['pillar.get']('global:managerip', '') %}
+{%- set MANAGER = salt['grains.get']('master') %}
 {% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
 output {
 	redis {
 		host => '{{ MANAGER }}'
+		port => 6379
 		data_type => 'list'
 		key => 'logstash:unparsed'
 		congestion_interval => 1
--- a/salt/nginx/files/navigator_config.json
+++ b/salt/nginx/files/navigator_config.json
@@ -1,4 +1,4 @@
-{%- set ip = salt['pillar.get']('global:managerip', '') %}
+{%- set URL_BASE = salt['pillar.get']('manager:url_base', '') %}

 {
    "enterprise_attack_url": "https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json",
@@ -16,7 +16,7 @@

    "domain": "mitre-enterprise",

-  "custom_context_menu_items": [ {"label": "view related plays","url": "  https://{{ip}}/playbook/projects/detection-playbooks/issues?utf8=%E2%9C%93&set_filter=1&sort=id%3Adesc&f%5B%5D=cf_15&op%5Bcf_15%5D=%3D&f%5B%5D=&c%5B%5D=status&c%5B%5D=cf_10&c%5B%5D=cf_13&c%5B%5D=cf_18&c%5B%5D=cf_19&c%5B%5D=cf_1&c%5B%5D=updated_on&v%5Bcf_15%5D%5B%5D=~Technique_ID~"}],
+  "custom_context_menu_items": [ {"label": "view related plays","url": "  https://{{URL_BASE}}/playbook/projects/detection-playbooks/issues?utf8=%E2%9C%93&set_filter=1&sort=id%3Adesc&f%5B%5D=cf_15&op%5Bcf_15%5D=%3D&f%5B%5D=&c%5B%5D=status&c%5B%5D=cf_10&c%5B%5D=cf_13&c%5B%5D=cf_18&c%5B%5D=cf_19&c%5B%5D=cf_1&c%5B%5D=updated_on&v%5Bcf_15%5D%5B%5D=~Technique_ID~"}],

 "default_layers": {
     "enabled": true,
--- a/salt/redis/etc/redis.conf
+++ b/salt/redis/etc/redis.conf
--- a/salt/redis/init.sls
+++ b/salt/redis/init.sls
@@ -53,10 +53,14 @@ so-redis:
    - user: socore
    - port_bindings:
      - 0.0.0.0:6379:6379
+      - 0.0.0.0:9696:9696
    - binds:
      - /opt/so/log/redis:/var/log/redis:rw
      - /opt/so/conf/redis/etc/redis.conf:/usr/local/etc/redis/redis.conf:ro
      - /opt/so/conf/redis/working:/redis:rw
+      - /etc/pki/redis.crt:/certs/redis.crt:ro
+      - /etc/pki/redis.key:/certs/redis.key:ro
+      - /etc/pki/ca.crt:/certs/ca.crt:ro
    - entrypoint: "redis-server /usr/local/etc/redis/redis.conf"
    - watch:
      - file: /opt/so/conf/redis/etc
--- a/salt/salt/master.defaults.yaml
+++ b/salt/salt/master.defaults.yaml
@@ -2,4 +2,4 @@
 # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions
 salt:
  master:
-    version: 3001
+    version: 3001.1
--- a/salt/salt/minion.defaults.yaml
+++ b/salt/salt/minion.defaults.yaml
@@ -2,4 +2,4 @@
 # When updating the salt version, also update the version in securityonion-builds/images/iso-task/Dockerfile and saltify function in so-functions
 salt:
  minion:
-    version: 3001
+    version: 3001.1
--- a/salt/ssl/init.sls
+++ b/salt/ssl/init.sls
@@ -216,6 +216,41 @@ miniokeyperms:
    - mode: 640
    - group: 939

+/etc/pki/redis.key:
+  x509.private_key_managed:
+    - CN: {{ manager }}
+    - bits: 4096
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/redis.key') -%}
+    - prereq:
+      - x509: /etc/pki/redis.crt
+    {%- endif %}
+
+# Create a cert for the docker registry
+/etc/pki/redis.crt:
+  x509.certificate_managed:
+    - ca_server: {{ ca_server }}
+    - signing_policy: registry
+    - public_key: /etc/pki/redis.key
+    - CN: {{ manager }}
+    - days_remaining: 0
+    - days_valid: 820
+    - backup: True
+    - unless:
+      # https://github.com/saltstack/salt/issues/52167
+      # Will trigger 5 days (432000 sec) from cert expiration
+      - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/redis.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+
+rediskeyperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/redis.key
+    - mode: 640
+    - group: 939
+
 /etc/pki/managerssl.key:
  x509.private_key_managed:
    - CN: {{ manager }}