Merge branch 'dev' of https://github.com/Security-Onion-Solutions/securityonion into dev

2025-12-27 03:13:15 +01:00 · 2021-06-22 09:52:44 -04:00
parent 85d7e75fb1 2ca8da0710
commit 6e780164ea
15 changed files with 58 additions and 41 deletions
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -68,6 +68,7 @@ base:

  '*_heavynode':
    - zeeklogs
+    - elasticsearch.auth
    - global
    - minions.{{ grains.id }}

@@ -89,6 +90,7 @@ base:
    - logstash
    - logstash.search
    - elasticsearch.search
+    - elasticsearch.auth
    - global
    - minions.{{ grains.id }}
    - data.nodestab
--- a/salt/common/tools/sbin/so-elastic-auth
+++ b/salt/common/tools/sbin/so-elastic-auth
@@ -45,7 +45,7 @@ if [[ "$authEnable" == "true" ]]; then
    if grep -q "argon" "$ES_USERS_FILE"; then
      echo ""
      echo "IMPORTANT: The following users will need to change their password, after logging into SOC, in order to access Kibana:"
-      grep argon "$ES_USERS" | cut -d ":" -f 1
+      grep argon "$ES_USERS_FILE" | cut -d ":" -f 1
    fi
  else
    echo "Auth is already enabled."
--- a/salt/common/tools/sbin/so-elasticsearch-templates-load
+++ b/salt/common/tools/sbin/so-elasticsearch-templates-load
@@ -1,8 +1,5 @@
-{%- set mainint = salt['pillar.get']('host:mainint') %}
-{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}
-
 #!/bin/bash
-# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,6 +14,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

+{%- set mainint = salt['pillar.get']('host:mainint') %}
+{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}
+
 default_conf_dir=/opt/so/conf
 ELASTICSEARCH_HOST="{{ MYIP }}"
 ELASTICSEARCH_PORT=9200
--- a/salt/common/tools/sbin/so-filebeat-module-setup
+++ b/salt/common/tools/sbin/so-filebeat-module-setup
@@ -2,7 +2,7 @@
 {%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}

 #!/bin/bash
-# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
--- a/salt/common/tools/sbin/so-user
+++ b/salt/common/tools/sbin/so-user
@@ -201,7 +201,7 @@ function syncElastic() {
    # Generate the new users file
    echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \
      "from identity_credential_identifiers ici, identity_credentials ic " \
-      "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \
+      "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \
      "order by ici.identifier;" | \
      sqlite3 "$databasePath" | \
      jq -r '.user + ":" + .data.hashed_password' \
@@ -212,7 +212,7 @@ function syncElastic() {
    
    echo "select 'superuser:' || ici.identifier " \
      "from identity_credential_identifiers ici, identity_credentials ic " \
-      "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \
+      "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \
      "order by ici.identifier;" | \
      sqlite3 "$databasePath" \
      >> "$rolesTmpFile"
@@ -226,9 +226,9 @@ function syncElastic() {
    mv "${rolesTmpFile}" "${elasticRolesFile}"

    if [[ -z "$SKIP_STATE_APPLY" ]]; then
-      echo "Elastic state will be re-applied to affected minions in the background."
+      echo "Elastic state will be re-applied to affected minions. This may take several minutes..."
      echo "Applying elastic state to elastic minions at $(date)" >> /opt/so/log/soc/sync.log 2>&1
-      salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 &
+      salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1
    fi
  else
    echo "Newly generated users/roles files are incomplete; aborting."
@@ -236,8 +236,9 @@ function syncElastic() {
 }

 function syncAll() {
-  if [[ -n "$STALE_MIN" && -f "$databasePath" ]]; then
-    staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${STALE_MIN} minutes');" \
+  if [[ -z "$FORCE_SYNC" && -f "$databasePath" && -f "$elasticUsersFile" ]]; then
+    usersFileAgeSecs=$(echo $(($(date +%s) - $(date +%s -r "$elasticUsersFile"))))
+    staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${usersFileAgeSecs} seconds');" \
      | sqlite3 "$databasePath")
    if [[ "$staleCount" == "0" ]]; then
      return 1
--- a/salt/elastalert/files/modules/so/playbook-es.py
+++ b/salt/elastalert/files/modules/so/playbook-es.py
@@ -12,7 +12,7 @@ class PlaybookESAlerter(Alerter):
    Use matched data to create alerts in elasticsearch
    """

-    required_options = set(['play_title','play_url','sigma_level','elasticsearch_host'])
+    required_options = set(['play_title','play_url','sigma_level'])

    def alert(self, matches):
       for match in matches:
@@ -21,11 +21,11 @@ class PlaybookESAlerter(Alerter):
            headers = {"Content-Type": "application/json"}

            creds = None
-            if 'elasticsearch_user' in self.rule and 'elasticsearch_pass' in self.rule:
-                creds = (self.rule['elasticsearch_user'], self.rule['elasticsearch_pass'])
+            if 'es_username' in self.rule and 'es_password' in self.rule:
+                creds = (self.rule['es_username'], self.rule['es_password'])

            payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
-            url = f"https://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"
+            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/so-playbook-alerts-{today}/_doc/"
            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
                            
    def get_info(self):
--- a/salt/elasticsearch/files/scripts/so-catrust
+++ b/salt/elasticsearch/files/scripts/so-catrust
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
--- a/salt/elasticsearch/files/so-elasticsearch-pipelines
+++ b/salt/elasticsearch/files/so-elasticsearch-pipelines
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
--- a/salt/patch/os/schedule.sls
+++ b/salt/patch/os/schedule.sls
@@ -1,12 +1,12 @@
 {% if salt['pillar.get']('patch:os:schedule_name') %}
-  {% set patch_os_pillar = salt['pillar.get']('patch:os') %}
-  {% set schedule_name = patch_os_pillar.schedule_name %}
-  {% set splay = patch_os_pillar.get('splay', 300) %}
+    {% set patch_os_pillar = salt['pillar.get']('patch:os') %}
+    {% set schedule_name = patch_os_pillar.schedule_name %}
+    {% set splay = patch_os_pillar.get('splay', 300) %}

-  {% if schedule_name != 'manual' and schedule_name != 'auto' %}
-    {% import_yaml "patch/os/schedules/"~schedule_name~".yml" as os_schedule %}
+    {% if schedule_name != 'manual' and schedule_name != 'auto' %}
+      {% import_yaml "patch/os/schedules/"~schedule_name~".yml" as os_schedule %}

-    {% if patch_os_pillar.enabled %}
+      {% if patch_os_pillar.enabled %}

 patch_os_schedule:
  schedule.present:
@@ -14,28 +14,28 @@ patch_os_schedule:
    - job_args:
      - patch.os
    - when:
-      {% for days in os_schedule.patch.os.schedule %}
-        {% for day, times in days.items() %}
-          {% for time in times %}
+        {% for days in os_schedule.patch.os.schedule %}
+          {% for day, times in days.items() %}
+            {% for time in times %}
        - {{day}} {{time}}
+            {% endfor %}
          {% endfor %}
        {% endfor %}
-      {% endfor %}
    - splay: {{splay}}
    - return_job: True

-    {% else %}
+      {% else %}

 disable_patch_os_schedule:
  schedule.disabled:
    - name: patch_os_schedule

-    {% endif %}
+      {% endif %}


-  {% elif schedule_name == 'auto' %}
+    {% elif schedule_name == 'auto' %}

-    {% if patch_os_pillar.enabled %}
+      {% if patch_os_pillar.enabled %}

 patch_os_schedule:
  schedule.present:
@@ -46,21 +46,21 @@ patch_os_schedule:
    - splay: {{splay}}
    - return_job: True

-    {% else %}
+      {% else %}

 disable_patch_os_schedule:
  schedule.disabled:
    - name: patch_os_schedule

-    {% endif %}
+      {% endif %}

-  {% elif schedule_name == 'manual' %}
+    {% elif schedule_name == 'manual' %}

 remove_patch_os_schedule:
  schedule.absent:
    - name: patch_os_schedule

-  {% endif %}
+    {% endif %}

 {% else %}

--- a/salt/registry/init.sls
+++ b/salt/registry/init.sls
@@ -43,6 +43,10 @@ so-dockerregistry:
      - /nsm/docker-registry/docker:/var/lib/registry/docker:rw
      - /etc/pki/registry.crt:/etc/pki/registry.crt:ro
      - /etc/pki/registry.key:/etc/pki/registry.key:ro
+    - timeout: 180
+    - retry:
+        attempts: 5
+        interval: 30

 append_so-dockerregistry_so-status.conf:
  file.append:
@@ -55,4 +59,4 @@ append_so-dockerregistry_so-status.conf:
  test.fail_without_changes:
    - name: {{sls}}_state_not_allowed

-{% endif %}
+{% endif %}
--- a/salt/salt/minion-check.sls
+++ b/salt/salt/minion-check.sls
@@ -1,6 +1,6 @@
 include:
  - salt.minion-state-apply-test
-  
+
 state-apply-test:
  schedule.present:
    - name: salt-minion-state-apply-test
@@ -16,4 +16,4 @@ state-apply-test:
  cron.present:
    - identifier: so-salt-minion-check
    - user: root
-    - minute: '*/5'
+    - minute: '*/5'
--- a/salt/soc/files/kratos/kratos.yaml
+++ b/salt/soc/files/kratos/kratos.yaml
@@ -1,10 +1,16 @@
 {%- set WEBACCESS = salt['pillar.get']('global:url_base', '') -%}
 {%- set KRATOSKEY = salt['pillar.get']('kratos:kratoskey', '') -%}
+{%- set SESSIONTIMEOUT = salt['pillar.get']('kratos:sessiontimeout', '24h') -%}
+
+session:
+  lifespan: {{ SESSIONTIMEOUT }}

 selfservice:
  methods:
    password:
      enabled: true
+      config:
+        haveibeenpwned_enabled: false

  flows:
    settings:
--- a/salt/soc/init.sls
+++ b/salt/soc/init.sls
@@ -62,10 +62,13 @@ soccustom:
    - mode: 600
    - template: jinja

+# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate'
+# is in the minion config. That line is added before the final highstate during setup
 sosyncusers:
  cron.present:
    - user: root
-    - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin STALE_MIN=1 /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
+    - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log'
+    - onlyif: "grep 'startup_states: highstate' /etc/salt/minion"

 so-soc:
  docker_container.running:
--- a/salt/soctopus/init.sls
+++ b/salt/soctopus/init.sls
@@ -43,7 +43,6 @@ playbookrulesdir:
    - name: /opt/so/rules/elastalert/playbook
    - user: 939
    - group: 939
-    - mode: 660
    - makedirs: True

 playbookrulessync:
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -2107,6 +2107,7 @@ saltify() {
 		{
 			if [[ ! $is_iso ]]; then
 			  yum -y install salt-minion-3003\
+			  	httpd-tools\
 			   	python3\
 					python36-docker\
 					python36-dateutil\
@@ -2132,6 +2133,7 @@ saltify() {
 		fi
 		
 		local pkg_arr=(
+			'apache2-utils'
 			'ca-certificates'
 			'curl'
 			'software-properties-common'