diff --git a/pillar/top.sls b/pillar/top.sls index ff4cb5787..cb170c0af 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -68,6 +68,7 @@ base: '*_heavynode': - zeeklogs + - elasticsearch.auth - global - minions.{{ grains.id }} @@ -89,6 +90,7 @@ base: - logstash - logstash.search - elasticsearch.search + - elasticsearch.auth - global - minions.{{ grains.id }} - data.nodestab diff --git a/salt/common/tools/sbin/so-elastic-auth b/salt/common/tools/sbin/so-elastic-auth index 663dbb9f6..6631badf2 100755 --- a/salt/common/tools/sbin/so-elastic-auth +++ b/salt/common/tools/sbin/so-elastic-auth @@ -45,7 +45,7 @@ if [[ "$authEnable" == "true" ]]; then if grep -q "argon" "$ES_USERS_FILE"; then echo "" echo "IMPORTANT: The following users will need to change their password, after logging into SOC, in order to access Kibana:" - grep argon "$ES_USERS" | cut -d ":" -f 1 + grep argon "$ES_USERS_FILE" | cut -d ":" -f 1 fi else echo "Auth is already enabled." diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load index 30ab66b48..c416a3ce2 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-load +++ b/salt/common/tools/sbin/so-elasticsearch-templates-load @@ -1,8 +1,5 @@ -{%- set mainint = salt['pillar.get']('host:mainint') %} -{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} - #!/bin/bash -# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,6 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +{%- set mainint = salt['pillar.get']('host:mainint') %} +{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} + default_conf_dir=/opt/so/conf ELASTICSEARCH_HOST="{{ MYIP }}" ELASTICSEARCH_PORT=9200 diff --git a/salt/common/tools/sbin/so-filebeat-module-setup b/salt/common/tools/sbin/so-filebeat-module-setup index 4f9811ca7..ef35a9b17 100755 --- a/salt/common/tools/sbin/so-filebeat-module-setup +++ b/salt/common/tools/sbin/so-filebeat-module-setup @@ -2,7 +2,7 @@ {%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} #!/bin/bash -# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 0f44a3227..f0c064d03 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -201,7 +201,7 @@ function syncElastic() { # Generate the new users file echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \ "from identity_credential_identifiers ici, identity_credentials ic " \ - "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ + "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \ "order by ici.identifier;" | \ sqlite3 "$databasePath" | \ jq -r '.user + ":" + .data.hashed_password' \ @@ -212,7 +212,7 @@ function syncElastic() { echo "select 'superuser:' || ici.identifier " \ "from identity_credential_identifiers ici, identity_credentials ic " \ - "where ici.identity_credential_id=ic.id and ic.config like '%hashed_password%' " \ + "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \ "order by ici.identifier;" | \ sqlite3 "$databasePath" \ >> "$rolesTmpFile" @@ -226,9 +226,9 @@ function syncElastic() { mv "${rolesTmpFile}" "${elasticRolesFile}" if [[ -z "$SKIP_STATE_APPLY" ]]; then - echo "Elastic state will be re-applied to affected minions in the background." + echo "Elastic state will be re-applied to affected minions. This may take several minutes..." echo "Applying elastic state to elastic minions at $(date)" >> /opt/so/log/soc/sync.log 2>&1 - salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 & + salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply elasticsearch queue=True >> /opt/so/log/soc/sync.log 2>&1 fi else echo "Newly generated users/roles files are incomplete; aborting." @@ -236,8 +236,9 @@ function syncElastic() { } function syncAll() { - if [[ -n "$STALE_MIN" && -f "$databasePath" ]]; then - staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${STALE_MIN} minutes');" \ + if [[ -z "$FORCE_SYNC" && -f "$databasePath" && -f "$elasticUsersFile" ]]; then + usersFileAgeSecs=$(echo $(($(date +%s) - $(date +%s -r "$elasticUsersFile")))) + staleCount=$(echo "select count(*) from identity_credentials where updated_at >= Datetime('now', '-${usersFileAgeSecs} seconds');" \ | sqlite3 "$databasePath") if [[ "$staleCount" == "0" ]]; then return 1 diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index 5b1835bac..bae967001 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -12,7 +12,7 @@ class PlaybookESAlerter(Alerter): Use matched data to create alerts in elasticsearch """ - required_options = set(['play_title','play_url','sigma_level','elasticsearch_host']) + required_options = set(['play_title','play_url','sigma_level']) def alert(self, matches): for match in matches: @@ -21,11 +21,11 @@ class PlaybookESAlerter(Alerter): headers = {"Content-Type": "application/json"} creds = None - if 'elasticsearch_user' in self.rule and 'elasticsearch_pass' in self.rule: - creds = (self.rule['elasticsearch_user'], self.rule['elasticsearch_pass']) + if 'es_username' in self.rule and 'es_password' in self.rule: + creds = (self.rule['es_username'], self.rule['es_password']) payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp} - url = f"https://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/" + url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/so-playbook-alerts-{today}/_doc/" requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) def get_info(self): diff --git a/salt/elasticsearch/files/scripts/so-catrust b/salt/elasticsearch/files/scripts/so-catrust index d49a29ce4..c157d9691 100644 --- a/salt/elasticsearch/files/scripts/so-catrust +++ b/salt/elasticsearch/files/scripts/so-catrust @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index 5d103963e..c1ff88397 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/salt/patch/os/schedule.sls b/salt/patch/os/schedule.sls index a91e61dfe..4ad9a454e 100644 --- a/salt/patch/os/schedule.sls +++ b/salt/patch/os/schedule.sls @@ -1,12 +1,12 @@ {% if salt['pillar.get']('patch:os:schedule_name') %} - {% set patch_os_pillar = salt['pillar.get']('patch:os') %} - {% set schedule_name = patch_os_pillar.schedule_name %} - {% set splay = patch_os_pillar.get('splay', 300) %} + {% set patch_os_pillar = salt['pillar.get']('patch:os') %} + {% set schedule_name = patch_os_pillar.schedule_name %} + {% set splay = patch_os_pillar.get('splay', 300) %} - {% if schedule_name != 'manual' and schedule_name != 'auto' %} - {% import_yaml "patch/os/schedules/"~schedule_name~".yml" as os_schedule %} + {% if schedule_name != 'manual' and schedule_name != 'auto' %} + {% import_yaml "patch/os/schedules/"~schedule_name~".yml" as os_schedule %} - {% if patch_os_pillar.enabled %} + {% if patch_os_pillar.enabled %} patch_os_schedule: schedule.present: @@ -14,28 +14,28 @@ patch_os_schedule: - job_args: - patch.os - when: - {% for days in os_schedule.patch.os.schedule %} - {% for day, times in days.items() %} - {% for time in times %} + {% for days in os_schedule.patch.os.schedule %} + {% for day, times in days.items() %} + {% for time in times %} - {{day}} {{time}} + {% endfor %} {% endfor %} {% endfor %} - {% endfor %} - splay: {{splay}} - return_job: True - {% else %} + {% else %} disable_patch_os_schedule: schedule.disabled: - name: patch_os_schedule - {% endif %} + {% endif %} - {% elif schedule_name == 'auto' %} + {% elif schedule_name == 'auto' %} - {% if patch_os_pillar.enabled %} + {% if patch_os_pillar.enabled %} patch_os_schedule: schedule.present: @@ -46,21 +46,21 @@ patch_os_schedule: - splay: {{splay}} - return_job: True - {% else %} + {% else %} disable_patch_os_schedule: schedule.disabled: - name: patch_os_schedule - {% endif %} + {% endif %} - {% elif schedule_name == 'manual' %} + {% elif schedule_name == 'manual' %} remove_patch_os_schedule: schedule.absent: - name: patch_os_schedule - {% endif %} + {% endif %} {% else %} diff --git a/salt/registry/init.sls b/salt/registry/init.sls index 1cec55fd2..b59465fb7 100644 --- a/salt/registry/init.sls +++ b/salt/registry/init.sls @@ -43,6 +43,10 @@ so-dockerregistry: - /nsm/docker-registry/docker:/var/lib/registry/docker:rw - /etc/pki/registry.crt:/etc/pki/registry.crt:ro - /etc/pki/registry.key:/etc/pki/registry.key:ro + - timeout: 180 + - retry: + attempts: 5 + interval: 30 append_so-dockerregistry_so-status.conf: file.append: @@ -55,4 +59,4 @@ append_so-dockerregistry_so-status.conf: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} diff --git a/salt/salt/minion-check.sls b/salt/salt/minion-check.sls index e8a0c2639..66ab732e2 100644 --- a/salt/salt/minion-check.sls +++ b/salt/salt/minion-check.sls @@ -1,6 +1,6 @@ include: - salt.minion-state-apply-test - + state-apply-test: schedule.present: - name: salt-minion-state-apply-test @@ -16,4 +16,4 @@ state-apply-test: cron.present: - identifier: so-salt-minion-check - user: root - - minute: '*/5' \ No newline at end of file + - minute: '*/5' diff --git a/salt/soc/files/kratos/kratos.yaml b/salt/soc/files/kratos/kratos.yaml index a0a72b3ab..b1174af58 100644 --- a/salt/soc/files/kratos/kratos.yaml +++ b/salt/soc/files/kratos/kratos.yaml @@ -1,10 +1,16 @@ {%- set WEBACCESS = salt['pillar.get']('global:url_base', '') -%} {%- set KRATOSKEY = salt['pillar.get']('kratos:kratoskey', '') -%} +{%- set SESSIONTIMEOUT = salt['pillar.get']('kratos:sessiontimeout', '24h') -%} + +session: + lifespan: {{ SESSIONTIMEOUT }} selfservice: methods: password: enabled: true + config: + haveibeenpwned_enabled: false flows: settings: diff --git a/salt/soc/init.sls b/salt/soc/init.sls index 01b57c8ce..b8cdb09ba 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -62,10 +62,13 @@ soccustom: - mode: 600 - template: jinja +# we dont want this added too early in setup, so we add the onlyif to verify 'startup_states: highstate' +# is in the minion config. That line is added before the final highstate during setup sosyncusers: cron.present: - user: root - - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin STALE_MIN=1 /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' + - name: 'PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin /usr/sbin/so-user sync &>> /opt/so/log/soc/sync.log' + - onlyif: "grep 'startup_states: highstate' /etc/salt/minion" so-soc: docker_container.running: diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index 1c7c92434..c2c8dc1ac 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -43,7 +43,6 @@ playbookrulesdir: - name: /opt/so/rules/elastalert/playbook - user: 939 - group: 939 - - mode: 660 - makedirs: True playbookrulessync: diff --git a/setup/so-functions b/setup/so-functions index 13438b1ba..ff019953e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2107,6 +2107,7 @@ saltify() { { if [[ ! $is_iso ]]; then yum -y install salt-minion-3003\ + httpd-tools\ python3\ python36-docker\ python36-dateutil\ @@ -2132,6 +2133,7 @@ saltify() { fi local pkg_arr=( + 'apache2-utils' 'ca-certificates' 'curl' 'software-properties-common'