Merge pull request #2828 from Security-Onion-Solutions/delta

adjust timeout for ssl states and pillarize ElastAlert

salt/ca/init.sls

@@ -42,6 +42,9 @@ pki_private_key:
     - replace: False
     - require:
       - file: /etc/pki
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 x509_pem_entries:
   module.run:

salt/elastalert/defaults.yaml

@@ -0,0 +1,48 @@
+elastalert:
+  config:
+    rules_folder: /opt/elastalert/rules/
+    scan_subdirectories: true
+    disable_rules_on_error: false
+    run_every:
+      minutes: 3
+    buffer_time:
+      minutes: 10
+    old_query_limit:
+      minutes: 5
+    es_host: {{salt['pillar.get']('manager:mainip', '')}}
+    es_port: {{salt['pillar.get']('manager:es_port', '')}}
+    es_conn_timeout: 55
+    max_query_size: 5000
+    #aws_region: us-east-1
+    #profile: test
+    #es_url_prefix: elasticsearch
+    #use_ssl: True
+    #verify_certs: True
+    #es_send_get_body_as: GET
+    #es_username: someusername
+    #es_password: somepassword
+    writeback_index: elastalert_status
+    alert_time_limit:
+      days: 2
+    index_settings:
+      shards: 1
+      replicas: 0
+    logging:
+      version: 1
+      incremental: false
+      disable_existing_loggers: false
+      formatters:
+        logline:
+          format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s'
+      handlers:
+        file:
+          class: logging.FileHandler
+          formatter: logline
+          level: INFO
+          filename: /var/log/elastalert/elastalert.log
+      loggers:
+        '':
+          level: INFO
+          handlers:
+            - file
+          propagate: false

salt/elastalert/elastalert_config.map.jinja

@@ -0,0 +1,4 @@
+{% import_yaml 'elastalert/defaults.yaml' as elastalert_defaults with context %}
+{% set elastalert_pillar = salt['pillar.get']('elastalert:config', {}) %}
+
+{% do salt['defaults.merge'](elastalert_defaults.elastalert.config, elastalert_pillar, in_place=True) %}

salt/elastalert/files/elastalert_config.yaml

@@ -1,110 +0,0 @@
-{% set esip = salt['pillar.get']('manager:mainip', '') %}
-{% set esport = salt['pillar.get']('manager:es_port', '') %}
-# This is the folder that contains the rule yaml files
-# Any .yaml file will be loaded as a rule
-rules_folder: /opt/elastalert/rules/
-
-# Sets whether or not ElastAlert should recursively descend
-# the rules directory - true or false
-scan_subdirectories: true
-
-# Do not disable a rule when an uncaught exception is thrown -
-# This setting should be tweaked once the following issue has been fixed
-# https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/98
-disable_rules_on_error: false
-
-# How often ElastAlert will query Elasticsearch
-# The unit can be anything from weeks to seconds
-run_every:
-  minutes: 3
-
-# ElastAlert will buffer results from the most recent
-# period of time, in case some log sources are not in real time
-buffer_time:
-  minutes: 10
-
-# The maximum time between queries for ElastAlert to start at the most recently
-# run query. When ElastAlert starts, for each rule, it will search elastalert_metadata
-# for the most recently run query and start from that time, unless it is older than
-# old_query_limit, in which case it will start from the present time. The default is one week.
-old_query_limit:
-  minutes: 5
-
-# The Elasticsearch hostname for metadata writeback
-# Note that every rule can have its own Elasticsearch host
-es_host: {{ esip }}
-
-# The Elasticsearch port
-es_port: {{ esport }}
-
-# Sets timeout for connecting to and reading from es_host
-es_conn_timeout: 55
-
-# The maximum number of documents that will be downloaded from Elasticsearch in
-# a single query. The default is 10,000, and if you expect to get near this number,
-# consider using use_count_query for the rule. If this limit is reached, ElastAlert
-# will scroll through pages the size of max_query_size until processing all results.
-max_query_size: 5000
-
-# The AWS region to use. Set this when using AWS-managed elasticsearch
-#aws_region: us-east-1
-
-# The AWS profile to use. Use this if you are using an aws-cli profile.
-# See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
-# for details
-#profile: test
-
-# Optional URL prefix for Elasticsearch
-#es_url_prefix: elasticsearch
-
-# Connect with TLS to Elasticsearch
-#use_ssl: True
-
-# Verify TLS certificates
-#verify_certs: True
-
-# GET request with body is the default option for Elasticsearch.
-# If it fails for some reason, you can pass 'GET', 'POST' or 'source'.
-# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport
-# for details
-#es_send_get_body_as: GET
-
-# Option basic-auth username and password for Elasticsearch
-#es_username: someusername
-#es_password: somepassword
-
-# The index on es_host which is used for metadata storage
-# This can be a unmapped index, but it is recommended that you run
-# elastalert-create-index to set a mapping
-writeback_index: elastalert_status
-
-# If an alert fails for some reason, ElastAlert will retry
-# sending the alert until this time period has elapsed
-alert_time_limit:
-  days: 2
-
-index_settings:
-  shards: 1
-  replicas: 0
-
-logging:
-  version: 1
-  incremental: false
-  disable_existing_loggers: false
-  formatters:
-    logline:
-      format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s'
-
-  handlers:
-    file:
-      class : logging.FileHandler
-      formatter: logline
-      level: INFO
-      filename: /var/log/elastalert/elastalert.log
-
-  loggers:
-    '':
-      level: INFO
-      handlers:
-        - file
-      propagate: false

salt/elastalert/files/elastalert_config.yaml.jinja

@@ -0,0 +1 @@
+{{ elastalert_config | yaml(False) }}

salt/elastalert/init.sls

@@ -15,6 +15,8 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 
+{% from 'elastalert/elastalert_config.map.jinja' import elastalert_defaults as elastalert_config with context %}
+
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set MANAGER = salt['grains.get']('master') %}
@@ -92,7 +94,9 @@ elastasomodulesync:
 elastaconf:
   file.managed:
     - name: /opt/so/conf/elastalert/elastalert_config.yaml
-    - source: salt://elastalert/files/elastalert_config.yaml
+    - source: salt://elastalert/files/elastalert_config.yaml.jinja
+    - context:
+        elastalert_config: {{ elastalert_config.elastalert.config }}
     - user: 933
     - group: 933
     - template: jinja
@@ -119,6 +123,8 @@ so-elastalert:
       - {{MANAGER_URL}}:{{MANAGER_IP}}
     - require:
       - module: wait_for_elasticsearch
+    - watch:
+      - file: elastaconf
 
 append_so-elastalert_so-status.conf:
   file.append:

salt/ssl/init.sls

@@ -67,6 +67,9 @@ removeesp12dir:
     - prereq:
       - x509: /etc/pki/influxdb.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 # Create a cert for the talking to influxdb
 /etc/pki/influxdb.crt:
@@ -82,6 +85,9 @@ removeesp12dir:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/influxdb.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 influxkeyperms:
   file.managed:
@@ -104,6 +110,9 @@ influxkeyperms:
     - prereq:
       - x509: /etc/pki/redis.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 /etc/pki/redis.crt:
   x509.certificate_managed:
@@ -118,6 +127,9 @@ influxkeyperms:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/redis.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 rediskeyperms:
   file.managed:
@@ -140,6 +152,9 @@ rediskeyperms:
     - prereq:
       - x509: /etc/pki/filebeat.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 # Request a cert and drop it where it needs to go to be distributed
 /etc/pki/filebeat.crt:
@@ -159,6 +174,9 @@ rediskeyperms:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
   cmd.run:
     - name: "/usr/bin/openssl pkcs8 -in /etc/pki/filebeat.key -topk8 -out /etc/pki/filebeat.p8 -nocrypt"
     - onchanges:
@@ -213,6 +231,9 @@ fbcrtlink:
     - prereq:
       - x509: /etc/pki/registry.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 # Create a cert for the docker registry
 /etc/pki/registry.crt:
@@ -228,6 +249,9 @@ fbcrtlink:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/registry.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 regkeyperms:
   file.managed:
@@ -248,6 +272,9 @@ regkeyperms:
     - prereq:
       - x509: /etc/pki/minio.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 # Create a cert for minio
 /etc/pki/minio.crt:
@@ -263,6 +290,9 @@ regkeyperms:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/minio.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 miniokeyperms:
   file.managed:
@@ -284,6 +314,9 @@ miniokeyperms:
     - prereq:
       - x509: /etc/pki/elasticsearch.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 /etc/pki/elasticsearch.crt:
   x509.certificate_managed:
@@ -298,6 +331,9 @@ miniokeyperms:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
   cmd.run:
     - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:"
     - onchanges:
@@ -329,6 +365,9 @@ elasticp12perms:
     - prereq:
       - x509: /etc/pki/managerssl.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 # Create a cert for the reverse proxy
 /etc/pki/managerssl.crt:
@@ -345,6 +384,9 @@ elasticp12perms:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 msslkeyperms:
   file.managed:
@@ -366,6 +408,9 @@ msslkeyperms:
     - prereq:
       - x509: /etc/pki/fleet.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 /etc/pki/fleet.crt:
   x509.certificate_managed:
@@ -379,6 +424,9 @@ msslkeyperms:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 fleetkeyperms:
   file.managed:
@@ -407,6 +455,9 @@ fbcertdir:
     - prereq:
       - x509: /opt/so/conf/filebeat/etc/pki/filebeat.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 # Request a cert and drop it where it needs to go to be distributed
 /opt/so/conf/filebeat/etc/pki/filebeat.crt:
@@ -426,6 +477,9 @@ fbcertdir:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /opt/so/conf/filebeat/etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 # Convert the key to pkcs#8 so logstash will work correctly.
 filebeatpkcs:
@@ -465,6 +519,9 @@ chownfilebeatp8:
     - prereq:
       - x509: /etc/pki/managerssl.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 # Create a cert for the reverse proxy
 /etc/pki/managerssl.crt:
@@ -481,6 +538,9 @@ chownfilebeatp8:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 msslkeyperms:
   file.managed:
@@ -502,6 +562,9 @@ msslkeyperms:
     - prereq:
       - x509: /etc/pki/fleet.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 /etc/pki/fleet.crt:
   x509.certificate_managed:
@@ -515,6 +578,9 @@ msslkeyperms:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 fleetkeyperms:
   file.managed:
@@ -539,6 +605,9 @@ fleetkeyperms:
     - prereq:
       - x509: /etc/pki/elasticsearch.crt
     {%- endif %}
+    - timeout: 30
+    - retry: 5
+    - interval: 30
 
 /etc/pki/elasticsearch.crt:
   x509.certificate_managed:
@@ -553,6 +622,9 @@ fleetkeyperms:
       # https://github.com/saltstack/salt/issues/52167
       # Will trigger 5 days (432000 sec) from cert expiration
       - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]'
+    - timeout: 30
+    - retry: 5
+    - interval: 30
   cmd.run:
     - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:"
     - onchanges: