diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-list b/salt/common/tools/sbin/so-elasticsearch-templates-list index f0cbe36e7..d4d080938 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-list +++ b/salt/common/tools/sbin/so-elasticsearch-templates-list @@ -17,7 +17,11 @@ {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} . /usr/sbin/so-common if [ "$1" == "" ]; then + {% if grains['role'] in ['so-node','so-heavynode'] %} + curl -s -k kttps://{{ NODEIP }}:9200/_template/* | jq 'keys' + {% else %} curl -s {{ NODEIP }}:9200/_template/* | jq 'keys' + {% endif %} else curl -s {{ NODEIP }}:9200/_template/$1 | jq fi diff --git a/salt/common/tools/sbin/so-elasticsearch-templates-load b/salt/common/tools/sbin/so-elasticsearch-templates-load index 61ff48330..292ade995 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates-load +++ b/salt/common/tools/sbin/so-elasticsearch-templates-load @@ -30,7 +30,11 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do + {% if grains['role'] in ['so-node','so-heavynode'] %} + curl -k --output /dev/null --silent --head --fail https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + {% else %} curl --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + {% endif %} if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index 514054359..eed62da24 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -27,7 +27,11 @@ echo -n "Waiting for ElasticSearch..." COUNT=0 ELASTICSEARCH_CONNECTED="no" while [[ "$COUNT" -le 240 ]]; do + {% if grains['role'] in ['so-node','so-heavynode'] %} + curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + {% else %} curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" + {% endif %} if [ $? -eq 0 ]; then ELASTICSEARCH_CONNECTED="yes" echo "connected!" @@ -47,7 +51,11 @@ fi cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." +{% if grains['role'] in ['so-node','so-heavynode'] %} +for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done +{% else %} for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done +{% endif %} echo cd - >/dev/null diff --git a/salt/elasticsearch/files/sotls.yml b/salt/elasticsearch/files/sotls.yml index 07e6446ba..2fc577337 100644 --- a/salt/elasticsearch/files/sotls.yml +++ b/salt/elasticsearch/files/sotls.yml @@ -10,4 +10,8 @@ ciphers: - TLS_RSA_WITH_AES_128_CBC_SHA256 - TLS_RSA_WITH_AES_256_GCM_SHA384 transport.encrypted: true +{%- if grains['role'] in ['so-node','so-heavynode'] %} +http.encrypted: true +{%- else %} http.encrypted: false +{%- endif %} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index c43edba23..1406df02c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -150,6 +150,7 @@ sotls: - source: salt://elasticsearch/files/sotls.yml - user: 930 - group: 939 + - template: jinja #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} @@ -228,6 +229,7 @@ so-elasticsearch-pipelines-file: - user: 930 - group: 939 - mode: 754 + - template: jinja so-elasticsearch-pipelines: cmd.run: @@ -242,6 +244,7 @@ so-elasticsearch-templates: cmd.run: - name: /usr/sbin/so-elasticsearch-templates-load - cwd: /opt/so + - template: jinja {% endif %} {% else %} diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 4c05f2241..6d6a181ac 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -416,6 +416,7 @@ role: manager: portgroups: - {{ portgroups.elasticsearch_node }} + - {{ portgroups.elasticsearch_rest }} dockernet: portgroups: - {{ portgroups.elasticsearch_node }} @@ -454,6 +455,7 @@ role: manager: portgroups: - {{ portgroups.elasticsearch_node }} + - {{ portgroups.elasticsearch_rest }} dockernet: portgroups: - {{ portgroups.elasticsearch_node }} diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index 98a842b2d..2a3babcbd 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -13,6 +13,10 @@ output { template_name => "so-zeek" template => "/templates/so-zeek-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 315c892e2..fdb969865 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -13,6 +13,10 @@ output { template_name => "so-import" template => "/templates/so-import-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 889a3567f..a9ca4c60d 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -12,6 +12,10 @@ output { template_name => "so-flow" template => "/templates/so-flow-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 96d2ae5ba..9da6c5b14 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -12,6 +12,10 @@ output { template_name => "so-ids" template => "/templates/so-ids-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index ee5c57c5a..d57611cb7 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -13,6 +13,10 @@ output { template_name => "so-syslog" template => "/templates/so-syslog-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index a9e5ac64d..ee0718029 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -13,6 +13,10 @@ output { template_name => "so-osquery" template => "/templates/so-osquery-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 3ad4a5722..8227aab01 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -12,6 +12,10 @@ output { template_name => "so-firewall" template => "/templates/so-firewall-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index e65952cca..93bfd7020 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -12,6 +12,10 @@ output { index => "so-ids-%{+YYYY.MM.dd}" template_name => "so-ids" template => "/templates/so-ids-template.json" + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index 10700733e..4d26d491a 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -13,6 +13,10 @@ output { template_name => "so-beats" template => "/templates/so-beats-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 7ebe6afbd..63e20c59a 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -13,6 +13,10 @@ output { template_name => "so-ossec" template => "/templates/so-ossec-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index cdc340b39..193057a53 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -13,6 +13,10 @@ output { template_name => "so-strelka" template => "/templates/so-strelka-template.json" template_overwrite => true + {%- if grains['role'] in ['so-node','so-heavynode'] %} + ssl => true + ssl_certificate_verification => false + {%- endif %} } } } diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 2c574616a..f6a6eca5b 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -24,6 +24,13 @@ }, "elastic": { "hostUrl": "http://{{ MANAGERIP }}:9200", + {%- if salt['pillar.get']('nodestab', {}) %} + "remoteHostUrls": [ + {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} + "https://{{ SN.split('_')|first }}:9200"{{ "," if not loop.last }} + {%- endfor %} + ], + {%- endif %} "username": "", "password": "", "verifyCert": false diff --git a/salt/soc/init.sls b/salt/soc/init.sls index c9dfc322d..a4f99d92d 100644 --- a/salt/soc/init.sls +++ b/salt/soc/init.sls @@ -56,6 +56,12 @@ so-soc: - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro - /opt/so/conf/soc/changes.json:/opt/sensoroni/html/changes.json:ro - /opt/so/log/soc/:/opt/sensoroni/logs/:rw + - extra_hosts: + {%- if salt['pillar.get']('nodestab', {}) %} + {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} + - {{ SN.split('_')|first }}:{{ SNDATA.ip }} + {%- endfor %} + {%- endif %} - port_bindings: - 0.0.0.0:9822:9822 - watch: