Merge pull request #1505 from Security-Onion-Solutions/experimental

Fix Cross Cluster Search Acks

salt/common/tools/sbin/so-elasticsearch-templates-list

@@ -17,7 +17,11 @@
 {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
 . /usr/sbin/so-common
 if [ "$1" == "" ]; then
+    {% if grains['role'] in ['so-node','so-heavynode'] %}
+    curl -s -k kttps://{{ NODEIP }}:9200/_template/* | jq 'keys'
+    {% else %}
     curl -s {{ NODEIP }}:9200/_template/* | jq 'keys'
+    {% endif %}
 else
     curl -s {{ NODEIP }}:9200/_template/$1 | jq
 fi

salt/common/tools/sbin/so-elasticsearch-templates-load

@@ -30,7 +30,11 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
+    {% if grains['role'] in ['so-node','so-heavynode'] %}
+    curl -k --output /dev/null --silent --head --fail https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+    {% else %}
 	curl --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"

salt/elasticsearch/files/so-elasticsearch-pipelines

@@ -27,7 +27,11 @@ echo -n "Waiting for ElasticSearch..."
 COUNT=0
 ELASTICSEARCH_CONNECTED="no"
 while [[ "$COUNT" -le 240 ]]; do
+    {% if grains['role'] in ['so-node','so-heavynode'] %}
+	curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+	{% else %}
 	curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+	{% endif %}
 	if [ $? -eq 0 ]; then
 		ELASTICSEARCH_CONNECTED="yes"
 		echo "connected!"
@@ -47,7 +51,11 @@ fi
 cd ${ELASTICSEARCH_INGEST_PIPELINES}
 
 echo "Loading pipelines..."
+{% if grains['role'] in ['so-node','so-heavynode'] %}
+for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
+{% else %}
 for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
+{% endif %}
 echo
 
 cd - >/dev/null

salt/elasticsearch/files/sotls.yml

@@ -10,4 +10,8 @@ ciphers:
 - TLS_RSA_WITH_AES_128_CBC_SHA256
 - TLS_RSA_WITH_AES_256_GCM_SHA384
 transport.encrypted: true
+{%- if grains['role'] in ['so-node','so-heavynode'] %}
+http.encrypted: true
+{%- else %}
 http.encrypted: false
+{%- endif %}

salt/elasticsearch/init.sls

@@ -150,6 +150,7 @@ sotls:
     - source: salt://elasticsearch/files/sotls.yml
     - user: 930
     - group: 939
+    - template: jinja
 
 #sync templates to /opt/so/conf/elasticsearch/templates
 {% for TEMPLATE in TEMPLATES %}
@@ -228,6 +229,7 @@ so-elasticsearch-pipelines-file:
     - user: 930
     - group: 939
     - mode: 754
+    - template: jinja
 
 so-elasticsearch-pipelines:
  cmd.run:
@@ -242,6 +244,7 @@ so-elasticsearch-templates:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-templates-load
     - cwd: /opt/so
+    - template: jinja
 {% endif %}
 
 {% else %}

salt/firewall/assigned_hostgroups.map.yaml

@@ -416,6 +416,7 @@ role:
           manager:
             portgroups:
               - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.elasticsearch_rest }}
           dockernet:
             portgroups:
               - {{ portgroups.elasticsearch_node }}
@@ -454,6 +455,7 @@ role:
           manager:
             portgroups:
               - {{ portgroups.elasticsearch_node }}
+              - {{ portgroups.elasticsearch_rest }}
           dockernet:
             portgroups:
               - {{ portgroups.elasticsearch_node }}

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -13,6 +13,10 @@ output {
       template_name => "so-zeek"
       template => "/templates/so-zeek-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9002_output_import.conf.jinja

@@ -13,6 +13,10 @@ output {
       template_name => "so-import"
       template => "/templates/so-import-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja

@@ -12,6 +12,10 @@ output {
       template_name => "so-flow"
       template => "/templates/so-flow-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja

@@ -12,6 +12,10 @@ output {
       template_name => "so-ids"
       template => "/templates/so-ids-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -13,6 +13,10 @@ output {
       template_name => "so-syslog"
       template => "/templates/so-syslog-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -13,6 +13,10 @@ output {
       template_name => "so-osquery"
       template => "/templates/so-osquery-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja

@@ -12,6 +12,10 @@ output {
       template_name => "so-firewall"
       template => "/templates/so-firewall-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -12,6 +12,10 @@ output {
       index => "so-ids-%{+YYYY.MM.dd}"
       template_name => "so-ids" 
       template => "/templates/so-ids-template.json"
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja

@@ -13,6 +13,10 @@ output {
       template_name => "so-beats"
       template => "/templates/so-beats-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -13,6 +13,10 @@ output {
       template_name => "so-ossec"
       template => "/templates/so-ossec-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja

@@ -13,6 +13,10 @@ output {
       template_name => "so-strelka"
       template => "/templates/so-strelka-template.json"
       template_overwrite => true
+      {%- if grains['role'] in ['so-node','so-heavynode'] %}
+      ssl => true
+      ssl_certificate_verification => false
+      {%- endif %}
     }
   }
 }

salt/soc/files/soc/soc.json

@@ -24,6 +24,13 @@
       },
       "elastic": {
         "hostUrl": "http://{{ MANAGERIP }}:9200",
+        {%- if salt['pillar.get']('nodestab', {}) %}
+        "remoteHostUrls": [
+        {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
+        "https://{{ SN.split('_')|first }}:9200"{{ "," if not loop.last }}
+        {%- endfor %}
+        ],
+        {%- endif %}
         "username": "",
         "password": "",
         "verifyCert": false

salt/soc/init.sls

@@ -56,6 +56,12 @@ so-soc:
       - /opt/so/conf/soc/soc.json:/opt/sensoroni/sensoroni.json:ro
       - /opt/so/conf/soc/changes.json:/opt/sensoroni/html/changes.json:ro
       - /opt/so/log/soc/:/opt/sensoroni/logs/:rw
+    - extra_hosts:
+    {%- if salt['pillar.get']('nodestab', {}) %}
+      {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
+      - {{ SN.split('_')|first }}:{{ SNDATA.ip }}
+      {%- endfor %}
+      {%- endif %}
     - port_bindings:
       - 0.0.0.0:9822:9822
     - watch: