From e659af346623132a9b9ad96fea0415c558c7f316 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 14:26:56 -0400 Subject: [PATCH 01/49] ES basic SSL --- salt/kibana/etc/kibana.yml | 5 ++ salt/logstash/init.sls | 3 +- .../config/so/9000_output_zeek.conf.jinja | 6 ++ .../config/so/9002_output_import.conf.jinja | 6 ++ .../config/so/9004_output_flow.conf.jinja | 6 ++ .../config/so/9033_output_snort.conf.jinja | 6 ++ .../config/so/9034_output_syslog.conf.jinja | 6 ++ .../config/so/9100_output_osquery.conf.jinja | 6 ++ .../config/so/9200_output_firewall.conf.jinja | 6 ++ .../config/so/9400_output_suricata.conf.jinja | 6 ++ .../config/so/9500_output_beats.conf.jinja | 6 ++ .../config/so/9600_output_ossec.conf.jinja | 6 ++ .../config/so/9700_output_strelka.conf.jinja | 6 ++ salt/soc/files/soc/soc.json | 5 ++ salt/ssl/init.sls | 76 ++++++++++++++++++- 15 files changed, 152 insertions(+), 3 deletions(-) diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 4d19b251b..89e568df9 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -1,10 +1,15 @@ --- # Default Kibana configuration from kibana-docker. {%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} server.name: kibana server.host: "0" server.basePath: /kibana +{% if FEATURES %} +elasticsearch.hosts: [ "https://{{ ES }}:9200" ] +{%- else %} elasticsearch.hosts: [ "http://{{ ES }}:9200" ] +{%- endif %} #kibana.index: ".kibana" #elasticsearch.username: elastic #elasticsearch.password: changeme diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 9f9a5c51b..07af6bbeb 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -167,7 +167,8 @@ so-logstash: - /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro - - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro + - /etc/pki/ca.cer:/ca/ca.crt:ro {%- if grains['role'] == 'so-eval' %} - /nsm/zeek:/nsm/zeek:ro - /nsm/suricata:/suricata:ro diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index f86bf946c..f9dbcccfa 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -3,11 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-zeek-%{+YYYY.MM.dd}" template_name => "so-zeek" template => "/templates/so-zeek-template.json" diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 52c9f034a..5be2c2640 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -3,11 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if "import" in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-import-%{+YYYY.MM.dd}" template_name => "so-import" template => "/templates/so-import-template.json" diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 740676367..f71cf5d52 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -3,10 +3,16 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [event_type] == "sflow" { elasticsearch { + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-flow-%{+YYYY.MM.dd}" template_name => "so-flow" template => "/templates/so-flow-template.json" diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index fed1ffdf5..f7a29415a 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -3,10 +3,16 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-ids-%{+YYYY.MM.dd}" template_name => "so-ids" template => "/templates/so-ids-template.json" diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 5087f41da..403ba1f2e 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -3,11 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "syslog" { elasticsearch { pipeline => "%{module}" + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-syslog-%{+YYYY.MM.dd}" template_name => "so-syslog" template => "/templates/so-syslog-template.json" diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 01436cf5f..a8c8910d9 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -3,11 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "osquery" { elasticsearch { pipeline => "%{module}.%{dataset}" + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-osquery-%{+YYYY.MM.dd}" template_name => "so-osquery" template => "/templates/so-osquery-template.json" diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index a295b5f7a..8f006c90e 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -3,10 +3,16 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if "firewall" in [tags] { elasticsearch { + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-firewall-%{+YYYY.MM.dd}" template_name => "so-firewall" template => "/templates/so-firewall-template.json" diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index ace7cccf1..35f9f35b4 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -3,11 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-ids-%{+YYYY.MM.dd}" template_name => "so-ids" template => "/templates/so-ids-template.json" diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index ed513f597..e923e5044 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -3,11 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { pipeline => "beats.common" + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-beats-%{+YYYY.MM.dd}" template_name => "so-beats" template => "/templates/so-beats-template.json" diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 14a9bc1d1..080c8e4e1 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -3,11 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "ossec" { elasticsearch { pipeline => "%{module}.%{dataset}" + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-ossec-%{+YYYY.MM.dd}" template_name => "so-ossec" template => "/templates/so-ossec-template.json" diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index 0e6977e29..8e5230af6 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -3,11 +3,17 @@ {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} output { if [module] =~ "strelka" { elasticsearch { pipeline => "%{module}.%{dataset}" + {%- if FEATURES %} + hosts => "https://{{ ES }}" + cacert => '/ca/ca.crt' + {%- else %} hosts => "{{ ES }}" + {%- endif %} index => "so-strelka-%{+YYYY.MM.dd}" template_name => "so-strelka" template => "/templates/so-strelka-template.json" diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 999819356..86bad6cf4 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -1,5 +1,6 @@ {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') -%} {%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') -%} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} { "logFilename": "/opt/sensoroni/logs/sensoroni-server.log", "server": { @@ -15,7 +16,11 @@ "hostUrl": "http://{{ MANAGERIP }}:4434/" }, "elastic": { + {%- if FEATURES %} + "hostUrl": "https://{{ MANAGERIP }}:9200", + {%- else %} "hostUrl": "http://{{ MANAGERIP }}:9200", + {%- endif %} "username": "", "password": "", "verifyCert": false diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 9691c861f..595910b1b 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -194,7 +194,7 @@ regkeyperms: - x509: /etc/pki/minio.crt {%- endif %} -# Create a cert for the docker registry +# Create a cert for minio /etc/pki/minio.crt: x509.certificate_managed: - ca_server: {{ ca_server }} @@ -229,6 +229,41 @@ miniokeyperms: - x509: /etc/pki/redis.crt {%- endif %} +# Create a cert for elasticsearch +/etc/pki/elasticsearch.crt: + x509.certificate_managed: + - ca_server: {{ ca_server }} + - signing_policy: registry + - public_key: /etc/pki/ealsticsearch.key + - CN: {{ manager }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - unless: + # https://github.com/saltstack/salt/issues/52167 + # Will trigger 5 days (432000 sec) from cert expiration + - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' + +miniokeyperms: + file.managed: + - replace: False + - name: /etc/pki/elasticsearch.key + - mode: 640 + - group: 939 + +/etc/pki/elasticsearch.key: + x509.private_key_managed: + - CN: {{ manager }} + - bits: 4096 + - days_remaining: 0 + - days_valid: 820 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} + - prereq: + - x509: /etc/pki/elasticsearch.crt + {%- endif %} + # Create a cert for the docker registry /etc/pki/redis.crt: x509.certificate_managed: @@ -457,4 +492,41 @@ fleetkeyperms: - mode: 640 - group: 939 -{% endif %} \ No newline at end of file +{% endif %} + +{% if grains['role'] in ['so-search', 'so-heavynode'] %} +# Create a cert for elasticsearch +/etc/pki/elasticsearch.crt: + x509.certificate_managed: + - ca_server: {{ ca_server }} + - signing_policy: registry + - public_key: /etc/pki/ealsticsearch.key + - CN: {{ manager }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - unless: + # https://github.com/saltstack/salt/issues/52167 + # Will trigger 5 days (432000 sec) from cert expiration + - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' + +miniokeyperms: + file.managed: + - replace: False + - name: /etc/pki/elasticsearch.key + - mode: 640 + - group: 939 + +/etc/pki/elasticsearch.key: + x509.private_key_managed: + - CN: {{ manager }} + - bits: 4096 + - days_remaining: 0 + - days_valid: 820 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} + - prereq: + - x509: /etc/pki/elasticsearch.crt + {%- endif %} +{%- endif %} \ No newline at end of file From 523e42bec83ed9eed5dd1a376526f4373c4864bd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 14:40:11 -0400 Subject: [PATCH 02/49] Fix ssl state --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 595910b1b..700083be6 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -244,7 +244,7 @@ miniokeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' -miniokeyperms: +ealstickeyperms: file.managed: - replace: False - name: /etc/pki/elasticsearch.key From 788864310c380c03d66a1aae379437eb70d820a7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 14:52:20 -0400 Subject: [PATCH 03/49] Fix ssl state --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 700083be6..9677bdda2 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -234,7 +234,7 @@ miniokeyperms: x509.certificate_managed: - ca_server: {{ ca_server }} - signing_policy: registry - - public_key: /etc/pki/ealsticsearch.key + - public_key: /etc/pki/elasticsearch.key - CN: {{ manager }} - days_remaining: 0 - days_valid: 820 From 28806513d9e4788b0a16720966ce948c6be19c12 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 20:53:56 -0400 Subject: [PATCH 04/49] Logstash logic fix --- salt/logstash/init.sls | 2 +- salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9002_output_import.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja | 2 +- .../logstash/pipelines/config/so/9100_output_osquery.conf.jinja | 2 +- .../pipelines/config/so/9200_output_firewall.conf.jinja | 2 +- .../pipelines/config/so/9400_output_suricata.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja | 2 +- .../logstash/pipelines/config/so/9700_output_strelka.conf.jinja | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 07af6bbeb..1a85a081d 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -17,7 +17,7 @@ {% set MANAGER = salt['grains.get']('master') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} -{% if FEATURES %} +{%- if FEATURES is sameas true %} {% set FEATURES = "-features" %} {% else %} {% set FEATURES = '' %} diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index f9dbcccfa..e075918f6 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -8,7 +8,7 @@ output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 5be2c2640..ae0a619fe 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -8,7 +8,7 @@ output { if "import" in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index f71cf5d52..c888a9752 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -7,7 +7,7 @@ output { if [event_type] == "sflow" { elasticsearch { - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index f7a29415a..daddd4b0a 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -7,7 +7,7 @@ output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 403ba1f2e..d554adf16 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -8,7 +8,7 @@ output { if [module] =~ "syslog" { elasticsearch { pipeline => "%{module}" - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index a8c8910d9..c1e6ae59f 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -8,7 +8,7 @@ output { if [module] =~ "osquery" { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 8f006c90e..14e741b9d 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -7,7 +7,7 @@ output { if "firewall" in [tags] { elasticsearch { - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 35f9f35b4..a684e2412 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -8,7 +8,7 @@ output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index e923e5044..321566bac 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -8,7 +8,7 @@ output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { pipeline => "beats.common" - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 080c8e4e1..4af0839c4 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -8,7 +8,7 @@ output { if [module] =~ "ossec" { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index 8e5230af6..a0e9950de 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -8,7 +8,7 @@ output { if [module] =~ "strelka" { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES %} + {%- if FEATURES is sameas true %} hosts => "https://{{ ES }}" cacert => '/ca/ca.crt' {%- else %} From 92cc176b6d8ae0a7302486ac1a42cbda586ec05b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 20:59:41 -0400 Subject: [PATCH 05/49] Fix features logic in all states that use it --- salt/elasticsearch/init.sls | 2 +- salt/filebeat/init.sls | 2 +- salt/kibana/init.sls | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 5f87a430c..2a675cc45 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -17,7 +17,7 @@ {% set MANAGER = salt['grains.get']('master') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} -{% if FEATURES %} +{%- if FEATURES is sameas true %} {% set FEATURES = "-features" %} {% else %} {% set FEATURES = '' %} diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index a4fa36b14..ee7c5ae10 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -16,7 +16,7 @@ {% set MANAGER = salt['grains.get']('master') %} {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} -{% if FEATURES %} +{%- if FEATURES is sameas true %} {% set FEATURES = "-features" %} {% else %} {% set FEATURES = '' %} diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index a1dccd137..8711d47d1 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -2,7 +2,7 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} -{% if FEATURES %} +{%- if FEATURES is sameas true %} {% set FEATURES = "-features" %} {% else %} {% set FEATURES = '' %} From e7cd527d4934cdfcf6b0c4312573a47c8cf2a281 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 21:18:03 -0400 Subject: [PATCH 06/49] Enable SSL in elastic --- salt/elasticsearch/files/elasticsearch.yml | 12 ++++++++++++ salt/elasticsearch/init.sls | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 4d5d5b2e4..0f5e9e59f 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -5,6 +5,7 @@ {%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername', '') %} {%- endif %} {%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} cluster.name: "{{ ESCLUSTERNAME }}" network.host: 0.0.0.0 @@ -22,6 +23,17 @@ cluster.routing.allocation.disk.threshold_enabled: true cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% +{%- if FEATURES is sameas true %} +xpack.security.enabled: true +xpack.security.http.ssl.enabled: true +xpack.security.transport.ssl.enabled: true +xpack.security.http.ssl.key: /ca/elasticsearch.key +xpack.security.http.ssl.certificate: /ca/elasticsearch.crt +xpack.security.http.ssl.certificate_authorities: /ca/ca.crt +xpack.security.transport.ssl.key: /ca/elasticsearch.key +xpack.security.transport.ssl.certificate: /ca/elasticsearch.crt +xpack.security.transport.ssl.certificate_authorities: /ca/ca.crt +{%- endif %} node.attr.box_type: {{ NODE_ROUTE_TYPE }} node.name: {{ ESCLUSTERNAME }} script.max_compilations_rate: 1000/1m diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 2a675cc45..d343f19c1 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -192,6 +192,11 @@ so-elasticsearch: - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro + {%- if FEATURES is sameas true %} + - /etc/pki/ca.crt:/ca/ca.cert:ro + - /etc/pki/elasticsearch.key:/ca/elasticsearch.key:ro + - /etc/pki/elasticsearch.crt:/ca/elasticsearch.crt:ro + {%- endif %} - watch: - file: cacertz From e28619604cda6a437fd2cc5c1101ea5c5a377341 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 21:26:00 -0400 Subject: [PATCH 07/49] Change certs path on elstic --- salt/elasticsearch/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index d343f19c1..6819f4796 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -193,9 +193,9 @@ so-elasticsearch: - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro {%- if FEATURES is sameas true %} - - /etc/pki/ca.crt:/ca/ca.cert:ro - - /etc/pki/elasticsearch.key:/ca/elasticsearch.key:ro - - /etc/pki/elasticsearch.crt:/ca/elasticsearch.crt:ro + - /etc/pki/ca.crt:/usr/share/elasticsearch/ca/ca.cert:ro + - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/ca/elasticsearch.key:ro + - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/ca/elasticsearch.crt:ro {%- endif %} - watch: - file: cacertz From cf5c29d01c00089c099ea224a4500a1d9e338809 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 21:30:53 -0400 Subject: [PATCH 08/49] Change certs path on elstic --- salt/elasticsearch/files/elasticsearch.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 0f5e9e59f..cbfede50e 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -27,12 +27,12 @@ cluster.routing.allocation.disk.watermark.flood_stage: 98% xpack.security.enabled: true xpack.security.http.ssl.enabled: true xpack.security.transport.ssl.enabled: true -xpack.security.http.ssl.key: /ca/elasticsearch.key -xpack.security.http.ssl.certificate: /ca/elasticsearch.crt -xpack.security.http.ssl.certificate_authorities: /ca/ca.crt -xpack.security.transport.ssl.key: /ca/elasticsearch.key -xpack.security.transport.ssl.certificate: /ca/elasticsearch.crt -xpack.security.transport.ssl.certificate_authorities: /ca/ca.crt +xpack.security.http.ssl.key: /usr/share/elasticsearch/ca/elasticsearch.key +xpack.security.http.ssl.certificate: /usr/share/elasticsearch/ca/elasticsearch.crt +xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/ca/ca.crt +xpack.security.transport.ssl.key: /usr/share/elasticsearch/ca/elasticsearch.key +xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/ca/elasticsearch.crt +xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/ca/ca.crt {%- endif %} node.attr.box_type: {{ NODE_ROUTE_TYPE }} node.name: {{ ESCLUSTERNAME }} From 08d544e527ef60e41cf3846d43cf603457edb528 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 21:44:45 -0400 Subject: [PATCH 09/49] Fix SSL perms --- salt/ssl/init.sls | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 9677bdda2..71daecfc6 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -216,7 +216,8 @@ miniokeyperms: - mode: 640 - group: 939 -/etc/pki/redis.key: +# Create a cert for elasticsearch +/etc/pki/elasticsearch.key: x509.private_key_managed: - CN: {{ manager }} - bits: 4096 @@ -224,12 +225,11 @@ miniokeyperms: - days_valid: 820 - backup: True - new: True - {% if salt['file.file_exists']('/etc/pki/redis.key') -%} + {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} - prereq: - - x509: /etc/pki/redis.crt + - x509: /etc/pki/elasticsearch.crt {%- endif %} -# Create a cert for elasticsearch /etc/pki/elasticsearch.crt: x509.certificate_managed: - ca_server: {{ ca_server }} @@ -249,9 +249,10 @@ ealstickeyperms: - replace: False - name: /etc/pki/elasticsearch.key - mode: 640 - - group: 939 + - group: 930 -/etc/pki/elasticsearch.key: +# Create a cert for Redis encryption +/etc/pki/redis.key: x509.private_key_managed: - CN: {{ manager }} - bits: 4096 @@ -259,12 +260,11 @@ ealstickeyperms: - days_valid: 820 - backup: True - new: True - {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} + {% if salt['file.file_exists']('/etc/pki/redis.key') -%} - prereq: - - x509: /etc/pki/elasticsearch.crt + - x509: /etc/pki/redis.crt {%- endif %} -# Create a cert for the docker registry /etc/pki/redis.crt: x509.certificate_managed: - ca_server: {{ ca_server }} From 811da5732a8beaf4206b032327ecedea34bb8c89 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 21:51:29 -0400 Subject: [PATCH 10/49] Elastic logic fix --- salt/elasticsearch/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 6819f4796..a983b809b 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -18,9 +18,9 @@ {% set FEATURES = salt['pillar.get']('elastic:features', False) %} {%- if FEATURES is sameas true %} - {% set FEATURES = "-features" %} + {% set FEATUREZ = "-features" %} {% else %} - {% set FEATURES = '' %} + {% set FEATUREZ = '' %} {% endif %} {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} @@ -168,7 +168,7 @@ eslogdir: so-elasticsearch: docker_container.running: - - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}{{ FEATURES }} + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}{{ FEATUREZ }} - hostname: elasticsearch - name: so-elasticsearch - user: elasticsearch From cdda46ce587fc112635f3b4d9d4d6fdaa31b8e8c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 21:54:36 -0400 Subject: [PATCH 11/49] ca typeo --- salt/elasticsearch/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index a983b809b..ef846ec5e 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -193,7 +193,7 @@ so-elasticsearch: - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro {%- if FEATURES is sameas true %} - - /etc/pki/ca.crt:/usr/share/elasticsearch/ca/ca.cert:ro + - /etc/pki/ca.crt:/usr/share/elasticsearch/ca/ca.crt:ro - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/ca/elasticsearch.key:ro - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/ca/elasticsearch.crt:ro {%- endif %} From 6d2be9af7e82b34add972f87d686b84572340b17 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 21:58:44 -0400 Subject: [PATCH 12/49] Things like this are why I hate Java --- salt/elasticsearch/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index ef846ec5e..b3f570c21 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -193,9 +193,9 @@ so-elasticsearch: - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro {%- if FEATURES is sameas true %} - - /etc/pki/ca.crt:/usr/share/elasticsearch/ca/ca.crt:ro - - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/ca/elasticsearch.key:ro - - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/ca/elasticsearch.crt:ro + - /etc/pki/ca.crt:/etc/elasticsearch/ca/ca.crt:ro + - /etc/pki/elasticsearch.key:/etc/elasticsearch/ca/elasticsearch.key:ro + - /etc/pki/elasticsearch.crt:/etc/elasticsearch/ca/elasticsearch.crt:ro {%- endif %} - watch: - file: cacertz From 31ab1e8ed8d6ba2f7fb53388e03ffe7aa6d02587 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 22:03:24 -0400 Subject: [PATCH 13/49] Things like this are why I hate Java --- salt/elasticsearch/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index b3f570c21..3d407f3fd 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -193,9 +193,9 @@ so-elasticsearch: - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro {%- if FEATURES is sameas true %} - - /etc/pki/ca.crt:/etc/elasticsearch/ca/ca.crt:ro - - /etc/pki/elasticsearch.key:/etc/elasticsearch/ca/elasticsearch.key:ro - - /etc/pki/elasticsearch.crt:/etc/elasticsearch/ca/elasticsearch.crt:ro + - /etc/pki/ca.crt:/etc/elasticsearch/ca.crt:ro + - /etc/pki/elasticsearch.key:/etc/elasticsearch/elasticsearch.key:ro + - /etc/pki/elasticsearch.crt:/etc/elasticsearch/elasticsearch.crt:ro {%- endif %} - watch: - file: cacertz From d00231af066d9f4b8f4506995237288cde919dab Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 22:05:46 -0400 Subject: [PATCH 14/49] Things like this are why I hate Java --- salt/elasticsearch/files/elasticsearch.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index cbfede50e..f54195467 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -27,12 +27,12 @@ cluster.routing.allocation.disk.watermark.flood_stage: 98% xpack.security.enabled: true xpack.security.http.ssl.enabled: true xpack.security.transport.ssl.enabled: true -xpack.security.http.ssl.key: /usr/share/elasticsearch/ca/elasticsearch.key -xpack.security.http.ssl.certificate: /usr/share/elasticsearch/ca/elasticsearch.crt -xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/ca/ca.crt -xpack.security.transport.ssl.key: /usr/share/elasticsearch/ca/elasticsearch.key -xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/ca/elasticsearch.crt -xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/ca/ca.crt +xpack.security.http.ssl.key: /etc/elasticsearch/elasticsearch.key +xpack.security.http.ssl.certificate: /etc/elasticsearch/elasticsearch.crt +xpack.security.http.ssl.certificate_authorities: /etc/elasticsearch/ca.crt +xpack.security.transport.ssl.key: /etc/elasticsearch/elasticsearch.key +xpack.security.transport.ssl.certificate: /etc/elasticsearch/elasticsearch.crt +xpack.security.transport.ssl.certificate_authorities: /etc/elasticsearch/ca.crt {%- endif %} node.attr.box_type: {{ NODE_ROUTE_TYPE }} node.name: {{ ESCLUSTERNAME }} From 6007a6c4d8373a239ccf1955a88391fc267e6785 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 22:10:03 -0400 Subject: [PATCH 15/49] Things like this are why I hate Java --- salt/elasticsearch/files/elasticsearch.yml | 12 ++++++------ salt/elasticsearch/init.sls | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index f54195467..cb1526eba 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -27,12 +27,12 @@ cluster.routing.allocation.disk.watermark.flood_stage: 98% xpack.security.enabled: true xpack.security.http.ssl.enabled: true xpack.security.transport.ssl.enabled: true -xpack.security.http.ssl.key: /etc/elasticsearch/elasticsearch.key -xpack.security.http.ssl.certificate: /etc/elasticsearch/elasticsearch.crt -xpack.security.http.ssl.certificate_authorities: /etc/elasticsearch/ca.crt -xpack.security.transport.ssl.key: /etc/elasticsearch/elasticsearch.key -xpack.security.transport.ssl.certificate: /etc/elasticsearch/elasticsearch.crt -xpack.security.transport.ssl.certificate_authorities: /etc/elasticsearch/ca.crt +xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key +xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt +xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt +xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key +xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt +xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt {%- endif %} node.attr.box_type: {{ NODE_ROUTE_TYPE }} node.name: {{ ESCLUSTERNAME }} diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 3d407f3fd..802957bd2 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -193,9 +193,9 @@ so-elasticsearch: - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro {%- if FEATURES is sameas true %} - - /etc/pki/ca.crt:/etc/elasticsearch/ca.crt:ro - - /etc/pki/elasticsearch.key:/etc/elasticsearch/elasticsearch.key:ro - - /etc/pki/elasticsearch.crt:/etc/elasticsearch/elasticsearch.crt:ro + - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro + - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro + - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro {%- endif %} - watch: - file: cacertz From c3d8c599cc19b09ccbebd1f63f48aa7259c6145d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 22:13:17 -0400 Subject: [PATCH 16/49] Turn off user auth --- salt/elasticsearch/files/elasticsearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index cb1526eba..f3b6bf1f5 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -24,7 +24,7 @@ cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% {%- if FEATURES is sameas true %} -xpack.security.enabled: true +xpack.security.enabled: false xpack.security.http.ssl.enabled: true xpack.security.transport.ssl.enabled: true xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key From 52cc56bebbfdcfebc29c185dbd2c4efba9423d9a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 22:56:15 -0400 Subject: [PATCH 17/49] Add transport hostname --- salt/elasticsearch/files/elasticsearch.yml | 4 ++-- salt/elasticsearch/init.sls | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index f3b6bf1f5..f8e62c701 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -16,7 +16,7 @@ discovery.zen.minimum_master_nodes: 1 # This is a test -- if this is here, then the volume is mounted correctly. path.logs: /var/log/elasticsearch action.destructive_requires_name: true -transport.bind_host: 0.0.0.0 +transport.bind_host: {{ grains.host }} transport.publish_host: {{ NODEIP }} transport.publish_port: 9300 cluster.routing.allocation.disk.threshold_enabled: true @@ -25,7 +25,7 @@ cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% {%- if FEATURES is sameas true %} xpack.security.enabled: false -xpack.security.http.ssl.enabled: true +xpack.security.http.ssl.enabled: false xpack.security.transport.ssl.enabled: true xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 802957bd2..6686054ef 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -172,6 +172,8 @@ so-elasticsearch: - hostname: elasticsearch - name: so-elasticsearch - user: elasticsearch + - extra_hosts: + - {{ grains.host }} - environment: - discovery.type=single-node #- bootstrap.memory_lock=true From 730e389aae9e3887c9b310077cb3a858da47c5a4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 22:57:49 -0400 Subject: [PATCH 18/49] Add transport hostname --- salt/elasticsearch/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 6686054ef..4d92291ae 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -173,7 +173,7 @@ so-elasticsearch: - name: so-elasticsearch - user: elasticsearch - extra_hosts: - - {{ grains.host }} + - {{ grains.host }}:127.0.0.1 - environment: - discovery.type=single-node #- bootstrap.memory_lock=true From ac3f490299b5ab4974bf948bba11506bbb3daa7d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 23:02:03 -0400 Subject: [PATCH 19/49] Add transport hostname --- salt/elasticsearch/files/elasticsearch.yml | 2 +- salt/elasticsearch/init.sls | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index f8e62c701..b26e759a5 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -17,7 +17,7 @@ discovery.zen.minimum_master_nodes: 1 path.logs: /var/log/elasticsearch action.destructive_requires_name: true transport.bind_host: {{ grains.host }} -transport.publish_host: {{ NODEIP }} +transport.publish_host: {{ grains.host }} transport.publish_port: 9300 cluster.routing.allocation.disk.threshold_enabled: true cluster.routing.allocation.disk.watermark.low: 95% diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 4d92291ae..738f7928b 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -16,6 +16,8 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set FEATURES = salt['pillar.get']('elastic:features', False) %} +{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%} + {%- if FEATURES is sameas true %} {% set FEATUREZ = "-features" %} @@ -173,7 +175,7 @@ so-elasticsearch: - name: so-elasticsearch - user: elasticsearch - extra_hosts: - - {{ grains.host }}:127.0.0.1 + - {{ grains.host }}:{{ NODEIP }} - environment: - discovery.type=single-node #- bootstrap.memory_lock=true From 59292425c0999c821258d5e718f961fd13844669 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 10 Aug 2020 23:03:54 -0400 Subject: [PATCH 20/49] Add transport hostname --- salt/elasticsearch/files/elasticsearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index b26e759a5..625d8c8d9 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -16,7 +16,7 @@ discovery.zen.minimum_master_nodes: 1 # This is a test -- if this is here, then the volume is mounted correctly. path.logs: /var/log/elasticsearch action.destructive_requires_name: true -transport.bind_host: {{ grains.host }} +transport.bind_host: 0.0.0.0 transport.publish_host: {{ grains.host }} transport.publish_port: 9300 cluster.routing.allocation.disk.threshold_enabled: true From 32f8ea3158d4ff0fdca567da4f46784e8a18b14d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 10:02:00 -0400 Subject: [PATCH 21/49] Removes https from rest port --- salt/kibana/etc/kibana.yml | 6 +----- salt/soc/files/soc/soc.json | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/salt/kibana/etc/kibana.yml b/salt/kibana/etc/kibana.yml index 89e568df9..4bcc22016 100644 --- a/salt/kibana/etc/kibana.yml +++ b/salt/kibana/etc/kibana.yml @@ -1,15 +1,11 @@ --- # Default Kibana configuration from kibana-docker. {%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} +{%- set FEATURES = salt['pillar.get']('elastic:features', False) %} server.name: kibana server.host: "0" server.basePath: /kibana -{% if FEATURES %} -elasticsearch.hosts: [ "https://{{ ES }}:9200" ] -{%- else %} elasticsearch.hosts: [ "http://{{ ES }}:9200" ] -{%- endif %} #kibana.index: ".kibana" #elasticsearch.username: elastic #elasticsearch.password: changeme diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 86bad6cf4..b44733cb1 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -16,11 +16,7 @@ "hostUrl": "http://{{ MANAGERIP }}:4434/" }, "elastic": { - {%- if FEATURES %} - "hostUrl": "https://{{ MANAGERIP }}:9200", - {%- else %} "hostUrl": "http://{{ MANAGERIP }}:9200", - {%- endif %} "username": "", "password": "", "verifyCert": false From d94120947963b3ca3b0dde7139142a0040a0eb44 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 10:17:28 -0400 Subject: [PATCH 22/49] Walk nodes tab --- salt/elasticsearch/init.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 738f7928b..846bb63f9 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -176,6 +176,12 @@ so-elasticsearch: - user: elasticsearch - extra_hosts: - {{ grains.host }}:{{ NODEIP }} + {%- if ismanager %} + {%- if salt['pillar.get']('nodestab', {}) %} + {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} + - {{ SN }}:{{ SNDATA.ip }} + {%- endif %} + {%- endif %} - environment: - discovery.type=single-node #- bootstrap.memory_lock=true From b84d7d818f430677ca69ae06a3300e5ce84766fa Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 10:20:02 -0400 Subject: [PATCH 23/49] Fix for loop --- salt/elasticsearch/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 846bb63f9..1df063ae6 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -180,6 +180,7 @@ so-elasticsearch: {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {{ SN }}:{{ SNDATA.ip }} + {%- endfor %} {%- endif %} {%- endif %} - environment: From 7e0249c3772571609491fb94a80ccdc7e101904f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 10:28:21 -0400 Subject: [PATCH 24/49] ES cleanup --- salt/elasticsearch/init.sls | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 1df063ae6..c93b6a900 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -185,12 +185,8 @@ so-elasticsearch: {%- endif %} - environment: - discovery.type=single-node - #- bootstrap.memory_lock=true - #- cluster.name={{ esclustername }} - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} - #- http.host=0.0.0.0 - #- transport.host=127.0.0.1 - - ulimits: + ulimits: - memlock=-1:-1 - nofile=65536:65536 - nproc=4096 From a5131da5c9e52cdb42834207b60c42059d8dacf2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 11:07:34 -0400 Subject: [PATCH 25/49] fix ssl certs for SN --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 71daecfc6..0336ee84b 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -494,7 +494,7 @@ fleetkeyperms: {% endif %} -{% if grains['role'] in ['so-search', 'so-heavynode'] %} +{% if grains['role'] in ['so-node', 'so-heavynode'] %} # Create a cert for elasticsearch /etc/pki/elasticsearch.crt: x509.certificate_managed: From 32c407231ffdab96c012b92af98b2acbd86a711c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 11:08:49 -0400 Subject: [PATCH 26/49] fix ssl certs for SN --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 0336ee84b..0fabe832d 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -500,7 +500,7 @@ fleetkeyperms: x509.certificate_managed: - ca_server: {{ ca_server }} - signing_policy: registry - - public_key: /etc/pki/ealsticsearch.key + - public_key: /etc/pki/elasticsearch.key - CN: {{ manager }} - days_remaining: 0 - days_valid: 820 From cbba473c2d687638d7e96610ed64916bd72639e2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 11:10:27 -0400 Subject: [PATCH 27/49] fix ssl certs for SN --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 0fabe832d..6751c4b15 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -515,7 +515,7 @@ miniokeyperms: - replace: False - name: /etc/pki/elasticsearch.key - mode: 640 - - group: 939 + - group: 930 /etc/pki/elasticsearch.key: x509.private_key_managed: From 05a05b5e9b391317acd280b3b1275f0d500daf0e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 11:15:57 -0400 Subject: [PATCH 28/49] use hostname for cross cluster --- salt/utility/bin/crossthestreams | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index d21e3c1a4..6301a4f71 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -1,6 +1,8 @@ #!/bin/bash {% set ES = salt['pillar.get']('manager:mainip', '') %} {%- set MANAGER = salt['grains.get']('master') %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} + # Wait for ElasticSearch to come up, so that we can query for version infromation echo -n "Waiting for ElasticSearch..." @@ -35,6 +37,10 @@ echo "Applying cross cluster search config..." {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} + {%- if FEATURES is sameas true %} +curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN }}:9300"]}}}}}' + {%- else %} curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}' + {%- endif %} {%- endfor %} {%- endif %} From 348f7f39cc4ea84cec77ba4f925bef774d59f910 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 11:37:53 -0400 Subject: [PATCH 29/49] strip node suffix --- salt/elasticsearch/init.sls | 2 +- salt/utility/bin/crossthestreams | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index c93b6a900..28db606f1 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -179,7 +179,7 @@ so-elasticsearch: {%- if ismanager %} {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - - {{ SN }}:{{ SNDATA.ip }} + - {{ SN.split('_')|first }}:{{ SNDATA.ip }} {%- endfor %} {%- endif %} {%- endif %} diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index 6301a4f71..9c398ae6d 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -38,7 +38,7 @@ echo "Applying cross cluster search config..." {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {%- if FEATURES is sameas true %} -curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN }}:9300"]}}}}}' +curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN.split('_')|first }}": {"skip_unavailable": "true", "seeds": ["{{ SN }}:9300"]}}}}}' {%- else %} curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}' {%- endif %} From 95367f8d236102e7c0bcd5738158022231dcaf3e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 12:00:58 -0400 Subject: [PATCH 30/49] Fix cross cluster --- salt/utility/bin/crossthestreams | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index 9c398ae6d..a057e261b 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -38,7 +38,7 @@ echo "Applying cross cluster search config..." {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} {%- if FEATURES is sameas true %} -curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN.split('_')|first }}": {"skip_unavailable": "true", "seeds": ["{{ SN }}:9300"]}}}}}' +curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' {%- else %} curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}' {%- endif %} From 362749ca85a77f21447fe42387bb57fe7063da3a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 13:00:42 -0400 Subject: [PATCH 31/49] Make hostnames default in cross cluster --- salt/utility/bin/crossthestreams | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/utility/bin/crossthestreams b/salt/utility/bin/crossthestreams index a057e261b..e67ce9f57 100644 --- a/salt/utility/bin/crossthestreams +++ b/salt/utility/bin/crossthestreams @@ -37,10 +37,6 @@ echo "Applying cross cluster search config..." {%- if salt['pillar.get']('nodestab', {}) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} - {%- if FEATURES is sameas true %} curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}' - {%- else %} -curl -XPUT http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SNDATA.ip }}:9300"]}}}}}' - {%- endif %} {%- endfor %} {%- endif %} From 8daf11f085e2da3e309935bd81fa8bf50149667d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 13:58:28 -0400 Subject: [PATCH 32/49] Fix logstash outputs --- .../logstash/pipelines/config/so/9000_output_zeek.conf.jinja | 5 ----- .../pipelines/config/so/9002_output_import.conf.jinja | 5 ----- .../logstash/pipelines/config/so/9004_output_flow.conf.jinja | 5 ----- .../pipelines/config/so/9033_output_snort.conf.jinja | 5 ----- .../pipelines/config/so/9034_output_syslog.conf.jinja | 5 ----- .../pipelines/config/so/9100_output_osquery.conf.jinja | 5 ----- .../pipelines/config/so/9200_output_firewall.conf.jinja | 5 ----- .../pipelines/config/so/9400_output_suricata.conf.jinja | 5 ----- .../pipelines/config/so/9500_output_beats.conf.jinja | 5 ----- .../pipelines/config/so/9600_output_ossec.conf.jinja | 5 ----- .../pipelines/config/so/9700_output_strelka.conf.jinja | 5 ----- 11 files changed, 55 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index e075918f6..98a842b2d 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -8,12 +8,7 @@ output { if [module] =~ "zeek" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-zeek-%{+YYYY.MM.dd}" template_name => "so-zeek" template => "/templates/so-zeek-template.json" diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index ae0a619fe..315c892e2 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -8,12 +8,7 @@ output { if "import" in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-import-%{+YYYY.MM.dd}" template_name => "so-import" template => "/templates/so-import-template.json" diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index c888a9752..889a3567f 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -7,12 +7,7 @@ output { if [event_type] == "sflow" { elasticsearch { - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-flow-%{+YYYY.MM.dd}" template_name => "so-flow" template => "/templates/so-flow-template.json" diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index daddd4b0a..96d2ae5ba 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -7,12 +7,7 @@ output { if [event_type] == "ids" and "import" not in [tags] { elasticsearch { - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-ids-%{+YYYY.MM.dd}" template_name => "so-ids" template => "/templates/so-ids-template.json" diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index d554adf16..ee5c57c5a 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -8,12 +8,7 @@ output { if [module] =~ "syslog" { elasticsearch { pipeline => "%{module}" - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-syslog-%{+YYYY.MM.dd}" template_name => "so-syslog" template => "/templates/so-syslog-template.json" diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index c1e6ae59f..a9e5ac64d 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -8,12 +8,7 @@ output { if [module] =~ "osquery" { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-osquery-%{+YYYY.MM.dd}" template_name => "so-osquery" template => "/templates/so-osquery-template.json" diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 14e741b9d..f8aa07b1b 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -7,12 +7,7 @@ output { if "firewall" in [tags] { elasticsearch { - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-firewall-%{+YYYY.MM.dd}" template_name => "so-firewall" template => "/templates/so-firewall-template.json" diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index a684e2412..e65952cca 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -8,12 +8,7 @@ output { if [module] =~ "suricata" and "import" not in [tags] { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-ids-%{+YYYY.MM.dd}" template_name => "so-ids" template => "/templates/so-ids-template.json" diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index 321566bac..10700733e 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -8,12 +8,7 @@ output { if "beat-ext" in [tags] and "import" not in [tags] { elasticsearch { pipeline => "beats.common" - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-beats-%{+YYYY.MM.dd}" template_name => "so-beats" template => "/templates/so-beats-template.json" diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 4af0839c4..89d1a9466 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -8,12 +8,7 @@ output { if [module] =~ "ossec" { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-ossec-%{+YYYY.MM.dd}" template_name => "so-ossec" template => "/templates/so-ossec-template.json" diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index a0e9950de..cdc340b39 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -8,12 +8,7 @@ output { if [module] =~ "strelka" { elasticsearch { pipeline => "%{module}.%{dataset}" - {%- if FEATURES is sameas true %} - hosts => "https://{{ ES }}" - cacert => '/ca/ca.crt' - {%- else %} hosts => "{{ ES }}" - {%- endif %} index => "so-strelka-%{+YYYY.MM.dd}" template_name => "so-strelka" template => "/templates/so-strelka-template.json" From f553a8e27aaae8067e0d9f7f0a7d73abf802a6f1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 14:40:34 -0400 Subject: [PATCH 33/49] anon user hack --- salt/elasticsearch/files/elasticsearch.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 625d8c8d9..54b1d9a94 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -24,7 +24,7 @@ cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% {%- if FEATURES is sameas true %} -xpack.security.enabled: false +xpack.security.enabled: true xpack.security.http.ssl.enabled: false xpack.security.transport.ssl.enabled: true xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key @@ -33,6 +33,11 @@ xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt +xpack.security.authc: + anonymous: + username: anonymous_user + roles: elasticsearch + authz_exception: true {%- endif %} node.attr.box_type: {{ NODE_ROUTE_TYPE }} node.name: {{ ESCLUSTERNAME }} From 42c9653669752b4af5f9dc7707c622a5712aea55 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 14:45:55 -0400 Subject: [PATCH 34/49] anon user hack --- salt/elasticsearch/files/elasticsearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 54b1d9a94..c1052035a 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -36,7 +36,7 @@ xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/c xpack.security.authc: anonymous: username: anonymous_user - roles: elasticsearch + roles: superuser authz_exception: true {%- endif %} node.attr.box_type: {{ NODE_ROUTE_TYPE }} From 5f30c947c95fc3df00559731cace19ffd1297652 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 15:12:23 -0400 Subject: [PATCH 35/49] SSL intraca --- salt/elasticsearch/init.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 28db606f1..6aa1257bf 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -200,9 +200,13 @@ so-elasticsearch: - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro {%- if FEATURES is sameas true %} + {%- if grains['role'] in ['so-node','so-heavynode'] %} + - /etc/ssl/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro + {%- else %} - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro + {%- endif %} {%- endif %} - watch: - file: cacertz From e8b61a3828a3f91e6f05bc91c00d59c80a79dc6a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 15:14:29 -0400 Subject: [PATCH 36/49] SSL intraca --- salt/elasticsearch/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 6aa1257bf..66bd0ec21 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -204,9 +204,9 @@ so-elasticsearch: - /etc/ssl/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro {%- else %} - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro + {%- endif %} - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro - {%- endif %} {%- endif %} - watch: - file: cacertz From a817465318bc39688997633988a0b919b9c70050 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 15:25:09 -0400 Subject: [PATCH 37/49] SSL intraca --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 6751c4b15..6d8674c92 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -501,7 +501,7 @@ fleetkeyperms: - ca_server: {{ ca_server }} - signing_policy: registry - public_key: /etc/pki/elasticsearch.key - - CN: {{ manager }} + - CN: {{ HOSTNAME }} - days_remaining: 0 - days_valid: 820 - backup: True From 5a0aae5fe7ce53f598d98c01c8c8b1f4ba0d1d6f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 15:34:07 -0400 Subject: [PATCH 38/49] SSL intraca --- salt/ssl/init.sls | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 6d8674c92..a0cade9f6 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -496,6 +496,18 @@ fleetkeyperms: {% if grains['role'] in ['so-node', 'so-heavynode'] %} # Create a cert for elasticsearch +/etc/pki/elasticsearch.key: + x509.private_key_managed: + - CN: {{ manager }} + - bits: 4096 + - days_remaining: 0 + - days_valid: 820 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} + - prereq: + - x509: /etc/pki/elasticsearch.crt + /etc/pki/elasticsearch.crt: x509.certificate_managed: - ca_server: {{ ca_server }} @@ -516,17 +528,5 @@ miniokeyperms: - name: /etc/pki/elasticsearch.key - mode: 640 - group: 930 - -/etc/pki/elasticsearch.key: - x509.private_key_managed: - - CN: {{ manager }} - - bits: 4096 - - days_remaining: 0 - - days_valid: 820 - - backup: True - - new: True - {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} - - prereq: - - x509: /etc/pki/elasticsearch.crt {%- endif %} {%- endif %} \ No newline at end of file From f8621333239bc70dace93928ea80241af9325153 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 15:37:55 -0400 Subject: [PATCH 39/49] SSL intraca --- salt/elasticsearch/init.sls | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 66bd0ec21..28db606f1 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -200,11 +200,7 @@ so-elasticsearch: - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro {%- if FEATURES is sameas true %} - {%- if grains['role'] in ['so-node','so-heavynode'] %} - - /etc/ssl/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro - {%- else %} - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro - {%- endif %} - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro {%- endif %} From 65d535d893f1ec4081c633e1811f7ca4c9532b05 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 15:45:17 -0400 Subject: [PATCH 40/49] SSL intraca --- salt/elasticsearch/files/elasticsearch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index c1052035a..1398e03a0 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -33,6 +33,7 @@ xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt +ssl.verification_mode: none xpack.security.authc: anonymous: username: anonymous_user From 0f7074a4997b792b248dd9747ac8a6b529b09aef Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 11 Aug 2020 15:49:04 -0400 Subject: [PATCH 41/49] SSL intraca --- salt/elasticsearch/files/elasticsearch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 1398e03a0..6f49c9584 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -33,7 +33,7 @@ xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt -ssl.verification_mode: none +xpack.security.http.ssl.client_authentication: none xpack.security.authc: anonymous: username: anonymous_user From 32083132e56587fae2f68f8acb7c8656040f8b0b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 Aug 2020 11:10:36 -0400 Subject: [PATCH 42/49] Back out some ES settings --- salt/elasticsearch/files/elasticsearch.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 6f49c9584..411f5bdf5 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -24,21 +24,22 @@ cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% {%- if FEATURES is sameas true %} -xpack.security.enabled: true +xpack.security.enabled: false xpack.security.http.ssl.enabled: false -xpack.security.transport.ssl.enabled: true +xpack.security.transport.ssl.enabled: false xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt -xpack.security.http.ssl.client_authentication: none -xpack.security.authc: - anonymous: - username: anonymous_user - roles: superuser - authz_exception: true +#xpack.security.transport.ssl.verification_mode: none +#xpack.security.http.ssl.client_authentication: none +#xpack.security.authc: +# anonymous: +# username: anonymous_user +# roles: superuser +# authz_exception: true {%- endif %} node.attr.box_type: {{ NODE_ROUTE_TYPE }} node.name: {{ ESCLUSTERNAME }} From 69e7285e302c2800ab90a837db78fb029004fd06 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 Aug 2020 12:44:55 -0400 Subject: [PATCH 43/49] Fix a bug where minio passwrods cause issues --- salt/elasticsearch/files/scripts/so-catrust | 2 +- salt/elasticsearch/files/sotls.yaml | 12 ++++++++++++ salt/elasticsearch/init.sls | 2 -- setup/so-functions | 4 ++-- 4 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 salt/elasticsearch/files/sotls.yaml diff --git a/salt/elasticsearch/files/scripts/so-catrust b/salt/elasticsearch/files/scripts/so-catrust index 02ea12726..aee83a379 100644 --- a/salt/elasticsearch/files/scripts/so-catrust +++ b/salt/elasticsearch/files/scripts/so-catrust @@ -20,7 +20,7 @@ . /usr/sbin/so-common # Check to see if we have extracted the ca cert. if [ ! -f /opt/so/saltstack/local/salt/common/cacerts ]; then - docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-elasticsearchca --user root --entrypoint keytool {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }} -keystore /etc/pki/ca-trust/extracted/java/cacerts -alias SOSCA -import -file /etc/pki/ca.crt -storepass changeit -noprompt + docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }} -keystore /etc/pki/ca-trust/extracted/java/cacerts -alias SOSCA -import -file /etc/pki/ca.crt -storepass changeit -noprompt docker cp so-elasticsearchca:/etc/pki/ca-trust/extracted/java/cacerts /opt/so/saltstack/local/salt/common/cacerts docker cp so-elasticsearchca:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /opt/so/saltstack/local/salt/common/tls-ca-bundle.pem docker rm so-elasticsearchca diff --git a/salt/elasticsearch/files/sotls.yaml b/salt/elasticsearch/files/sotls.yaml new file mode 100644 index 000000000..1b6353856 --- /dev/null +++ b/salt/elasticsearch/files/sotls.yaml @@ -0,0 +1,12 @@ +keystore.path: /etc/pki/ca-trust/extracted/java/sokeys +keystore.password: changeit +keystore.algorithm: SunX509 +truststore.path: /etc/pki/ca-trust/extracted/java/cacerts +truststore.password: changeit +truststore.algorithm: PKIX +protocols: +- TLSv1.2 +ciphers: +- TLS_RSA_WITH_AES_128_CBC_SHA256 +transport.encrypted: true +http.encrypted: false \ No newline at end of file diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 28db606f1..5bc9ddbb6 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -199,11 +199,9 @@ so-elasticsearch: - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro - {%- if FEATURES is sameas true %} - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro - {%- endif %} - watch: - file: cacertz diff --git a/setup/so-functions b/setup/so-functions index db8e3d6f1..e9574fa10 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1138,8 +1138,8 @@ minio_generate_keys() { local charSet="[:graph:]" - ACCESS_KEY=$(tr -cd "$charSet" < /dev/urandom | tr -d \' | tr -d \" | head -c 20) - ACCESS_SECRET=$(tr -cd "$charSet" < /dev/urandom | tr -d \' | tr -d \" | head -c 40) + ACCESS_KEY=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1) + ACCESS_SECRET=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 40 | head -n 1) } From 683799d07734e13f3949534957acb09373fa9d19 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 Aug 2020 15:02:54 -0400 Subject: [PATCH 44/49] Convert ES cert to p12 --- salt/ssl/init.sls | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index a0cade9f6..9e0c1d9e8 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -243,7 +243,11 @@ miniokeyperms: # https://github.com/saltstack/salt/issues/52167 # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - + cmd.run: + - name: "/usr/bin/openssl pkcs12 -in /etc/pki/elasticsearch.key -topk12 -out /etc/pki/elasticsearch.p12 -nocrypt" + - onchanges: + - x509: /etc/pki/elasticsearch.key + ealstickeyperms: file.managed: - replace: False @@ -507,7 +511,7 @@ fleetkeyperms: {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%} - prereq: - x509: /etc/pki/elasticsearch.crt - + /etc/pki/elasticsearch.crt: x509.certificate_managed: - ca_server: {{ ca_server }} @@ -521,6 +525,10 @@ fleetkeyperms: # https://github.com/saltstack/salt/issues/52167 # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' + cmd.run: + - name: "/usr/bin/openssl pkcs12 -in /etc/pki/elasticsearch.key -topk12 -out /etc/pki/elasticsearch.p12 -nocrypt" + - onchanges: + - x509: /etc/pki/elasticsearch.key miniokeyperms: file.managed: From daaffd518562f1a85bad7366c76cae79c49371ed Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 Aug 2020 15:05:33 -0400 Subject: [PATCH 45/49] Convert ES cert to p12 --- salt/ssl/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 9e0c1d9e8..2cb435ffc 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -244,10 +244,10 @@ miniokeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' cmd.run: - - name: "/usr/bin/openssl pkcs12 -in /etc/pki/elasticsearch.key -topk12 -out /etc/pki/elasticsearch.p12 -nocrypt" + - name: "/usr/bin/openssl pkcs12 -in /etc/pki/elasticsearch.key -export -out /etc/pki/elasticsearch.p12 -nocrypt" - onchanges: - x509: /etc/pki/elasticsearch.key - + ealstickeyperms: file.managed: - replace: False @@ -526,7 +526,7 @@ fleetkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' cmd.run: - - name: "/usr/bin/openssl pkcs12 -in /etc/pki/elasticsearch.key -topk12 -out /etc/pki/elasticsearch.p12 -nocrypt" + - name: "/usr/bin/openssl pkcs12 -in /etc/pki/elasticsearch.key -export -out /etc/pki/elasticsearch.p12 -nocrypt" - onchanges: - x509: /etc/pki/elasticsearch.key From 82821fbb256056843ab5d827e8683c13bc954231 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 Aug 2020 15:09:52 -0400 Subject: [PATCH 46/49] Convert ES cert to p12 --- salt/ssl/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 2cb435ffc..3dd509861 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -244,7 +244,7 @@ miniokeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' cmd.run: - - name: "/usr/bin/openssl pkcs12 -in /etc/pki/elasticsearch.key -export -out /etc/pki/elasticsearch.p12 -nocrypt" + - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nocrypt" - onchanges: - x509: /etc/pki/elasticsearch.key @@ -526,7 +526,7 @@ fleetkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' cmd.run: - - name: "/usr/bin/openssl pkcs12 -in /etc/pki/elasticsearch.key -export -out /etc/pki/elasticsearch.p12 -nocrypt" + - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12" - onchanges: - x509: /etc/pki/elasticsearch.key From 7e3e4d0f54d41725b294385a5535ea0049cf6a43 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 Aug 2020 15:16:12 -0400 Subject: [PATCH 47/49] Convert ES cert to p12 --- salt/ssl/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 3dd509861..a5cae35b8 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -244,7 +244,7 @@ miniokeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' cmd.run: - - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nocrypt" + - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: - x509: /etc/pki/elasticsearch.key @@ -526,7 +526,7 @@ fleetkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' cmd.run: - - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12" + - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: - x509: /etc/pki/elasticsearch.key From 9980d0284473eee7bc8d51c74c8f0fae791e6785 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 12 Aug 2020 15:38:19 -0400 Subject: [PATCH 48/49] Elastic Transport TLSgit add . --- salt/elasticsearch/files/sotls.yaml | 2 +- salt/elasticsearch/init.sls | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/files/sotls.yaml b/salt/elasticsearch/files/sotls.yaml index 1b6353856..6fee1e8e2 100644 --- a/salt/elasticsearch/files/sotls.yaml +++ b/salt/elasticsearch/files/sotls.yaml @@ -1,4 +1,4 @@ -keystore.path: /etc/pki/ca-trust/extracted/java/sokeys +keystore.path: /usr/share/elasticsearch/config/sokeys keystore.password: changeit keystore.algorithm: SunX509 truststore.path: /etc/pki/ca-trust/extracted/java/cacerts diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 5bc9ddbb6..7cb887b05 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -139,6 +139,13 @@ esyml: - group: 939 - template: jinja +sotls: + file.managed: + - name: /opt/so/conf/elasticsearch/sotls.yml + - source: salt://elasticsearch/files/sotls.yml + - user: 930 + - group: 939 + #sync templates to /opt/so/conf/elasticsearch/templates {% for TEMPLATE in TEMPLATES %} es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: @@ -200,8 +207,9 @@ so-elasticsearch: - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro - - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro - - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro + - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro + - /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro + - watch: - file: cacertz From 5640faef13bfcda794ea27b565683556a06f3349 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 12 Aug 2020 16:34:59 -0400 Subject: [PATCH 49/49] Kernel consoleblank is causing whiptail progress screen to appear to hang #1084 --- setup/so-setup | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 1e49b325f..3924e4a8e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -118,7 +118,22 @@ if [ "$OS" == ubuntu ]; then update-alternatives --set newt-palette /etc/newt/palette.original >> $setup_log 2>&1 fi -setterm -blank 0 >> $setup_log 2>&1 +# Kernel consoleblank is causing whiptail progress screen to appear to hang #1084 +# https://github.com/Security-Onion-Solutions/securityonion/issues/1084 +if [ "$automated" == no ]; then + TTY=$(tty) + echo "Setup is running on TTY $TTY" >> $setup_log 2>&1 + if echo $TTY | grep -q "/dev/tty"; then + CONSOLEBLANK=$(cat /sys/module/kernel/parameters/consoleblank) + echo "Kernel consoleblank value before: $CONSOLEBLANK" >> $setup_log 2>&1 + if [ $CONSOLEBLANK -gt 0 ]; then + echo "Running 'setterm -blank 0' for TTY $TTY" >> $setup_log 2>&1 + TERM=linux setterm -blank 0 >$TTY <$TTY + CONSOLEBLANK=$(cat /sys/module/kernel/parameters/consoleblank) + echo "Kernel consoleblank value after: $CONSOLEBLANK" >> $setup_log 2>&1 + fi + fi +fi if [ "$setup_type" == 'iso' ] || (whiptail_you_sure); then true