diff --git a/pillar/elasticsearch/search.sls b/pillar/elasticsearch/search.sls new file mode 100644 index 000000000..9ff97de5b --- /dev/null +++ b/pillar/elasticsearch/search.sls @@ -0,0 +1,13 @@ +elasticsearch: + templates: + - so/so-beats-template.json.jinja + - so/so-common-template.json + - so/so-firewall-template.json.jinja + - so/so-flow-template.json.jinja + - so/so-ids-template.json.jinja + - so/so-import-template.json.jinja + - so/so-osquery-template.json.jinja + - so/so-ossec-template.json.jinja + - so/so-strelka-template.json.jinja + - so/so-syslog-template.json.jinja + - so/so-zeek-template.json.jinja diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls deleted file mode 100644 index fcdd13bb7..000000000 --- a/pillar/logstash/eval.sls +++ /dev/null @@ -1,29 +0,0 @@ -logstash: - pipelines: - eval: - config: - - so/0800_input_eval.conf - - so/1002_preprocess_json.conf - - so/1033_preprocess_snort.conf - - so/7100_osquery_wel.conf - - so/8999_postprocess_rename_type.conf - - so/9000_output_bro.conf.jinja - - so/9002_output_import.conf.jinja - - so/9033_output_snort.conf.jinja - - so/9100_output_osquery.conf.jinja - - so/9400_output_suricata.conf.jinja - - so/9500_output_beats.conf.jinja - - so/9600_output_ossec.conf.jinja - - so/9700_output_strelka.conf.jinja - templates: - - so/so-beats-template.json.jinja - - so/so-common-template.json - - so/so-firewall-template.json.jinja - - so/so-flow-template.json.jinja - - so/so-ids-template.json.jinja - - so/so-import-template.json.jinja - - so/so-osquery-template.json.jinja - - so/so-ossec-template.json.jinja - - so/so-strelka-template.json.jinja - - so/so-syslog-template.json.jinja - - so/so-zeek-template.json.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 9c069fd20..486deb408 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -11,15 +11,3 @@ logstash: - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja - templates: - - so/so-beats-template.json.jinja - - so/so-common-template.json - - so/so-firewall-template.json.jinja - - so/so-flow-template.json.jinja - - so/so-ids-template.json.jinja - - so/so-import-template.json.jinja - - so/so-osquery-template.json.jinja - - so/so-ossec-template.json.jinja - - so/so-strelka-template.json.jinja - - so/so-syslog-template.json.jinja - - so/so-zeek-template.json.jinja diff --git a/pillar/top.sls b/pillar/top.sls index 6eba800a9..e3ae34f28 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -11,6 +11,7 @@ base: - logstash - logstash.manager - logstash.search + - elasticsearch.search '*_sensor': - static @@ -41,6 +42,7 @@ base: - logstash - logstash.manager - logstash.search + - elasticsearch.search - data.* - brologs - secrets @@ -75,4 +77,5 @@ base: - static - logstash - logstash.search + - elasticsearch.search - minions.{{ grains.id }} diff --git a/salt/common/tools/sbin/so-elasticsearch-templates b/salt/common/tools/sbin/so-elasticsearch-templates index 6b3e19d30..dfbf07c42 100755 --- a/salt/common/tools/sbin/so-elasticsearch-templates +++ b/salt/common/tools/sbin/so-elasticsearch-templates @@ -15,13 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -default_salt_dir=/opt/so/saltstack/default +default_conf_dir=/opt/so/conf ELASTICSEARCH_HOST="{{ MANAGERIP}}" ELASTICSEARCH_PORT=9200 #ELASTICSEARCH_AUTH="" # Define a default directory to load pipelines from -ELASTICSEARCH_TEMPLATES="$default_salt_dir/salt/logstash/pipelines/templates/so/" +ELASTICSEARCH_TEMPLATES="$default_conf_dir/elasticsearch/templates/" # Wait for ElasticSearch to initialize echo -n "Waiting for ElasticSearch..." diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index 78fa5a10e..d9d6fc0f0 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -19,7 +19,7 @@ } } }, - { "set": { "field": "observer.name", "value": "{{agent.name}}", + { "set": { "field": "observer.name", "value": "{{agent.name}}" }}, { "remove": { "field": ["host", "path", "message", "scan.exiftool.keys"], "ignore_missing": true } }, { "pipeline": { "name": "common" } } ] diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 7a791c0d2..7e09ed6c1 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -31,6 +31,8 @@ {% set esheap = salt['pillar.get']('elasticsearch:esheap', '') %} {% endif %} +{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} + vm.max_map_count: sysctl.present: - value: 262144 @@ -63,6 +65,13 @@ esingestdir: - group: 939 - makedirs: True +estemplatedir: + file.directory: + - name: /opt/so/conf/elasticsearch/templates + - user: 930 + - group: 939 + - makedirs: True + esingestconf: file.recurse: - name: /opt/so/conf/elasticsearch/ingest @@ -86,6 +95,21 @@ esyml: - group: 939 - template: jinja +#sync templates to /opt/so/conf/elasticsearch/templates +{% for TEMPLATE in TEMPLATES %} +es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: + file.managed: + - source: salt://elasticsearch/templates/{{TEMPLATE}} + {% if 'jinja' in TEMPLATE.split('.')[-1] %} + - name: /opt/so/conf/elasticsearch/templates/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} + - template: jinja + {% else %} + - name: /opt/so/conf/elasticsearch/templates/{{TEMPLATE.split('/')[1]}} + {% endif %} + - user: 930 + - group: 939 +{% endfor %} + nsmesdir: file.directory: - name: /nsm/elasticsearch diff --git a/salt/logstash/pipelines/templates/custom/place_custom_template_in_local b/salt/elasticsearch/templates/custom/place_custom_template_in_local similarity index 100% rename from salt/logstash/pipelines/templates/custom/place_custom_template_in_local rename to salt/elasticsearch/templates/custom/place_custom_template_in_local diff --git a/salt/logstash/pipelines/templates/so/so-beats-template.json.jinja b/salt/elasticsearch/templates/so/so-beats-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-beats-template.json.jinja rename to salt/elasticsearch/templates/so/so-beats-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json similarity index 100% rename from salt/logstash/pipelines/templates/so/so-common-template.json rename to salt/elasticsearch/templates/so/so-common-template.json diff --git a/salt/logstash/pipelines/templates/so/so-firewall-template.json.jinja b/salt/elasticsearch/templates/so/so-firewall-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-firewall-template.json.jinja rename to salt/elasticsearch/templates/so/so-firewall-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-flow-template.json.jinja b/salt/elasticsearch/templates/so/so-flow-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-flow-template.json.jinja rename to salt/elasticsearch/templates/so/so-flow-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-ids-template.json.jinja b/salt/elasticsearch/templates/so/so-ids-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-ids-template.json.jinja rename to salt/elasticsearch/templates/so/so-ids-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-import-template.json.jinja b/salt/elasticsearch/templates/so/so-import-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-import-template.json.jinja rename to salt/elasticsearch/templates/so/so-import-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-osquery-template.json.jinja b/salt/elasticsearch/templates/so/so-osquery-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-osquery-template.json.jinja rename to salt/elasticsearch/templates/so/so-osquery-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-ossec-template.json.jinja b/salt/elasticsearch/templates/so/so-ossec-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-ossec-template.json.jinja rename to salt/elasticsearch/templates/so/so-ossec-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-strelka-template.json.jinja b/salt/elasticsearch/templates/so/so-strelka-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-strelka-template.json.jinja rename to salt/elasticsearch/templates/so/so-strelka-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-syslog-template.json.jinja b/salt/elasticsearch/templates/so/so-syslog-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-syslog-template.json.jinja rename to salt/elasticsearch/templates/so/so-syslog-template.json.jinja diff --git a/salt/logstash/pipelines/templates/so/so-zeek-template.json.jinja b/salt/elasticsearch/templates/so/so-zeek-template.json.jinja similarity index 100% rename from salt/logstash/pipelines/templates/so/so-zeek-template.json.jinja rename to salt/elasticsearch/templates/so/so-zeek-template.json.jinja diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 87f348744..dbf345822 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -36,7 +36,6 @@ {% endif %} {% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %} -{% set TEMPLATES = salt['pillar.get']('logstash:templates', {}) %} {% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} # Create the logstash group @@ -94,21 +93,6 @@ ls_pipeline_{{PL}}: {% endfor %} -#sync templates to /opt/so/conf/logstash/etc -{% for TEMPLATE in TEMPLATES %} -ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: - file.managed: - - source: salt://logstash/pipelines/templates/{{TEMPLATE}} - {% if 'jinja' in TEMPLATE.split('.')[-1] %} - - name: /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} - - template: jinja - {% else %} - - name: /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1]}} - {% endif %} - - user: 931 - - group: 939 -{% endfor %} - lspipelinesyml: file.managed: - name: /opt/so/conf/logstash/etc/pipelines.yml @@ -126,12 +110,6 @@ lsetcsync: - group: 939 - template: jinja - clean: True -{% if TEMPLATES %} - - require: - {% for TEMPLATE in TEMPLATES %} - - file: ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }} - {% endfor %} -{% endif %} - exclude_pat: pipelines* # Create the import directory @@ -171,13 +149,7 @@ so-logstash: - {{ BINDING }} {% endfor %} - binds: -{% for TEMPLATE in TEMPLATES %} - {% if 'jinja' in TEMPLATE.split('.')[-1] %} - - /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}:/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}:ro - {% else %} - - /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1]}}:/{{TEMPLATE.split('/')[1]}}:ro - {% endif %} -{% endfor %} + - /opt/so/conf/elasticsearch/templates/:/templates/:ro - /opt/so/conf/logstash/etc/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro - /opt/so/conf/logstash/etc/logstash.yml:/usr/share/logstash/config/logstash.yml:ro - /opt/so/conf/logstash/etc/pipelines.yml:/usr/share/logstash/config/pipelines.yml @@ -206,7 +178,4 @@ so-logstash: - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} {% endfor %} -{% for TEMPLATE in TEMPLATES %} - - file: ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }} -{% endfor %} -# - file: /opt/so/conf/logstash/rulesets + - file: /opt/so/conf/elasticsearch/templates/* \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index 54a30f272..f86bf946c 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -10,7 +10,7 @@ output { hosts => "{{ ES }}" index => "so-zeek-%{+YYYY.MM.dd}" template_name => "so-zeek" - template => "/so-zeek-template.json" + template => "/templates/so-zeek-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 563e5984e..52c9f034a 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -10,7 +10,7 @@ output { hosts => "{{ ES }}" index => "so-import-%{+YYYY.MM.dd}" template_name => "so-import" - template => "/so-import-template.json" + template => "/templates/so-import-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 007713811..740676367 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -9,7 +9,7 @@ output { hosts => "{{ ES }}" index => "so-flow-%{+YYYY.MM.dd}" template_name => "so-flow" - template => "/so-flow-template.json" + template => "/templates/so-flow-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index 065653f01..fed1ffdf5 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -9,7 +9,7 @@ output { hosts => "{{ ES }}" index => "so-ids-%{+YYYY.MM.dd}" template_name => "so-ids" - template => "/so-ids-template.json" + template => "/templates/so-ids-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index cd7e44d74..5087f41da 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -10,7 +10,7 @@ output { hosts => "{{ ES }}" index => "so-syslog-%{+YYYY.MM.dd}" template_name => "so-syslog" - template => "/so-syslog-template.json" + template => "/templates/so-syslog-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 3b99a7afa..01436cf5f 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -10,7 +10,7 @@ output { hosts => "{{ ES }}" index => "so-osquery-%{+YYYY.MM.dd}" template_name => "so-osquery" - template => "/so-osquery-template.json" + template => "/templates/so-osquery-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 9407fe79e..a295b5f7a 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -9,7 +9,7 @@ output { hosts => "{{ ES }}" index => "so-firewall-%{+YYYY.MM.dd}" template_name => "so-firewall" - template => "/so-firewall-template.json" + template => "/templates/so-firewall-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index d3026aa20..ace7cccf1 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -10,7 +10,7 @@ output { hosts => "{{ ES }}" index => "so-ids-%{+YYYY.MM.dd}" template_name => "so-ids" - template => "/so-ids-template.json" + template => "/templates/so-ids-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja index 6874e5e76..ed513f597 100644 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja @@ -10,7 +10,7 @@ output { hosts => "{{ ES }}" index => "so-beats-%{+YYYY.MM.dd}" template_name => "so-beats" - template => "/so-beats-template.json" + template => "/templates/so-beats-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja index 77610d9e0..14a9bc1d1 100644 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja @@ -10,7 +10,7 @@ output { hosts => "{{ ES }}" index => "so-ossec-%{+YYYY.MM.dd}" template_name => "so-ossec" - template => "/so-ossec-template.json" + template => "/templates/so-ossec-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja index b92e2a3d9..9fd074f3f 100644 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja @@ -10,7 +10,7 @@ output { hosts => "{{ ES }}" index => "so-strelka-%{+YYYY.MM.dd}" template_name => "so-strelka" - template => "/so-strelka-template.json" + template => "/templates/so-strelka-template.json" template_overwrite => true } }