From 5ed3dbff39da067376789fa2b84652394efe2ab3 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 17 Mar 2020 15:15:01 +0000 Subject: [PATCH 1/6] Fix template name --- salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9002_output_import.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja | 2 +- .../pipelines/config/so/9030_output_greensql.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja | 2 +- salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja | 2 +- .../logstash/pipelines/config/so/9100_output_osquery.conf.jinja | 2 +- .../pipelines/config/so/9200_output_firewall.conf.jinja | 2 +- .../logstash/pipelines/config/so/9300_output_windows.conf.jinja | 2 +- .../pipelines/config/so/9301_output_dns_windows.conf.jinja | 2 +- .../pipelines/config/so/9400_output_suricata.conf.jinja | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja b/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja index 0fc30c4b0..824f29e34 100644 --- a/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja +++ b/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja @@ -21,7 +21,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "so-switch-%{+YYYY.MM.dd}" - template => "/so-template.json" + template => "/so-common-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja index 2b7db9370..b32794f5b 100644 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja @@ -20,7 +20,7 @@ output { hosts => "{{ ES }}" index => "so-import-%{+YYYY.MM.dd}" template_name => "logstash" - template => "/so-template.json" + template => "/so-common-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja index 2fd427129..f830ffeb3 100644 --- a/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja +++ b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja @@ -21,7 +21,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "so-flow-%{+YYYY.MM.dd}" - template => "/so-template.json" + template => "/so-common-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja b/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja index f7f3d8060..0772841a8 100644 --- a/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja +++ b/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja @@ -20,7 +20,7 @@ output { #stdout { codec => rubydebug } elasticsearch { hosts => "{{ ES }}" - template => "/so-template.json" + template => "/so-common-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja b/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja index 7de501bf8..fc664564c 100644 --- a/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja +++ b/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja @@ -19,7 +19,7 @@ output { if [event_type] == "esxi" and "test_data" not in [tags] { elasticsearch { hosts => "{{ ES }}" - template => "/so-template.json" + template => "/so-common-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja b/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja index 544e62856..1007d5f58 100644 --- a/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja +++ b/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja @@ -19,7 +19,7 @@ output { if [event_type] == "greensql" and "test_data" not in [tags] { elasticsearch { hosts => "{{ ES }}" - template => "/so-template.json" + template => "/so-common-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja b/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja index 7de10b974..f17616060 100644 --- a/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja @@ -20,7 +20,7 @@ output { #stdout { codec => rubydebug } elasticsearch { hosts => "{{ ES }}" - template => "/so-template.json" + template => "/so-common-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja b/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja index bb3ec0714..9a6668619 100644 --- a/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja +++ b/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja @@ -20,7 +20,7 @@ output { #stdout { codec => rubydebug } elasticsearch { hosts => "{{ ES }}" - template => "/so-template.json" + template => "/so-common-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja index dc9c5f7e1..4ea9cfe12 100644 --- a/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja +++ b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja @@ -22,7 +22,7 @@ output { hosts => "{{ ES }}" index => "so-ids-%{+YYYY.MM.dd}" template_name => "logstash" - template => "/so-template.json" + template => "/so-common-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja index 33b841c08..35d3cf7dc 100644 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja @@ -21,7 +21,7 @@ output { hosts => "{{ ES }}" index => "so-syslog-%{+YYYY.MM.dd}" template_name => "logstash" - template => "/so-template.json" + template => "/so-common-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja index 63fd3c25b..ca9c90215 100644 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja @@ -13,7 +13,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "so-osquery-%{+YYYY.MM.dd}" - template => "/so-template.json" + template => "/so-common-template.json" } } } \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja index 17e774976..2c9796b5f 100644 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja @@ -22,7 +22,7 @@ output { hosts => "{{ ES }}" index => "so-firewall-%{+YYYY.MM.dd}" template_name => "logstash" - template => "/so-template.json" + template => "/so-common-template.json" template_overwrite => true } } diff --git a/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja b/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja index 9779d01a5..8b8a9299f 100644 --- a/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja +++ b/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja @@ -21,7 +21,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "so-windows-%{+YYYY.MM.dd}" - template => "/so-template.json" + template => "/so-common-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja b/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja index dc6bbbda4..fec703b1c 100644 --- a/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja +++ b/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja @@ -21,7 +21,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "so-%{+YYYY.MM.dd}" - template => "/so-template.json" + template => "/so-common-template.json" } } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index a85fba758..7b587242a 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -21,7 +21,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "so-ids-%{+YYYY.MM.dd}" - template => "/so-template.json" + template => "/so-common-template.json" } } } From faea67c9cf45d2949331dcae36626fca6b3a7200 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 17 Mar 2020 15:17:13 +0000 Subject: [PATCH 2/6] update env vars --- salt/elasticsearch/init.sls | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 07d75abfb..575d8162c 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -114,11 +114,12 @@ so-elasticsearch: - name: so-elasticsearch - user: elasticsearch - environment: - - bootstrap.memory_lock=true - - cluster.name={{ esclustername }} + - discovery.type=single-node + #- bootstrap.memory_lock=true + #- cluster.name={{ esclustername }} - ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} - - http.host=0.0.0.0 - - transport.host=127.0.0.1 + #- http.host=0.0.0.0 + #- transport.host=127.0.0.1 - ulimits: - memlock=-1:-1 - nofile=65536:65536 From 8bea56eccfb7ae34a43c812805f7dd4f01465534 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 17 Mar 2020 15:17:52 +0000 Subject: [PATCH 3/6] change prospectors to inputs --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index e350b5798..5affc2411 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -71,7 +71,7 @@ filebeat.modules: #=========================== Filebeat prospectors ============================= # List of prospectors to fetch data. -filebeat.prospectors: +filebeat.inputs: #------------------------------ Log prospector -------------------------------- {%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" %} {%- if BROVER != 'SURICATA' %} From 488858f8bcea7be39323b9f12377aaa40c313bfb Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 17 Mar 2020 15:19:08 +0000 Subject: [PATCH 4/6] remove beat field removal --- salt/elasticsearch/files/ingest/common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index f9d6199ba..de83ef4cf 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -39,7 +39,7 @@ { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_missing": true } }, { "remove": { - "field": [ "index_name_prefix", "beat"], + "field": [ "index_name_prefix"], "ignore_failure": false } } From b80e7fedcbe9d9219506eead477c4581639c5afb Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 17 Mar 2020 15:20:31 +0000 Subject: [PATCH 5/6] remove agent field for non-Wazuh logs --- salt/elasticsearch/files/ingest/ossec.alert | 1 + salt/elasticsearch/files/ingest/suricata.common | 2 +- salt/elasticsearch/files/ingest/zeek.common | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/ossec.alert b/salt/elasticsearch/files/ingest/ossec.alert index 39362c4ed..23d374fdc 100644 --- a/salt/elasticsearch/files/ingest/ossec.alert +++ b/salt/elasticsearch/files/ingest/ossec.alert @@ -2,6 +2,7 @@ "description" : "ossec", "processors" : [ { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "remove": { "field": [ "agent" ], "ignore_missing": true, "ignore_failure": false } }, { "rename": { "field": "message2.agent", "target_field": "agent", "ignore_missing": true } }, { "rename": { "field": "message2.data", "target_field": "data", "ignore_missing": true } }, { "rename": { "field": "message2.decoder", "target_field": "decoder", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 79876d366..52d9372a2 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -8,7 +8,7 @@ { "rename":{ "field": "message2.src_port", "target_field": "source.port", "ignore_failure": true } }, { "rename":{ "field": "message2.dest_ip", "target_field": "destination.ip", "ignore_failure": true } }, { "rename":{ "field": "message2.dest_port", "target_field": "destination.port", "ignore_failure": true } }, - { "remove": { "field": ["message2"], "ignore_failure": true } }, + { "remove": { "field": ["message2", "agent"], "ignore_failure": true } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common index 85902ffa5..4c062c1c3 100644 --- a/salt/elasticsearch/files/ingest/zeek.common +++ b/salt/elasticsearch/files/ingest/zeek.common @@ -16,7 +16,7 @@ { "rename": { "field": "message2.id.resp_p", "target_field": "destination.port", "ignore_missing": true } }, { "set": { "field": "server.port", "value": "{{destination.port}}" } }, { "date": { "field": "message2.ts", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "ignore_failure": true } }, - { "remove": { "field": ["message2.ts", "path"], "ignore_failure": true } }, + { "remove": { "field": ["message2.ts", "path", "agent"], "ignore_failure": true } }, { "pipeline": { "name": "common" } } ] } From a5ff21c5284ca7263ad579cc6c7bbd12d7ac8b01 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 17 Mar 2020 15:20:46 +0000 Subject: [PATCH 6/6] remove agent field for non-Wazuh logs --- salt/elasticsearch/files/ingest/strelka | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/strelka b/salt/elasticsearch/files/ingest/strelka index 8652fb912..39783c2ce 100644 --- a/salt/elasticsearch/files/ingest/strelka +++ b/salt/elasticsearch/files/ingest/strelka @@ -6,7 +6,7 @@ { "rename": { "field": "message2.scan", "target_field": "scan", "ignore_missing": true } }, { "rename": { "field": "message2.request", "target_field": "request", "ignore_missing": true } }, { "rename": { "field": "scan.hash", "target_field": "file.hash", "ignore_missing": true } }, - { "remove": { "field": ["host", "path"], "ignore_missing": true } }, + { "remove": { "field": ["host", "path", "agent"], "ignore_missing": true } }, { "pipeline": { "name": "common" } } ] }