From 80525ee73688b1e2f4ce414e04493798bdf6ace3 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 8 Jul 2021 12:29:50 -0400 Subject: [PATCH 01/57] [wip] Add logscan pipeline --- pillar/logstash/search.sls | 1 + salt/elasticsearch/files/ingest/logscan | 21 +++++++++++++++ salt/filebeat/etc/filebeat.yml | 13 +++++++++ .../config/so/9800_output_logscan.conf.jinja | 27 +++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/logscan create mode 100644 salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 10fab2ed1..2f287f533 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -13,3 +13,4 @@ logstash: - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja + - so/9800_ouput_logscan.conf.jinja diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan new file mode 100644 index 000000000..936d316f7 --- /dev/null +++ b/salt/elasticsearch/files/ingest/logscan @@ -0,0 +1,21 @@ +{ + "description": "logscan", + "processors": [ + { "set": { "target_field": "event.severity", "value": 2 } }, + { "rename": { "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, + { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "rename": { "field": "source_ip", "target_field": "source.ip" } }, + { "set": { "if": "model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, + { "set": { "if": "model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, + { "set": { "if": "model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, + { "set": { "if": "model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, + { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, + { "script": { "lang": "painless", "source": "logscan.attempts.succeeded.amount = logscan.attempts.total.amount - logscan.attempts.failed.amount" , "ignore_failure": true} }, + { "rename": { "field": "avg_failure_interval", "target_field": "logscan.attempts.failed.avg_interval", "ignore_missing": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 2a86b486f..751186119 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -111,6 +111,19 @@ filebeat.inputs: fields: ["source", "prospector", "input", "offset", "beat"] fields_under_root: true +- type: log + paths: + - /opt/so/log/logscan/alerts.log + fields: + module: logscan + dataset: alert + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + clean_removed: true + close_removed: false + {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} {%- if ZEEKVER != 'SURICATA' %} {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja new file mode 100644 index 000000000..6fcf05a70 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -0,0 +1,27 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} + +output { + if [module] =~ logscan { + elasticsearch { + id => "logscan_pipeline" + pipeline => "%{module}" + hosts => "{{ ES }}" + {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + {% endif %} + index => "so-%{[event][module]}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} From bac7ef71d826b3c786b239e2a1c289f114105071 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Jul 2021 10:55:11 -0400 Subject: [PATCH 02/57] Add logscan.source.ips field --- salt/elasticsearch/files/ingest/logscan | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index 936d316f7..68c6aae44 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -8,6 +8,8 @@ { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, { "rename": { "field": "source_ip", "target_field": "source.ip" } }, + { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}" } }, + { "rename": { "field": "source_ips", "target_field": "logscan.source.ips" } }, { "set": { "if": "model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, { "set": { "if": "model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, { "set": { "if": "model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, From e059c25ebce24e8f9bf5abe8052d02dd39ecb4ea Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 13 Jul 2021 11:05:05 -0400 Subject: [PATCH 03/57] [fix][wip] Fix pipeline parsing errors --- salt/elasticsearch/files/ingest/logscan | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index 68c6aae44..6cd32dcbc 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -1,22 +1,23 @@ { "description": "logscan", "processors": [ - { "set": { "target_field": "event.severity", "value": 2 } }, + { "set": { "field": "event.severity", "value": 2 } }, { "rename": { "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, - { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, - { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, - { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, - { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, { "rename": { "field": "source_ip", "target_field": "source.ip" } }, { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}" } }, { "rename": { "field": "source_ips", "target_field": "logscan.source.ips" } }, - { "set": { "if": "model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, - { "set": { "if": "model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, - { "set": { "if": "model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, - { "set": { "if": "model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "set": { "if": "ctx.model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, + { "set": { "if": "ctx.model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "rename": { "field": "model", "target_field": "logscan.model" } } { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, - { "script": { "lang": "painless", "source": "logscan.attempts.succeeded.amount = logscan.attempts.total.amount - logscan.attempts.failed.amount" , "ignore_failure": true} }, + { "script": { "lang": "painless", "source": "ctx.logscan.attempts.succeeded.amount = ctx.logscan.attempts.total.amount - ctx.logscan.attempts.failed.amount" , "ignore_failure": true} }, { "rename": { "field": "avg_failure_interval", "target_field": "logscan.attempts.failed.avg_interval", "ignore_missing": true } }, { "pipeline": { "name": "common" } } ] From 115e0a6fee52cf9a789209d0668b97050f2e72bf Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 13 Jul 2021 12:04:10 -0400 Subject: [PATCH 04/57] [fix] Add missing comma --- salt/elasticsearch/files/ingest/logscan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index 6cd32dcbc..d199161a5 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -14,7 +14,7 @@ { "set": { "if": "ctx.model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, { "set": { "if": "ctx.model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, { "set": { "if": "ctx.model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, - { "rename": { "field": "model", "target_field": "logscan.model" } } + { "rename": { "field": "model", "target_field": "logscan.model" } }, { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.logscan.attempts.succeeded.amount = ctx.logscan.attempts.total.amount - ctx.logscan.attempts.failed.amount" , "ignore_failure": true} }, From e7a6172d7e439d2d2fe4adfe951e8b84e2d23720 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 13 Jul 2021 14:07:27 -0400 Subject: [PATCH 05/57] [fix] Add single quotes to strings --- salt/elasticsearch/files/ingest/logscan | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index d199161a5..9a094f4c6 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -10,10 +10,10 @@ { "rename": { "field": "source_ip", "target_field": "source.ip" } }, { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}" } }, { "rename": { "field": "source_ips", "target_field": "logscan.source.ips" } }, - { "set": { "if": "ctx.model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, - { "set": { "if": "ctx.model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, - { "set": { "if": "ctx.model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, - { "set": { "if": "ctx.model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "set": { "if": "ctx.model == 'kff'", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'kff'", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, + { "set": { "if": "ctx.model == 'kl'", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'kl'", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, { "rename": { "field": "model", "target_field": "logscan.model" } }, { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, From e41811fbd01824999ff5012e13ec2a9dc8dba1b0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 13 Jul 2021 15:14:13 -0400 Subject: [PATCH 06/57] [fix] Typo --- pillar/logstash/search.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 2f287f533..55b2070ce 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -13,4 +13,4 @@ logstash: - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja - - so/9800_ouput_logscan.conf.jinja + - so/9800_output_logscan.conf.jinja From 818f912a905a8d15cfab13c9dfe22800fe7237cf Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 14 Jul 2021 10:13:14 -0400 Subject: [PATCH 07/57] [fix] Remove indent --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 39257c991..68fc32d7c 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -119,7 +119,7 @@ filebeat.inputs: module: logscan dataset: alert processors: - - drop_fields: + - drop_fields: fields: ["source", "prospector", "input", "offset", "beat"] fields_under_root: true clean_removed: true From e8ba4bdc6cb37fbaeeeef75d5407230d782bd045 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 16 Jul 2021 14:07:23 -0400 Subject: [PATCH 08/57] Add quotes to string --- .../logstash/pipelines/config/so/9800_output_logscan.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja index 6fcf05a70..a0e5bd570 100644 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -7,7 +7,7 @@ {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { - if [module] =~ logscan { + if [module] =~ "logscan" { elasticsearch { id => "logscan_pipeline" pipeline => "%{module}" From 9bf1d3e0c6321c243e44d6b883e41060b6b500dd Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 16 Jul 2021 14:59:44 -0400 Subject: [PATCH 09/57] Misc fixes --- salt/elasticsearch/files/ingest/logscan | 9 ++++++--- salt/filebeat/etc/filebeat.yml | 2 +- .../pipelines/config/so/9800_output_logscan.conf.jinja | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index 9a094f4c6..3a3debabc 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -2,14 +2,17 @@ "description": "logscan", "processors": [ { "set": { "field": "event.severity", "value": 2 } }, + { "json": { "field": "message", "add_to_root": true, "ignore_failure": true } }, { "rename": { "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, - { "rename": { "field": "source_ip", "target_field": "source.ip" } }, - { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}" } }, - { "rename": { "field": "source_ips", "target_field": "logscan.source.ips" } }, + { "remove": { "field": "start_time", "ignore_missing": true } }, + { "remove": { "field": "end_time", "ignore_missing": true } }, + { "rename": { "field": "source_ip", "target_field": "source.ip", "ignore_missing": true } }, + { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, + { "rename": { "field": "source_ips", "target_field": "logscan.source.ips", "ignore_missing": true } }, { "set": { "if": "ctx.model == 'kff'", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, { "set": { "if": "ctx.model == 'kff'", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, { "set": { "if": "ctx.model == 'kl'", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 68fc32d7c..f5ba5dc74 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -114,7 +114,7 @@ filebeat.inputs: - type: log paths: - - /opt/so/log/logscan/alerts.log + - /logs/logscan/alerts.log fields: module: logscan dataset: alert diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja index a0e5bd570..a1c621484 100644 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -16,7 +16,7 @@ output { user => "{{ ES_USER }}" password => "{{ ES_PASS }}" {% endif %} - index => "so-%{[event][module]}" + index => "so-logscan" template_name => "so-common" template => "/templates/so-common-template.json" template_overwrite => true From 09165daab842dc9481f23ded252e8cefa48ef68d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Jul 2021 09:10:33 -0400 Subject: [PATCH 10/57] Several Suricata things --- HOTFIX | 2 +- salt/elasticsearch/files/ingest/suricata.fileinfo | 1 + salt/suricata/suricata_meta.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/HOTFIX b/HOTFIX index 31f81a97c..d3f5a12fa 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -ECSFIX HEAVYNODE_SSL_LOGSTASH_REDIS_PIPELINES FBPIPELINE CURATORAUTH + diff --git a/salt/elasticsearch/files/ingest/suricata.fileinfo b/salt/elasticsearch/files/ingest/suricata.fileinfo index d5147fb40..fe9e4b109 100644 --- a/salt/elasticsearch/files/ingest/suricata.fileinfo +++ b/salt/elasticsearch/files/ingest/suricata.fileinfo @@ -13,6 +13,7 @@ { "rename": { "field": "message2.fileinfo.size", "target_field": "file.size", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.state", "target_field": "file.state", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.stored", "target_field": "file.saved", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.sha256", "target_field": "hash.sha256", "ignore_missing": true } }, { "set": { "if": "ctx.network?.protocol != null", "field": "file.source", "value": "{{network.protocol}}" } }, { "pipeline": { "name": "common" } } ] diff --git a/salt/suricata/suricata_meta.yaml b/salt/suricata/suricata_meta.yaml index 90b220000..1c3855501 100644 --- a/salt/suricata/suricata_meta.yaml +++ b/salt/suricata/suricata_meta.yaml @@ -7,7 +7,7 @@ suricata: dir: /nsm/extracted #write-fileinfo: "yes" #force-filestore: "yes" - #stream-depth: 0 + stream-depth: 0 #max-open-files: 1000 #force-hash: [sha1, md5] xff: From b8e3a45a7efcb727676bb2706fa64cc52fc6901a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 23 Jul 2021 08:53:45 -0400 Subject: [PATCH 11/57] [wip] Add logscan state Do not add state to top file or setup yet, script will be written to enable the feature shortly --- salt/allowed_states.map.jinja | 20 ++++++++---- salt/logscan/files/logscan.conf | 6 ++++ salt/logscan/init.sls | 57 +++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 salt/logscan/files/logscan.conf create mode 100644 salt/logscan/init.sls diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 665fdbe3d..12d8f99ec 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -45,7 +45,8 @@ 'schedule', 'soctopus', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-heavynode': [ 'ca', @@ -75,7 +76,8 @@ 'logstash', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-fleet': [ 'ca', @@ -108,7 +110,8 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-manager': [ 'salt.master', @@ -127,7 +130,8 @@ 'utility', 'schedule', 'soctopus', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-managersearch': [ 'salt.master', @@ -146,7 +150,8 @@ 'utility', 'schedule', 'soctopus', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-node': [ 'ca', @@ -178,7 +183,8 @@ 'schedule', 'soctopus', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-sensor': [ 'ca', @@ -296,4 +302,4 @@ {% endif %} {# all nodes can always run salt.minion state #} -{% do allowed_states.append('salt.minion') %} \ No newline at end of file +{% do allowed_states.append('salt.minion') %} diff --git a/salt/logscan/files/logscan.conf b/salt/logscan/files/logscan.conf new file mode 100644 index 000000000..9b3df8027 --- /dev/null +++ b/salt/logscan/files/logscan.conf @@ -0,0 +1,6 @@ +[global] +ts_format = iso8601 +scan_interval = 30s + +[kratos] +log_path = kratos/kratos.log diff --git a/salt/logscan/init.sls b/salt/logscan/init.sls new file mode 100644 index 000000000..51badbcdd --- /dev/null +++ b/salt/logscan/init.sls @@ -0,0 +1,57 @@ +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} + +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{% set MANAGER = salt['grains.get']('master') %} +{% set logscan_cpu_period = salt['pillar.get']('logscan:cpu_period', 10000) %} + +logscan_data_dir: + file.directory: + - name: /nsm/logscan/data + - user: 939 + - group: 939 + - makedirs: True + +logscan_conf_dir: + file.directory: + - name: /opt/so/conf/logscan + - user: 939 + - group: 939 + - makedirs: True + +logscan_conf: + file.managed: + - name: /opt/so/conf/logscan/logscan.conf + - source: salt://logscan/files/logscan.conf + - user: 939 + - group: 939 + - mode: 600 + - template: jinja + +logscan_log_dir: + file.directory: + - name: /opt/so/log/logscan + - user: 939 + - group: 939 + +so-logscan: + docker_container.running: + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logscan:{{ VERSION }} + - hostname: logscan + - name: so-logscan + - binds: + - /nsm/logscan/data:/logscan/data:rw + - /opt/so/conf/logscan.conf:/logscan/logscan.conf:ro + - /opt/so/log/logscan:/logscan/output:rw + - /opt/so/log:/logscan/logs:ro + - cpu_period: {{ logscan_cpu_period }} + + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} From 64945cec16b6610db863e55c031bf686ef4466d5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 26 Jul 2021 14:24:10 -0400 Subject: [PATCH 12/57] [wip] Initial work to enable/disable "learn" modules --- salt/common/tools/sbin/so-learn | 229 +++++++++++++++++++ salt/common/tools/sbin/so-rule | 4 +- salt/{logscan => learn}/files/logscan.conf | 0 salt/learn/init.sls | 19 ++ salt/{logscan/init.sls => learn/logscan.sls} | 12 - salt/top.sls | 6 +- setup/so-setup | 6 + 7 files changed, 260 insertions(+), 16 deletions(-) create mode 100644 salt/common/tools/sbin/so-learn rename salt/{logscan => learn}/files/logscan.conf (100%) create mode 100644 salt/learn/init.sls rename salt/{logscan/init.sls => learn/logscan.sls} (84%) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn new file mode 100644 index 000000000..8b5c2473f --- /dev/null +++ b/salt/common/tools/sbin/so-learn @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import signal +import sys +import os +import re +import subprocess +import argparse +import textwrap +from typing import List + +import yaml + +minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' +salt_proc = subprocess.CompletedProcess = None + +# Temp store of modules, will likely be broken out into salt +learn_modules = [ + { 'name': 'logscan', 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } +] + + +def sigint_handler(*_): + print('Exiting gracefully on Ctrl-C') + if salt_proc is not None: salt_proc.send_signal(signal.SIGINT) + sys.exit(0) + + +def find_minion_pillar() -> str: + regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$' + + result = [] + for root, _, files in os.walk(minion_pillar_dir): + for f_minion_id in files: + if re.search(regex, f_minion_id): + result.append(os.path.join(root, f_minion_id)) + + if len(result) == 0: + print('Could not find manager-type pillar (eval, standalone, manager, managersearch, import). Are you running this script on the manager?', file=sys.stderr) + sys.exit(3) + elif len(result) > 1: + res_str = ', '.join(f'\"{result}\"') + print('(This should not happen, the system is in an error state if you see this message.)\n', file=sys.stderr) + print('More than one manager-type pillar exists, minion id\'s listed below:', file=sys.stderr) + print(f' {res_str}', file=sys.stderr) + sys.exit(3) + else: + return result[0] + + +def read_pillar(pillar: str): + try: + with open(pillar, 'r') as f: + loaded_yaml = yaml.safe_load(f.read()) + if loaded_yaml is None: + print(f'Could not parse {pillar}', file=sys.stderr) + sys.exit(3) + return loaded_yaml + except: + print(f'Could not open {pillar}', file=sys.stderr) + sys.exit(3) + + +def write_pillar(pillar: str, content: dict): + try: + with open(pillar, 'w') as f: + return yaml.dump(content, f, default_flow_style=False) + except: + print(f'Could not open {pillar}', file=sys.stderr) + sys.exit(3) + + +def create_pillar_if_not_exist(pillar:str, content: dict): + if content.get('learn', {}).get('modules') is None: + content['learn']['modules'] = learn_modules + write_pillar(pillar, content) + + +def apply(module_list: List): + return_code = 0 + for module in module_list: + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True'] + print(f'Applying salt state for {module} module...') + cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + if cmd.returncode != 0: + print(f'[ERROR] Failed to apply salt state for {module}') + return_code = cmd.returncode + return return_code + + +def check_apply(args: dict): + if args.apply: + print('Configuration updated. Applying changes:') + return apply(args.modules) + else: + if not hasattr(args, 'yes'): + message = 'Configuration updated. Would you like to apply your changes now? (y/N) ' + answer = input(message) + while answer.lower() not in [ 'y', 'n', '' ]: + answer = input(message) + if answer.lower() in [ 'n', '' ]: + return 0 + else: + print('Applying changes:') + return apply(args.modules) + else: + return 0 + + +def enable_disable_modules(args, enable: bool): + pillar_modules = args.pillar_dict.get('learn', {}).get('modules') + + if 'all' in args.modules: + for module in pillar_modules: + module['enabled'] = enable + args.pillar_dict.update() + write_pillar(args.pillar, args.pillar_dict) + else: + write_needed = False + for module in args.modules: + if module in pillar_modules: + pillar_modules[module]['enabled'] = enable + write_needed = enable + if write_needed: + args.pillar_dict.update() + write_pillar(args.pillarm, args.pillar_dict) + + check_apply(args) + + +def enable_modules(args): + enable_disable_modules(args, enable=True) + + +def disable_modules(args): + enable_disable_modules(args, enable=False) + + +def list_modules(*_): + print('Available ML modules:') + for module in learn_modules: + + print(f' - { module["name"] } : {module["description"]}') + + +def main(): + beta_str = 'BETA - SUBJECT TO CHANGE\n' + + apply_help='After ACTION the chosen modules, apply any necessary salt states.' + enable_apply_help = apply_help.replace('ACTION', 'enabling') + disable_apply_help = apply_help.replace('ACTION', 'disabling') + + yes_help = 'Accept apply prompt.' + + signal.signal(signal.SIGINT, sigint_handler) + + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) + + main_parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter) + + subcommand_desc = textwrap.dedent( + """\ + enable Enable one or more ML modules. + disable Disable one or more ML modules. + list List all available ML modules. + """ + ) + + subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='', dest='command') + + module_help_str = 'One or more ML modules, which can be listed using \'so-learn list\'. Use the keyword \'all\' to apply the action to all available modules.' + + enable = subparsers.add_parser('enable') + enable.set_defaults(func=enable_modules) + enable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) + enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help) + enable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) + + + disable = subparsers.add_parser('disable') + disable.set_defaults(func=disable_modules) + disable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) + disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help) + disable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) + + + + list = subparsers.add_parser('list') + list.set_defaults(func=list_modules) + + args = main_parser.parse_args(sys.argv[1:]) + + # args.pillar = find_minion_pillar() + + # args.pillar_dict = read_pillar(args.pillar) + + # create_pillar_if_not_exist(args.pillar, args.pillar_dict) + + if hasattr(args, 'func'): + exit_code = args.func(args) + else: + if args.command is None: + print(beta_str) + main_parser.print_help() + sys.exit(0) + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index 07d6cd2cc..bbb916807 100755 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -349,9 +349,7 @@ def sigint_handler(*_): def main(): signal.signal(signal.SIGINT, sigint_handler) - if os.geteuid() != 0: - print_err('You must run this script as root') - sys.exit(1) + apply_help='After updating rule configuration, apply the idstools state.' diff --git a/salt/logscan/files/logscan.conf b/salt/learn/files/logscan.conf similarity index 100% rename from salt/logscan/files/logscan.conf rename to salt/learn/files/logscan.conf diff --git a/salt/learn/init.sls b/salt/learn/init.sls new file mode 100644 index 000000000..581f3f073 --- /dev/null +++ b/salt/learn/init.sls @@ -0,0 +1,19 @@ +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} + +{% set module_list = salt['pillar.get']('learn:modules', [] ) %} + +{% if len(module_list) != 0 %}} +include: +{% for module in module_list %} + - .{{ module }} +{% endfor %} +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/logscan/init.sls b/salt/learn/logscan.sls similarity index 84% rename from salt/logscan/init.sls rename to salt/learn/logscan.sls index 51badbcdd..43318d81a 100644 --- a/salt/logscan/init.sls +++ b/salt/learn/logscan.sls @@ -1,6 +1,3 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} - {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} @@ -46,12 +43,3 @@ so-logscan: - /opt/so/log/logscan:/logscan/output:rw - /opt/so/log:/logscan/logs:ro - cpu_period: {{ logscan_cpu_period }} - - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/top.sls b/salt/top.sls index 24be8283c..32069ab66 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -156,6 +156,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_manager and G@saltversion:{{saltversion}}': - match: compound @@ -218,6 +219,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_standalone and G@saltversion:{{saltversion}}': - match: compound @@ -292,6 +294,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_searchnode and G@saltversion:{{saltversion}}': - match: compound @@ -366,7 +369,6 @@ base: {%- if FILEBEAT %} - filebeat {%- endif %} - - utility - schedule {%- if FLEETMANAGER or FLEETNODE %} @@ -388,6 +390,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_heavynode and G@saltversion:{{saltversion}}': - match: compound @@ -478,3 +481,4 @@ base: - schedule - docker_clean - pipeline.load + - learn diff --git a/setup/so-setup b/setup/so-setup index 68490657f..bfca9b2bd 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -962,6 +962,12 @@ else set_progress_str 99 'Waiting for TheHive to start up' check_hive_init >> $setup_log 2>&1 fi + + if [[ -n $LEARN_LOGSCAN_ENABLE ]]; then + set_progress_str 99 'Enabling logscan' + so-learn enable logscan --apply -y >> $setup_log 2>&1 + fi + } | whiptail_gauge_post_setup "Running post-installation steps..." whiptail_setup_complete From f31dc5abc7c46b8c076adabc0275cd1a504a0543 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 09:49:59 -0400 Subject: [PATCH 13/57] Add learn to allowed states --- salt/allowed_states.map.jinja | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 12d8f99ec..1aa79474e 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -46,7 +46,8 @@ 'soctopus', 'tcpreplay', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-heavynode': [ 'ca', @@ -111,7 +112,8 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-manager': [ 'salt.master', @@ -131,7 +133,8 @@ 'schedule', 'soctopus', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-managersearch': [ 'salt.master', @@ -151,7 +154,8 @@ 'schedule', 'soctopus', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-node': [ 'ca', @@ -184,7 +188,8 @@ 'soctopus', 'tcpreplay', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-sensor': [ 'ca', From 8d56fc71fa6b242ceca2720dce2160bb108e1780 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 09:53:24 -0400 Subject: [PATCH 14/57] Fix jinja length calculation --- salt/learn/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/learn/init.sls b/salt/learn/init.sls index 581f3f073..a511c1074 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -3,7 +3,7 @@ {% set module_list = salt['pillar.get']('learn:modules', [] ) %} -{% if len(module_list) != 0 %}} +{% if module_list|length != 0 %}} include: {% for module in module_list %} - .{{ module }} From 455719936bdbffaef594c9744ab3f5c59e746a3d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 09:53:35 -0400 Subject: [PATCH 15/57] Uncomment required lines in so-learn --- salt/common/tools/sbin/so-learn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 8b5c2473f..a033c7db5 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -208,11 +208,11 @@ def main(): args = main_parser.parse_args(sys.argv[1:]) - # args.pillar = find_minion_pillar() + args.pillar = find_minion_pillar() - # args.pillar_dict = read_pillar(args.pillar) + args.pillar_dict = read_pillar(args.pillar) - # create_pillar_if_not_exist(args.pillar, args.pillar_dict) + create_pillar_if_not_exist(args.pillar, args.pillar_dict) if hasattr(args, 'func'): exit_code = args.func(args) From e2abe8840f0b99c14910c512d6a95c96a0a55f8f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 10:12:19 -0400 Subject: [PATCH 16/57] Fix directory in logscan state --- salt/learn/logscan.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/learn/logscan.sls b/salt/learn/logscan.sls index 43318d81a..833c40bd2 100644 --- a/salt/learn/logscan.sls +++ b/salt/learn/logscan.sls @@ -20,7 +20,7 @@ logscan_conf_dir: logscan_conf: file.managed: - name: /opt/so/conf/logscan/logscan.conf - - source: salt://logscan/files/logscan.conf + - source: salt://learn/files/logscan.conf - user: 939 - group: 939 - mode: 600 From 91accb0bc666c179c014b7108a599184952c44a1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 10:12:32 -0400 Subject: [PATCH 17/57] [wip] Fixing so-learn script --- salt/common/tools/sbin/so-learn | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index a033c7db5..4734b3fad 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -88,8 +88,12 @@ def write_pillar(pillar: str, content: dict): def create_pillar_if_not_exist(pillar:str, content: dict): if content.get('learn', {}).get('modules') is None: + content['learn'] = {} content['learn']['modules'] = learn_modules write_pillar(pillar, content) + + content.update() + return content def apply(module_list: List): @@ -99,7 +103,7 @@ def apply(module_list: List): print(f'Applying salt state for {module} module...') cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) if cmd.returncode != 0: - print(f'[ERROR] Failed to apply salt state for {module}') + print(f'[ERROR] Failed to apply salt state for {module} module.') return_code = cmd.returncode return return_code @@ -135,13 +139,18 @@ def enable_disable_modules(args, enable: bool): write_needed = False for module in args.modules: if module in pillar_modules: - pillar_modules[module]['enabled'] = enable - write_needed = enable + if pillar_modules[module]['enabled'] == enable: + action_str = 'enabled' if enable else 'disabled' + print(f'{module} module already {action_str}.', file=sys.stderr) + else: + pillar_modules[module]['enabled'] = enable + write_needed = enable if write_needed: args.pillar_dict.update() write_pillar(args.pillarm, args.pillar_dict) - check_apply(args) + salt_proc = check_apply(args) + return salt_proc def enable_modules(args): @@ -155,8 +164,8 @@ def disable_modules(args): def list_modules(*_): print('Available ML modules:') for module in learn_modules: - print(f' - { module["name"] } : {module["description"]}') + return 0 def main(): @@ -194,25 +203,18 @@ def main(): enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help) enable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) - disable = subparsers.add_parser('disable') disable.set_defaults(func=disable_modules) disable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help) disable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) - - list = subparsers.add_parser('list') list.set_defaults(func=list_modules) args = main_parser.parse_args(sys.argv[1:]) - args.pillar = find_minion_pillar() - - args.pillar_dict = read_pillar(args.pillar) - - create_pillar_if_not_exist(args.pillar, args.pillar_dict) + args.pillar_dict = create_pillar_if_not_exist(args.pillar, read_pillar(args.pillar)) if hasattr(args, 'func'): exit_code = args.func(args) From cf9121dfc2015555ecaffdf9a8e1720b545f7b1e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 14:13:16 -0400 Subject: [PATCH 18/57] Actually download so-learn container --- salt/common/tools/sbin/so-image-common | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 9b6e2174a..ad16379bb 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -37,6 +37,7 @@ container_list() { "so-idstools" "so-kibana" "so-kratos" + "so-learn" "so-nginx" "so-pcaptools" "so-soc" @@ -58,6 +59,7 @@ container_list() { "so-influxdb" "so-kibana" "so-kratos" + "so-learn" "so-logstash" "so-mysql" "so-nginx" From 7ef5b39b048079d84fb29a930e21d937341b77da Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 14:28:00 -0400 Subject: [PATCH 19/57] [wip] Fix `'Nonetype' object is not callable` error --- salt/common/tools/sbin/so-learn | 54 +++++++++++++++------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 4734b3fad..85b70730d 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from typing import List import signal import sys @@ -23,12 +24,10 @@ import re import subprocess import argparse import textwrap -from typing import List - import yaml minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' -salt_proc = subprocess.CompletedProcess = None +salt_proc: subprocess.CompletedProcess = None # Temp store of modules, will likely be broken out into salt learn_modules = [ @@ -80,19 +79,21 @@ def read_pillar(pillar: str): def write_pillar(pillar: str, content: dict): try: with open(pillar, 'w') as f: - return yaml.dump(content, f, default_flow_style=False) + yaml.dump(content, f, default_flow_style=False, sort_keys=False) except: print(f'Could not open {pillar}', file=sys.stderr) sys.exit(3) def create_pillar_if_not_exist(pillar:str, content: dict): - if content.get('learn', {}).get('modules') is None: - content['learn'] = {} - content['learn']['modules'] = learn_modules - write_pillar(pillar, content) + pillar_dict = content + + if pillar_dict.get('learn', {}).get('modules') is None: + pillar_dict['learn'] = {} + pillar_dict['learn']['modules'] = learn_modules + content.update() + write_pillar(pillar, content) - content.update() return content @@ -100,11 +101,11 @@ def apply(module_list: List): return_code = 0 for module in module_list: salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True'] - print(f'Applying salt state for {module} module...') - cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) - if cmd.returncode != 0: - print(f'[ERROR] Failed to apply salt state for {module} module.') - return_code = cmd.returncode + print(f' Applying salt state for {module} module...') + salt_proc = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + if salt_proc.returncode != 0: + print(f' [ERROR] Failed to apply salt state for {module} module.') + return_code = salt_proc.returncode return return_code @@ -113,18 +114,15 @@ def check_apply(args: dict): print('Configuration updated. Applying changes:') return apply(args.modules) else: - if not hasattr(args, 'yes'): - message = 'Configuration updated. Would you like to apply your changes now? (y/N) ' + message = 'Configuration updated. Would you like to apply your changes now? (y/N) ' + answer = input(message) + while answer.lower() not in [ 'y', 'n', '' ]: answer = input(message) - while answer.lower() not in [ 'y', 'n', '' ]: - answer = input(message) - if answer.lower() in [ 'n', '' ]: - return 0 - else: - print('Applying changes:') - return apply(args.modules) - else: + if answer.lower() in [ 'n', '' ]: return 0 + else: + print('Applying changes:') + return apply(args.modules) def enable_disable_modules(args, enable: bool): @@ -149,8 +147,8 @@ def enable_disable_modules(args, enable: bool): args.pillar_dict.update() write_pillar(args.pillarm, args.pillar_dict) - salt_proc = check_apply(args) - return salt_proc + cmd_ret = check_apply(args) + return cmd_ret def enable_modules(args): @@ -175,8 +173,6 @@ def main(): enable_apply_help = apply_help.replace('ACTION', 'enabling') disable_apply_help = apply_help.replace('ACTION', 'disabling') - yes_help = 'Accept apply prompt.' - signal.signal(signal.SIGINT, sigint_handler) if os.geteuid() != 0: @@ -201,13 +197,11 @@ def main(): enable.set_defaults(func=enable_modules) enable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help) - enable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) disable = subparsers.add_parser('disable') disable.set_defaults(func=disable_modules) disable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help) - disable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) list = subparsers.add_parser('list') list.set_defaults(func=list_modules) From dd0e407935f7992dbcff32cf7ba4f914224e20f0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 15:06:38 -0400 Subject: [PATCH 20/57] Use correct container name --- salt/common/tools/sbin/so-image-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index ad16379bb..cea0f6507 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -37,7 +37,7 @@ container_list() { "so-idstools" "so-kibana" "so-kratos" - "so-learn" + "so-logscan" "so-nginx" "so-pcaptools" "so-soc" @@ -59,7 +59,7 @@ container_list() { "so-influxdb" "so-kibana" "so-kratos" - "so-learn" + "so-logscan" "so-logstash" "so-mysql" "so-nginx" From 5894b85bd1956f0656252e72a7049b23b733471b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 10:57:53 -0400 Subject: [PATCH 21/57] Remove broken yaml dump arg, rename metavars --- salt/common/tools/sbin/so-learn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 85b70730d..53e2b5020 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -79,7 +79,7 @@ def read_pillar(pillar: str): def write_pillar(pillar: str, content: dict): try: with open(pillar, 'w') as f: - yaml.dump(content, f, default_flow_style=False, sort_keys=False) + yaml.dump(content, f, default_flow_style=False) except: print(f'Could not open {pillar}', file=sys.stderr) sys.exit(3) @@ -195,12 +195,12 @@ def main(): enable = subparsers.add_parser('enable') enable.set_defaults(func=enable_modules) - enable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) + enable.add_argument('modules', metavar='ML_MODULE', nargs='+', help=module_help_str) enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help) disable = subparsers.add_parser('disable') disable.set_defaults(func=disable_modules) - disable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) + disable.add_argument('modules', metavar='ML_MODULE', nargs='+', help=module_help_str) disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help) list = subparsers.add_parser('list') From 211a841cdbd397b61fea9cc096fcaf1c25ea99ac Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 11:40:19 -0400 Subject: [PATCH 22/57] Fix file path in bind mount for logscan --- salt/learn/logscan.sls | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/learn/logscan.sls b/salt/learn/logscan.sls index 833c40bd2..1e21e3b2a 100644 --- a/salt/learn/logscan.sls +++ b/salt/learn/logscan.sls @@ -1,7 +1,7 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% set logscan_cpu_period = salt['pillar.get']('logscan:cpu_period', 10000) %} +{% set logscan_cpu_period = salt['pillar.get']('learn:modules:logscan:cpu_period', 20000) %} logscan_data_dir: file.directory: @@ -24,7 +24,6 @@ logscan_conf: - user: 939 - group: 939 - mode: 600 - - template: jinja logscan_log_dir: file.directory: @@ -39,7 +38,7 @@ so-logscan: - name: so-logscan - binds: - /nsm/logscan/data:/logscan/data:rw - - /opt/so/conf/logscan.conf:/logscan/logscan.conf:ro + - /opt/so/conf/logscan/logscan.conf:/logscan/logscan.conf:ro - /opt/so/log/logscan:/logscan/output:rw - /opt/so/log:/logscan/logs:ro - cpu_period: {{ logscan_cpu_period }} From d53e989c55b5a222fb87fb88de00fb1c61f7ceab Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 11:52:10 -0400 Subject: [PATCH 23/57] Add ability to set cpu_period per module --- salt/common/tools/sbin/so-learn | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 53e2b5020..39004b052 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -25,14 +25,28 @@ import subprocess import argparse import textwrap import yaml +import multiprocessing minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' salt_proc: subprocess.CompletedProcess = None # Temp store of modules, will likely be broken out into salt -learn_modules = [ - { 'name': 'logscan', 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } -] +def get_learn_modules(): + return [ + { 'name': 'logscan', 'cpu_period': get_cpu_period(fraction=0.25), 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } + ] + + +def get_cpu_period(fraction: float): + multiplier = 10000 + + num_cores = multiprocessing.cpu_count() + if num_cores <= 2: + fraction = 1. + + num_used_cores = int(num_cores * fraction) + cpu_period = num_used_cores * multiplier + return cpu_period def sigint_handler(*_): @@ -90,7 +104,7 @@ def create_pillar_if_not_exist(pillar:str, content: dict): if pillar_dict.get('learn', {}).get('modules') is None: pillar_dict['learn'] = {} - pillar_dict['learn']['modules'] = learn_modules + pillar_dict['learn']['modules'] = get_learn_modules() content.update() write_pillar(pillar, content) From 2560a9b78c3087f782a11486ba42a5979d797b81 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 11:58:58 -0400 Subject: [PATCH 24/57] [wip] Change learn:modules to dictionary --- salt/common/tools/sbin/so-learn | 6 +++--- salt/learn/init.sls | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 39004b052..e444bc442 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -32,9 +32,9 @@ salt_proc: subprocess.CompletedProcess = None # Temp store of modules, will likely be broken out into salt def get_learn_modules(): - return [ - { 'name': 'logscan', 'cpu_period': get_cpu_period(fraction=0.25), 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } - ] + return { + 'logscan': { 'cpu_period': get_cpu_period(fraction=0.25), 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } + } def get_cpu_period(fraction: float): diff --git a/salt/learn/init.sls b/salt/learn/init.sls index a511c1074..ee0b5e91b 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -1,12 +1,12 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% set module_list = salt['pillar.get']('learn:modules', [] ) %} +{% set module_dict = salt['pillar.get']('learn:modules', [] ) %} {% if module_list|length != 0 %}} include: -{% for module in module_list %} - - .{{ module }} +{% for module, _ in module_dict %} + - 'learn.{{ module }}' {% endfor %} {% endif %} From e1785dbd9a74896ed92310d5f3a7f641773a8a89 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 12:00:53 -0400 Subject: [PATCH 25/57] Fix typo --- salt/common/tools/sbin/so-learn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index e444bc442..be5e6ee02 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -159,7 +159,7 @@ def enable_disable_modules(args, enable: bool): write_needed = enable if write_needed: args.pillar_dict.update() - write_pillar(args.pillarm, args.pillar_dict) + write_pillar(args.pillar, args.pillar_dict) cmd_ret = check_apply(args) return cmd_ret From c53da9b1fffc296490a72fd4c263c43747ec7db9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 12:04:40 -0400 Subject: [PATCH 26/57] Fix wrong variables in learn init.sls --- salt/learn/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/learn/init.sls b/salt/learn/init.sls index ee0b5e91b..32f8647b9 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -3,9 +3,9 @@ {% set module_dict = salt['pillar.get']('learn:modules', [] ) %} -{% if module_list|length != 0 %}} +{% if module_dict.items()|length != 0 %}} include: -{% for module, _ in module_dict %} +{% for module, _ in module_dict.items() %} - 'learn.{{ module }}' {% endfor %} {% endif %} From 44551ea9eeaf91aff80b54e7004f966e4ee241ac Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 13:31:48 -0400 Subject: [PATCH 27/57] Fix so-learn list --- salt/common/tools/sbin/so-learn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index be5e6ee02..9a307712b 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -175,8 +175,8 @@ def disable_modules(args): def list_modules(*_): print('Available ML modules:') - for module in learn_modules: - print(f' - { module["name"] } : {module["description"]}') + for module, details in get_learn_modules().items(): + print(f' - { module } : {details["description"]}') return 0 From 9e92f6da3d339f61b29a90fab498ed31af399e87 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 14:25:20 -0400 Subject: [PATCH 28/57] Add container to so-status when enabling/disabling ml module --- salt/common/tools/sbin/so-learn | 44 ++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 9a307712b..ea5a18b2c 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -28,6 +28,7 @@ import yaml import multiprocessing minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' +so_status_conf = '/opt/so/conf/so-status/so-status.conf' salt_proc: subprocess.CompletedProcess = None # Temp store of modules, will likely be broken out into salt @@ -79,8 +80,8 @@ def find_minion_pillar() -> str: def read_pillar(pillar: str): try: - with open(pillar, 'r') as f: - loaded_yaml = yaml.safe_load(f.read()) + with open(pillar, 'r') as pillar_file: + loaded_yaml = yaml.safe_load(pillar_file.read()) if loaded_yaml is None: print(f'Could not parse {pillar}', file=sys.stderr) sys.exit(3) @@ -92,13 +93,31 @@ def read_pillar(pillar: str): def write_pillar(pillar: str, content: dict): try: - with open(pillar, 'w') as f: - yaml.dump(content, f, default_flow_style=False) + with open(pillar, 'w') as pillar_file: + yaml.dump(content, pillar_file, default_flow_style=False) except: print(f'Could not open {pillar}', file=sys.stderr) sys.exit(3) +def mod_so_status(action: str, items: str): + with open(so_status_conf, 'a+') as conf: + conf.seek(0) + containers = conf.readlines() + + for item in items: + if f'so-{item}' in containers: + if action == 'remove': containers.remove(f'so-{item}') + if action == 'add': pass + else: + if action == 'remove': pass + if action == 'add': containers.append(f'so-{item}') + + conf.seek(0) + conf.truncate(0) + conf.writelines(containers) + + def create_pillar_if_not_exist(pillar:str, content: dict): pillar_dict = content @@ -141,21 +160,26 @@ def check_apply(args: dict): def enable_disable_modules(args, enable: bool): pillar_modules = args.pillar_dict.get('learn', {}).get('modules') + pillar_mod_names = args.pillar_dict.get('learn', {}).get('modules').keys() + action_str = 'add' if enable else 'remove' + if 'all' in args.modules: - for module in pillar_modules: - module['enabled'] = enable + for module, details in pillar_modules.items(): + details['enabled'] = enable + mod_so_status(action_str, module) args.pillar_dict.update() write_pillar(args.pillar, args.pillar_dict) else: write_needed = False for module in args.modules: - if module in pillar_modules: + if module in pillar_mod_names: if pillar_modules[module]['enabled'] == enable: - action_str = 'enabled' if enable else 'disabled' - print(f'{module} module already {action_str}.', file=sys.stderr) + state_str = 'enabled' if enable else 'disabled' + print(f'{module} module already {state_str}.', file=sys.stderr) else: pillar_modules[module]['enabled'] = enable + mod_so_status(action_str, module) write_needed = enable if write_needed: args.pillar_dict.update() @@ -167,10 +191,12 @@ def enable_disable_modules(args, enable: bool): def enable_modules(args): enable_disable_modules(args, enable=True) + mod_so_status('add', args.modules) def disable_modules(args): enable_disable_modules(args, enable=False) + mod_so_status('remove', args.modules) def list_modules(*_): From e38219aa2e4b88f827aeccccc9bda6134f01b163 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 14:35:02 -0400 Subject: [PATCH 29/57] Fix learn init.sls typo --- salt/learn/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/learn/init.sls b/salt/learn/init.sls index 32f8647b9..51a3fc973 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -3,7 +3,7 @@ {% set module_dict = salt['pillar.get']('learn:modules', [] ) %} -{% if module_dict.items()|length != 0 %}} +{% if module_dict.items()|length != 0 %} include: {% for module, _ in module_dict.items() %} - 'learn.{{ module }}' From b30f771fa2166c77869f526a03f09cdb3acec650 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 14:59:26 -0400 Subject: [PATCH 30/57] Set write_needed flag correctly, include newline in so-status.conf string --- salt/common/tools/sbin/so-learn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index ea5a18b2c..8103dc09b 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -106,12 +106,12 @@ def mod_so_status(action: str, items: str): containers = conf.readlines() for item in items: - if f'so-{item}' in containers: - if action == 'remove': containers.remove(f'so-{item}') + if f'so-{item}\n' in containers: + if action == 'remove': containers.remove(f'so-{item}\n') if action == 'add': pass else: if action == 'remove': pass - if action == 'add': containers.append(f'so-{item}') + if action == 'add': containers.append(f'so-{item}\n') conf.seek(0) conf.truncate(0) @@ -180,7 +180,7 @@ def enable_disable_modules(args, enable: bool): else: pillar_modules[module]['enabled'] = enable mod_so_status(action_str, module) - write_needed = enable + write_needed = True if write_needed: args.pillar_dict.update() write_pillar(args.pillar, args.pillar_dict) From ba265d94f45cbb7b2ec2440606d7d4ff72aee180 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 15:14:28 -0400 Subject: [PATCH 31/57] Change default value in learn init to a dict where approriate --- salt/learn/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/learn/init.sls b/salt/learn/init.sls index 51a3fc973..fb5b89802 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -1,7 +1,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% set module_dict = salt['pillar.get']('learn:modules', [] ) %} +{% set module_dict = salt['pillar.get']('learn:modules', {} ) %} {% if module_dict.items()|length != 0 %} include: From b2a83018ba0fe44d77af77b57415264ac19f1a4b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 15:14:54 -0400 Subject: [PATCH 32/57] Remove or run logscan based on enabled bool --- salt/learn/logscan.sls | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/learn/logscan.sls b/salt/learn/logscan.sls index 1e21e3b2a..cc8bb2996 100644 --- a/salt/learn/logscan.sls +++ b/salt/learn/logscan.sls @@ -2,6 +2,14 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set logscan_cpu_period = salt['pillar.get']('learn:modules:logscan:cpu_period', 20000) %} +{% set enabled = salt['pillar.get']('learn:modules:logscan:enabled', False) %} + +{% if enabled %} + {% set container_action = 'running' %} +{% else %} + {% set container_action = 'absent'%} +{% endif %} + logscan_data_dir: file.directory: @@ -32,7 +40,8 @@ logscan_log_dir: - group: 939 so-logscan: - docker_container.running: + docker_container.{{ container_action }}: + {% if container_action == 'running' %} - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logscan:{{ VERSION }} - hostname: logscan - name: so-logscan @@ -42,3 +51,6 @@ so-logscan: - /opt/so/log/logscan:/logscan/output:rw - /opt/so/log:/logscan/logs:ro - cpu_period: {{ logscan_cpu_period }} + {% else %} + - force: true + {% endif %} From d71967ea1db09c37cc25b2e7e4304219321a2bdb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 10:28:39 -0400 Subject: [PATCH 33/57] Fix incorrect writing of so-status.conf --- salt/common/tools/sbin/so-learn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 8103dc09b..815d78ebf 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -100,7 +100,7 @@ def write_pillar(pillar: str, content: dict): sys.exit(3) -def mod_so_status(action: str, items: str): +def mod_so_status(action: str, items: List): with open(so_status_conf, 'a+') as conf: conf.seek(0) containers = conf.readlines() @@ -112,6 +112,8 @@ def mod_so_status(action: str, items: str): else: if action == 'remove': pass if action == 'add': containers.append(f'so-{item}\n') + + [containers.remove(c_name) for c_name in containers if c_name == '\n'] # remove extra newlines conf.seek(0) conf.truncate(0) From 01bb94514c32eb390f3e107f1fb71c8cd32e7dcb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 11:05:48 -0400 Subject: [PATCH 34/57] Correct mod_so_status to only act on single string --- salt/common/tools/sbin/so-learn | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 815d78ebf..f4a8ef90e 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -100,18 +100,18 @@ def write_pillar(pillar: str, content: dict): sys.exit(3) -def mod_so_status(action: str, items: List): +def mod_so_status(action: str, item: str): with open(so_status_conf, 'a+') as conf: conf.seek(0) containers = conf.readlines() - - for item in items: - if f'so-{item}\n' in containers: - if action == 'remove': containers.remove(f'so-{item}\n') - if action == 'add': pass - else: - if action == 'remove': pass - if action == 'add': containers.append(f'so-{item}\n') + + if f'so-{item}\n' in containers: + if action == 'remove': containers.remove(f'so-{item}\n') + if action == 'add': pass + else: + if action == 'remove': pass + if action == 'add': containers.append(f'so-{item}\n') + [containers.remove(c_name) for c_name in containers if c_name == '\n'] # remove extra newlines @@ -193,12 +193,10 @@ def enable_disable_modules(args, enable: bool): def enable_modules(args): enable_disable_modules(args, enable=True) - mod_so_status('add', args.modules) def disable_modules(args): enable_disable_modules(args, enable=False) - mod_so_status('remove', args.modules) def list_modules(*_): From b9980c9d30e547f5ae32e8e8afe15f89f5e78932 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 13:09:09 -0400 Subject: [PATCH 35/57] Fix pipeline name --- .../logstash/pipelines/config/so/9800_output_logscan.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja index a1c621484..38ee29b69 100644 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -10,7 +10,7 @@ output { if [module] =~ "logscan" { elasticsearch { id => "logscan_pipeline" - pipeline => "%{module}" + pipeline => "logscan" hosts => "{{ ES }}" {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" From 33bd6aed20d42d941ed08cd1b83cea4b34c5aceb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 14:41:15 -0400 Subject: [PATCH 36/57] Fix logscan pipeline on eval * Rename logscan pipeline to logscan.alert * Add module to indices array in filebeat.yml --- salt/elasticsearch/files/ingest/{logscan => logscan.alert} | 0 salt/filebeat/etc/filebeat.yml | 3 +++ .../pipelines/config/so/9800_output_logscan.conf.jinja | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) rename salt/elasticsearch/files/ingest/{logscan => logscan.alert} (100%) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan.alert similarity index 100% rename from salt/elasticsearch/files/ingest/logscan rename to salt/elasticsearch/files/ingest/logscan.alert diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index f5ba5dc74..3c482e274 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -307,6 +307,9 @@ output.elasticsearch: - index: "so-strelka" when.contains: module: "strelka" + - index: "so-logscan" + when.contains: + module: "logscan" setup.template.enabled: false {%- else %} diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja index 38ee29b69..86944d155 100644 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -10,7 +10,7 @@ output { if [module] =~ "logscan" { elasticsearch { id => "logscan_pipeline" - pipeline => "logscan" + pipeline => "logscan.alert" hosts => "{{ ES }}" {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" From 2a6277c0c3bfab781799cd685ae99396011eca1d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 15:46:39 -0400 Subject: [PATCH 37/57] Fix field names in logscan pipeline --- salt/elasticsearch/files/ingest/logscan.alert | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/files/ingest/logscan.alert b/salt/elasticsearch/files/ingest/logscan.alert index 3a3debabc..88772c16c 100644 --- a/salt/elasticsearch/files/ingest/logscan.alert +++ b/salt/elasticsearch/files/ingest/logscan.alert @@ -11,12 +11,14 @@ { "remove": { "field": "start_time", "ignore_missing": true } }, { "remove": { "field": "end_time", "ignore_missing": true } }, { "rename": { "field": "source_ip", "target_field": "source.ip", "ignore_missing": true } }, - { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, - { "rename": { "field": "source_ips", "target_field": "logscan.source.ips", "ignore_missing": true } }, - { "set": { "if": "ctx.model == 'kff'", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, - { "set": { "if": "ctx.model == 'kff'", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, - { "set": { "if": "ctx.model == 'kl'", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, - { "set": { "if": "ctx.model == 'kl'", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "append": { "field": "logscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, + { "rename": { "field": "top_source_ips", "target_field": "logscan.source.ips", "ignore_missing": true } }, + { "set": { "if": "ctx.model == 'k1'", "field": "rule.name", "value": "LOGSCAN K1 MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'k1'", "field": "rule.description", "value": "High number of logins from single IP in 1 minute window" } }, + { "set": { "if": "ctx.model == 'k5'", "field": "rule.name", "value": "LOGSCAN K5 MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'k5'", "field": "rule.description", "value": "High ratio of login failures from single IP in 5 minute window" } }, + { "set": { "if": "ctx.model == 'k60'", "field": "rule.name", "value": "LOGSCAN K60 MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'k60'", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, { "rename": { "field": "model", "target_field": "logscan.model" } }, { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, From 4f39cd1d7f80516134e6b603b82c35cafb3d07df Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 16:02:02 -0400 Subject: [PATCH 38/57] Add logscan dynamic object to so-common template mappings --- salt/elasticsearch/templates/so/so-common-template.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 8afac271c..777bf3f53 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -313,6 +313,10 @@ "type":"object", "dynamic": true }, + "logscan": { + "type": "object", + "dynamic": true + }, "manager":{ "type":"object", "dynamic": true From 8a49039b854749ab6f76635cf4eeb35bb092c579 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 2 Aug 2021 09:50:49 -0400 Subject: [PATCH 39/57] Only append source.ip to logscan.source.ips if it's been created --- salt/elasticsearch/files/ingest/logscan.alert | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/logscan.alert b/salt/elasticsearch/files/ingest/logscan.alert index 88772c16c..7473060a7 100644 --- a/salt/elasticsearch/files/ingest/logscan.alert +++ b/salt/elasticsearch/files/ingest/logscan.alert @@ -11,8 +11,8 @@ { "remove": { "field": "start_time", "ignore_missing": true } }, { "remove": { "field": "end_time", "ignore_missing": true } }, { "rename": { "field": "source_ip", "target_field": "source.ip", "ignore_missing": true } }, - { "append": { "field": "logscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, { "rename": { "field": "top_source_ips", "target_field": "logscan.source.ips", "ignore_missing": true } }, + { "append": { "if": "ctx.source != null", "field": "logscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, { "set": { "if": "ctx.model == 'k1'", "field": "rule.name", "value": "LOGSCAN K1 MODEL THRESHOLD" } }, { "set": { "if": "ctx.model == 'k1'", "field": "rule.description", "value": "High number of logins from single IP in 1 minute window" } }, { "set": { "if": "ctx.model == 'k5'", "field": "rule.name", "value": "LOGSCAN K5 MODEL THRESHOLD" } }, From 757091beeb6573d73df6aca44130f4c0d39b9817 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 2 Aug 2021 10:35:39 -0400 Subject: [PATCH 40/57] Add log_level to logscan.conf --- salt/learn/files/logscan.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/learn/files/logscan.conf b/salt/learn/files/logscan.conf index 9b3df8027..d7aa30734 100644 --- a/salt/learn/files/logscan.conf +++ b/salt/learn/files/logscan.conf @@ -1,6 +1,7 @@ [global] ts_format = iso8601 scan_interval = 30s +log_level = info [kratos] log_path = kratos/kratos.log From d3b170c6df21c0ba9a00b41928aa613a804a5122 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 2 Aug 2021 12:37:37 -0400 Subject: [PATCH 41/57] Add logscan automation file + fix enable command in setup --- setup/automation/standalone-iso-logscan | 77 +++++++++++++++++++++++++ setup/so-setup | 2 +- 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 setup/automation/standalone-iso-logscan diff --git a/setup/automation/standalone-iso-logscan b/setup/automation/standalone-iso-logscan new file mode 100644 index 000000000..d83ad73db --- /dev/null +++ b/setup/automation/standalone-iso-logscan @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=2 +BASICSURI=2 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=standalone +install_type=STANDALONE +LEARN_LOGSCAN_ENABLE=true +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/so-setup b/setup/so-setup index bfca9b2bd..958d8aea1 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -965,7 +965,7 @@ else if [[ -n $LEARN_LOGSCAN_ENABLE ]]; then set_progress_str 99 'Enabling logscan' - so-learn enable logscan --apply -y >> $setup_log 2>&1 + so-learn enable logscan --apply >> $setup_log 2>&1 fi } | whiptail_gauge_post_setup "Running post-installation steps..." From 25bf25eae68528e8148ca709548214cc274d1c8e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 3 Aug 2021 15:24:32 -0400 Subject: [PATCH 42/57] Allowed states remove typo'd logscan --- salt/allowed_states.map.jinja | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 1aa79474e..f5f77d8f3 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -46,7 +46,6 @@ 'soctopus', 'tcpreplay', 'docker_clean', - 'logscsan', 'learn' ], 'so-heavynode': [ @@ -78,7 +77,7 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'logscsan' + 'learn' ], 'so-fleet': [ 'ca', @@ -112,7 +111,6 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'logscsan', 'learn' ], 'so-manager': [ @@ -133,7 +131,6 @@ 'schedule', 'soctopus', 'docker_clean', - 'logscsan', 'learn' ], 'so-managersearch': [ @@ -154,7 +151,6 @@ 'schedule', 'soctopus', 'docker_clean', - 'logscsan', 'learn' ], 'so-node': [ @@ -188,7 +184,6 @@ 'soctopus', 'tcpreplay', 'docker_clean', - 'logscsan', 'learn' ], 'so-sensor': [ From 9e5d3aa28675155cf10d7fb26ad95a574607f16c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 3 Aug 2021 15:25:53 -0400 Subject: [PATCH 43/57] Fix removed root check in so-rule --- salt/common/tools/sbin/so-rule | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index bbb916807..bd1556f58 100755 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -349,7 +349,9 @@ def sigint_handler(*_): def main(): signal.signal(signal.SIGINT, sigint_handler) - + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) apply_help='After updating rule configuration, apply the idstools state.' From ef59cb47dd22c700a9c6103aeb8c2d2b90979b0f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 3 Aug 2021 15:26:57 -0400 Subject: [PATCH 44/57] Use print_err function --- salt/common/tools/sbin/so-rule | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index bd1556f58..07d6cd2cc 100755 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -350,7 +350,7 @@ def main(): signal.signal(signal.SIGINT, sigint_handler) if os.geteuid() != 0: - print('You must run this script as root', file=sys.stderr) + print_err('You must run this script as root') sys.exit(1) apply_help='After updating rule configuration, apply the idstools state.' From 2bc88e77507ea09ec317ce4e7d9b53040ac006df Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 3 Aug 2021 15:29:37 -0400 Subject: [PATCH 45/57] Remove learn from allowed states for helixsensor --- salt/allowed_states.map.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index f5f77d8f3..a51ea434b 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -76,8 +76,7 @@ 'logstash', 'schedule', 'tcpreplay', - 'docker_clean', - 'learn' + 'docker_clean' ], 'so-fleet': [ 'ca', From 52cb72ba67cf40a18822c8de3c2e5aef841789b3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 4 Aug 2021 09:40:34 -0400 Subject: [PATCH 46/57] dont run curator on searchnode if truecluster is enabled - https://github.com/Security-Onion-Solutions/securityonion/issues/2806 --- salt/curator/init.sls | 21 ++++++++++++++++++++- salt/curator/map.jinja | 12 ++++++++++++ salt/top.sls | 11 ----------- 3 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 salt/curator/map.jinja diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 48a10b4b8..24d9f8c0d 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -6,6 +6,7 @@ {% set MANAGER = salt['grains.get']('master') %} {% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone'] %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} + {% from "curator/map.jinja" import CURATOROPTIONS with context %} # Curator # Create the group curatorgroup: @@ -118,8 +119,10 @@ so-curatordeletecron: - dayweek: '*' so-curator: - docker_container.running: + docker_container.{{ CURATOROPTIONS.status }}: + {% if CURATOROPTIONS.status == 'running' %} - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-curator:{{ VERSION }} + - start: {{ CURATOROPTIONS.start }} - hostname: curator - name: so-curator - user: curator @@ -129,11 +132,27 @@ so-curator: - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro - /opt/so/conf/curator/action/:/etc/curator/action:ro - /opt/so/log/curator:/var/log/curator:rw + {% else %} + - force: True + {% endif %} append_so-curator_so-status.conf: file.append: - name: /opt/so/conf/so-status/so-status.conf - text: so-curator + - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf + + {% if not CURATOROPTIONS.start %} +so-curator_so-status.disabled: + file.comment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-curator$ + {% else %} +delete_so-curator_so-status.disabled: + file.uncomment: + - name: /opt/so/conf/so-status/so-status.conf + - regex: ^so-curator$ + {% endif %} # Begin Curator Cron Jobs diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja new file mode 100644 index 000000000..2dc2c7311 --- /dev/null +++ b/salt/curator/map.jinja @@ -0,0 +1,12 @@ +{% set CURATOROPTIONS = {} %} +{% set ENABLED = salt['pillar.get']('curator:enabled', True) %} +{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} + +# don't start the docker container if searchnode and true clustering is enabled or disabled via pillar +{% if (grains.id.split('_')|last == 'searchnode' and TRUECLUSTER) or not ENABLED %} + {% do CURATOROPTIONS.update({'start': False}) %} + {% do CURATOROPTIONS.update({'status': 'absent'}) %} +{% else %} + {% do CURATOROPTIONS.update({'start': True}) %} + {% do CURATOROPTIONS.update({'status': 'running'}) %} +{% endif %} diff --git a/salt/top.sls b/salt/top.sls index 24be8283c..86a2f19c1 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -11,7 +11,6 @@ {% set FILEBEAT = salt['pillar.get']('filebeat:enabled', True) %} {% set KIBANA = salt['pillar.get']('kibana:enabled', True) %} {% set LOGSTASH = salt['pillar.get']('logstash:enabled', True) %} -{% set CURATOR = salt['pillar.get']('curator:enabled', True) %} {% set REDIS = salt['pillar.get']('redis:enabled', True) %} {% set STRELKA = salt['pillar.get']('strelka:enabled', '0') %} {% import_yaml 'salt/minion.defaults.yaml' as saltversion %} @@ -127,9 +126,7 @@ base: {%- if FILEBEAT %} - filebeat {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if ELASTALERT %} - elastalert {%- endif %} @@ -265,9 +262,7 @@ base: {%- if FILEBEAT %} - filebeat {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if ELASTALERT %} - elastalert {%- endif %} @@ -310,9 +305,7 @@ base: {%- if LOGSTASH %} - logstash {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if FILEBEAT %} - filebeat {%- endif %} @@ -354,9 +347,7 @@ base: {%- if REDIS %} - redis {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if KIBANA %} - kibana {%- endif %} @@ -409,9 +400,7 @@ base: {%- if REDIS %} - redis {%- endif %} - {%- if CURATOR %} - curator - {%- endif %} {%- if FILEBEAT %} - filebeat {%- endif %} From 771688a70fed289338720f7fa0efd64c8ec828f4 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 5 Aug 2021 07:34:07 -0400 Subject: [PATCH 47/57] fix typo --- salt/common/tools/sbin/so-tcpreplay | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index 3c1ce8fb9..e85474c67 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -31,7 +31,7 @@ if [[ $# -lt 1 ]]; then echo "Usage: $0 " echo echo "All PCAPs must be placed in the /opt/so/samples directory unless replaying" - echo "a sample pcap that is included in the so-tcpreplay image. Those PCAP sampes" + echo "a sample pcap that is included in the so-tcpreplay image. Those PCAP samples" echo "are located in the /opt/samples directory inside of the image." echo echo "Customer provided PCAP example:" From 727a3742f53231c94ebdf63ebf45a0a052f79a95 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 5 Aug 2021 09:50:51 -0400 Subject: [PATCH 48/57] run only on manager if truecluster enabled --- salt/allowed_states.map.jinja | 4 ++-- salt/curator/init.sls | 2 +- salt/curator/map.jinja | 4 ++-- salt/top.sls | 1 + 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 665fdbe3d..26a3ee84b 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -237,7 +237,7 @@ {% do allowed_states.append('kibana') %} {% endif %} - {% if CURATOR and grains.role in ['so-eval', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode'] %} + {% if CURATOR and grains.role in ['so-eval', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-manager'] %} {% do allowed_states.append('curator') %} {% endif %} @@ -296,4 +296,4 @@ {% endif %} {# all nodes can always run salt.minion state #} -{% do allowed_states.append('salt.minion') %} \ No newline at end of file +{% do allowed_states.append('salt.minion') %} diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 24d9f8c0d..fe3204105 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -4,7 +4,7 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone'] %} +{% if grains['role'] in ['so-eval', 'so-node', 'so-managersearch', 'so-heavynode', 'so-standalone', 'so-manager'] %} {% from 'elasticsearch/auth.map.jinja' import ELASTICAUTH with context %} {% from "curator/map.jinja" import CURATOROPTIONS with context %} # Curator diff --git a/salt/curator/map.jinja b/salt/curator/map.jinja index 2dc2c7311..e67587cae 100644 --- a/salt/curator/map.jinja +++ b/salt/curator/map.jinja @@ -2,8 +2,8 @@ {% set ENABLED = salt['pillar.get']('curator:enabled', True) %} {% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %} -# don't start the docker container if searchnode and true clustering is enabled or disabled via pillar -{% if (grains.id.split('_')|last == 'searchnode' and TRUECLUSTER) or not ENABLED %} +# don't start the docker container if searchnode and true clustering is enabled or curator disabled via pillar or true cluster not enabled and manager +{% if not ENABLED or (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %} {% do CURATOROPTIONS.update({'start': False}) %} {% do CURATOROPTIONS.update({'status': 'absent'}) %} {% else %} diff --git a/salt/top.sls b/salt/top.sls index 86a2f19c1..0b5322bcc 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -188,6 +188,7 @@ base: {%- if KIBANA %} - kibana {%- endif %} + - curator {%- if ELASTALERT %} - elastalert {%- endif %} From 853a986082e45cd0df66dd6afb85c980bbce3b23 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 5 Aug 2021 10:36:18 -0400 Subject: [PATCH 49/57] add reqs to docker add manager to so-curator-closed-delete-delte --- salt/curator/files/bin/so-curator-closed-delete-delete | 2 +- salt/curator/init.sls | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index 7dd7b82e7..9730b72fb 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -4,7 +4,7 @@ {%- if grains['role'] in ['so-node', 'so-heavynode'] %} {%- set ELASTICSEARCH_HOST = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- set ELASTICSEARCH_PORT = salt['pillar.get']('elasticsearch:es_port', '') -%} -{%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %} +{%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %} {%- set ELASTICSEARCH_HOST = salt['pillar.get']('manager:mainip', '') -%} {%- set ELASTICSEARCH_PORT = salt['pillar.get']('manager:es_port', '') -%} {%- endif -%} diff --git a/salt/curator/init.sls b/salt/curator/init.sls index fe3204105..35d5b2f1b 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -132,6 +132,10 @@ so-curator: - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro - /opt/so/conf/curator/action/:/etc/curator/action:ro - /opt/so/log/curator:/var/log/curator:rw + - require: + - file: actionconfs + - file: curconf + - file: curlogdir {% else %} - force: True {% endif %} From dd1769fbefd60e862de82a5400f7b9b0bd69f4ea Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 5 Aug 2021 11:02:09 -0400 Subject: [PATCH 50/57] Only check for logscan on manager-type and import --- salt/filebeat/etc/filebeat.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 3c482e274..0c27e3c1b 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -112,6 +112,7 @@ filebeat.inputs: fields: ["source", "prospector", "input", "offset", "beat"] fields_under_root: true +{%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager', 'so-managersearch', 'so-import'] %} - type: log paths: - /logs/logscan/alerts.log @@ -124,6 +125,7 @@ filebeat.inputs: fields_under_root: true clean_removed: true close_removed: false +{%- endif %} {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} {%- if ZEEKVER != 'SURICATA' %} From ac4faf673d15492eccedf3a97fab9dedce335ead Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 5 Aug 2021 11:11:59 -0400 Subject: [PATCH 51/57] add so-manager to curator.yml --- salt/curator/files/curator.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/curator.yml b/salt/curator/files/curator.yml index 956f60491..15ecb8cb1 100644 --- a/salt/curator/files/curator.yml +++ b/salt/curator/files/curator.yml @@ -1,6 +1,6 @@ {% if grains['role'] in ['so-node', 'so-heavynode'] %} {%- set elasticsearch = salt['pillar.get']('elasticsearch:mainip', '') -%} -{% elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %} +{% elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %} {%- set elasticsearch = salt['pillar.get']('manager:mainip', '') -%} {%- endif %} {%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} From e72ad9eb5a7f113e68182b80bb8f856357cc7942 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 5 Aug 2021 11:54:49 -0400 Subject: [PATCH 52/57] allow curator --- salt/allowed_states.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 26a3ee84b..0c9de34b0 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -237,7 +237,7 @@ {% do allowed_states.append('kibana') %} {% endif %} - {% if CURATOR and grains.role in ['so-eval', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-manager'] %} + {% if grains.role in ['so-eval', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-manager'] %} {% do allowed_states.append('curator') %} {% endif %} From 3b01f6431e152d33349ae63645fe69dcc8ffd250 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 6 Aug 2021 09:43:58 -0400 Subject: [PATCH 53/57] Add logscan to logrotate config --- salt/common/files/log-rotate.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/files/log-rotate.conf b/salt/common/files/log-rotate.conf index 061b76271..35c6fd724 100644 --- a/salt/common/files/log-rotate.conf +++ b/salt/common/files/log-rotate.conf @@ -22,6 +22,7 @@ /opt/so/log/salt/so-salt-minion-check /opt/so/log/salt/minion /opt/so/log/salt/master +/opt/so/log/logscan/*.log { {{ logrotate_conf | indent(width=4) }} } From ad3b6cf6297cfc8fa99a761c2288f58ae96e6245 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 9 Aug 2021 13:33:21 -0400 Subject: [PATCH 54/57] remove old dashboard dirs - https://github.com/Security-Onion-Solutions/securityonion/issues/4674 --- salt/grafana/init.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index dbb659217..3c98d6370 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -44,6 +44,12 @@ grafanadashdir: - group: 939 - makedirs: True +{% for type in ['eval','manager','managersearch','search_nodes','sensor_nodes','standalone'] %} +remove_dashboard_dir_{{type}}: + file.absent: + - name: /opt/so/conf/grafana/grafana_dashboards/{{type}} +{% endfor %} + grafana-dashboard-config: file.managed: - name: /opt/so/conf/grafana/etc/dashboards/dashboard.yml From 1415de858c806c5b9924fab98381453751c680de Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Aug 2021 10:16:14 -0400 Subject: [PATCH 55/57] delete old dashboard folders via api - https://github.com/Security-Onion-Solutions/securityonion/issues/4674 --- .../sbin/so-grafana-dashboard-folder-delete | 17 +++++++++++++++++ salt/grafana/init.sls | 5 +++++ 2 files changed, 22 insertions(+) create mode 100644 salt/common/tools/sbin/so-grafana-dashboard-folder-delete diff --git a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete new file mode 100644 index 000000000..60a998290 --- /dev/null +++ b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete @@ -0,0 +1,17 @@ +# this script is used to delete the default Grafana dashboard folders that existed prior to Grafana dashboard and Salt management changes in 2.3.70 + +folders=$(curl -X GET http://admin:{{salt['pillar.get'('secrets:grafana_admin')]}}@localhost:3000/api/folders | jq -r '.[] | @base64') +delfolder=("Manager" "Manager Search" "Sensor Nodes" "Search Nodes" "Standalone" "Eval Mode") + +for row in $folders; do + title=$(echo ${row} | base64 --decode | jq -r '.title') + uid=$(echo ${row} | base64 --decode | jq -r '.uid') + + if [[ " ${delfolder[@]} " =~ " ${title} " ]]; then + curl -X DELETE http://admin:{{salt['pillar.get'('secrets:grafana_admin')]}}@localhost:3000/api/folders/$uid + fi +done + +echo "so-grafana-dashboard-folder-delete has been run to delete default Grafana dashboard folders that existed prior to 2.3.70" > /opt/so/state/so-grafana-dashboard-folder-delete-complete + +exit 0 diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index 3c98d6370..1c1bdfb2e 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -88,6 +88,11 @@ grafana-config-files: - source: salt://grafana/etc/files - makedirs: True +so-grafana-dashboard-folder-delete: + cmd.run: + - name: /usr/sbin/so-grafana-dashboard-folder-delete + - unless: ls /opt/so/state/so-grafana-dashboard-folder-delete-complete + {% for dashboard in DASHBOARDS %} {{dashboard}}-dashboard: file.managed: From 3c1114403ee2b29afe8da8eeb62b09fd3eef4ac2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Aug 2021 10:25:05 -0400 Subject: [PATCH 56/57] fix the pillar.get --- salt/common/tools/sbin/so-grafana-dashboard-folder-delete | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete index 60a998290..3d7a2cfd3 100644 --- a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete +++ b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete @@ -1,6 +1,6 @@ # this script is used to delete the default Grafana dashboard folders that existed prior to Grafana dashboard and Salt management changes in 2.3.70 -folders=$(curl -X GET http://admin:{{salt['pillar.get'('secrets:grafana_admin')]}}@localhost:3000/api/folders | jq -r '.[] | @base64') +folders=$(curl -X GET http://admin:{{salt['pillar.get']('secrets:grafana_admin')]}}@localhost:3000/api/folders | jq -r '.[] | @base64') delfolder=("Manager" "Manager Search" "Sensor Nodes" "Search Nodes" "Standalone" "Eval Mode") for row in $folders; do @@ -8,7 +8,7 @@ for row in $folders; do uid=$(echo ${row} | base64 --decode | jq -r '.uid') if [[ " ${delfolder[@]} " =~ " ${title} " ]]; then - curl -X DELETE http://admin:{{salt['pillar.get'('secrets:grafana_admin')]}}@localhost:3000/api/folders/$uid + curl -X DELETE http://admin:{{salt['pillar.get']('secrets:grafana_admin')]}}@localhost:3000/api/folders/$uid fi done From 9e48a5b57b9000f89e74410fcf278e61ea8be3d0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Aug 2021 10:29:29 -0400 Subject: [PATCH 57/57] fix the pillar.get --- salt/common/tools/sbin/so-grafana-dashboard-folder-delete | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete index 3d7a2cfd3..f3338de84 100644 --- a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete +++ b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete @@ -1,6 +1,6 @@ # this script is used to delete the default Grafana dashboard folders that existed prior to Grafana dashboard and Salt management changes in 2.3.70 -folders=$(curl -X GET http://admin:{{salt['pillar.get']('secrets:grafana_admin')]}}@localhost:3000/api/folders | jq -r '.[] | @base64') +folders=$(curl -X GET http://admin:{{salt['pillar.get']('secrets:grafana_admin')}}@localhost:3000/api/folders | jq -r '.[] | @base64') delfolder=("Manager" "Manager Search" "Sensor Nodes" "Search Nodes" "Standalone" "Eval Mode") for row in $folders; do @@ -8,7 +8,7 @@ for row in $folders; do uid=$(echo ${row} | base64 --decode | jq -r '.uid') if [[ " ${delfolder[@]} " =~ " ${title} " ]]; then - curl -X DELETE http://admin:{{salt['pillar.get']('secrets:grafana_admin')]}}@localhost:3000/api/folders/$uid + curl -X DELETE http://admin:{{salt['pillar.get']('secrets:grafana_admin')}}@localhost:3000/api/folders/$uid fi done