From 80525ee73688b1e2f4ce414e04493798bdf6ace3 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 8 Jul 2021 12:29:50 -0400 Subject: [PATCH 01/60] [wip] Add logscan pipeline --- pillar/logstash/search.sls | 1 + salt/elasticsearch/files/ingest/logscan | 21 +++++++++++++++ salt/filebeat/etc/filebeat.yml | 13 +++++++++ .../config/so/9800_output_logscan.conf.jinja | 27 +++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/logscan create mode 100644 salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 10fab2ed1..2f287f533 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -13,3 +13,4 @@ logstash: - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja + - so/9800_ouput_logscan.conf.jinja diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan new file mode 100644 index 000000000..936d316f7 --- /dev/null +++ b/salt/elasticsearch/files/ingest/logscan @@ -0,0 +1,21 @@ +{ + "description": "logscan", + "processors": [ + { "set": { "target_field": "event.severity", "value": 2 } }, + { "rename": { "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, + { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "rename": { "field": "source_ip", "target_field": "source.ip" } }, + { "set": { "if": "model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, + { "set": { "if": "model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, + { "set": { "if": "model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, + { "set": { "if": "model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, + { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, + { "script": { "lang": "painless", "source": "logscan.attempts.succeeded.amount = logscan.attempts.total.amount - logscan.attempts.failed.amount" , "ignore_failure": true} }, + { "rename": { "field": "avg_failure_interval", "target_field": "logscan.attempts.failed.avg_interval", "ignore_missing": true } }, + { "pipeline": { "name": "common" } } + ] +} diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 2a86b486f..751186119 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -111,6 +111,19 @@ filebeat.inputs: fields: ["source", "prospector", "input", "offset", "beat"] fields_under_root: true +- type: log + paths: + - /opt/so/log/logscan/alerts.log + fields: + module: logscan + dataset: alert + processors: + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + clean_removed: true + close_removed: false + {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} {%- if ZEEKVER != 'SURICATA' %} {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja new file mode 100644 index 000000000..6fcf05a70 --- /dev/null +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -0,0 +1,27 @@ +{%- if grains['role'] == 'so-eval' -%} +{%- set ES = salt['pillar.get']('manager:mainip', '') -%} +{%- else %} +{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} +{%- endif %} +{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} +{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} + +output { + if [module] =~ logscan { + elasticsearch { + id => "logscan_pipeline" + pipeline => "%{module}" + hosts => "{{ ES }}" + {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} + user => "{{ ES_USER }}" + password => "{{ ES_PASS }}" + {% endif %} + index => "so-%{[event][module]}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } +} From bac7ef71d826b3c786b239e2a1c289f114105071 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 9 Jul 2021 10:55:11 -0400 Subject: [PATCH 02/60] Add logscan.source.ips field --- salt/elasticsearch/files/ingest/logscan | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index 936d316f7..68c6aae44 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -8,6 +8,8 @@ { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, { "rename": { "field": "source_ip", "target_field": "source.ip" } }, + { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}" } }, + { "rename": { "field": "source_ips", "target_field": "logscan.source.ips" } }, { "set": { "if": "model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, { "set": { "if": "model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, { "set": { "if": "model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, From e059c25ebce24e8f9bf5abe8052d02dd39ecb4ea Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 13 Jul 2021 11:05:05 -0400 Subject: [PATCH 03/60] [fix][wip] Fix pipeline parsing errors --- salt/elasticsearch/files/ingest/logscan | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index 68c6aae44..6cd32dcbc 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -1,22 +1,23 @@ { "description": "logscan", "processors": [ - { "set": { "target_field": "event.severity", "value": 2 } }, + { "set": { "field": "event.severity", "value": 2 } }, { "rename": { "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, - { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, - { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, - { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, - { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failures": true } }, + { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, + { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, { "rename": { "field": "source_ip", "target_field": "source.ip" } }, { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}" } }, { "rename": { "field": "source_ips", "target_field": "logscan.source.ips" } }, - { "set": { "if": "model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, - { "set": { "if": "model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, - { "set": { "if": "model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, - { "set": { "if": "model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "set": { "if": "ctx.model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, + { "set": { "if": "ctx.model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "rename": { "field": "model", "target_field": "logscan.model" } } { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, - { "script": { "lang": "painless", "source": "logscan.attempts.succeeded.amount = logscan.attempts.total.amount - logscan.attempts.failed.amount" , "ignore_failure": true} }, + { "script": { "lang": "painless", "source": "ctx.logscan.attempts.succeeded.amount = ctx.logscan.attempts.total.amount - ctx.logscan.attempts.failed.amount" , "ignore_failure": true} }, { "rename": { "field": "avg_failure_interval", "target_field": "logscan.attempts.failed.avg_interval", "ignore_missing": true } }, { "pipeline": { "name": "common" } } ] From 115e0a6fee52cf9a789209d0668b97050f2e72bf Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 13 Jul 2021 12:04:10 -0400 Subject: [PATCH 04/60] [fix] Add missing comma --- salt/elasticsearch/files/ingest/logscan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index 6cd32dcbc..d199161a5 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -14,7 +14,7 @@ { "set": { "if": "ctx.model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, { "set": { "if": "ctx.model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, { "set": { "if": "ctx.model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, - { "rename": { "field": "model", "target_field": "logscan.model" } } + { "rename": { "field": "model", "target_field": "logscan.model" } }, { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.logscan.attempts.succeeded.amount = ctx.logscan.attempts.total.amount - ctx.logscan.attempts.failed.amount" , "ignore_failure": true} }, From e7a6172d7e439d2d2fe4adfe951e8b84e2d23720 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 13 Jul 2021 14:07:27 -0400 Subject: [PATCH 05/60] [fix] Add single quotes to strings --- salt/elasticsearch/files/ingest/logscan | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index d199161a5..9a094f4c6 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -10,10 +10,10 @@ { "rename": { "field": "source_ip", "target_field": "source.ip" } }, { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}" } }, { "rename": { "field": "source_ips", "target_field": "logscan.source.ips" } }, - { "set": { "if": "ctx.model == kff", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, - { "set": { "if": "ctx.model == kff", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, - { "set": { "if": "ctx.model == kl", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, - { "set": { "if": "ctx.model == kl", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "set": { "if": "ctx.model == 'kff'", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'kff'", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, + { "set": { "if": "ctx.model == 'kl'", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'kl'", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, { "rename": { "field": "model", "target_field": "logscan.model" } }, { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, From e41811fbd01824999ff5012e13ec2a9dc8dba1b0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 13 Jul 2021 15:14:13 -0400 Subject: [PATCH 06/60] [fix] Typo --- pillar/logstash/search.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 2f287f533..55b2070ce 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -13,4 +13,4 @@ logstash: - so/9500_output_beats.conf.jinja - so/9600_output_ossec.conf.jinja - so/9700_output_strelka.conf.jinja - - so/9800_ouput_logscan.conf.jinja + - so/9800_output_logscan.conf.jinja From 818f912a905a8d15cfab13c9dfe22800fe7237cf Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 14 Jul 2021 10:13:14 -0400 Subject: [PATCH 07/60] [fix] Remove indent --- salt/filebeat/etc/filebeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 39257c991..68fc32d7c 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -119,7 +119,7 @@ filebeat.inputs: module: logscan dataset: alert processors: - - drop_fields: + - drop_fields: fields: ["source", "prospector", "input", "offset", "beat"] fields_under_root: true clean_removed: true From e8ba4bdc6cb37fbaeeeef75d5407230d782bd045 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 16 Jul 2021 14:07:23 -0400 Subject: [PATCH 08/60] Add quotes to string --- .../logstash/pipelines/config/so/9800_output_logscan.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja index 6fcf05a70..a0e5bd570 100644 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -7,7 +7,7 @@ {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} output { - if [module] =~ logscan { + if [module] =~ "logscan" { elasticsearch { id => "logscan_pipeline" pipeline => "%{module}" From 9bf1d3e0c6321c243e44d6b883e41060b6b500dd Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 16 Jul 2021 14:59:44 -0400 Subject: [PATCH 09/60] Misc fixes --- salt/elasticsearch/files/ingest/logscan | 9 ++++++--- salt/filebeat/etc/filebeat.yml | 2 +- .../pipelines/config/so/9800_output_logscan.conf.jinja | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan index 9a094f4c6..3a3debabc 100644 --- a/salt/elasticsearch/files/ingest/logscan +++ b/salt/elasticsearch/files/ingest/logscan @@ -2,14 +2,17 @@ "description": "logscan", "processors": [ { "set": { "field": "event.severity", "value": 2 } }, + { "json": { "field": "message", "add_to_root": true, "ignore_failure": true } }, { "rename": { "field": "@timestamp", "target_field": "event.ingested", "ignore_missing": true } }, { "date": { "field": "timestamp", "target_field": "event.created", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, { "date": { "field": "start_time", "target_field": "@timestamp", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, { "date": { "field": "start_time", "target_field": "event.start", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, { "date": { "field": "end_time", "target_field": "event.end", "formats": [ "ISO8601", "UNIX" ], "ignore_failure": true } }, - { "rename": { "field": "source_ip", "target_field": "source.ip" } }, - { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}" } }, - { "rename": { "field": "source_ips", "target_field": "logscan.source.ips" } }, + { "remove": { "field": "start_time", "ignore_missing": true } }, + { "remove": { "field": "end_time", "ignore_missing": true } }, + { "rename": { "field": "source_ip", "target_field": "source.ip", "ignore_missing": true } }, + { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, + { "rename": { "field": "source_ips", "target_field": "logscan.source.ips", "ignore_missing": true } }, { "set": { "if": "ctx.model == 'kff'", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, { "set": { "if": "ctx.model == 'kff'", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, { "set": { "if": "ctx.model == 'kl'", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 68fc32d7c..f5ba5dc74 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -114,7 +114,7 @@ filebeat.inputs: - type: log paths: - - /opt/so/log/logscan/alerts.log + - /logs/logscan/alerts.log fields: module: logscan dataset: alert diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja index a0e5bd570..a1c621484 100644 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -16,7 +16,7 @@ output { user => "{{ ES_USER }}" password => "{{ ES_PASS }}" {% endif %} - index => "so-%{[event][module]}" + index => "so-logscan" template_name => "so-common" template => "/templates/so-common-template.json" template_overwrite => true From 09165daab842dc9481f23ded252e8cefa48ef68d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 21 Jul 2021 09:10:33 -0400 Subject: [PATCH 10/60] Several Suricata things --- HOTFIX | 2 +- salt/elasticsearch/files/ingest/suricata.fileinfo | 1 + salt/suricata/suricata_meta.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/HOTFIX b/HOTFIX index 31f81a97c..d3f5a12fa 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -ECSFIX HEAVYNODE_SSL_LOGSTASH_REDIS_PIPELINES FBPIPELINE CURATORAUTH + diff --git a/salt/elasticsearch/files/ingest/suricata.fileinfo b/salt/elasticsearch/files/ingest/suricata.fileinfo index d5147fb40..fe9e4b109 100644 --- a/salt/elasticsearch/files/ingest/suricata.fileinfo +++ b/salt/elasticsearch/files/ingest/suricata.fileinfo @@ -13,6 +13,7 @@ { "rename": { "field": "message2.fileinfo.size", "target_field": "file.size", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.state", "target_field": "file.state", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.stored", "target_field": "file.saved", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.sha256", "target_field": "hash.sha256", "ignore_missing": true } }, { "set": { "if": "ctx.network?.protocol != null", "field": "file.source", "value": "{{network.protocol}}" } }, { "pipeline": { "name": "common" } } ] diff --git a/salt/suricata/suricata_meta.yaml b/salt/suricata/suricata_meta.yaml index 90b220000..1c3855501 100644 --- a/salt/suricata/suricata_meta.yaml +++ b/salt/suricata/suricata_meta.yaml @@ -7,7 +7,7 @@ suricata: dir: /nsm/extracted #write-fileinfo: "yes" #force-filestore: "yes" - #stream-depth: 0 + stream-depth: 0 #max-open-files: 1000 #force-hash: [sha1, md5] xff: From b8e3a45a7efcb727676bb2706fa64cc52fc6901a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 23 Jul 2021 08:53:45 -0400 Subject: [PATCH 11/60] [wip] Add logscan state Do not add state to top file or setup yet, script will be written to enable the feature shortly --- salt/allowed_states.map.jinja | 20 ++++++++---- salt/logscan/files/logscan.conf | 6 ++++ salt/logscan/init.sls | 57 +++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 salt/logscan/files/logscan.conf create mode 100644 salt/logscan/init.sls diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 665fdbe3d..12d8f99ec 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -45,7 +45,8 @@ 'schedule', 'soctopus', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-heavynode': [ 'ca', @@ -75,7 +76,8 @@ 'logstash', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-fleet': [ 'ca', @@ -108,7 +110,8 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-manager': [ 'salt.master', @@ -127,7 +130,8 @@ 'utility', 'schedule', 'soctopus', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-managersearch': [ 'salt.master', @@ -146,7 +150,8 @@ 'utility', 'schedule', 'soctopus', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-node': [ 'ca', @@ -178,7 +183,8 @@ 'schedule', 'soctopus', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'logscsan' ], 'so-sensor': [ 'ca', @@ -296,4 +302,4 @@ {% endif %} {# all nodes can always run salt.minion state #} -{% do allowed_states.append('salt.minion') %} \ No newline at end of file +{% do allowed_states.append('salt.minion') %} diff --git a/salt/logscan/files/logscan.conf b/salt/logscan/files/logscan.conf new file mode 100644 index 000000000..9b3df8027 --- /dev/null +++ b/salt/logscan/files/logscan.conf @@ -0,0 +1,6 @@ +[global] +ts_format = iso8601 +scan_interval = 30s + +[kratos] +log_path = kratos/kratos.log diff --git a/salt/logscan/init.sls b/salt/logscan/init.sls new file mode 100644 index 000000000..51badbcdd --- /dev/null +++ b/salt/logscan/init.sls @@ -0,0 +1,57 @@ +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} + +{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} +{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{% set MANAGER = salt['grains.get']('master') %} +{% set logscan_cpu_period = salt['pillar.get']('logscan:cpu_period', 10000) %} + +logscan_data_dir: + file.directory: + - name: /nsm/logscan/data + - user: 939 + - group: 939 + - makedirs: True + +logscan_conf_dir: + file.directory: + - name: /opt/so/conf/logscan + - user: 939 + - group: 939 + - makedirs: True + +logscan_conf: + file.managed: + - name: /opt/so/conf/logscan/logscan.conf + - source: salt://logscan/files/logscan.conf + - user: 939 + - group: 939 + - mode: 600 + - template: jinja + +logscan_log_dir: + file.directory: + - name: /opt/so/log/logscan + - user: 939 + - group: 939 + +so-logscan: + docker_container.running: + - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logscan:{{ VERSION }} + - hostname: logscan + - name: so-logscan + - binds: + - /nsm/logscan/data:/logscan/data:rw + - /opt/so/conf/logscan.conf:/logscan/logscan.conf:ro + - /opt/so/log/logscan:/logscan/output:rw + - /opt/so/log:/logscan/logs:ro + - cpu_period: {{ logscan_cpu_period }} + + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} From 64945cec16b6610db863e55c031bf686ef4466d5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 26 Jul 2021 14:24:10 -0400 Subject: [PATCH 12/60] [wip] Initial work to enable/disable "learn" modules --- salt/common/tools/sbin/so-learn | 229 +++++++++++++++++++ salt/common/tools/sbin/so-rule | 4 +- salt/{logscan => learn}/files/logscan.conf | 0 salt/learn/init.sls | 19 ++ salt/{logscan/init.sls => learn/logscan.sls} | 12 - salt/top.sls | 6 +- setup/so-setup | 6 + 7 files changed, 260 insertions(+), 16 deletions(-) create mode 100644 salt/common/tools/sbin/so-learn rename salt/{logscan => learn}/files/logscan.conf (100%) create mode 100644 salt/learn/init.sls rename salt/{logscan/init.sls => learn/logscan.sls} (84%) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn new file mode 100644 index 000000000..8b5c2473f --- /dev/null +++ b/salt/common/tools/sbin/so-learn @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import signal +import sys +import os +import re +import subprocess +import argparse +import textwrap +from typing import List + +import yaml + +minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' +salt_proc = subprocess.CompletedProcess = None + +# Temp store of modules, will likely be broken out into salt +learn_modules = [ + { 'name': 'logscan', 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } +] + + +def sigint_handler(*_): + print('Exiting gracefully on Ctrl-C') + if salt_proc is not None: salt_proc.send_signal(signal.SIGINT) + sys.exit(0) + + +def find_minion_pillar() -> str: + regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$' + + result = [] + for root, _, files in os.walk(minion_pillar_dir): + for f_minion_id in files: + if re.search(regex, f_minion_id): + result.append(os.path.join(root, f_minion_id)) + + if len(result) == 0: + print('Could not find manager-type pillar (eval, standalone, manager, managersearch, import). Are you running this script on the manager?', file=sys.stderr) + sys.exit(3) + elif len(result) > 1: + res_str = ', '.join(f'\"{result}\"') + print('(This should not happen, the system is in an error state if you see this message.)\n', file=sys.stderr) + print('More than one manager-type pillar exists, minion id\'s listed below:', file=sys.stderr) + print(f' {res_str}', file=sys.stderr) + sys.exit(3) + else: + return result[0] + + +def read_pillar(pillar: str): + try: + with open(pillar, 'r') as f: + loaded_yaml = yaml.safe_load(f.read()) + if loaded_yaml is None: + print(f'Could not parse {pillar}', file=sys.stderr) + sys.exit(3) + return loaded_yaml + except: + print(f'Could not open {pillar}', file=sys.stderr) + sys.exit(3) + + +def write_pillar(pillar: str, content: dict): + try: + with open(pillar, 'w') as f: + return yaml.dump(content, f, default_flow_style=False) + except: + print(f'Could not open {pillar}', file=sys.stderr) + sys.exit(3) + + +def create_pillar_if_not_exist(pillar:str, content: dict): + if content.get('learn', {}).get('modules') is None: + content['learn']['modules'] = learn_modules + write_pillar(pillar, content) + + +def apply(module_list: List): + return_code = 0 + for module in module_list: + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True'] + print(f'Applying salt state for {module} module...') + cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + if cmd.returncode != 0: + print(f'[ERROR] Failed to apply salt state for {module}') + return_code = cmd.returncode + return return_code + + +def check_apply(args: dict): + if args.apply: + print('Configuration updated. Applying changes:') + return apply(args.modules) + else: + if not hasattr(args, 'yes'): + message = 'Configuration updated. Would you like to apply your changes now? (y/N) ' + answer = input(message) + while answer.lower() not in [ 'y', 'n', '' ]: + answer = input(message) + if answer.lower() in [ 'n', '' ]: + return 0 + else: + print('Applying changes:') + return apply(args.modules) + else: + return 0 + + +def enable_disable_modules(args, enable: bool): + pillar_modules = args.pillar_dict.get('learn', {}).get('modules') + + if 'all' in args.modules: + for module in pillar_modules: + module['enabled'] = enable + args.pillar_dict.update() + write_pillar(args.pillar, args.pillar_dict) + else: + write_needed = False + for module in args.modules: + if module in pillar_modules: + pillar_modules[module]['enabled'] = enable + write_needed = enable + if write_needed: + args.pillar_dict.update() + write_pillar(args.pillarm, args.pillar_dict) + + check_apply(args) + + +def enable_modules(args): + enable_disable_modules(args, enable=True) + + +def disable_modules(args): + enable_disable_modules(args, enable=False) + + +def list_modules(*_): + print('Available ML modules:') + for module in learn_modules: + + print(f' - { module["name"] } : {module["description"]}') + + +def main(): + beta_str = 'BETA - SUBJECT TO CHANGE\n' + + apply_help='After ACTION the chosen modules, apply any necessary salt states.' + enable_apply_help = apply_help.replace('ACTION', 'enabling') + disable_apply_help = apply_help.replace('ACTION', 'disabling') + + yes_help = 'Accept apply prompt.' + + signal.signal(signal.SIGINT, sigint_handler) + + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) + + main_parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter) + + subcommand_desc = textwrap.dedent( + """\ + enable Enable one or more ML modules. + disable Disable one or more ML modules. + list List all available ML modules. + """ + ) + + subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='', dest='command') + + module_help_str = 'One or more ML modules, which can be listed using \'so-learn list\'. Use the keyword \'all\' to apply the action to all available modules.' + + enable = subparsers.add_parser('enable') + enable.set_defaults(func=enable_modules) + enable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) + enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help) + enable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) + + + disable = subparsers.add_parser('disable') + disable.set_defaults(func=disable_modules) + disable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) + disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help) + disable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) + + + + list = subparsers.add_parser('list') + list.set_defaults(func=list_modules) + + args = main_parser.parse_args(sys.argv[1:]) + + # args.pillar = find_minion_pillar() + + # args.pillar_dict = read_pillar(args.pillar) + + # create_pillar_if_not_exist(args.pillar, args.pillar_dict) + + if hasattr(args, 'func'): + exit_code = args.func(args) + else: + if args.command is None: + print(beta_str) + main_parser.print_help() + sys.exit(0) + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index 07d6cd2cc..bbb916807 100755 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -349,9 +349,7 @@ def sigint_handler(*_): def main(): signal.signal(signal.SIGINT, sigint_handler) - if os.geteuid() != 0: - print_err('You must run this script as root') - sys.exit(1) + apply_help='After updating rule configuration, apply the idstools state.' diff --git a/salt/logscan/files/logscan.conf b/salt/learn/files/logscan.conf similarity index 100% rename from salt/logscan/files/logscan.conf rename to salt/learn/files/logscan.conf diff --git a/salt/learn/init.sls b/salt/learn/init.sls new file mode 100644 index 000000000..581f3f073 --- /dev/null +++ b/salt/learn/init.sls @@ -0,0 +1,19 @@ +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} + +{% set module_list = salt['pillar.get']('learn:modules', [] ) %} + +{% if len(module_list) != 0 %}} +include: +{% for module in module_list %} + - .{{ module }} +{% endfor %} +{% endif %} + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} diff --git a/salt/logscan/init.sls b/salt/learn/logscan.sls similarity index 84% rename from salt/logscan/init.sls rename to salt/learn/logscan.sls index 51badbcdd..43318d81a 100644 --- a/salt/logscan/init.sls +++ b/salt/learn/logscan.sls @@ -1,6 +1,3 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} - {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} @@ -46,12 +43,3 @@ so-logscan: - /opt/so/log/logscan:/logscan/output:rw - /opt/so/log:/logscan/logs:ro - cpu_period: {{ logscan_cpu_period }} - - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/top.sls b/salt/top.sls index 24be8283c..32069ab66 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -156,6 +156,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_manager and G@saltversion:{{saltversion}}': - match: compound @@ -218,6 +219,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_standalone and G@saltversion:{{saltversion}}': - match: compound @@ -292,6 +294,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_searchnode and G@saltversion:{{saltversion}}': - match: compound @@ -366,7 +369,6 @@ base: {%- if FILEBEAT %} - filebeat {%- endif %} - - utility - schedule {%- if FLEETMANAGER or FLEETNODE %} @@ -388,6 +390,7 @@ base: {%- endif %} - docker_clean - pipeline.load + - learn '*_heavynode and G@saltversion:{{saltversion}}': - match: compound @@ -478,3 +481,4 @@ base: - schedule - docker_clean - pipeline.load + - learn diff --git a/setup/so-setup b/setup/so-setup index 68490657f..bfca9b2bd 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -962,6 +962,12 @@ else set_progress_str 99 'Waiting for TheHive to start up' check_hive_init >> $setup_log 2>&1 fi + + if [[ -n $LEARN_LOGSCAN_ENABLE ]]; then + set_progress_str 99 'Enabling logscan' + so-learn enable logscan --apply -y >> $setup_log 2>&1 + fi + } | whiptail_gauge_post_setup "Running post-installation steps..." whiptail_setup_complete From f31dc5abc7c46b8c076adabc0275cd1a504a0543 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 09:49:59 -0400 Subject: [PATCH 13/60] Add learn to allowed states --- salt/allowed_states.map.jinja | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 12d8f99ec..1aa79474e 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -46,7 +46,8 @@ 'soctopus', 'tcpreplay', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-heavynode': [ 'ca', @@ -111,7 +112,8 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-manager': [ 'salt.master', @@ -131,7 +133,8 @@ 'schedule', 'soctopus', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-managersearch': [ 'salt.master', @@ -151,7 +154,8 @@ 'schedule', 'soctopus', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-node': [ 'ca', @@ -184,7 +188,8 @@ 'soctopus', 'tcpreplay', 'docker_clean', - 'logscsan' + 'logscsan', + 'learn' ], 'so-sensor': [ 'ca', From 8d56fc71fa6b242ceca2720dce2160bb108e1780 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 09:53:24 -0400 Subject: [PATCH 14/60] Fix jinja length calculation --- salt/learn/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/learn/init.sls b/salt/learn/init.sls index 581f3f073..a511c1074 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -3,7 +3,7 @@ {% set module_list = salt['pillar.get']('learn:modules', [] ) %} -{% if len(module_list) != 0 %}} +{% if module_list|length != 0 %}} include: {% for module in module_list %} - .{{ module }} From 455719936bdbffaef594c9744ab3f5c59e746a3d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 09:53:35 -0400 Subject: [PATCH 15/60] Uncomment required lines in so-learn --- salt/common/tools/sbin/so-learn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 8b5c2473f..a033c7db5 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -208,11 +208,11 @@ def main(): args = main_parser.parse_args(sys.argv[1:]) - # args.pillar = find_minion_pillar() + args.pillar = find_minion_pillar() - # args.pillar_dict = read_pillar(args.pillar) + args.pillar_dict = read_pillar(args.pillar) - # create_pillar_if_not_exist(args.pillar, args.pillar_dict) + create_pillar_if_not_exist(args.pillar, args.pillar_dict) if hasattr(args, 'func'): exit_code = args.func(args) From e2abe8840f0b99c14910c512d6a95c96a0a55f8f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 10:12:19 -0400 Subject: [PATCH 16/60] Fix directory in logscan state --- salt/learn/logscan.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/learn/logscan.sls b/salt/learn/logscan.sls index 43318d81a..833c40bd2 100644 --- a/salt/learn/logscan.sls +++ b/salt/learn/logscan.sls @@ -20,7 +20,7 @@ logscan_conf_dir: logscan_conf: file.managed: - name: /opt/so/conf/logscan/logscan.conf - - source: salt://logscan/files/logscan.conf + - source: salt://learn/files/logscan.conf - user: 939 - group: 939 - mode: 600 From 91accb0bc666c179c014b7108a599184952c44a1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 10:12:32 -0400 Subject: [PATCH 17/60] [wip] Fixing so-learn script --- salt/common/tools/sbin/so-learn | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index a033c7db5..4734b3fad 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -88,8 +88,12 @@ def write_pillar(pillar: str, content: dict): def create_pillar_if_not_exist(pillar:str, content: dict): if content.get('learn', {}).get('modules') is None: + content['learn'] = {} content['learn']['modules'] = learn_modules write_pillar(pillar, content) + + content.update() + return content def apply(module_list: List): @@ -99,7 +103,7 @@ def apply(module_list: List): print(f'Applying salt state for {module} module...') cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) if cmd.returncode != 0: - print(f'[ERROR] Failed to apply salt state for {module}') + print(f'[ERROR] Failed to apply salt state for {module} module.') return_code = cmd.returncode return return_code @@ -135,13 +139,18 @@ def enable_disable_modules(args, enable: bool): write_needed = False for module in args.modules: if module in pillar_modules: - pillar_modules[module]['enabled'] = enable - write_needed = enable + if pillar_modules[module]['enabled'] == enable: + action_str = 'enabled' if enable else 'disabled' + print(f'{module} module already {action_str}.', file=sys.stderr) + else: + pillar_modules[module]['enabled'] = enable + write_needed = enable if write_needed: args.pillar_dict.update() write_pillar(args.pillarm, args.pillar_dict) - check_apply(args) + salt_proc = check_apply(args) + return salt_proc def enable_modules(args): @@ -155,8 +164,8 @@ def disable_modules(args): def list_modules(*_): print('Available ML modules:') for module in learn_modules: - print(f' - { module["name"] } : {module["description"]}') + return 0 def main(): @@ -194,25 +203,18 @@ def main(): enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help) enable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) - disable = subparsers.add_parser('disable') disable.set_defaults(func=disable_modules) disable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help) disable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) - - list = subparsers.add_parser('list') list.set_defaults(func=list_modules) args = main_parser.parse_args(sys.argv[1:]) - args.pillar = find_minion_pillar() - - args.pillar_dict = read_pillar(args.pillar) - - create_pillar_if_not_exist(args.pillar, args.pillar_dict) + args.pillar_dict = create_pillar_if_not_exist(args.pillar, read_pillar(args.pillar)) if hasattr(args, 'func'): exit_code = args.func(args) From cf9121dfc2015555ecaffdf9a8e1720b545f7b1e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 14:13:16 -0400 Subject: [PATCH 18/60] Actually download so-learn container --- salt/common/tools/sbin/so-image-common | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 9b6e2174a..ad16379bb 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -37,6 +37,7 @@ container_list() { "so-idstools" "so-kibana" "so-kratos" + "so-learn" "so-nginx" "so-pcaptools" "so-soc" @@ -58,6 +59,7 @@ container_list() { "so-influxdb" "so-kibana" "so-kratos" + "so-learn" "so-logstash" "so-mysql" "so-nginx" From 7ef5b39b048079d84fb29a930e21d937341b77da Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 14:28:00 -0400 Subject: [PATCH 19/60] [wip] Fix `'Nonetype' object is not callable` error --- salt/common/tools/sbin/so-learn | 54 +++++++++++++++------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 4734b3fad..85b70730d 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from typing import List import signal import sys @@ -23,12 +24,10 @@ import re import subprocess import argparse import textwrap -from typing import List - import yaml minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' -salt_proc = subprocess.CompletedProcess = None +salt_proc: subprocess.CompletedProcess = None # Temp store of modules, will likely be broken out into salt learn_modules = [ @@ -80,19 +79,21 @@ def read_pillar(pillar: str): def write_pillar(pillar: str, content: dict): try: with open(pillar, 'w') as f: - return yaml.dump(content, f, default_flow_style=False) + yaml.dump(content, f, default_flow_style=False, sort_keys=False) except: print(f'Could not open {pillar}', file=sys.stderr) sys.exit(3) def create_pillar_if_not_exist(pillar:str, content: dict): - if content.get('learn', {}).get('modules') is None: - content['learn'] = {} - content['learn']['modules'] = learn_modules - write_pillar(pillar, content) + pillar_dict = content + + if pillar_dict.get('learn', {}).get('modules') is None: + pillar_dict['learn'] = {} + pillar_dict['learn']['modules'] = learn_modules + content.update() + write_pillar(pillar, content) - content.update() return content @@ -100,11 +101,11 @@ def apply(module_list: List): return_code = 0 for module in module_list: salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True'] - print(f'Applying salt state for {module} module...') - cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) - if cmd.returncode != 0: - print(f'[ERROR] Failed to apply salt state for {module} module.') - return_code = cmd.returncode + print(f' Applying salt state for {module} module...') + salt_proc = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) + if salt_proc.returncode != 0: + print(f' [ERROR] Failed to apply salt state for {module} module.') + return_code = salt_proc.returncode return return_code @@ -113,18 +114,15 @@ def check_apply(args: dict): print('Configuration updated. Applying changes:') return apply(args.modules) else: - if not hasattr(args, 'yes'): - message = 'Configuration updated. Would you like to apply your changes now? (y/N) ' + message = 'Configuration updated. Would you like to apply your changes now? (y/N) ' + answer = input(message) + while answer.lower() not in [ 'y', 'n', '' ]: answer = input(message) - while answer.lower() not in [ 'y', 'n', '' ]: - answer = input(message) - if answer.lower() in [ 'n', '' ]: - return 0 - else: - print('Applying changes:') - return apply(args.modules) - else: + if answer.lower() in [ 'n', '' ]: return 0 + else: + print('Applying changes:') + return apply(args.modules) def enable_disable_modules(args, enable: bool): @@ -149,8 +147,8 @@ def enable_disable_modules(args, enable: bool): args.pillar_dict.update() write_pillar(args.pillarm, args.pillar_dict) - salt_proc = check_apply(args) - return salt_proc + cmd_ret = check_apply(args) + return cmd_ret def enable_modules(args): @@ -175,8 +173,6 @@ def main(): enable_apply_help = apply_help.replace('ACTION', 'enabling') disable_apply_help = apply_help.replace('ACTION', 'disabling') - yes_help = 'Accept apply prompt.' - signal.signal(signal.SIGINT, sigint_handler) if os.geteuid() != 0: @@ -201,13 +197,11 @@ def main(): enable.set_defaults(func=enable_modules) enable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help) - enable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) disable = subparsers.add_parser('disable') disable.set_defaults(func=disable_modules) disable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help) - disable.add_argument('--yes', '-y', action='store_const', const=True, required=False, help=yes_help) list = subparsers.add_parser('list') list.set_defaults(func=list_modules) From dd0e407935f7992dbcff32cf7ba4f914224e20f0 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 28 Jul 2021 15:06:38 -0400 Subject: [PATCH 20/60] Use correct container name --- salt/common/tools/sbin/so-image-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index ad16379bb..cea0f6507 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -37,7 +37,7 @@ container_list() { "so-idstools" "so-kibana" "so-kratos" - "so-learn" + "so-logscan" "so-nginx" "so-pcaptools" "so-soc" @@ -59,7 +59,7 @@ container_list() { "so-influxdb" "so-kibana" "so-kratos" - "so-learn" + "so-logscan" "so-logstash" "so-mysql" "so-nginx" From 5894b85bd1956f0656252e72a7049b23b733471b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 10:57:53 -0400 Subject: [PATCH 21/60] Remove broken yaml dump arg, rename metavars --- salt/common/tools/sbin/so-learn | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 85b70730d..53e2b5020 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -79,7 +79,7 @@ def read_pillar(pillar: str): def write_pillar(pillar: str, content: dict): try: with open(pillar, 'w') as f: - yaml.dump(content, f, default_flow_style=False, sort_keys=False) + yaml.dump(content, f, default_flow_style=False) except: print(f'Could not open {pillar}', file=sys.stderr) sys.exit(3) @@ -195,12 +195,12 @@ def main(): enable = subparsers.add_parser('enable') enable.set_defaults(func=enable_modules) - enable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) + enable.add_argument('modules', metavar='ML_MODULE', nargs='+', help=module_help_str) enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help) disable = subparsers.add_parser('disable') disable.set_defaults(func=disable_modules) - disable.add_argument('modules', metavar='ML_MODULES', nargs='+', help=module_help_str) + disable.add_argument('modules', metavar='ML_MODULE', nargs='+', help=module_help_str) disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help) list = subparsers.add_parser('list') From 211a841cdbd397b61fea9cc096fcaf1c25ea99ac Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 11:40:19 -0400 Subject: [PATCH 22/60] Fix file path in bind mount for logscan --- salt/learn/logscan.sls | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/learn/logscan.sls b/salt/learn/logscan.sls index 833c40bd2..1e21e3b2a 100644 --- a/salt/learn/logscan.sls +++ b/salt/learn/logscan.sls @@ -1,7 +1,7 @@ {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -{% set logscan_cpu_period = salt['pillar.get']('logscan:cpu_period', 10000) %} +{% set logscan_cpu_period = salt['pillar.get']('learn:modules:logscan:cpu_period', 20000) %} logscan_data_dir: file.directory: @@ -24,7 +24,6 @@ logscan_conf: - user: 939 - group: 939 - mode: 600 - - template: jinja logscan_log_dir: file.directory: @@ -39,7 +38,7 @@ so-logscan: - name: so-logscan - binds: - /nsm/logscan/data:/logscan/data:rw - - /opt/so/conf/logscan.conf:/logscan/logscan.conf:ro + - /opt/so/conf/logscan/logscan.conf:/logscan/logscan.conf:ro - /opt/so/log/logscan:/logscan/output:rw - /opt/so/log:/logscan/logs:ro - cpu_period: {{ logscan_cpu_period }} From d53e989c55b5a222fb87fb88de00fb1c61f7ceab Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 11:52:10 -0400 Subject: [PATCH 23/60] Add ability to set cpu_period per module --- salt/common/tools/sbin/so-learn | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 53e2b5020..39004b052 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -25,14 +25,28 @@ import subprocess import argparse import textwrap import yaml +import multiprocessing minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' salt_proc: subprocess.CompletedProcess = None # Temp store of modules, will likely be broken out into salt -learn_modules = [ - { 'name': 'logscan', 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } -] +def get_learn_modules(): + return [ + { 'name': 'logscan', 'cpu_period': get_cpu_period(fraction=0.25), 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } + ] + + +def get_cpu_period(fraction: float): + multiplier = 10000 + + num_cores = multiprocessing.cpu_count() + if num_cores <= 2: + fraction = 1. + + num_used_cores = int(num_cores * fraction) + cpu_period = num_used_cores * multiplier + return cpu_period def sigint_handler(*_): @@ -90,7 +104,7 @@ def create_pillar_if_not_exist(pillar:str, content: dict): if pillar_dict.get('learn', {}).get('modules') is None: pillar_dict['learn'] = {} - pillar_dict['learn']['modules'] = learn_modules + pillar_dict['learn']['modules'] = get_learn_modules() content.update() write_pillar(pillar, content) From 2560a9b78c3087f782a11486ba42a5979d797b81 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 11:58:58 -0400 Subject: [PATCH 24/60] [wip] Change learn:modules to dictionary --- salt/common/tools/sbin/so-learn | 6 +++--- salt/learn/init.sls | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 39004b052..e444bc442 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -32,9 +32,9 @@ salt_proc: subprocess.CompletedProcess = None # Temp store of modules, will likely be broken out into salt def get_learn_modules(): - return [ - { 'name': 'logscan', 'cpu_period': get_cpu_period(fraction=0.25), 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } - ] + return { + 'logscan': { 'cpu_period': get_cpu_period(fraction=0.25), 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' } + } def get_cpu_period(fraction: float): diff --git a/salt/learn/init.sls b/salt/learn/init.sls index a511c1074..ee0b5e91b 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -1,12 +1,12 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% set module_list = salt['pillar.get']('learn:modules', [] ) %} +{% set module_dict = salt['pillar.get']('learn:modules', [] ) %} {% if module_list|length != 0 %}} include: -{% for module in module_list %} - - .{{ module }} +{% for module, _ in module_dict %} + - 'learn.{{ module }}' {% endfor %} {% endif %} From e1785dbd9a74896ed92310d5f3a7f641773a8a89 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 12:00:53 -0400 Subject: [PATCH 25/60] Fix typo --- salt/common/tools/sbin/so-learn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index e444bc442..be5e6ee02 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -159,7 +159,7 @@ def enable_disable_modules(args, enable: bool): write_needed = enable if write_needed: args.pillar_dict.update() - write_pillar(args.pillarm, args.pillar_dict) + write_pillar(args.pillar, args.pillar_dict) cmd_ret = check_apply(args) return cmd_ret From c53da9b1fffc296490a72fd4c263c43747ec7db9 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 12:04:40 -0400 Subject: [PATCH 26/60] Fix wrong variables in learn init.sls --- salt/learn/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/learn/init.sls b/salt/learn/init.sls index ee0b5e91b..32f8647b9 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -3,9 +3,9 @@ {% set module_dict = salt['pillar.get']('learn:modules', [] ) %} -{% if module_list|length != 0 %}} +{% if module_dict.items()|length != 0 %}} include: -{% for module, _ in module_dict %} +{% for module, _ in module_dict.items() %} - 'learn.{{ module }}' {% endfor %} {% endif %} From 44551ea9eeaf91aff80b54e7004f966e4ee241ac Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 13:31:48 -0400 Subject: [PATCH 27/60] Fix so-learn list --- salt/common/tools/sbin/so-learn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index be5e6ee02..9a307712b 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -175,8 +175,8 @@ def disable_modules(args): def list_modules(*_): print('Available ML modules:') - for module in learn_modules: - print(f' - { module["name"] } : {module["description"]}') + for module, details in get_learn_modules().items(): + print(f' - { module } : {details["description"]}') return 0 From 9e92f6da3d339f61b29a90fab498ed31af399e87 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 14:25:20 -0400 Subject: [PATCH 28/60] Add container to so-status when enabling/disabling ml module --- salt/common/tools/sbin/so-learn | 44 ++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 9a307712b..ea5a18b2c 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -28,6 +28,7 @@ import yaml import multiprocessing minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' +so_status_conf = '/opt/so/conf/so-status/so-status.conf' salt_proc: subprocess.CompletedProcess = None # Temp store of modules, will likely be broken out into salt @@ -79,8 +80,8 @@ def find_minion_pillar() -> str: def read_pillar(pillar: str): try: - with open(pillar, 'r') as f: - loaded_yaml = yaml.safe_load(f.read()) + with open(pillar, 'r') as pillar_file: + loaded_yaml = yaml.safe_load(pillar_file.read()) if loaded_yaml is None: print(f'Could not parse {pillar}', file=sys.stderr) sys.exit(3) @@ -92,13 +93,31 @@ def read_pillar(pillar: str): def write_pillar(pillar: str, content: dict): try: - with open(pillar, 'w') as f: - yaml.dump(content, f, default_flow_style=False) + with open(pillar, 'w') as pillar_file: + yaml.dump(content, pillar_file, default_flow_style=False) except: print(f'Could not open {pillar}', file=sys.stderr) sys.exit(3) +def mod_so_status(action: str, items: str): + with open(so_status_conf, 'a+') as conf: + conf.seek(0) + containers = conf.readlines() + + for item in items: + if f'so-{item}' in containers: + if action == 'remove': containers.remove(f'so-{item}') + if action == 'add': pass + else: + if action == 'remove': pass + if action == 'add': containers.append(f'so-{item}') + + conf.seek(0) + conf.truncate(0) + conf.writelines(containers) + + def create_pillar_if_not_exist(pillar:str, content: dict): pillar_dict = content @@ -141,21 +160,26 @@ def check_apply(args: dict): def enable_disable_modules(args, enable: bool): pillar_modules = args.pillar_dict.get('learn', {}).get('modules') + pillar_mod_names = args.pillar_dict.get('learn', {}).get('modules').keys() + action_str = 'add' if enable else 'remove' + if 'all' in args.modules: - for module in pillar_modules: - module['enabled'] = enable + for module, details in pillar_modules.items(): + details['enabled'] = enable + mod_so_status(action_str, module) args.pillar_dict.update() write_pillar(args.pillar, args.pillar_dict) else: write_needed = False for module in args.modules: - if module in pillar_modules: + if module in pillar_mod_names: if pillar_modules[module]['enabled'] == enable: - action_str = 'enabled' if enable else 'disabled' - print(f'{module} module already {action_str}.', file=sys.stderr) + state_str = 'enabled' if enable else 'disabled' + print(f'{module} module already {state_str}.', file=sys.stderr) else: pillar_modules[module]['enabled'] = enable + mod_so_status(action_str, module) write_needed = enable if write_needed: args.pillar_dict.update() @@ -167,10 +191,12 @@ def enable_disable_modules(args, enable: bool): def enable_modules(args): enable_disable_modules(args, enable=True) + mod_so_status('add', args.modules) def disable_modules(args): enable_disable_modules(args, enable=False) + mod_so_status('remove', args.modules) def list_modules(*_): From e38219aa2e4b88f827aeccccc9bda6134f01b163 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 14:35:02 -0400 Subject: [PATCH 29/60] Fix learn init.sls typo --- salt/learn/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/learn/init.sls b/salt/learn/init.sls index 32f8647b9..51a3fc973 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -3,7 +3,7 @@ {% set module_dict = salt['pillar.get']('learn:modules', [] ) %} -{% if module_dict.items()|length != 0 %}} +{% if module_dict.items()|length != 0 %} include: {% for module, _ in module_dict.items() %} - 'learn.{{ module }}' From b30f771fa2166c77869f526a03f09cdb3acec650 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 14:59:26 -0400 Subject: [PATCH 30/60] Set write_needed flag correctly, include newline in so-status.conf string --- salt/common/tools/sbin/so-learn | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index ea5a18b2c..8103dc09b 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -106,12 +106,12 @@ def mod_so_status(action: str, items: str): containers = conf.readlines() for item in items: - if f'so-{item}' in containers: - if action == 'remove': containers.remove(f'so-{item}') + if f'so-{item}\n' in containers: + if action == 'remove': containers.remove(f'so-{item}\n') if action == 'add': pass else: if action == 'remove': pass - if action == 'add': containers.append(f'so-{item}') + if action == 'add': containers.append(f'so-{item}\n') conf.seek(0) conf.truncate(0) @@ -180,7 +180,7 @@ def enable_disable_modules(args, enable: bool): else: pillar_modules[module]['enabled'] = enable mod_so_status(action_str, module) - write_needed = enable + write_needed = True if write_needed: args.pillar_dict.update() write_pillar(args.pillar, args.pillar_dict) From ba265d94f45cbb7b2ec2440606d7d4ff72aee180 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 15:14:28 -0400 Subject: [PATCH 31/60] Change default value in learn init to a dict where approriate --- salt/learn/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/learn/init.sls b/salt/learn/init.sls index 51a3fc973..fb5b89802 100644 --- a/salt/learn/init.sls +++ b/salt/learn/init.sls @@ -1,7 +1,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% set module_dict = salt['pillar.get']('learn:modules', [] ) %} +{% set module_dict = salt['pillar.get']('learn:modules', {} ) %} {% if module_dict.items()|length != 0 %} include: From b2a83018ba0fe44d77af77b57415264ac19f1a4b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 29 Jul 2021 15:14:54 -0400 Subject: [PATCH 32/60] Remove or run logscan based on enabled bool --- salt/learn/logscan.sls | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/learn/logscan.sls b/salt/learn/logscan.sls index 1e21e3b2a..cc8bb2996 100644 --- a/salt/learn/logscan.sls +++ b/salt/learn/logscan.sls @@ -2,6 +2,14 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} {% set logscan_cpu_period = salt['pillar.get']('learn:modules:logscan:cpu_period', 20000) %} +{% set enabled = salt['pillar.get']('learn:modules:logscan:enabled', False) %} + +{% if enabled %} + {% set container_action = 'running' %} +{% else %} + {% set container_action = 'absent'%} +{% endif %} + logscan_data_dir: file.directory: @@ -32,7 +40,8 @@ logscan_log_dir: - group: 939 so-logscan: - docker_container.running: + docker_container.{{ container_action }}: + {% if container_action == 'running' %} - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logscan:{{ VERSION }} - hostname: logscan - name: so-logscan @@ -42,3 +51,6 @@ so-logscan: - /opt/so/log/logscan:/logscan/output:rw - /opt/so/log:/logscan/logs:ro - cpu_period: {{ logscan_cpu_period }} + {% else %} + - force: true + {% endif %} From d71967ea1db09c37cc25b2e7e4304219321a2bdb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 10:28:39 -0400 Subject: [PATCH 33/60] Fix incorrect writing of so-status.conf --- salt/common/tools/sbin/so-learn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 8103dc09b..815d78ebf 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -100,7 +100,7 @@ def write_pillar(pillar: str, content: dict): sys.exit(3) -def mod_so_status(action: str, items: str): +def mod_so_status(action: str, items: List): with open(so_status_conf, 'a+') as conf: conf.seek(0) containers = conf.readlines() @@ -112,6 +112,8 @@ def mod_so_status(action: str, items: str): else: if action == 'remove': pass if action == 'add': containers.append(f'so-{item}\n') + + [containers.remove(c_name) for c_name in containers if c_name == '\n'] # remove extra newlines conf.seek(0) conf.truncate(0) From 01bb94514c32eb390f3e107f1fb71c8cd32e7dcb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 11:05:48 -0400 Subject: [PATCH 34/60] Correct mod_so_status to only act on single string --- salt/common/tools/sbin/so-learn | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 815d78ebf..f4a8ef90e 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -100,18 +100,18 @@ def write_pillar(pillar: str, content: dict): sys.exit(3) -def mod_so_status(action: str, items: List): +def mod_so_status(action: str, item: str): with open(so_status_conf, 'a+') as conf: conf.seek(0) containers = conf.readlines() - - for item in items: - if f'so-{item}\n' in containers: - if action == 'remove': containers.remove(f'so-{item}\n') - if action == 'add': pass - else: - if action == 'remove': pass - if action == 'add': containers.append(f'so-{item}\n') + + if f'so-{item}\n' in containers: + if action == 'remove': containers.remove(f'so-{item}\n') + if action == 'add': pass + else: + if action == 'remove': pass + if action == 'add': containers.append(f'so-{item}\n') + [containers.remove(c_name) for c_name in containers if c_name == '\n'] # remove extra newlines @@ -193,12 +193,10 @@ def enable_disable_modules(args, enable: bool): def enable_modules(args): enable_disable_modules(args, enable=True) - mod_so_status('add', args.modules) def disable_modules(args): enable_disable_modules(args, enable=False) - mod_so_status('remove', args.modules) def list_modules(*_): From b9980c9d30e547f5ae32e8e8afe15f89f5e78932 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 13:09:09 -0400 Subject: [PATCH 35/60] Fix pipeline name --- .../logstash/pipelines/config/so/9800_output_logscan.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja index a1c621484..38ee29b69 100644 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -10,7 +10,7 @@ output { if [module] =~ "logscan" { elasticsearch { id => "logscan_pipeline" - pipeline => "%{module}" + pipeline => "logscan" hosts => "{{ ES }}" {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" From 33bd6aed20d42d941ed08cd1b83cea4b34c5aceb Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 14:41:15 -0400 Subject: [PATCH 36/60] Fix logscan pipeline on eval * Rename logscan pipeline to logscan.alert * Add module to indices array in filebeat.yml --- salt/elasticsearch/files/ingest/{logscan => logscan.alert} | 0 salt/filebeat/etc/filebeat.yml | 3 +++ .../pipelines/config/so/9800_output_logscan.conf.jinja | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) rename salt/elasticsearch/files/ingest/{logscan => logscan.alert} (100%) diff --git a/salt/elasticsearch/files/ingest/logscan b/salt/elasticsearch/files/ingest/logscan.alert similarity index 100% rename from salt/elasticsearch/files/ingest/logscan rename to salt/elasticsearch/files/ingest/logscan.alert diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index f5ba5dc74..3c482e274 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -307,6 +307,9 @@ output.elasticsearch: - index: "so-strelka" when.contains: module: "strelka" + - index: "so-logscan" + when.contains: + module: "logscan" setup.template.enabled: false {%- else %} diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja index 38ee29b69..86944d155 100644 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja @@ -10,7 +10,7 @@ output { if [module] =~ "logscan" { elasticsearch { id => "logscan_pipeline" - pipeline => "logscan" + pipeline => "logscan.alert" hosts => "{{ ES }}" {% if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %} user => "{{ ES_USER }}" From 2a6277c0c3bfab781799cd685ae99396011eca1d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 15:46:39 -0400 Subject: [PATCH 37/60] Fix field names in logscan pipeline --- salt/elasticsearch/files/ingest/logscan.alert | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/elasticsearch/files/ingest/logscan.alert b/salt/elasticsearch/files/ingest/logscan.alert index 3a3debabc..88772c16c 100644 --- a/salt/elasticsearch/files/ingest/logscan.alert +++ b/salt/elasticsearch/files/ingest/logscan.alert @@ -11,12 +11,14 @@ { "remove": { "field": "start_time", "ignore_missing": true } }, { "remove": { "field": "end_time", "ignore_missing": true } }, { "rename": { "field": "source_ip", "target_field": "source.ip", "ignore_missing": true } }, - { "append": { "field": "logsscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, - { "rename": { "field": "source_ips", "target_field": "logscan.source.ips", "ignore_missing": true } }, - { "set": { "if": "ctx.model == 'kff'", "field": "rule.name", "value": "LOGSCAN KFF MODEL THRESHOLD" } }, - { "set": { "if": "ctx.model == 'kff'", "field": "rule.description", "value": "High ratio of login failures in 5 minute window" } }, - { "set": { "if": "ctx.model == 'kl'", "field": "rule.name", "value": "LOGSCAN KL MODEL THRESHOLD" } }, - { "set": { "if": "ctx.model == 'kl'", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, + { "append": { "field": "logscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, + { "rename": { "field": "top_source_ips", "target_field": "logscan.source.ips", "ignore_missing": true } }, + { "set": { "if": "ctx.model == 'k1'", "field": "rule.name", "value": "LOGSCAN K1 MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'k1'", "field": "rule.description", "value": "High number of logins from single IP in 1 minute window" } }, + { "set": { "if": "ctx.model == 'k5'", "field": "rule.name", "value": "LOGSCAN K5 MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'k5'", "field": "rule.description", "value": "High ratio of login failures from single IP in 5 minute window" } }, + { "set": { "if": "ctx.model == 'k60'", "field": "rule.name", "value": "LOGSCAN K60 MODEL THRESHOLD" } }, + { "set": { "if": "ctx.model == 'k60'", "field": "rule.description", "value": "Large number of login failures in 1 hour window" } }, { "rename": { "field": "model", "target_field": "logscan.model" } }, { "rename": { "field": "num_attempts", "target_field": "logscan.attempts.total.amount", "ignore_missing": true } }, { "rename": { "field": "num_failed", "target_field": "logscan.attempts.failed.amount", "ignore_missing": true } }, From 4f39cd1d7f80516134e6b603b82c35cafb3d07df Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 30 Jul 2021 16:02:02 -0400 Subject: [PATCH 38/60] Add logscan dynamic object to so-common template mappings --- salt/elasticsearch/templates/so/so-common-template.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticsearch/templates/so/so-common-template.json b/salt/elasticsearch/templates/so/so-common-template.json index 8afac271c..777bf3f53 100644 --- a/salt/elasticsearch/templates/so/so-common-template.json +++ b/salt/elasticsearch/templates/so/so-common-template.json @@ -313,6 +313,10 @@ "type":"object", "dynamic": true }, + "logscan": { + "type": "object", + "dynamic": true + }, "manager":{ "type":"object", "dynamic": true From 8a49039b854749ab6f76635cf4eeb35bb092c579 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 2 Aug 2021 09:50:49 -0400 Subject: [PATCH 39/60] Only append source.ip to logscan.source.ips if it's been created --- salt/elasticsearch/files/ingest/logscan.alert | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/logscan.alert b/salt/elasticsearch/files/ingest/logscan.alert index 88772c16c..7473060a7 100644 --- a/salt/elasticsearch/files/ingest/logscan.alert +++ b/salt/elasticsearch/files/ingest/logscan.alert @@ -11,8 +11,8 @@ { "remove": { "field": "start_time", "ignore_missing": true } }, { "remove": { "field": "end_time", "ignore_missing": true } }, { "rename": { "field": "source_ip", "target_field": "source.ip", "ignore_missing": true } }, - { "append": { "field": "logscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, { "rename": { "field": "top_source_ips", "target_field": "logscan.source.ips", "ignore_missing": true } }, + { "append": { "if": "ctx.source != null", "field": "logscan.source.ips", "value": "{{{source.ip}}}", "ignore_failure": true } }, { "set": { "if": "ctx.model == 'k1'", "field": "rule.name", "value": "LOGSCAN K1 MODEL THRESHOLD" } }, { "set": { "if": "ctx.model == 'k1'", "field": "rule.description", "value": "High number of logins from single IP in 1 minute window" } }, { "set": { "if": "ctx.model == 'k5'", "field": "rule.name", "value": "LOGSCAN K5 MODEL THRESHOLD" } }, From 757091beeb6573d73df6aca44130f4c0d39b9817 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 2 Aug 2021 10:35:39 -0400 Subject: [PATCH 40/60] Add log_level to logscan.conf --- salt/learn/files/logscan.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/learn/files/logscan.conf b/salt/learn/files/logscan.conf index 9b3df8027..d7aa30734 100644 --- a/salt/learn/files/logscan.conf +++ b/salt/learn/files/logscan.conf @@ -1,6 +1,7 @@ [global] ts_format = iso8601 scan_interval = 30s +log_level = info [kratos] log_path = kratos/kratos.log From d3b170c6df21c0ba9a00b41928aa613a804a5122 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 2 Aug 2021 12:37:37 -0400 Subject: [PATCH 41/60] Add logscan automation file + fix enable command in setup --- setup/automation/standalone-iso-logscan | 77 +++++++++++++++++++++++++ setup/so-setup | 2 +- 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 setup/automation/standalone-iso-logscan diff --git a/setup/automation/standalone-iso-logscan b/setup/automation/standalone-iso-logscan new file mode 100644 index 000000000..d83ad73db --- /dev/null +++ b/setup/automation/standalone-iso-logscan @@ -0,0 +1,77 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +ALLOW_CIDR=0.0.0.0/0 +ALLOW_ROLE=a +BASICZEEK=2 +BASICSURI=2 +# BLOGS= +BNICS=eth1 +ZEEKVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=standalone +install_type=STANDALONE +LEARN_LOGSCAN_ENABLE=true +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MANAGERADV=BASIC +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MANAGER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +# SKIP_REBOOT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=0n10nus3r +WEBPASSWD2=0n10nus3r diff --git a/setup/so-setup b/setup/so-setup index bfca9b2bd..958d8aea1 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -965,7 +965,7 @@ else if [[ -n $LEARN_LOGSCAN_ENABLE ]]; then set_progress_str 99 'Enabling logscan' - so-learn enable logscan --apply -y >> $setup_log 2>&1 + so-learn enable logscan --apply >> $setup_log 2>&1 fi } | whiptail_gauge_post_setup "Running post-installation steps..." From 25bf25eae68528e8148ca709548214cc274d1c8e Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 3 Aug 2021 15:24:32 -0400 Subject: [PATCH 42/60] Allowed states remove typo'd logscan --- salt/allowed_states.map.jinja | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 1aa79474e..f5f77d8f3 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -46,7 +46,6 @@ 'soctopus', 'tcpreplay', 'docker_clean', - 'logscsan', 'learn' ], 'so-heavynode': [ @@ -78,7 +77,7 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'logscsan' + 'learn' ], 'so-fleet': [ 'ca', @@ -112,7 +111,6 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'logscsan', 'learn' ], 'so-manager': [ @@ -133,7 +131,6 @@ 'schedule', 'soctopus', 'docker_clean', - 'logscsan', 'learn' ], 'so-managersearch': [ @@ -154,7 +151,6 @@ 'schedule', 'soctopus', 'docker_clean', - 'logscsan', 'learn' ], 'so-node': [ @@ -188,7 +184,6 @@ 'soctopus', 'tcpreplay', 'docker_clean', - 'logscsan', 'learn' ], 'so-sensor': [ From 9e5d3aa28675155cf10d7fb26ad95a574607f16c Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 3 Aug 2021 15:25:53 -0400 Subject: [PATCH 43/60] Fix removed root check in so-rule --- salt/common/tools/sbin/so-rule | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index bbb916807..bd1556f58 100755 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -349,7 +349,9 @@ def sigint_handler(*_): def main(): signal.signal(signal.SIGINT, sigint_handler) - + if os.geteuid() != 0: + print('You must run this script as root', file=sys.stderr) + sys.exit(1) apply_help='After updating rule configuration, apply the idstools state.' From ef59cb47dd22c700a9c6103aeb8c2d2b90979b0f Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 3 Aug 2021 15:26:57 -0400 Subject: [PATCH 44/60] Use print_err function --- salt/common/tools/sbin/so-rule | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-rule b/salt/common/tools/sbin/so-rule index bd1556f58..07d6cd2cc 100755 --- a/salt/common/tools/sbin/so-rule +++ b/salt/common/tools/sbin/so-rule @@ -350,7 +350,7 @@ def main(): signal.signal(signal.SIGINT, sigint_handler) if os.geteuid() != 0: - print('You must run this script as root', file=sys.stderr) + print_err('You must run this script as root') sys.exit(1) apply_help='After updating rule configuration, apply the idstools state.' From 2bc88e77507ea09ec317ce4e7d9b53040ac006df Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 3 Aug 2021 15:29:37 -0400 Subject: [PATCH 45/60] Remove learn from allowed states for helixsensor --- salt/allowed_states.map.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index f5f77d8f3..a51ea434b 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -76,8 +76,7 @@ 'logstash', 'schedule', 'tcpreplay', - 'docker_clean', - 'learn' + 'docker_clean' ], 'so-fleet': [ 'ca', From 771688a70fed289338720f7fa0efd64c8ec828f4 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 5 Aug 2021 07:34:07 -0400 Subject: [PATCH 46/60] fix typo --- salt/common/tools/sbin/so-tcpreplay | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index 3c1ce8fb9..e85474c67 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -31,7 +31,7 @@ if [[ $# -lt 1 ]]; then echo "Usage: $0 " echo echo "All PCAPs must be placed in the /opt/so/samples directory unless replaying" - echo "a sample pcap that is included in the so-tcpreplay image. Those PCAP sampes" + echo "a sample pcap that is included in the so-tcpreplay image. Those PCAP samples" echo "are located in the /opt/samples directory inside of the image." echo echo "Customer provided PCAP example:" From dd1769fbefd60e862de82a5400f7b9b0bd69f4ea Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 5 Aug 2021 11:02:09 -0400 Subject: [PATCH 47/60] Only check for logscan on manager-type and import --- salt/filebeat/etc/filebeat.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 3c482e274..0c27e3c1b 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -112,6 +112,7 @@ filebeat.inputs: fields: ["source", "prospector", "input", "offset", "beat"] fields_under_root: true +{%- if grains['role'] in ['so-eval', 'so-standalone', 'so-manager', 'so-managersearch', 'so-import'] %} - type: log paths: - /logs/logscan/alerts.log @@ -124,6 +125,7 @@ filebeat.inputs: fields_under_root: true clean_removed: true close_removed: false +{%- endif %} {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} {%- if ZEEKVER != 'SURICATA' %} From 3b01f6431e152d33349ae63645fe69dcc8ffd250 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 6 Aug 2021 09:43:58 -0400 Subject: [PATCH 48/60] Add logscan to logrotate config --- salt/common/files/log-rotate.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/files/log-rotate.conf b/salt/common/files/log-rotate.conf index 061b76271..35c6fd724 100644 --- a/salt/common/files/log-rotate.conf +++ b/salt/common/files/log-rotate.conf @@ -22,6 +22,7 @@ /opt/so/log/salt/so-salt-minion-check /opt/so/log/salt/minion /opt/so/log/salt/master +/opt/so/log/logscan/*.log { {{ logrotate_conf | indent(width=4) }} } From ad3b6cf6297cfc8fa99a761c2288f58ae96e6245 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 9 Aug 2021 13:33:21 -0400 Subject: [PATCH 49/60] remove old dashboard dirs - https://github.com/Security-Onion-Solutions/securityonion/issues/4674 --- salt/grafana/init.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index dbb659217..3c98d6370 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -44,6 +44,12 @@ grafanadashdir: - group: 939 - makedirs: True +{% for type in ['eval','manager','managersearch','search_nodes','sensor_nodes','standalone'] %} +remove_dashboard_dir_{{type}}: + file.absent: + - name: /opt/so/conf/grafana/grafana_dashboards/{{type}} +{% endfor %} + grafana-dashboard-config: file.managed: - name: /opt/so/conf/grafana/etc/dashboards/dashboard.yml From 1415de858c806c5b9924fab98381453751c680de Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Aug 2021 10:16:14 -0400 Subject: [PATCH 50/60] delete old dashboard folders via api - https://github.com/Security-Onion-Solutions/securityonion/issues/4674 --- .../sbin/so-grafana-dashboard-folder-delete | 17 +++++++++++++++++ salt/grafana/init.sls | 5 +++++ 2 files changed, 22 insertions(+) create mode 100644 salt/common/tools/sbin/so-grafana-dashboard-folder-delete diff --git a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete new file mode 100644 index 000000000..60a998290 --- /dev/null +++ b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete @@ -0,0 +1,17 @@ +# this script is used to delete the default Grafana dashboard folders that existed prior to Grafana dashboard and Salt management changes in 2.3.70 + +folders=$(curl -X GET http://admin:{{salt['pillar.get'('secrets:grafana_admin')]}}@localhost:3000/api/folders | jq -r '.[] | @base64') +delfolder=("Manager" "Manager Search" "Sensor Nodes" "Search Nodes" "Standalone" "Eval Mode") + +for row in $folders; do + title=$(echo ${row} | base64 --decode | jq -r '.title') + uid=$(echo ${row} | base64 --decode | jq -r '.uid') + + if [[ " ${delfolder[@]} " =~ " ${title} " ]]; then + curl -X DELETE http://admin:{{salt['pillar.get'('secrets:grafana_admin')]}}@localhost:3000/api/folders/$uid + fi +done + +echo "so-grafana-dashboard-folder-delete has been run to delete default Grafana dashboard folders that existed prior to 2.3.70" > /opt/so/state/so-grafana-dashboard-folder-delete-complete + +exit 0 diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index 3c98d6370..1c1bdfb2e 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -88,6 +88,11 @@ grafana-config-files: - source: salt://grafana/etc/files - makedirs: True +so-grafana-dashboard-folder-delete: + cmd.run: + - name: /usr/sbin/so-grafana-dashboard-folder-delete + - unless: ls /opt/so/state/so-grafana-dashboard-folder-delete-complete + {% for dashboard in DASHBOARDS %} {{dashboard}}-dashboard: file.managed: From 3c1114403ee2b29afe8da8eeb62b09fd3eef4ac2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Aug 2021 10:25:05 -0400 Subject: [PATCH 51/60] fix the pillar.get --- salt/common/tools/sbin/so-grafana-dashboard-folder-delete | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete index 60a998290..3d7a2cfd3 100644 --- a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete +++ b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete @@ -1,6 +1,6 @@ # this script is used to delete the default Grafana dashboard folders that existed prior to Grafana dashboard and Salt management changes in 2.3.70 -folders=$(curl -X GET http://admin:{{salt['pillar.get'('secrets:grafana_admin')]}}@localhost:3000/api/folders | jq -r '.[] | @base64') +folders=$(curl -X GET http://admin:{{salt['pillar.get']('secrets:grafana_admin')]}}@localhost:3000/api/folders | jq -r '.[] | @base64') delfolder=("Manager" "Manager Search" "Sensor Nodes" "Search Nodes" "Standalone" "Eval Mode") for row in $folders; do @@ -8,7 +8,7 @@ for row in $folders; do uid=$(echo ${row} | base64 --decode | jq -r '.uid') if [[ " ${delfolder[@]} " =~ " ${title} " ]]; then - curl -X DELETE http://admin:{{salt['pillar.get'('secrets:grafana_admin')]}}@localhost:3000/api/folders/$uid + curl -X DELETE http://admin:{{salt['pillar.get']('secrets:grafana_admin')]}}@localhost:3000/api/folders/$uid fi done From 9e48a5b57b9000f89e74410fcf278e61ea8be3d0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Aug 2021 10:29:29 -0400 Subject: [PATCH 52/60] fix the pillar.get --- salt/common/tools/sbin/so-grafana-dashboard-folder-delete | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete index 3d7a2cfd3..f3338de84 100644 --- a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete +++ b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete @@ -1,6 +1,6 @@ # this script is used to delete the default Grafana dashboard folders that existed prior to Grafana dashboard and Salt management changes in 2.3.70 -folders=$(curl -X GET http://admin:{{salt['pillar.get']('secrets:grafana_admin')]}}@localhost:3000/api/folders | jq -r '.[] | @base64') +folders=$(curl -X GET http://admin:{{salt['pillar.get']('secrets:grafana_admin')}}@localhost:3000/api/folders | jq -r '.[] | @base64') delfolder=("Manager" "Manager Search" "Sensor Nodes" "Search Nodes" "Standalone" "Eval Mode") for row in $folders; do @@ -8,7 +8,7 @@ for row in $folders; do uid=$(echo ${row} | base64 --decode | jq -r '.uid') if [[ " ${delfolder[@]} " =~ " ${title} " ]]; then - curl -X DELETE http://admin:{{salt['pillar.get']('secrets:grafana_admin')]}}@localhost:3000/api/folders/$uid + curl -X DELETE http://admin:{{salt['pillar.get']('secrets:grafana_admin')}}@localhost:3000/api/folders/$uid fi done From 983549711c71da3509e7e7b8242804a4da9729ef Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 11 Aug 2021 13:47:31 -0400 Subject: [PATCH 53/60] Pull image if missing when enabling module in so-learn --- salt/common/tools/sbin/so-image-pull | 48 +++++++++++++++++++++ salt/common/tools/sbin/so-learn | 64 ++++++++++++++++++++++------ 2 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 salt/common/tools/sbin/so-image-pull diff --git a/salt/common/tools/sbin/so-image-pull b/salt/common/tools/sbin/so-image-pull new file mode 100644 index 000000000..f86d08ca2 --- /dev/null +++ b/salt/common/tools/sbin/so-image-pull @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common +. /usr/sbin/so-image-common + +usage() { + read -r -d '' message <<- EOM + usage: so-image-pull [-h] IMAGE [IMAGE ...] + + positional arguments: + IMAGE One or more 'so-' prefixed images to download and verify. + + optional arguments: + -h, --help Show this help message and exit. + EOM + echo "$message" + exit 1 +} + +if [[ $# -eq 0 || $# -gt 1 ]] || [[ $1 == '-h' || $1 == '--help' ]]; then + usage +fi + +TRUSTED_CONTAINERS=("$@") +set_version + +for image in "${TRUSTED_CONTAINERS[@]}"; do + if ! docker images | grep "$image" | grep ":5000" | grep -q "$VERSION"; then + update_docker_containers "$image" "" "" "" + else + echo "$image:$VERSION image exists." 1>&2 + fi +done diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index f4a8ef90e..d87649cd2 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from itertools import chain from typing import List import signal @@ -26,6 +27,7 @@ import argparse import textwrap import yaml import multiprocessing +import docker minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' so_status_conf = '/opt/so/conf/so-status/so-status.conf' @@ -112,7 +114,6 @@ def mod_so_status(action: str, item: str): if action == 'remove': pass if action == 'add': containers.append(f'so-{item}\n') - [containers.remove(c_name) for c_name in containers if c_name == '\n'] # remove extra newlines conf.seek(0) @@ -132,19 +133,57 @@ def create_pillar_if_not_exist(pillar:str, content: dict): return content -def apply(module_list: List): +def salt_call(module: str): return_code = 0 - for module in module_list: - salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True'] - print(f' Applying salt state for {module} module...') - salt_proc = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL) - if salt_proc.returncode != 0: - print(f' [ERROR] Failed to apply salt state for {module} module.') - return_code = salt_proc.returncode + salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True'] + + print(f' Applying salt state for {module} module...') + return_code = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL).returncode + if return_code != 0: + print(f' [ERROR] Failed to apply salt state for {module} module.') + return_code = salt_proc.returncode + return return_code -def check_apply(args: dict): +def pull_image(module: str): + container_basename = f'so-{module}' + + client = docker.from_env() + image_list = client.images.list(filters={ 'dangling': False }) + tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list)))) + basename_match = list(filter(lambda x: f'{container_basename}' in x, tag_list)) + local_registry_match = list(filter(lambda x: ':5000' in x, basename_match)) + + if len(local_registry_match) > 0: + print(f' Pulling missing image for {module}:') + pull_command = ['so-image-pull', container_basename] + + return_code = subprocess.run(pull_command, stdout=subprocess.DEVNULL).returncode + if return_code != 0: + print(f' [ERROR] Failed to pull image so-{module}, skipping state.') + + return return_code + + +def apply(module_list: List, enable: bool): + return_code = 0 + for module in module_list: + if enable: + temp_return = pull_image(module) + if temp_return == 0: + temp_return = salt_call(module) + else: + temp_return = salt_call(module) + + # Only update return_code if a command returned a non-zero return + if temp_return != 0: + return_code = temp_return + + return return_code + + +def check_apply(args: dict, enable: bool): if args.apply: print('Configuration updated. Applying changes:') return apply(args.modules) @@ -157,7 +196,7 @@ def check_apply(args: dict): return 0 else: print('Applying changes:') - return apply(args.modules) + return apply(args.modules, enable) def enable_disable_modules(args, enable: bool): @@ -170,6 +209,7 @@ def enable_disable_modules(args, enable: bool): for module, details in pillar_modules.items(): details['enabled'] = enable mod_so_status(action_str, module) + if enable: pull_image(module) args.pillar_dict.update() write_pillar(args.pillar, args.pillar_dict) else: @@ -187,7 +227,7 @@ def enable_disable_modules(args, enable: bool): args.pillar_dict.update() write_pillar(args.pillar, args.pillar_dict) - cmd_ret = check_apply(args) + cmd_ret = check_apply(args, enable) return cmd_ret From 64dfc6e1917a32b59f90d028f3bb03b8806f0a24 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 11 Aug 2021 16:33:45 -0400 Subject: [PATCH 54/60] Fix pull logic and properly hide output --- salt/common/tools/sbin/so-image-pull | 14 ++++++-- salt/common/tools/sbin/so-learn | 52 +++++++++++++--------------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/salt/common/tools/sbin/so-image-pull b/salt/common/tools/sbin/so-image-pull index f86d08ca2..cf312acec 100644 --- a/salt/common/tools/sbin/so-image-pull +++ b/salt/common/tools/sbin/so-image-pull @@ -32,6 +32,12 @@ usage() { exit 1 } +for arg; do + shift + [[ "$arg" = "--quiet" || "$arg" = "-q" ]] && quiet=true && continue + set -- "$@" "$arg" +done + if [[ $# -eq 0 || $# -gt 1 ]] || [[ $1 == '-h' || $1 == '--help' ]]; then usage fi @@ -41,8 +47,12 @@ set_version for image in "${TRUSTED_CONTAINERS[@]}"; do if ! docker images | grep "$image" | grep ":5000" | grep -q "$VERSION"; then - update_docker_containers "$image" "" "" "" + if [[ $quiet == true ]]; then + update_docker_containers "$image" "" "" "/dev/null" + else + update_docker_containers "$image" "" "" "" + fi else - echo "$image:$VERSION image exists." 1>&2 + echo "$image:$VERSION image exists." fi done diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index d87649cd2..1729e460d 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -28,10 +28,11 @@ import textwrap import yaml import multiprocessing import docker +import pty minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions' so_status_conf = '/opt/so/conf/so-status/so-status.conf' -salt_proc: subprocess.CompletedProcess = None +proc: subprocess.CompletedProcess = None # Temp store of modules, will likely be broken out into salt def get_learn_modules(): @@ -54,8 +55,8 @@ def get_cpu_period(fraction: float): def sigint_handler(*_): print('Exiting gracefully on Ctrl-C') - if salt_proc is not None: salt_proc.send_signal(signal.SIGINT) - sys.exit(0) + if proc is not None: proc.send_signal(signal.SIGINT) + sys.exit(1) def find_minion_pillar() -> str: @@ -134,14 +135,13 @@ def create_pillar_if_not_exist(pillar:str, content: dict): def salt_call(module: str): - return_code = 0 salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True'] print(f' Applying salt state for {module} module...') - return_code = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL).returncode + proc = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return_code = proc.returncode if return_code != 0: print(f' [ERROR] Failed to apply salt state for {module} module.') - return_code = salt_proc.returncode return return_code @@ -155,35 +155,31 @@ def pull_image(module: str): basename_match = list(filter(lambda x: f'{container_basename}' in x, tag_list)) local_registry_match = list(filter(lambda x: ':5000' in x, basename_match)) - if len(local_registry_match) > 0: - print(f' Pulling missing image for {module}:') - pull_command = ['so-image-pull', container_basename] - - return_code = subprocess.run(pull_command, stdout=subprocess.DEVNULL).returncode - if return_code != 0: - print(f' [ERROR] Failed to pull image so-{module}, skipping state.') + if len(local_registry_match) == 0: + print(f'Pulling missing image for {module} (may take several minutes) ...') + pull_command = ['so-image-pull', '--quiet', container_basename] + proc = subprocess.run(pull_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return_code = proc.returncode + if return_code != 0: + print(f' [ERROR] Failed to pull image so-{module}, skipping state.') + else: + return_code = 0 return return_code -def apply(module_list: List, enable: bool): +def apply(module_list: List): return_code = 0 for module in module_list: - if enable: - temp_return = pull_image(module) - if temp_return == 0: - temp_return = salt_call(module) - else: - temp_return = salt_call(module) - - # Only update return_code if a command returned a non-zero return - if temp_return != 0: - return_code = temp_return + salt_ret = salt_call(module) + # Only update return_code if the command returned a non-zero return + if salt_ret != 0: + return_code = salt_ret return return_code -def check_apply(args: dict, enable: bool): +def check_apply(args: dict): if args.apply: print('Configuration updated. Applying changes:') return apply(args.modules) @@ -196,7 +192,7 @@ def check_apply(args: dict, enable: bool): return 0 else: print('Applying changes:') - return apply(args.modules, enable) + return apply(args.modules) def enable_disable_modules(args, enable: bool): @@ -220,6 +216,8 @@ def enable_disable_modules(args, enable: bool): state_str = 'enabled' if enable else 'disabled' print(f'{module} module already {state_str}.', file=sys.stderr) else: + if enable and pull_image(module) != 0: + continue pillar_modules[module]['enabled'] = enable mod_so_status(action_str, module) write_needed = True @@ -227,7 +225,7 @@ def enable_disable_modules(args, enable: bool): args.pillar_dict.update() write_pillar(args.pillar, args.pillar_dict) - cmd_ret = check_apply(args, enable) + cmd_ret = check_apply(args) return cmd_ret From 4a31d6b3bc4fb00ce17ffb8ccfe98647a7054a84 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 11 Aug 2021 16:35:33 -0400 Subject: [PATCH 55/60] Specify images are also verified --- salt/common/tools/sbin/so-learn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 1729e460d..3ac1e2d4f 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -156,7 +156,7 @@ def pull_image(module: str): local_registry_match = list(filter(lambda x: ':5000' in x, basename_match)) if len(local_registry_match) == 0: - print(f'Pulling missing image for {module} (may take several minutes) ...') + print(f'Pulling and verifying missing image for {module} (may take several minutes) ...') pull_command = ['so-image-pull', '--quiet', container_basename] proc = subprocess.run(pull_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) From 3312a66e75c2a7bafe2bc3dafe4614bdc0444f32 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 11 Aug 2021 16:37:22 -0400 Subject: [PATCH 56/60] Fix indent --- salt/common/tools/sbin/so-learn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn index 3ac1e2d4f..273f1b8f4 100644 --- a/salt/common/tools/sbin/so-learn +++ b/salt/common/tools/sbin/so-learn @@ -162,7 +162,7 @@ def pull_image(module: str): proc = subprocess.run(pull_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) return_code = proc.returncode if return_code != 0: - print(f' [ERROR] Failed to pull image so-{module}, skipping state.') + print(f'[ERROR] Failed to pull image so-{module}, skipping state.') else: return_code = 0 return return_code From bf40a1038ee0702cc42eb46bad9b3f9336dd088a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 12 Aug 2021 10:32:27 -0400 Subject: [PATCH 57/60] Whiptail changes * Update wording of ip mask prompt + so-allow question for clarity * Remove old ip+mask prompts --- setup/so-whiptail | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 961924afa..780411841 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -959,33 +959,18 @@ whiptail_management_interface_gateway() { whiptail_management_interface_ip_mask() { [ -n "$TESTING" ] && return - manager_ip_mask=$(whiptail --title "$whiptail_title" --inputbox \ - "Enter your IPv4 address with CIDR mask (e.g. 192.168.1.2/24):" 10 60 "$1" 3>&1 1>&2 2>&3) + local msg + read -r -d '' msg <<- EOM + What IPv4 address would you like to assign to this Security Onion installation? + + Please enter the IPv4 address with CIDR mask + (e.g. 192.168.1.2/24): + EOM + + manager_ip_mask=$(whiptail --title "$whiptail_title" --inputbox "$msg" 12 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? - whiptail_check_exitstatus $exitstatus -} - -whiptail_management_interface_ip() { - - [ -n "$TESTING" ] && return - - MIP=$(whiptail --title "$whiptail_title" --inputbox \ - "Enter your IP address:" 10 60 X.X.X.X 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus -} - -whiptail_management_interface_mask() { - - [ -n "$TESTING" ] && return - - MMASK=$(whiptail --title "$whiptail_title" --inputbox \ - "Enter the bit mask for your subnet:" 10 60 24 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus + # whiptail_check_exitstatus $exitstatus } whiptail_management_nic() { @@ -1734,7 +1719,7 @@ whiptail_so_allow_yesno() { [ -n "$TESTING" ] && return whiptail --title "$whiptail_title" \ - --yesno "Do you want to run so-allow to allow access to the web tools?" \ + --yesno "Do you want to run so-allow to allow other machines to access this Security Onion installation via the web interface?" \ 8 75 } From 258cebda6e4ddf20a961250a5cf8024b8c6e8246 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 12 Aug 2021 15:01:45 -0400 Subject: [PATCH 58/60] Correct identity update payload to not have unsupported fields --- salt/common/tools/sbin/so-user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index f0c064d03..742c3ca5d 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -306,7 +306,7 @@ function updateStatus() { [[ $? != 0 ]] && fail "Unable to unlock credential record" fi - updatedJson=$(echo "$response" | jq ".traits.status = \"$status\" | del(.verifiable_addresses) | del(.id) | del(.schema_url)") + updatedJson=$(echo "$response" | jq ".traits.status = \"$status\" | del(.verifiable_addresses) | del(.id) | del(.schema_url) | del(.created_at) | del(.updated_at)") response=$(curl -Ss -XPUT -L ${kratosUrl}/identities/$identityId -d "$updatedJson") [[ $? != 0 ]] && fail "Unable to mark user as locked" From 86569b05990402cee97ef88b1d3a9b859b99ed16 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 12 Aug 2021 16:05:54 -0400 Subject: [PATCH 59/60] Make sbin script permissions consistent --- salt/common/tools/sbin/so-grafana-dashboard-folder-delete | 0 salt/common/tools/sbin/so-image-pull | 0 salt/common/tools/sbin/so-influxdb-drop-autogen | 0 salt/common/tools/sbin/so-learn | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 salt/common/tools/sbin/so-grafana-dashboard-folder-delete mode change 100644 => 100755 salt/common/tools/sbin/so-image-pull mode change 100644 => 100755 salt/common/tools/sbin/so-influxdb-drop-autogen mode change 100644 => 100755 salt/common/tools/sbin/so-learn diff --git a/salt/common/tools/sbin/so-grafana-dashboard-folder-delete b/salt/common/tools/sbin/so-grafana-dashboard-folder-delete old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-image-pull b/salt/common/tools/sbin/so-image-pull old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-influxdb-drop-autogen b/salt/common/tools/sbin/so-influxdb-drop-autogen old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn old mode 100644 new mode 100755 From fb4c2c35e3592ea004861076423072c657386657 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Fri, 13 Aug 2021 13:58:08 -0400 Subject: [PATCH 60/60] Remove so-logscan from so-image-common arrays --- salt/common/tools/sbin/so-image-common | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index cea0f6507..9b6e2174a 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -37,7 +37,6 @@ container_list() { "so-idstools" "so-kibana" "so-kratos" - "so-logscan" "so-nginx" "so-pcaptools" "so-soc" @@ -59,7 +58,6 @@ container_list() { "so-influxdb" "so-kibana" "so-kratos" - "so-logscan" "so-logstash" "so-mysql" "so-nginx"