From 747d62dae5748ca9122bc288e167f9ad22fadd8b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Feb 2021 09:44:24 -0500 Subject: [PATCH 1/5] Add acng to import installs for consistency --- salt/common/tools/sbin/so-image-common | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 9702da3f1..4e5aedc5f 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -30,6 +30,7 @@ container_list() { if [ $MANAGERCHECK == 'so-import' ]; then TRUSTED_CONTAINERS=( + "so-acng" "so-elasticsearch" "so-filebeat" "so-idstools" From 3467f30603bab452efc912904c0ca16f8d084a0d Mon Sep 17 00:00:00 2001 From: doug Date: Mon, 22 Feb 2021 10:27:24 -0500 Subject: [PATCH 2/5] Improve support for Suricata metadata #2200 --- .../files/ingest/{zeek.dns.tld => dns.tld} | 0 salt/elasticsearch/files/ingest/suricata.dns | 25 +++++++------- .../files/ingest/suricata.fileinfo | 13 +++---- salt/elasticsearch/files/ingest/suricata.tls | 34 +++++++++---------- salt/elasticsearch/files/ingest/zeek.dns | 2 +- 5 files changed, 38 insertions(+), 36 deletions(-) rename salt/elasticsearch/files/ingest/{zeek.dns.tld => dns.tld} (100%) diff --git a/salt/elasticsearch/files/ingest/zeek.dns.tld b/salt/elasticsearch/files/ingest/dns.tld similarity index 100% rename from salt/elasticsearch/files/ingest/zeek.dns.tld rename to salt/elasticsearch/files/ingest/dns.tld diff --git a/salt/elasticsearch/files/ingest/suricata.dns b/salt/elasticsearch/files/ingest/suricata.dns index a40107819..e0986c97c 100644 --- a/salt/elasticsearch/files/ingest/suricata.dns +++ b/salt/elasticsearch/files/ingest/suricata.dns @@ -2,18 +2,19 @@ "description" : "suricata.dns", "processors" : [ { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.type", "target_field": "dns.type", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.tx_id", "target_field": "dns.id", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.version", "target_field": "dns.version", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.rrname", "target_field": "dns.query.name", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.flags", "target_field": "dns.flags", "ignore_missing": true } }, + { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.type", "target_field": "dns.type", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.tx_id", "target_field": "dns.id", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.version", "target_field": "dns.version", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.rrname", "target_field": "dns.query.name", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.flags", "target_field": "dns.flags", "ignore_missing": true } }, { "rename": { "field": "message2.dns.qr", "target_field": "dns.qr", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.rd", "target_field": "dns.recursion.desired", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.ra", "target_field": "dns.recursion.available", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.rcode", "target_field": "dns.response.code", "ignore_missing": true } }, - { "rename": { "field": "message2.grouped.A", "target_field": "dns.answers.data", "ignore_missing": true } }, - { "rename": { "field": "message2.grouped.CNAME", "target_field": "dns.answers.name", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.rd", "target_field": "dns.recursion.desired", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.ra", "target_field": "dns.recursion.available", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.rcode", "target_field": "dns.response.code", "ignore_missing": true } }, + { "rename": { "field": "message2.grouped.A", "target_field": "dns.answers.data", "ignore_missing": true } }, + { "rename": { "field": "message2.grouped.CNAME", "target_field": "dns.answers.name", "ignore_missing": true } }, + { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, { "pipeline": { "name": "common" } } ] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/files/ingest/suricata.fileinfo b/salt/elasticsearch/files/ingest/suricata.fileinfo index 7b5bff14c..d5147fb40 100644 --- a/salt/elasticsearch/files/ingest/suricata.fileinfo +++ b/salt/elasticsearch/files/ingest/suricata.fileinfo @@ -2,17 +2,18 @@ "description" : "suricata.fileinfo", "processors" : [ { "set": { "field": "dataset", "value": "file" } }, - { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.fileinfo.filename", "target_field": "file.name", "ignore_missing": true } }, - { "rename": { "field": "message2.fileinfo.gaps", "target_field": "file.bytes.missing", "ignore_missing": true } }, - { "rename": { "field": "message2.fileinfo.magic", "target_field": "file.mime_type", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, + { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.filename", "target_field": "file.name", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.gaps", "target_field": "file.bytes.missing", "ignore_missing": true } }, + { "rename": { "field": "message2.fileinfo.magic", "target_field": "file.mime_type", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.md5", "target_field": "hash.md5", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.sha1", "target_field": "hash.sha1", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.sid", "target_field": "rule.uuid", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.size", "target_field": "file.size", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.state", "target_field": "file.state", "ignore_missing": true } }, { "rename": { "field": "message2.fileinfo.stored", "target_field": "file.saved", "ignore_missing": true } }, + { "set": { "if": "ctx.network?.protocol != null", "field": "file.source", "value": "{{network.protocol}}" } }, { "pipeline": { "name": "common" } } ] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/files/ingest/suricata.tls b/salt/elasticsearch/files/ingest/suricata.tls index 0dfc06eaa..6fb0aa5ad 100644 --- a/salt/elasticsearch/files/ingest/suricata.tls +++ b/salt/elasticsearch/files/ingest/suricata.tls @@ -1,22 +1,22 @@ { "description" : "suricata.tls", "processors" : [ - { "set": { "field": "dataset", "value": "ssl" } }, - { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.subject", "target_field": "ssl.certificate.subject", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.serial", "target_field": "ssl.certificate.serial", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.fingerprint", "target_field": "ssl.certificate.fingerprint", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.version", "target_field": "ssl.certificate.version", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.ja3.hash", "target_field": "hash.ja3", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.ja3.hash.string", "target_field": "hash.ja3_string", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.ja3s.hash", "target_field": "hash.ja3s", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.ja3s.hash.string", "target_field": "hash.ja3s_string", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.notbefore", "target_field": "x509.certificate.not_valid_before", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.notafter", "target_field": "x509.certificate.not_valid_after", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.sni", "target_field": "ssl.server_name", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.issuerdn", "target_field": "ssl.certificate.issuer", "ignore_missing": true } }, - { "rename": { "field": "message2.tls.session_resumed", "target_field": "ssl.session_resumed", "ignore_missing": true } }, + { "set": { "field": "dataset", "value": "ssl" } }, + { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, + { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.subject", "target_field": "ssl.certificate.subject", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.serial", "target_field": "ssl.certificate.serial", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.fingerprint", "target_field": "ssl.certificate.fingerprint", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.version", "target_field": "ssl.version", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.ja3.hash", "target_field": "hash.ja3", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.ja3.hash.string", "target_field": "hash.ja3_string", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.ja3s.hash", "target_field": "hash.ja3s", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.ja3s.hash.string", "target_field": "hash.ja3s_string", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.notbefore", "target_field": "x509.certificate.not_valid_before", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.notafter", "target_field": "x509.certificate.not_valid_after", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.sni", "target_field": "ssl.server_name", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.issuerdn", "target_field": "ssl.certificate.issuer", "ignore_missing": true } }, + { "rename": { "field": "message2.tls.session_resumed", "target_field": "ssl.session_resumed", "ignore_missing": true } }, { "pipeline": { "name": "common" } } ] -} \ No newline at end of file +} diff --git a/salt/elasticsearch/files/ingest/zeek.dns b/salt/elasticsearch/files/ingest/zeek.dns index 09ce7fd9f..d0c07492e 100644 --- a/salt/elasticsearch/files/ingest/zeek.dns +++ b/salt/elasticsearch/files/ingest/zeek.dns @@ -23,7 +23,7 @@ { "rename": { "field": "message2.TTLs", "target_field": "dns.ttls", "ignore_missing": true } }, { "rename": { "field": "message2.rejected", "target_field": "dns.query.rejected", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()", "ignore_failure": true } }, - { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "zeek.dns.tld" } }, + { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, { "pipeline": { "name": "zeek.common" } } ] } From 43e0c3a60b7eb0c60dd03005666a8a44588d1212 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Feb 2021 12:35:17 -0500 Subject: [PATCH 3/5] Apply action on PR only now that PRs are mandatory --- .github/workflows/leaktest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/leaktest.yml b/.github/workflows/leaktest.yml index e66a06fa8..590e220d0 100644 --- a/.github/workflows/leaktest.yml +++ b/.github/workflows/leaktest.yml @@ -1,6 +1,6 @@ name: leak-test -on: [push,pull_request] +on: [pull_request] jobs: build: From bcce205430c526c19f138c3c3e3b24a3f19cd243 Mon Sep 17 00:00:00 2001 From: doug Date: Mon, 22 Feb 2021 13:00:14 -0500 Subject: [PATCH 4/5] Improve support for Suricata metadata #2200 --- salt/elasticsearch/files/ingest/suricata.ftp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/elasticsearch/files/ingest/suricata.ftp b/salt/elasticsearch/files/ingest/suricata.ftp index 7d29fa708..492bd97e9 100644 --- a/salt/elasticsearch/files/ingest/suricata.ftp +++ b/salt/elasticsearch/files/ingest/suricata.ftp @@ -1,14 +1,14 @@ { "description" : "suricata.ftp", "processors" : [ - { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, - { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.reply", "target_field": "server.reply_message", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.completion_code", "target_field": "server.reply_code", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.reply_received", "target_field": "server.reply_received", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.command", "target_field": "ftp.command", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.command_data", "target_field": "ftp.command_data", "ignore_missing": true } }, - { "rename": { "field": "message2.ftp.dynamic_port", "target_field": "ftp.data_channel_destination.port", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, + { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.reply", "target_field": "server.reply_message", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.completion_code", "target_field": "server.reply_code", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.reply_received", "target_field": "server.reply_received", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.command", "target_field": "ftp.command", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.command_data", "target_field": "ftp.argument", "ignore_missing": true } }, + { "rename": { "field": "message2.ftp.dynamic_port", "target_field": "ftp.data_channel_destination.port", "ignore_missing": true } }, { "pipeline": { "name": "common" } } ] } From 71c7ffae3e7519df187d55b5bcc309a5a0271894 Mon Sep 17 00:00:00 2001 From: doug Date: Mon, 22 Feb 2021 13:49:29 -0500 Subject: [PATCH 5/5] Improve support for Suricata metadata #2200 --- salt/elasticsearch/files/ingest/suricata.dns | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/suricata.dns b/salt/elasticsearch/files/ingest/suricata.dns index e0986c97c..85229ee92 100644 --- a/salt/elasticsearch/files/ingest/suricata.dns +++ b/salt/elasticsearch/files/ingest/suricata.dns @@ -3,10 +3,11 @@ "processors" : [ { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, { "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } }, - { "rename": { "field": "message2.dns.type", "target_field": "dns.type", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.type", "target_field": "dns.query.type", "ignore_missing": true } }, { "rename": { "field": "message2.dns.tx_id", "target_field": "dns.id", "ignore_missing": true } }, { "rename": { "field": "message2.dns.version", "target_field": "dns.version", "ignore_missing": true } }, { "rename": { "field": "message2.dns.rrname", "target_field": "dns.query.name", "ignore_missing": true } }, + { "rename": { "field": "message2.dns.rrtype", "target_field": "dns.query.type_name", "ignore_missing": true } }, { "rename": { "field": "message2.dns.flags", "target_field": "dns.flags", "ignore_missing": true } }, { "rename": { "field": "message2.dns.qr", "target_field": "dns.qr", "ignore_missing": true } }, { "rename": { "field": "message2.dns.rd", "target_field": "dns.recursion.desired", "ignore_missing": true } },