From d40bbf6b090fce2fc922e6cf7d5e1f078195da46 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 29 Aug 2023 10:59:40 -0400 Subject: [PATCH 01/16] Add Apache templates --- salt/elasticsearch/defaults.yaml | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 3ea24c3fd..8ae75f984 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -286,6 +286,42 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so-logs-apache_x_access: + index_sorting: False + index_template: + index_patterns: + - "logs-apache.access-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-apache.access@package" + - "logs-apache.access@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-apache_x_error: + index_sorting: False + index_template: + index_patterns: + - "logs-apache.error-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-apache.error@package" + - "logs-apache.error@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false so-logs-auditd_x_log: index_sorting: False index_template: From f118e25e8c8e424cd80110f04eeac77e4470aa95 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 29 Aug 2023 11:00:31 -0400 Subject: [PATCH 02/16] Add Apache references --- salt/elasticsearch/soc_elasticsearch.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index a960facd1..1823337b5 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -201,7 +201,8 @@ elasticsearch: so-logs-windows_x_powershell: *indexSettings so-logs-windows_x_powershell_operational: *indexSettings so-logs-windows_x_sysmon_operational: *indexSettings - so-logs-auditd_x_log: *indexSettings + so-logs-apache_x_access: *indexSettings + so-logs-apache_x_error: *indexSettings so-logs-aws_x_cloudtrail: *indexSettings so-logs-aws_x_cloudwatch_logs: *indexSettings so-logs-aws_x_ec2_logs: *indexSettings From c01a9006a6609a0d928bea0c640fe997d42c415e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 29 Aug 2023 11:01:22 -0400 Subject: [PATCH 03/16] Add Apache package --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 77fa9dd31..55e70113f 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -26,6 +26,7 @@ elasticfleet: - stderr - stdout packages: + - apache - auditd - aws - azure From d2063c7e119d77f31139c6c9c94a5e5de1f18b3e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 29 Aug 2023 11:14:49 -0400 Subject: [PATCH 04/16] Add auditd reference back --- salt/elasticsearch/soc_elasticsearch.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 1823337b5..e4de29e00 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -203,6 +203,7 @@ elasticsearch: so-logs-windows_x_sysmon_operational: *indexSettings so-logs-apache_x_access: *indexSettings so-logs-apache_x_error: *indexSettings + so-logs-auditd_x_log: *indexSettings so-logs-aws_x_cloudtrail: *indexSettings so-logs-aws_x_cloudwatch_logs: *indexSettings so-logs-aws_x_ec2_logs: *indexSettings From 706a6e2d56ca045f97e2393d270e01a406334928 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 08:34:04 -0400 Subject: [PATCH 05/16] Make sure a data stream is created for syslog --- salt/elasticsearch/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8ae75f984..33362825f 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -4187,6 +4187,7 @@ elasticsearch: so-syslog: index_sorting: False index_template: + data_stream: {} index_patterns: - logs-syslog-so* template: From ce05f29dc4e436060a05cf02adfe3aa9578e3ee6 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 13:03:28 +0000 Subject: [PATCH 06/16] Add port_bindings for port 514 --- salt/docker/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index e39feaf06..a5d6c5d6d 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -178,6 +178,9 @@ docker: extra_env: [] 'so-elastic-agent': final_octet: 46 + port_bindings: + - 0.0.0.0:514:514/tcp + - 0.0.0.0:514:514/udp custom_bind_mounts: [] extra_hosts: [] extra_env: [] From 655eea2b007124d9abe0674b5281435817cf290d Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 13:03:56 +0000 Subject: [PATCH 07/16] Add port_bindings --- salt/elasticagent/enabled.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls index 963b8549b..7d0f401e9 100644 --- a/salt/elasticagent/enabled.sls +++ b/salt/elasticagent/enabled.sls @@ -31,6 +31,10 @@ so-elastic-agent: - {{ XTRAHOST }} {% endfor %} {% endif %} + - port_bindings: + {% for BINDING in DOCKER.containers['so-elastic-agent'].port_bindings %} + - {{ BINDING }} + {% endfor %} - binds: - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro - /opt/so/log/elasticagent:/usr/share/elastic-agent/logs From 0e22acc255cc62af53810156eabaf9471d8bbcae Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 13:04:32 +0000 Subject: [PATCH 08/16] Add tcp and udp integration --- .../files/elastic-agent.yml.jinja | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/salt/elasticagent/files/elastic-agent.yml.jinja b/salt/elasticagent/files/elastic-agent.yml.jinja index 92aacfa44..7d0b93344 100644 --- a/salt/elasticagent/files/elastic-agent.yml.jinja +++ b/salt/elasticagent/files/elastic-agent.yml.jinja @@ -430,3 +430,54 @@ inputs: exclude_files: - >- broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$ + - id: udp-udp-35051de0-46a5-11ee-8d5d-9f98c8182f60 + name: syslog-udp-514 + revision: 3 + type: udp + use_output: default + meta: + package: + name: udp + version: 1.10.0 + data_stream: + namespace: so + package_policy_id: 35051de0-46a5-11ee-8d5d-9f98c8182f60 + streams: + - id: udp-udp.generic-35051de0-46a5-11ee-8d5d-9f98c8182f60 + data_stream: + dataset: syslog + pipeline: syslog + host: '0.0.0.0:514' + max_message_size: 10KiB + processors: + - add_fields: + fields: + module: syslog + target: event + tags: + - syslog + - id: tcp-tcp-33d37bb0-46a5-11ee-8d5d-9f98c8182f60 + name: syslog-tcp-514 + revision: 3 + type: tcp + use_output: default + meta: + package: + name: tcp + version: 1.10.0 + data_stream: + namespace: so + package_policy_id: 33d37bb0-46a5-11ee-8d5d-9f98c8182f60 + streams: + - id: tcp-tcp.generic-33d37bb0-46a5-11ee-8d5d-9f98c8182f60 + data_stream: + dataset: syslog + pipeline: syslog + host: '0.0.0.0:514' + processors: + - add_fields: + fields: + module: syslog + target: event + tags: + - syslog From 60b0af5ab793fa1b6a592743f162e6905b797798 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 13:05:30 +0000 Subject: [PATCH 09/16] Allow external syslog --- salt/firewall/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index ff127c419..ecb4bad6b 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -1141,6 +1141,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: From fe690922de38a2d6ef9dd9da8afad312e39ad97e Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 19:16:05 +0000 Subject: [PATCH 10/16] Add analyzer configuration to the defaults file --- salt/sensoroni/defaults.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/salt/sensoroni/defaults.yaml b/salt/sensoroni/defaults.yaml index 4ccc11ce9..f53646ac2 100644 --- a/salt/sensoroni/defaults.yaml +++ b/salt/sensoroni/defaults.yaml @@ -8,3 +8,31 @@ sensoroni: node_checkin_interval_ms: 10000 sensoronikey: soc_host: + analyzers: + emailrep: + base_url: https://emailrep.io/ + api_key: + greynoise: + base_url: https://api.greynoise.io/ + api_key: + api_version: community + localfile: + file_path: [] + otx: + base_url: https://otx.alienvault.com/api/v1/ + api_key: + pulsedive: + base_url: https://pulsedive.com/api/ + api_key: + spamhaus: + lookup_host: zen.spamhaus.org + nameservers: [] + urlscan: + base_url: https://urlscan.io/api/v1/ + api_key: + enabled: False + visibility: public + timeout: 180 + virustotal: + base_url: https://www.virustotal.com/api/v3/search?query= + api_key: From 8cc19b0748c6804abe01abb8f7dd3df9dc23784f Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 19:16:38 +0000 Subject: [PATCH 11/16] Add analyzer configuration description --- salt/sensoroni/soc_sensoroni.yaml | 142 ++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index 8a35272ea..6a728ef9c 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -37,3 +37,145 @@ sensoroni: helpLink: sensoroni.html global: True advanced: True + analyzers: + emailrep: + api_key: + description: API key for the EmailRep analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the EmailRep analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + greynoise: + api_key: + description: API key for the GreyNoise analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + api_version: + description: API key for the GreyNoise analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + base_url: + description: Base URL for the GreyNoise analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + localfile: + file_path: + description: File path for the LocalFile analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: "[]string" + otx: + api_key: + description: API key for the OTX analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the OTX analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + pulsedive: + api_key: + description: API key for the Pulsedive analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the Pulsedive analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + spamhaus: + lookup_host: + description: Host to use for lookups. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + nameservers: + description: Nameservers used for queries. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedTypes: "[]string" + urlscan: + api_key: + description: API key for the Urlscan analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the Urlscan analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + enabled: + description: Analyzer enabled + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: bool + timeout: + description: Timeout for the Urlscan analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: int + visibility: + description: Type of visibility. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + virustotal: + api_key: + description: API key for the VirusTotal analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the VirusTotal analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string From 78915f900b8aad6ebb9e4038ec1f4b0ad916add6 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 15:37:30 -0400 Subject: [PATCH 12/16] Add fortigate package --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 55e70113f..979e795f7 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -41,6 +41,7 @@ elasticfleet: - fleet_server - fim - fortinet + - fortinet_fortigate - gcp - github - google_workspace From d090852895fb899fb9d029c57ae2c54e879a9722 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 15:40:40 -0400 Subject: [PATCH 13/16] Correct fortigate template name --- salt/elasticsearch/defaults.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 33362825f..cc2f5e1cd 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -970,18 +970,18 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-fortinet_x_fortigate: + so-logs-fortinet_fortigate_x_log: index_sorting: False index_template: index_patterns: - - "logs-fortinet.fortigate-*" + - "logs-fortinet_fortigate.log-*" template: settings: index: number_of_replicas: 0 composed_of: - - "logs-fortinet.fortigate@package" - - "logs-fortinet.fortigate@custom" + - "logs-fortinet_fortigate.log@package" + - "logs-fortinet_fortigate.log@custom" - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 501 From 21e91a753701b2672fedc9a5025982b05138fb6b Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 16:10:38 -0400 Subject: [PATCH 14/16] Fix api_version --- salt/sensoroni/soc_sensoroni.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index 6a728ef9c..2d1536191 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -62,7 +62,7 @@ sensoroni: advanced: True forcedType: string api_version: - description: API key for the GreyNoise analyzer. + description: API version for the GreyNoise analyzer. helpLink: sensoroni.html global: True sensitive: False From 41300af944c1c537ef9bf99bb2411d80bafdbb4e Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 16:30:32 -0400 Subject: [PATCH 15/16] Set global to false --- salt/sensoroni/soc_sensoroni.yaml | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index 2d1536191..eb63dbe25 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -42,14 +42,14 @@ sensoroni: api_key: description: API key for the EmailRep analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the EmailRep analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -57,21 +57,21 @@ sensoroni: api_key: description: API key for the GreyNoise analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string api_version: description: API version for the GreyNoise analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string base_url: description: Base URL for the GreyNoise analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -79,7 +79,7 @@ sensoroni: file_path: description: File path for the LocalFile analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: "[]string" @@ -87,14 +87,14 @@ sensoroni: api_key: description: API key for the OTX analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the OTX analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -102,14 +102,14 @@ sensoroni: api_key: description: API key for the Pulsedive analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the Pulsedive analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -117,14 +117,14 @@ sensoroni: lookup_host: description: Host to use for lookups. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string nameservers: description: Nameservers used for queries. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedTypes: "[]string" @@ -132,35 +132,35 @@ sensoroni: api_key: description: API key for the Urlscan analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the Urlscan analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string enabled: description: Analyzer enabled helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: bool timeout: description: Timeout for the Urlscan analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: int visibility: description: Type of visibility. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -168,14 +168,14 @@ sensoroni: api_key: description: API key for the VirusTotal analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the VirusTotal analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string From a615fc8e47f444ea9dc87390c897626b5b226216 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 30 Aug 2023 15:33:01 -0600 Subject: [PATCH 16/16] New Config Default: longRelayTimeoutMs Salt is getting a second timeout for operations known to take a long time such as sending and importing files. There's also an entry in soc_soc.yaml so the value can be changed in SOC's config page. --- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ff8b240ec..05543cd19 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1012,6 +1012,7 @@ soc: verifyCert: false salt: queueDir: /opt/sensoroni/queue + longRelayTimeoutMs: 120000 sostatus: refreshIntervalMs: 30000 offlineThresholdMs: 900000 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b2ed893f6..e94144069 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -111,6 +111,11 @@ soc: description: Duration (in milliseconds) that must elapse after a grid node fails to check-in before the node will be marked offline (fault). global: True advanced: True + salt: + longRelayTimeoutMs: + description: Duration (in milliseconds) to wait for a response from the Salt API when executing tasks known for being long running before giving up and showing an error on the SOC UI. + global: True + advanced: True client: enableReverseLookup: description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI.