diff --git a/example_config/intelmq/etc/BOTS b/example_config/intelmq/etc/BOTS index 0018e0c..f5e918b 100644 --- a/example_config/intelmq/etc/BOTS +++ b/example_config/intelmq/etc/BOTS @@ -41,6 +41,14 @@ "rate_limit": 300 } }, + "Kafka": { + "description": "Fetch data from the Apache Kafka distributed stream processing system.", + "module": "intelmq.bots.collectors.kafka.collector", + "parameters": { + "bootstrap_servers": "localhost:9092", + "topic": "" + } + }, "Mail Attachment Fetcher": { "description": "Monitor IMAP mailboxes and retrieve mail attachments.", "module": "intelmq.bots.collectors.mail.collector_mail_attach", @@ -100,6 +108,7 @@ "module": "intelmq.bots.collectors.http.collector_http", "parameters": { "extract_files": false, + "gpg_keyring": null, "http_password": null, "http_url": "", "http_url_formatting": false, @@ -107,10 +116,10 @@ "name": "__FEED__", "provider": "__PROVIDER__", "rate_limit": 3600, + "signature_url": null, + "signature_url_formatting": false, "ssl_client_certificate": null, - "verify_gpg_signatures": false, - "gpg_signature_suffix": ".asc", - "gpg_keyring": null + "verify_pgp_signatures": false } }, "URL Stream Fetcher": { @@ -242,12 +251,12 @@ "description": "Collect data from ESET's TAXII API", "module": "intelmq.bots.collectors.eset.collector", "parameters": { - "username": "", - "password": "", + "collection": "", "endpoint": "eti.eset.com", - "time_delta": 3600, + "password": "", "rate_limit": 3600, - "collection": "" + "time_delta": 3600, + "username": "" } }, "Github API": { @@ -305,6 +314,21 @@ "redis_cache_ttl": 604800 } }, + "Shadowserver Reports API": { + "description": "Connects to the Shadowserver API, requests a list of all the reports for a specific country and processes the ones that are new.", + "module": "intelmq.bots.collectors.shadowserver.collector_reports_api", + "parameters": { + "country": "", + "api_key": "", + "secret": "", + "types": "", + "rate_limit": 86400, + "redis_cache_db": 12, + "redis_cache_host": "127.0.0.1", + "redis_cache_port": 6379, + "redis_cache_ttl": 864000 + } + }, "Shodan Stream": { "description": "Collect the Shodan stream from the Shodan API.", "module": "intelmq.bots.collectors.shodan.collector_stream", @@ -407,6 +431,16 @@ "module": "intelmq.bots.parsers.ci_army.parser", "parameters": {} }, + "CZ.NIC HaaS": { + "description": "CZ.NIC HaaS Parser is the bot responsible to parse the report and sanitize the information.", + "module": "intelmq.bots.parsers.cznic.parser_haas", + "parameters": {} + }, + "CZ.NIC Proki": { + "description": "Parse the feed from malicious IP addresses on Czech networks.", + "module": "intelmq.bots.parsers.cznic.parser_proki", + "parameters": {} + }, "CertStream": { "description": "Parse the CertStream feed.", "module": "intelmq.bots.parsers.calidog.parser_certstream", @@ -427,11 +461,6 @@ "module": "intelmq.bots.parsers.cymru.parser_full_bogons", "parameters": {} }, - "CZ.NIC HaaS": { - "description": "CZ.NIC HaaS Parser is the bot responsible to parse the report and sanitize the information.", - "module": "intelmq.bots.parsers.cznic.parser_haas", - "parameters": {} - }, "DShield AS": { "description": "Parse the DShield AS.", "module": "intelmq.bots.parsers.dshield.parser_asn", @@ -481,6 +510,7 @@ "", "source.fqdn" ], + "compose_fields": null, "default_url_protocol": "http://", "delimiter": ",", "filter_text": null, @@ -532,6 +562,17 @@ "splitlines": false } }, + "Key-Value": { + "description": "Parse key=value strings.", + "module": "intelmq.bots.parsers.key_value.parser", + "parameters": { + "keys": {}, + "kv_separator": "=", + "pair_separator": " ", + "strip_quotes": true, + "timestamp_key": null + } + }, "MISP": { "description": "Parse MISP events.", "module": "intelmq.bots.parsers.misp.parser", @@ -542,11 +583,6 @@ "module": "intelmq.bots.parsers.malc0de.parser", "parameters": {} }, - "Malware Domain List": { - "description": "Parse the Malware Domain List feed.", - "module": "intelmq.bots.parsers.malwaredomainlist.parser", - "parameters": {} - }, "Malware Domains": { "description": "Parse the Malware Domains feed.", "module": "intelmq.bots.parsers.malwaredomains.parser", @@ -604,14 +640,22 @@ "module": "intelmq.bots.parsers.phishtank.parser", "parameters": {} }, - "ShadowServer": { - "description": "Parse all ShadowServer feeds.", + "Shadowserver CSV": { + "description": "Parse Shadowserver feeds in CSV format.", "module": "intelmq.bots.parsers.shadowserver.parser", "parameters": { "feedname": "", "overwrite": true } }, + "Shadowserver JSON": { + "description": "Parse all Shadowserver feeds in JSON format (data coming from the reports API).", + "module": "intelmq.bots.parsers.shadowserver.parser_json", + "parameters": { + "feedname": "", + "overwrite": true + } + }, "Shodan": { "description": "Parse Shodan data collected via the Shodan API.", "module": "intelmq.bots.parsers.shodan.parser", @@ -729,7 +773,7 @@ } }, "Deduplicator": { - "description": "Detection and drop exact duplicate messages. Message hashes are cached in the Redis datbase.", + "description": "Detection and drop exact duplicate messages. Message hashes are cached in the Redis database.", "module": "intelmq.bots.experts.deduplicator.expert", "parameters": { "filter_keys": "raw,time.observation", @@ -816,7 +860,8 @@ "module": "intelmq.bots.experts.gethostbyname.expert", "parameters": { "fallback_to_url": true, - "gaierrors_to_ignore": null + "gaierrors_to_ignore": null, + "overwrite": false } }, "IDEA Converter": { @@ -839,9 +884,9 @@ "module": "intelmq.bots.experts.maxmind_geoip.expert", "parameters": { "database": "/opt/intelmq/var/lib/bots/maxmind_geoip/GeoLite2-City.mmdb", + "license_key": "", "overwrite": false, - "use_registered": false, - "license_key": "" + "use_registered": false } }, "McAfee Active Response Lookup": { @@ -853,7 +898,7 @@ } }, "Modify": { - "description": "Perform arbitrary changes to event's fields based on regular-expression-based rules on different values. See docs/Bots.md for some examples.", + "description": "Perform arbitrary changes to event's fields based on regular-expression-based rules on different values. See the bot's documentation for some examples.", "module": "intelmq.bots.experts.modify.expert", "parameters": { "case_sensitive": true, @@ -900,9 +945,9 @@ "description": "Adds the Risk Score from RecordedFuture IPRisk associated with source.ip or destination.ip with a local database.", "module": "intelmq.bots.experts.recordedfuture_iprisk.expert", "parameters": { + "api_token": "", "database": "/opt/intelmq/var/lib/bots/recordedfuture_iprisk/rfiprisk.dat", - "overwrite": false, - "api_token": "" + "overwrite": false } }, "Reverse DNS": { @@ -925,11 +970,54 @@ "file": "/opt/intelmq/var/lib/bots/sieve/filter.sieve" } }, + "Splunk saved search": { + "description": "Enrich an event from Splunk search results.", + "module": "intelmq.bots.experts.splunk_saved_search.expert", + "parameters": { + "auth_token": "VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wIG92ZXIgdGhlIGxhenkgZG9nLgo=", + "multiple_result_handling": [ + "warn", + "use_first", + "send" + ], + "not_found": [ + "warn", + "send" + ], + "overwrite": null, + "result_fields": { + "result field": "event field" + }, + "retry_interval": 5, + "saved_search": "search_name", + "search_parameters": { + "event field": "search parameter" + }, + "url": "https://splunk:8089/" + } + }, "Taxonomy": { "description": "Apply the eCSIRT Taxonomy to all events.", "module": "intelmq.bots.experts.taxonomy.expert", "parameters": {} }, + "Threshold": { + "description": "Check if the number of similar messages during a specified time interval exceeds a set value.", + "module": "intelmq.bots.experts.threshold.expert", + "parameters": { + "add_keys": { + "comment": "Threshold reached" + }, + "filter_keys": "raw,time.observation", + "filter_type": "blacklist", + "redis_cache_db": "11", + "redis_cache_host": "127.0.0.1", + "redis_cache_password": null, + "redis_cache_port": "6379", + "threshold": 100, + "timeout": 3600 + } + }, "Tor Nodes": { "description": "Check if the IP address is a Tor Exit Node based on a local database of TOR nodes.", "module": "intelmq.bots.experts.tor_nodes.expert", @@ -939,7 +1027,7 @@ } }, "Wait": { - "description": "Wait for a some time or until a queue size is lower than a given numer.", + "description": "Wait for a some time or until a queue size is lower than a given number.", "module": "intelmq.bots.experts.wait.expert", "parameters": { "queue_db": 2, @@ -1119,24 +1207,24 @@ "description": "Request Tracker ticket creation bot. Create linked Investigation queue ticket if needed, according to the RTIR flow", "module": "intelmq.bots.outputs.rt.output", "parameters": { - "rt_uri": "http://localhost/REST/1.0", - "verify_cert": true, - "rt_user": "apiuser", - "rt_password": "", - "queue": "Incidents", - "description_attr": "event_description.text", "CF_mapping": { - "event_description.text": "Description", - "source.ip": "IP", - "classification.type": "Incident Type", "classification.taxonomy": "Classification", - "extra.incident.severity": "Incident Severity", + "classification.type": "Incident Type", + "event_description.text": "Description", "extra.incident.importance": "Importance", - "extra.organization.name": "Customer" + "extra.incident.severity": "Incident Severity", + "extra.organization.name": "Customer", + "source.ip": "IP" }, "create_investigation": false, + "description_attr": "event_description.text", + "final_status": "resolved", "investigation_fields": "time.source,time.observation,source.ip,source.port,source.fqdn,source.url,classification.taxonomy,classification.type,classification.identifier,event_description.url,event_description.text,malware.name,protocol.application,protocol.transport", - "final_status": "resolved" + "queue": "Incidents", + "rt_password": "", + "rt_uri": "http://localhost/REST/1.0", + "rt_user": "apiuser", + "verify_cert": true } }, "SMTP": { diff --git a/example_config/intelmq/etc/defaults.conf b/example_config/intelmq/etc/defaults.conf index 090150e..17350a8 100644 --- a/example_config/intelmq/etc/defaults.conf +++ b/example_config/intelmq/etc/defaults.conf @@ -2,7 +2,7 @@ "accuracy": 100, "destination_pipeline_broker": "redis", "destination_pipeline_db": 2, - "destination_pipeline_host": "redis", + "destination_pipeline_host": "127.0.0.1", "destination_pipeline_password": null, "destination_pipeline_port": 6379, "error_dump_message": true, @@ -30,12 +30,12 @@ "rate_limit": 0, "source_pipeline_broker": "redis", "source_pipeline_db": 2, - "source_pipeline_host": "redis", + "source_pipeline_host": "127.0.0.1", "source_pipeline_password": null, "source_pipeline_port": 6379, "ssl_ca_certificate": null, "statistics_database": 3, - "statistics_host": "redis", + "statistics_host": "127.0.0.1", "statistics_password": null, "statistics_port": 6379 } diff --git a/example_config/intelmq/etc/feeds.yaml b/example_config/intelmq/etc/feeds.yaml index 8ca7e89..e87fcba 100644 --- a/example_config/intelmq/etc/feeds.yaml +++ b/example_config/intelmq/etc/feeds.yaml @@ -571,7 +571,7 @@ providers: services on the router or tried to gain access to them. The list also contains a list of tags for each address which indicate what behaviour of the address was observed. - + The Turris Greylist feed provides PGP signatures for the provided files. You will need to import the public PGP key from the linked documentation page, currently available at @@ -579,13 +579,13 @@ providers: or from below. See the URL Fetcher Collector documentation for more information on PGP signature verification. - + PGP Public key: ``` -----BEGIN PGP PUBLIC KEY BLOCK----- Version: SKS 1.1.6 Comment: Hostname: pgp.mit.edu - + mQINBFRl7D8BEADaRFoDa/+r27Gtqrdn8sZL4aSYTU4Q3gDr3TfigK8H26Un/Y79a/DUL1o0 o8SRae3uwVcjJDHZ6KDnxThbqF7URfpuCcCYxOs8p/eu3dSueqEGTODHWF4ChIh2japJDc4t 3FQHbIh2e3GHotVqJGhvxMmWqBFoZ/mlWvhjs99FFBZ87qbUNk7l1UAGEXeWeECgz9nGox40 @@ -1219,24 +1219,6 @@ providers: revision: 2018-01-20 documentation: http://clean-mx.de/ public: no - Malware Domain List: - Blacklist: - description: No description provided by feed provider. - additional_information: - bots: - collector: - module: intelmq.bots.collectors.http.collector_http - parameters: - http_url: http://www.malwaredomainlist.com/updatescsv.php - rate_limit: 3600 - name: __FEED__ - provider: __PROVIDER__ - parser: - module: intelmq.bots.parsers.malwaredomainlist.parser - parameters: - revision: 2018-01-20 - documentation: http://www.malwaredomainlist.com/ - public: yes AnubisNetworks: Cyberfeed Stream: description: Fetches and parsers the Cyberfeed data stream. @@ -1444,7 +1426,7 @@ providers: revision: 2018-01-20 documentation: http://vxvault.net/ViriList.php public: yes - ShadowServer: + Shadowserver: Via IMAP: description: Shadowserver sends out a variety of reports (see https://www.shadowserver.org/wiki/pmwiki.php/Services/Reports). additional_information: The configuration retrieves the data from a e-mails via IMAP from the attachments. @@ -1752,10 +1734,10 @@ providers: listen 443 ssl http2; server_name [your host name]; client_max_body_size 50M; - + ssl_certificate [path to your key]; ssl_certificate_key [path to your certificate]; - + location /[your private url] { if ($http_authorization != '[your private password]') { return 403; @@ -1872,3 +1854,25 @@ providers: revision: 2020-06-30 documentation: https://www.eset.com/int/business/services/threat-intelligence/ public: no + Shodan: + Country Stream: + description: Collects the Shodan stream for one or multiple countries from the Shodan API. + additional_information: A Shodan account with streaming permissions is needed. + bots: + collector: + module: intelmq.bots.collectors.shodan.collector_stream + parameters: + api_key: + countries: + error_retry_delay: 0 + name: __FEED__ + provider: __PROVIDER__ + parser: + module: intelmq.bots.parsers.shodan.parser + parameters: + ignore_errors: false + error_retry_delay: 0 + minimal_mode: false + revision: 2021-03-22 + documentation: https://developer.shodan.io/api/stream + public: no diff --git a/example_config/intelmq/etc/pipeline.conf b/example_config/intelmq/etc/pipeline.conf index 1571db7..f9cd011 100644 --- a/example_config/intelmq/etc/pipeline.conf +++ b/example_config/intelmq/etc/pipeline.conf @@ -48,17 +48,6 @@ "malc0de-parser-queue" ] }, - "malware-domain-list-collector": { - "destination-queues": [ - "malware-domain-list-parser-queue" - ] - }, - "malware-domain-list-parser": { - "destination-queues": [ - "deduplicator-expert-queue" - ], - "source-queue": "malware-domain-list-parser-queue" - }, "spamhaus-drop-collector": { "destination-queues": [ "spamhaus-drop-parser-queue" diff --git a/example_config/intelmq/etc/runtime.conf b/example_config/intelmq/etc/runtime.conf index 13bde6a..60572b8 100644 --- a/example_config/intelmq/etc/runtime.conf +++ b/example_config/intelmq/etc/runtime.conf @@ -143,33 +143,6 @@ }, "run_mode": "continuous" }, - "malware-domain-list-collector": { - "bot_id": "malware-domain-list-collector", - "description": "Malware Domain List Collector is the bot responsible to get the report from source of information.", - "enabled": true, - "group": "Collector", - "groupname": "collectors", - "module": "intelmq.bots.collectors.http.collector_http", - "name": "Malware Domain List", - "parameters": { - "http_url": "http://www.malwaredomainlist.com/updatescsv.php", - "name": "Malware Domain List", - "provider": "Malware Domain List", - "rate_limit": 3600 - }, - "run_mode": "continuous" - }, - "malware-domain-list-parser": { - "bot_id": "malware-domain-list-parser", - "description": "Malware Domain List Parser is the bot responsible to parse the report and sanitize the information.", - "enabled": true, - "group": "Parser", - "groupname": "parsers", - "module": "intelmq.bots.parsers.malwaredomainlist.parser", - "name": "Malware Domain List", - "parameters": {}, - "run_mode": "continuous" - }, "spamhaus-drop-collector": { "bot_id": "spamhaus-drop-collector", "description": "",