From 454ca624ff1a97ca5c9c59afc6960d36354944ea Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Mon, 28 Nov 2022 18:24:28 -0300 Subject: [PATCH] remove example bots --- docker-compose-dev.yml | 2 +- example_bots/.keep | 1 + .../bots/collectors/noop/collector.py | 33 ---- .../intelmq/etc/manager/positions.conf | 54 ++++-- example_config/intelmq/etc/runtime.yaml | 172 +++++++++++++++++- 5 files changed, 203 insertions(+), 59 deletions(-) create mode 100644 example_bots/.keep delete mode 100644 example_bots/bots/collectors/noop/collector.py diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index b87a7b7..0fe852a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -25,7 +25,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./example_bots:/my_bots + - ./intelmq3-bots-example:/my_bots depends_on: - redis environment: diff --git a/example_bots/.keep b/example_bots/.keep new file mode 100644 index 0000000..b3f79f4 --- /dev/null +++ b/example_bots/.keep @@ -0,0 +1 @@ +Link your bots repository here diff --git a/example_bots/bots/collectors/noop/collector.py b/example_bots/bots/collectors/noop/collector.py deleted file mode 100644 index c9b5f22..0000000 --- a/example_bots/bots/collectors/noop/collector.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -SPDX-FileCopyrightText: 2022 Einar Lanfranco -SPDX-License-Identifier: AGPL-3.0-or-later - -Example Collector Bot for Demo purpose only. - -Document possible necessary configurations. -""" -import sys -import time -# imports for additional libraries and intelmq -from intelmq.lib.bot import CollectorBot - - -class NoOpCollectorBot(CollectorBot): - """Este bot no hace nada util""" - paso: str = "step" - cantidad: int = 5 - rate_limit: int = 3600 - - def process(self): - self.logger.info("Comenzando NOOP Collector") - time.sleep(self.cantidad) - for i in range(self.cantidad): - time.sleep(i) - self.logger.info(f'{self.paso}{i}') - report = self.new_report() - report.add("raw","bGEgbmFkYSBtaXNtYQ==") - report.add("feed.url", "http://noop.url") - self.send_message(report) - - -BOT = NoOpCollectorBot diff --git a/example_config/intelmq/etc/manager/positions.conf b/example_config/intelmq/etc/manager/positions.conf index 942a4c7..8d2e0f5 100644 --- a/example_config/intelmq/etc/manager/positions.conf +++ b/example_config/intelmq/etc/manager/positions.conf @@ -1,11 +1,39 @@ { + "Deduplicator-Expert-CISA": { + "x": 653, + "y": 308 + }, + "Filter-Expert-timebased": { + "x": 852, + "y": 380 + }, + "GenericCsv-Parser": { + "x": 579, + "y": 211 + }, + "GenericCsv-Parser-2": { + "x": 456, + "y": 362 + }, + "HTTP-Collector": { + "x": 206, + "y": 361 + }, + "NoOp-Collector": { + "x": 276, + "y": 220 + }, + "Telegram-Output": { + "x": 767, + "y": 510 + }, "cymru-whois-expert": { "x": -280, "y": 414 }, "deduplicator-expert": { - "x": 148, - "y": -107 + "x": 340, + "y": -113 }, "feodo-tracker-browse-collector": { "x": 136, @@ -15,9 +43,9 @@ "x": 97, "y": -287 }, - "file-output": { - "x": -413, - "y": 428 + "file-output-einar": { + "x": 1113, + "y": -93 }, "gethostbyname-1-expert": { "x": -125, @@ -32,16 +60,8 @@ "y": 24 }, "malc0de-windows-format-collector": { - "x": 433, - "y": 121 - }, - "malware-domain-list-collector": { - "x": 465, - "y": -198 - }, - "malware-domain-list-parser": { - "x": 283, - "y": -125 + "x": 697, + "y": -202 }, "spamhaus-drop-collector": { "x": -137, @@ -56,8 +76,8 @@ "y": 30 }, "url2fqdn-expert": { - "x": -75, - "y": 179 + "x": -57, + "y": 39 }, "settings": { "physics": false, diff --git a/example_config/intelmq/etc/runtime.yaml b/example_config/intelmq/etc/runtime.yaml index 0cfad2b..77fc7d1 100644 --- a/example_config/intelmq/etc/runtime.yaml +++ b/example_config/intelmq/etc/runtime.yaml @@ -1,3 +1,152 @@ +Deduplicator-Expert-CISA: + bot_id: Deduplicator-Expert-CISA + description: Detection and drop exact duplicate messages. Message hashes are cached + in the Redis database + enabled: true + group: Expert + groupname: experts + module: intelmq.bots.experts.deduplicator.expert + name: Deduplicator + parameters: + bypass: false + destination_queues: + _default: [Filter-Expert-timebased-queue] + filter_keys: '' + filter_type: blacklist + redis_cache_db: 6 + redis_cache_host: 127.0.0.1 + redis_cache_password: '' + redis_cache_port: 6379 + redis_cache_ttl: 86400 + run_mode: continuous +Filter-Expert-timebased: + bot_id: Filter-Expert-timebased + description: Filter events, supports named paths for splitting the message flow + enabled: true + group: Expert + module: intelmq.bots.experts.filter.expert + name: Filter + parameters: + destination_queues: + _default: [Telegram-Output-queue] + filter_action: '' + filter_key: time.source + filter_regex: '' + filter_value: '' + not_after: '' + not_before: 15 days + run_mode: continuous +GenericCsv-Parser: + bot_id: GenericCsv-Parser + description: 'Parse generic CSV data. Ignoring lines starting with character #. + URLs without protocol can be prefixed with a default value.' + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.generic.parser_csv + name: GenericCsv + parameters: + column_regex_search: '' + columns: '' + columns_required: '' + compose_fields: {} + data_type: '' + default_url_protocol: http:// + delimiter: ',' + destination_queues: {} + filter_text: '' + filter_type: '' + skip_header: false + time_format: '' + type: '' + type_translation: {} + run_mode: continuous +GenericCsv-Parser-2: + bot_id: GenericCsv-Parser-2 + description: 'Parse generic CSV data. Ignoring lines starting with character #. + URLs without protocol can be prefixed with a default value.' + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.generic.parser_csv + name: GenericCsv + parameters: + column_regex_search: '' + columns: extra.cveID,extra.vendorProject,extra.product,extra.vulnerabilityName,time.source,extra.shortDescription,extra.requiredAction,extra.dueDate,extra.notes + compose_fields: {} + data_type: '' + default_url_protocol: http:// + delimiter: ',' + destination_queues: + _default: [Deduplicator-Expert-CISA-queue, Filter-Expert-timebased-queue] + filter_text: '' + filter_type: '' + skip_header: true + time_format: '' + type: '' + type_translation: {} + run_mode: continuous +HTTP-Collector: + bot_id: HTTP-Collector + description: Fetch reports from an URL + enabled: true + group: Collector + groupname: collectors + module: intelmq.bots.collectors.http.collector_http + name: HTTP + parameters: + code: '' + destination_queues: + _default: [GenericCsv-Parser-2-queue] + documentation: '' + extract_files: false + gpg_keyring: '' + http_header: {} + http_password: '' + http_url: https://www.cisa.gov/sites/default/files/csv/known_exploited_vulnerabilities.csv + http_url_formatting: false + http_username: '' + provider: '' + rate_limit: 3600 + signature_url: '' + signature_url_formatting: false + ssl_client_cert: '' + ssl_client_certificate: '' + verify_pgp_signatures: false + run_mode: continuous +NoOp-Collector: + bot_id: NoOp-Collector + description: Este bot no hace nada util + enabled: true + group: Collector + groupname: collectors + module: intelmq.bots.collectors.otherexample.collector + name: NoOp + parameters: + cantidad: 5 + code: '' + destination_queues: + _default: [GenericCsv-Parser-queue] + documentation: '' + paso: step + provider: '' + rate_limit: 3600 + run_mode: continuous +Telegram-Output: + bot_id: Telegram-Output + description: Send events to a REST API listener through HTTP POST + enabled: true + group: Output + groupname: outputs + module: intelmq.bots.outputs.telegram.output + name: Telegram + parameters: + chat_id: 145090811 + destination_queues: {} + message: 'Nuevo CVE siendo explotado: {ev[extra.cveID]}:{ev[extra.vulnerabilityName]}. + Agregado {ev[time.source]}. Detalles: Producto{ev[extra.product]} ' + token: 5985070241:AAFhp3Hrl-_sv67lZoibhW2uBTqpPfBRJiE + run_mode: continuous cymru-whois-expert: bot_id: cymru-whois-expert description: Cymru Whois (IP to ASN) is the bot responsible to add network information @@ -9,7 +158,7 @@ cymru-whois-expert: name: Cymru Whois parameters: destination_queues: - _default: [file-output-queue] + _default: [file-output-einar-queue] overwrite: true redis_cache_db: 5 redis_cache_host: 127.0.0.1 @@ -38,10 +187,12 @@ deduplicator-expert: redis_cache_ttl: 86400 run_mode: continuous feodo-tracker-browse-collector: + bot_id: feodo-tracker-browse-collector description: Generic URL Fetcher is the bot responsible to get the report from an URL. enabled: true group: Collector + groupname: collectors module: intelmq.bots.collectors.http.collector_http name: URL Fetcher parameters: @@ -58,10 +209,12 @@ feodo-tracker-browse-collector: ssl_client_certificate: null run_mode: continuous feodo-tracker-browse-parser: + bot_id: feodo-tracker-browse-parser description: HTML Table Parser is a bot configurable to parse different html table data. enabled: true group: Parser + groupname: parsers module: intelmq.bots.parsers.html_table.parser name: HTML Table parameters: @@ -80,16 +233,19 @@ feodo-tracker-browse-parser: time_format: null type: c2-server run_mode: continuous -file-output: - bot_id: file-output +file-output-einar: + bot_id: file-output-einar description: File is the bot responsible to send events to a file. enabled: true group: Output groupname: outputs module: intelmq.bots.outputs.file.output name: File - parameters: {file: /opt/intelmq/var/lib/bots/file-output/events.txt, hierarchical_output: false, - single_key: null} + parameters: + destination_queues: {} + file: /opt/intelmq/var/lib/bots/file-output/events.txt + hierarchical_output: false + single_key: '' run_mode: continuous gethostbyname-1-expert: bot_id: gethostbyname-1-expert @@ -115,9 +271,9 @@ gethostbyname-2-expert: destination_queues: _default: [cymru-whois-expert-queue] run_mode: continuous -global: {destination_pipeline_broker: redis, process_manager: intelmq, source_pipeline_broker: redis, - ssl_ca_certificate: null, statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null, - statistics_port: 6379, destination_pipeline_host: redis, source_pipeline_host: redis} +global: {destination_pipeline_broker: redis, destination_pipeline_host: redis, process_manager: intelmq, + source_pipeline_broker: redis, source_pipeline_host: redis, ssl_ca_certificate: null, + statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null, statistics_port: 6379} malc0de-parser: bot_id: malc0de-parser description: Malc0de Parser is the bot responsible to parse the IP Blacklist and