remove example bots

docker-compose-dev.yml

@@ -25,7 +25,7 @@ services:
            - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
            - ./intelmq_logs:/opt/intelmq/var/log
            - ./intelmq_output:/opt/intelmq/var/lib/bots
-           - ./example_bots:/my_bots
+           - ./intelmq3-bots-example:/my_bots
         depends_on: 
             - redis
         environment:

example_bots/.keep

@@ -0,0 +1 @@
+Link your bots repository here

example_bots/bots/collectors/noop/collector.py

@@ -1,33 +0,0 @@
-"""
-SPDX-FileCopyrightText: 2022 Einar Lanfranco
-SPDX-License-Identifier: AGPL-3.0-or-later
-
-Example Collector Bot for Demo purpose only.
-
-Document possible necessary configurations.
-"""
-import sys
-import time
-# imports for additional libraries and intelmq
-from intelmq.lib.bot import CollectorBot
-
-
-class NoOpCollectorBot(CollectorBot):
-    """Este bot no hace nada util"""
-    paso: str = "step"
-    cantidad: int = 5
-    rate_limit: int = 3600
-
-    def process(self):
-        self.logger.info("Comenzando NOOP Collector")
-        time.sleep(self.cantidad)
-        for i in range(self.cantidad):
-            time.sleep(i)
-            self.logger.info(f'{self.paso}{i}')
-        report = self.new_report()
-        report.add("raw","bGEgbmFkYSBtaXNtYQ==")
-        report.add("feed.url", "http://noop.url")
-        self.send_message(report)
-     
-
-BOT = NoOpCollectorBot

example_config/intelmq/etc/manager/positions.conf

@@ -1,11 +1,39 @@
 {
+    "Deduplicator-Expert-CISA": {
+        "x": 653,
+        "y": 308
+    },
+    "Filter-Expert-timebased": {
+        "x": 852,
+        "y": 380
+    },
+    "GenericCsv-Parser": {
+        "x": 579,
+        "y": 211
+    },
+    "GenericCsv-Parser-2": {
+        "x": 456,
+        "y": 362
+    },
+    "HTTP-Collector": {
+        "x": 206,
+        "y": 361
+    },
+    "NoOp-Collector": {
+        "x": 276,
+        "y": 220
+    },
+    "Telegram-Output": {
+        "x": 767,
+        "y": 510
+    },
     "cymru-whois-expert": {
         "x": -280,
         "y": 414
     },
     "deduplicator-expert": {
-        "x": 148,
+        "x": 340,
-        "y": -107
+        "y": -113
     },
     "feodo-tracker-browse-collector": {
         "x": 136,
@@ -15,9 +43,9 @@
         "x": 97,
         "y": -287
     },
-    "file-output": {
+    "file-output-einar": {
-        "x": -413,
+        "x": 1113,
-        "y": 428
+        "y": -93
     },
     "gethostbyname-1-expert": {
         "x": -125,
@@ -32,16 +60,8 @@
         "y": 24
     },
     "malc0de-windows-format-collector": {
-        "x": 433,
+        "x": 697,
-        "y": 121
+        "y": -202
-    },
-    "malware-domain-list-collector": {
-        "x": 465,
-        "y": -198
-    },
-    "malware-domain-list-parser": {
-        "x": 283,
-        "y": -125
     },
     "spamhaus-drop-collector": {
         "x": -137,
@@ -56,8 +76,8 @@
         "y": 30
     },
     "url2fqdn-expert": {
-        "x": -75,
+        "x": -57,
-        "y": 179
+        "y": 39
     },
     "settings": {
         "physics": false,

example_config/intelmq/etc/runtime.yaml

@@ -1,3 +1,152 @@
+Deduplicator-Expert-CISA:
+  bot_id: Deduplicator-Expert-CISA
+  description: Detection and drop exact duplicate messages. Message hashes are cached
+    in the Redis database
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.deduplicator.expert
+  name: Deduplicator
+  parameters:
+    bypass: false
+    destination_queues:
+      _default: [Filter-Expert-timebased-queue]
+    filter_keys: ''
+    filter_type: blacklist
+    redis_cache_db: 6
+    redis_cache_host: 127.0.0.1
+    redis_cache_password: ''
+    redis_cache_port: 6379
+    redis_cache_ttl: 86400
+  run_mode: continuous
+Filter-Expert-timebased:
+  bot_id: Filter-Expert-timebased
+  description: Filter events, supports named paths for splitting the message flow
+  enabled: true
+  group: Expert
+  module: intelmq.bots.experts.filter.expert
+  name: Filter
+  parameters:
+    destination_queues:
+      _default: [Telegram-Output-queue]
+    filter_action: ''
+    filter_key: time.source
+    filter_regex: ''
+    filter_value: ''
+    not_after: ''
+    not_before: 15 days
+  run_mode: continuous
+GenericCsv-Parser:
+  bot_id: GenericCsv-Parser
+  description: 'Parse generic CSV data. Ignoring lines starting with character #.
+    URLs without protocol can be prefixed with a default value.'
+  enabled: true
+  group: Parser
+  groupname: parsers
+  module: intelmq.bots.parsers.generic.parser_csv
+  name: GenericCsv
+  parameters:
+    column_regex_search: ''
+    columns: ''
+    columns_required: ''
+    compose_fields: {}
+    data_type: ''
+    default_url_protocol: http://
+    delimiter: ','
+    destination_queues: {}
+    filter_text: ''
+    filter_type: ''
+    skip_header: false
+    time_format: ''
+    type: ''
+    type_translation: {}
+  run_mode: continuous
+GenericCsv-Parser-2:
+  bot_id: GenericCsv-Parser-2
+  description: 'Parse generic CSV data. Ignoring lines starting with character #.
+    URLs without protocol can be prefixed with a default value.'
+  enabled: true
+  group: Parser
+  groupname: parsers
+  module: intelmq.bots.parsers.generic.parser_csv
+  name: GenericCsv
+  parameters:
+    column_regex_search: ''
+    columns: extra.cveID,extra.vendorProject,extra.product,extra.vulnerabilityName,time.source,extra.shortDescription,extra.requiredAction,extra.dueDate,extra.notes
+    compose_fields: {}
+    data_type: ''
+    default_url_protocol: http://
+    delimiter: ','
+    destination_queues:
+      _default: [Deduplicator-Expert-CISA-queue, Filter-Expert-timebased-queue]
+    filter_text: ''
+    filter_type: ''
+    skip_header: true
+    time_format: ''
+    type: ''
+    type_translation: {}
+  run_mode: continuous
+HTTP-Collector:
+  bot_id: HTTP-Collector
+  description: Fetch reports from an URL
+  enabled: true
+  group: Collector
+  groupname: collectors
+  module: intelmq.bots.collectors.http.collector_http
+  name: HTTP
+  parameters:
+    code: ''
+    destination_queues:
+      _default: [GenericCsv-Parser-2-queue]
+    documentation: ''
+    extract_files: false
+    gpg_keyring: ''
+    http_header: {}
+    http_password: ''
+    http_url: https://www.cisa.gov/sites/default/files/csv/known_exploited_vulnerabilities.csv
+    http_url_formatting: false
+    http_username: ''
+    provider: ''
+    rate_limit: 3600
+    signature_url: ''
+    signature_url_formatting: false
+    ssl_client_cert: ''
+    ssl_client_certificate: ''
+    verify_pgp_signatures: false
+  run_mode: continuous
+NoOp-Collector:
+  bot_id: NoOp-Collector
+  description: Este bot no hace nada util
+  enabled: true
+  group: Collector
+  groupname: collectors
+  module: intelmq.bots.collectors.otherexample.collector
+  name: NoOp
+  parameters:
+    cantidad: 5
+    code: ''
+    destination_queues:
+      _default: [GenericCsv-Parser-queue]
+    documentation: ''
+    paso: step
+    provider: ''
+    rate_limit: 3600
+  run_mode: continuous
+Telegram-Output:
+  bot_id: Telegram-Output
+  description: Send events to a REST API listener through HTTP POST
+  enabled: true
+  group: Output
+  groupname: outputs
+  module: intelmq.bots.outputs.telegram.output
+  name: Telegram
+  parameters:
+    chat_id: 145090811
+    destination_queues: {}
+    message: 'Nuevo CVE siendo explotado: {ev[extra.cveID]}:{ev[extra.vulnerabilityName]}.
+      Agregado {ev[time.source]}. Detalles: Producto{ev[extra.product]} '
+    token: 5985070241:AAFhp3Hrl-_sv67lZoibhW2uBTqpPfBRJiE
+  run_mode: continuous
 cymru-whois-expert:
   bot_id: cymru-whois-expert
   description: Cymru Whois (IP to ASN) is the bot responsible to add network information
@@ -9,7 +158,7 @@ cymru-whois-expert:
   name: Cymru Whois
   parameters:
     destination_queues:
-      _default: [file-output-queue]
+      _default: [file-output-einar-queue]
     overwrite: true
     redis_cache_db: 5
     redis_cache_host: 127.0.0.1
@@ -38,10 +187,12 @@ deduplicator-expert:
     redis_cache_ttl: 86400
   run_mode: continuous
 feodo-tracker-browse-collector:
+  bot_id: feodo-tracker-browse-collector
   description: Generic URL Fetcher is the bot responsible to get the report from an
     URL.
   enabled: true
   group: Collector
+  groupname: collectors
   module: intelmq.bots.collectors.http.collector_http
   name: URL Fetcher
   parameters:
@@ -58,10 +209,12 @@ feodo-tracker-browse-collector:
     ssl_client_certificate: null
   run_mode: continuous
 feodo-tracker-browse-parser:
+  bot_id: feodo-tracker-browse-parser
   description: HTML Table Parser is a bot configurable to parse different html table
     data.
   enabled: true
   group: Parser
+  groupname: parsers
   module: intelmq.bots.parsers.html_table.parser
   name: HTML Table
   parameters:
@@ -80,16 +233,19 @@ feodo-tracker-browse-parser:
     time_format: null
     type: c2-server
   run_mode: continuous
-file-output:
+file-output-einar:
-  bot_id: file-output
+  bot_id: file-output-einar
   description: File is the bot responsible to send events to a file.
   enabled: true
   group: Output
   groupname: outputs
   module: intelmq.bots.outputs.file.output
   name: File
-  parameters: {file: /opt/intelmq/var/lib/bots/file-output/events.txt, hierarchical_output: false,
+  parameters:
-    single_key: null}
+    destination_queues: {}
+    file: /opt/intelmq/var/lib/bots/file-output/events.txt
+    hierarchical_output: false
+    single_key: ''
   run_mode: continuous
 gethostbyname-1-expert:
   bot_id: gethostbyname-1-expert
@@ -115,9 +271,9 @@ gethostbyname-2-expert:
     destination_queues:
       _default: [cymru-whois-expert-queue]
   run_mode: continuous
-global: {destination_pipeline_broker: redis, process_manager: intelmq, source_pipeline_broker: redis,
+global: {destination_pipeline_broker: redis, destination_pipeline_host: redis, process_manager: intelmq,
-  ssl_ca_certificate: null, statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null,
+  source_pipeline_broker: redis, source_pipeline_host: redis, ssl_ca_certificate: null,
-  statistics_port: 6379, destination_pipeline_host: redis, source_pipeline_host: redis}
+  statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null, statistics_port: 6379}
 malc0de-parser:
   bot_id: malc0de-parser
   description: Malc0de Parser is the bot responsible to parse the IP Blacklist and