From 77d05fbc1feb9ff9abf14e1c19f00c43e5888071 Mon Sep 17 00:00:00 2001 From: Jeremias Pretto Date: Tue, 23 Mar 2021 11:57:27 -0300 Subject: [PATCH] Features: - Support for bot development in intelmq 2.3.1 version - Add mail handler - Add start botnet at boot option --- .docker/intelmq-full-dev/Dockerfile | 19 ++++++-- .docker/intelmq-full-dev/entrypoint | 48 ------------------ .docker/intelmq-full-dev/entrypoint_dev.sh | 14 ++++++ .docker/intelmq-full-dev/merge_BOTS.py | 33 +++++++++++++ .docker/intelmq-full-dev/update | 9 ---- .docker/intelmq-full-dev/update.sh | 16 ++++++ docker-compose-dev.yml | 57 ++++++++++++++++++++++ mybots/BOTS | 17 +++++++ mybots/bots/experts/example/expert.py | 14 ++++++ 9 files changed, 165 insertions(+), 62 deletions(-) delete mode 100644 .docker/intelmq-full-dev/entrypoint create mode 100755 .docker/intelmq-full-dev/entrypoint_dev.sh create mode 100644 .docker/intelmq-full-dev/merge_BOTS.py delete mode 100644 .docker/intelmq-full-dev/update create mode 100755 .docker/intelmq-full-dev/update.sh create mode 100644 docker-compose-dev.yml create mode 100644 mybots/BOTS create mode 100644 mybots/bots/experts/example/expert.py diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index 10a3fdf..8b18930 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -3,12 +3,21 @@ FROM certat/intelmq-full:1.0 MAINTAINER Einar MAINTAINER Jeremias +USER root -#ADD dev_intelmq /opt/dev_intelmq -ADD entrypoint /usr/bin/entrypoint -ADD update /usr/bin/update +ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh +ADD update.sh /opt/dev/update.sh +ADD merge_BOTS.py /opt/dev/merge_BOTS.py -RUN chmod +x /usr/bin/entrypoint /usr/bin/update +# Merge bots for merge_BOTS.py +RUN python3 -m pip install jsonmerge -ENTRYPOINT ["entrypoint"] + +# Permission denied when installing new bots +RUN chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info + +USER intelmq:intelmq +ENV PATH="/opt/intelmq/.local/bin:${PATH}" + +ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"] diff --git a/.docker/intelmq-full-dev/entrypoint b/.docker/intelmq-full-dev/entrypoint deleted file mode 100644 index 0219273..0000000 --- a/.docker/intelmq-full-dev/entrypoint +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -function check_config_files { - - echo "Checking for configuration files" - for i in $(find /intelmq-bots/etc/ -name "*.conf"); do - if [[ ! -f /opt/intelmq/etc/$(basename $i) ]]; then - cp $i /opt/intelmq/etc/; - fi; - done; - if [[ ! -f /opt/intelmq/etc/manager/positions.conf ]]; then - cp -a /intelmq-bots/etc/manager /opt/intelmq/etc/; - fi; - chown -R intelmq.www-data /opt/intelmq/etc/; - - } - -if [[ ! -z "${DEV}" ]]; then - update -else - echo "Mixing bots" - if [[ ! -z "${REPO_UPDATE}" ]]; then - rm -fr /intelmq-bots - git clone ${REPO_UPDATE} /intelmq-bots - cp -a intelmq-bots/bots/BOTS /opt/intelmq/etc/ - check_config_files - update - else - check_config_files - fi -fi - -if [ "${LOG_MAIL_ENABLED}" = "true" ]; then - sed -i "s/return\ logger/### Code added to fix unexistent mail handler ###\n mail_handler=logging.handlers.SMTPHandler(mailhost = ('${LOG_MAIL_MAILHOST}', ${LOG_MAIL_PORT}),fromaddr = '${LOG_MAIL_FROMADDR}',toaddrs = ['${LOG_MAIL_TOADDR}'],subject = '${LOG_MAIL_SUBJECT}',credentials = ${LOG_MAIL_CREDENTIALS}, secure = ${LOG_MAIL_SECURE} )\n mail_handler.setLevel(${LOG_MAIL_LEVEL})\n mail_handler.setFormatter(logging.Formatter(LOG_FORMAT))\n logger.addHandler(mail_handler)\n aux_logger = logger\n return aux_logger\n ### End code added to fix unexistent mail handler ###/g" /opt/dev_intelmq/intelmq/lib/utils.py -fi - - -if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then - su - intelmq -s /bin/bash -c 'intelmqctl start' -fi - - -# Requirements TeamCymru Bots -sed -i 's/MinProtocol = TLSv1.2/MinProtocol = TLSv1.0/' /etc/ssl/openssl.cnf - - -/opt/entrypoint.sh - diff --git a/.docker/intelmq-full-dev/entrypoint_dev.sh b/.docker/intelmq-full-dev/entrypoint_dev.sh new file mode 100755 index 0000000..be94d63 --- /dev/null +++ b/.docker/intelmq-full-dev/entrypoint_dev.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +/opt/dev/update.sh + +if [ "${LOG_MAIL_ENABLED}" = "true" ]; then + sed -i "s/return\ logger/### Code added to fix unexistent mail handler ###\n mail_handler=logging.handlers.SMTPHandler(mailhost = ('${LOG_MAIL_MAILHOST}', ${LOG_MAIL_PORT}),fromaddr = '${LOG_MAIL_FROMADDR}',toaddrs = ['${LOG_MAIL_TOADDR}'],subject = '${LOG_MAIL_SUBJECT}',credentials = ${LOG_MAIL_CREDENTIALS}, secure = ${LOG_MAIL_SECURE} )\n mail_handler.setLevel(${LOG_MAIL_LEVEL})\n mail_handler.setFormatter(logging.Formatter(LOG_FORMAT))\n logger.addHandler(mail_handler)\n aux_logger = logger\n return aux_logger\n ### End code added to fix unexistent mail handler ###/g" /opt/intelmq/intelmq/lib/utils.py +fi + +if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then + intelmqctl start +fi + + +/opt/entrypoint.sh \ No newline at end of file diff --git a/.docker/intelmq-full-dev/merge_BOTS.py b/.docker/intelmq-full-dev/merge_BOTS.py new file mode 100644 index 0000000..095d4bd --- /dev/null +++ b/.docker/intelmq-full-dev/merge_BOTS.py @@ -0,0 +1,33 @@ +import json +from jsonmerge import merge +from collections import OrderedDict + +file1="/opt/dev/mybots/BOTS" +file2="/opt/intelmq/intelmq/bots/BOTS" + + +with open(file1, 'r') as f: + j1 = json.load(f) +with open(file2, 'r') as f: + j2 = json.load(f) + +def sortOD(od): + res = OrderedDict() + for k, v in sorted(od.items()): + if isinstance(v, dict): + res[k] = sortOD(v) + else: + res[k] = v + return res + + +merged = sortOD(merge(j1,j2)) + +desired_order_list = ['Collector', 'Parser', 'Expert', 'Output'] +reordered_dict = {k: merged[k] for k in desired_order_list} + +reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list}) + +with open(file2, 'w') as f: + json.dump(reordered_dict, f, indent=4) + diff --git a/.docker/intelmq-full-dev/update b/.docker/intelmq-full-dev/update deleted file mode 100644 index 144e78d..0000000 --- a/.docker/intelmq-full-dev/update +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -cp -a /mybots/bots/* /opt/intelmq/dev_intelmq/intelmq/bots/ -cd /opt/dev_intelmq && pip3 install -e . -echo "Install requirements for bots in dev repository" -for file in $(find /intelmq-bots/ -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done -chown -R intelmq:intelmq /opt/intelmq/ -chown -R intelmq.www-data /opt/intelmq/etc/; -chmod -R g+w /opt/intelmq -su - intelmq -s /bin/bash -c 'intelmqctl upgrade-config' diff --git a/.docker/intelmq-full-dev/update.sh b/.docker/intelmq-full-dev/update.sh new file mode 100755 index 0000000..5ac1252 --- /dev/null +++ b/.docker/intelmq-full-dev/update.sh @@ -0,0 +1,16 @@ +#!/bin/bash +echo "Installing requirements for bots in dev repository" +for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done + +echo "Merge your BOTS file with BOTS" +python3 /opt/dev/merge_BOTS.py + +echo "Copying BOTS" +cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/ +cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS + + +echo "Installing new BOTS" +cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user + + diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000..21a29cf --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,57 @@ +version: "3" +services: + redis: + image: redis:latest + volumes: + - ./example_config/redis/redis.conf:/usr/local/etc/redis/redis.conf + command: + - redis-server + - /usr/local/etc/redis/redis.conf + restart: always + networks: + - intelmq-internal + nginx: + image: certat/intelmq-nginx:latest + restart: always + ports: + - 1337:80 + volumes: + - ./intelmq-manager/html:/www + depends_on: + - intelmq + networks: + - intelmq-internal + intelmq: + build: .docker/intelmq-full-dev + volumes: + - ./example_config/intelmq/etc/:/opt/intelmq/etc/ + - ./example_config/intelmq-api:/opt/intelmq-api/config + - ./intelmq_logs:/opt/intelmq/var/log + - ./intelmq_output:/opt/intelmq/var/lib/bots + - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot + - ./mybots:/opt/dev/mybots + depends_on: + - redis + environment: + INTELMQ_PIPELINE_DRIVER: "redis" + INTELMQ_PIPELINE_HOST: redis + INTELMQ_REDIS_CACHE_HOST: redis + # Mail handler + LOG_MAIL_ENABLED: false + LOG_MAIL_LEVEL: "logging.ERROR" + LOG_MAIL_MAILHOST: "mail.example.unlp.edu.ar" + LOG_MAIL_PORT: 25 + LOG_MAIL_FROMADDR: "intelmq@examplefeeds.unlp.edu.ar" + LOG_MAIL_TOADDR: "support@example.unlp.edu.ar" + LOG_MAIL_SUBJECT: "[INTELMQ] Application Error" + LOG_MAIL_CREDENTIALS: None #tuple (username, password) + LOG_MAIL_SECURE: None + # Start botnet at boot + ENABLE_BOTNET_AT_BOOT: true + networks: + - intelmq-internal + + +networks: + intelmq-internal: + driver: bridge \ No newline at end of file diff --git a/mybots/BOTS b/mybots/BOTS new file mode 100644 index 0000000..185295d --- /dev/null +++ b/mybots/BOTS @@ -0,0 +1,17 @@ +{ + "Collector": { + }, + "Parser": { + }, + "Expert": { + "Example": { + "description": "Example own bot.", + "module": "intelmq.bots.experts.example.expert", + "parameters": { + } + } + }, + "Output": { + + } +} \ No newline at end of file diff --git a/mybots/bots/experts/example/expert.py b/mybots/bots/experts/example/expert.py new file mode 100644 index 0000000..c39a6e2 --- /dev/null +++ b/mybots/bots/experts/example/expert.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +from intelmq.lib.bot import Bot + + +class ExampleExpertBot(Bot): + + def init(self): + pass + + def process(self): + pass + + +BOT = ExampleExpertBot