Features:

- Support for bot development in intelmq 2.3.1 version
 - Add mail handler
 - Add start botnet at boot option

.docker/intelmq-full-dev/Dockerfile

@@ -3,12 +3,21 @@ FROM certat/intelmq-full:1.0
 MAINTAINER Einar <elanfranco@cert.unlp.edu.ar>
 MAINTAINER Jeremias <jpretto@cert.unlp.edu.ar>
 
+USER root
 
-#ADD dev_intelmq /opt/dev_intelmq
+ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh
-ADD entrypoint /usr/bin/entrypoint
+ADD update.sh /opt/dev/update.sh
-ADD update /usr/bin/update
+ADD merge_BOTS.py /opt/dev/merge_BOTS.py
 
-RUN chmod +x /usr/bin/entrypoint /usr/bin/update
+# Merge bots for merge_BOTS.py
+RUN python3 -m pip install jsonmerge
 
-ENTRYPOINT ["entrypoint"]
+
+# Permission denied when installing new bots
+RUN chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
+
+USER intelmq:intelmq
+ENV PATH="/opt/intelmq/.local/bin:${PATH}"
+
+ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"]
 

.docker/intelmq-full-dev/entrypoint

@@ -1,48 +0,0 @@
-#!/bin/bash
-
-function check_config_files {
-
-	echo "Checking for configuration files"
-	for i in $(find /intelmq-bots/etc/ -name "*.conf"); do
-		if [[ ! -f /opt/intelmq/etc/$(basename $i) ]]; then
-			cp $i /opt/intelmq/etc/;
-		fi;
-	done;
-	if [[ ! -f /opt/intelmq/etc/manager/positions.conf ]]; then
-			cp -a /intelmq-bots/etc/manager /opt/intelmq/etc/;
-	fi;
-	chown -R intelmq.www-data /opt/intelmq/etc/;
-
- }
-
-if [[ ! -z "${DEV}" ]]; then
-  update
-else
-	echo "Mixing bots"
-	if [[ ! -z "${REPO_UPDATE}" ]]; then
-		rm -fr /intelmq-bots
-		git clone ${REPO_UPDATE} /intelmq-bots
-		cp -a intelmq-bots/bots/BOTS /opt/intelmq/etc/
-		check_config_files
-		update
-	else
-		check_config_files
-	fi
-fi
-
-if [ "${LOG_MAIL_ENABLED}" = "true" ]; then
-    sed -i "s/return\ logger/### Code added to fix unexistent mail handler ###\n    mail_handler=logging.handlers.SMTPHandler(mailhost = ('${LOG_MAIL_MAILHOST}', ${LOG_MAIL_PORT}),fromaddr = '${LOG_MAIL_FROMADDR}',toaddrs = ['${LOG_MAIL_TOADDR}'],subject = '${LOG_MAIL_SUBJECT}',credentials = ${LOG_MAIL_CREDENTIALS}, secure = ${LOG_MAIL_SECURE} )\n    mail_handler.setLevel(${LOG_MAIL_LEVEL})\n    mail_handler.setFormatter(logging.Formatter(LOG_FORMAT))\n    logger.addHandler(mail_handler)\n    aux_logger = logger\n    return aux_logger\n    ### End code added to fix unexistent mail handler ###/g" /opt/dev_intelmq/intelmq/lib/utils.py
-fi
-
-
-if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
-	su - intelmq -s /bin/bash -c 'intelmqctl start'
-fi
-
-
-# Requirements TeamCymru Bots
-sed -i 's/MinProtocol = TLSv1.2/MinProtocol = TLSv1.0/' /etc/ssl/openssl.cnf
-
-
-/opt/entrypoint.sh
-

.docker/intelmq-full-dev/entrypoint_dev.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+/opt/dev/update.sh
+
+if [ "${LOG_MAIL_ENABLED}" = "true" ]; then
+    sed -i "s/return\ logger/### Code added to fix unexistent mail handler ###\n    mail_handler=logging.handlers.SMTPHandler(mailhost = ('${LOG_MAIL_MAILHOST}', ${LOG_MAIL_PORT}),fromaddr = '${LOG_MAIL_FROMADDR}',toaddrs = ['${LOG_MAIL_TOADDR}'],subject = '${LOG_MAIL_SUBJECT}',credentials = ${LOG_MAIL_CREDENTIALS}, secure = ${LOG_MAIL_SECURE} )\n    mail_handler.setLevel(${LOG_MAIL_LEVEL})\n    mail_handler.setFormatter(logging.Formatter(LOG_FORMAT))\n    logger.addHandler(mail_handler)\n    aux_logger = logger\n    return aux_logger\n    ### End code added to fix unexistent mail handler ###/g" /opt/intelmq/intelmq/lib/utils.py
+fi
+
+if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
+	intelmqctl start
+fi
+
+
+/opt/entrypoint.sh

.docker/intelmq-full-dev/merge_BOTS.py

@@ -0,0 +1,33 @@
+import json
+from jsonmerge import merge
+from collections import OrderedDict
+
+file1="/opt/dev/mybots/BOTS"
+file2="/opt/intelmq/intelmq/bots/BOTS"
+
+
+with open(file1, 'r') as f:
+    j1 = json.load(f)
+with open(file2, 'r') as f:
+    j2 = json.load(f)
+
+def sortOD(od):
+    res = OrderedDict()
+    for k, v in sorted(od.items()):
+        if isinstance(v, dict):
+            res[k] = sortOD(v)
+        else:
+            res[k] = v
+    return res
+
+
+merged = sortOD(merge(j1,j2))
+
+desired_order_list = ['Collector', 'Parser', 'Expert', 'Output']
+reordered_dict = {k: merged[k] for k in desired_order_list}
+
+reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list})
+
+with open(file2, 'w') as f:
+    json.dump(reordered_dict, f, indent=4)
+

.docker/intelmq-full-dev/update

@@ -1,9 +0,0 @@
-#!/bin/bash
-cp -a /mybots/bots/* /opt/intelmq/dev_intelmq/intelmq/bots/
-cd /opt/dev_intelmq && pip3 install -e .
-echo "Install requirements for bots in dev repository"
-for file in $(find /intelmq-bots/ -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
-chown -R intelmq:intelmq /opt/intelmq/
-chown -R intelmq.www-data /opt/intelmq/etc/;
-chmod -R g+w  /opt/intelmq
-su - intelmq -s /bin/bash -c 'intelmqctl upgrade-config'

.docker/intelmq-full-dev/update.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+echo "Installing requirements for bots in dev repository"
+for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
+
+echo "Merge your BOTS file with BOTS"
+python3 /opt/dev/merge_BOTS.py
+
+echo "Copying BOTS"
+cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
+cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
+
+
+echo "Installing new BOTS"
+cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user
+
+

docker-compose-dev.yml

@@ -0,0 +1,57 @@
+version: "3"
+services:
+    redis:
+        image: redis:latest
+        volumes:
+            - ./example_config/redis/redis.conf:/usr/local/etc/redis/redis.conf
+        command:
+            - redis-server
+            - /usr/local/etc/redis/redis.conf
+        restart: always
+        networks: 
+            - intelmq-internal
+    nginx:
+        image: certat/intelmq-nginx:latest
+        restart: always
+        ports: 
+            - 1337:80
+        volumes: 
+            - ./intelmq-manager/html:/www
+        depends_on: 
+            - intelmq
+        networks: 
+            - intelmq-internal
+    intelmq:
+        build: .docker/intelmq-full-dev
+        volumes: 
+            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+            - ./example_config/intelmq-api:/opt/intelmq-api/config
+            - ./intelmq_logs:/opt/intelmq/var/log
+            - ./intelmq_output:/opt/intelmq/var/lib/bots
+            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
+            - ./mybots:/opt/dev/mybots
+        depends_on: 
+            - redis
+        environment: 
+            INTELMQ_PIPELINE_DRIVER: "redis"
+            INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_REDIS_CACHE_HOST: redis
+            # Mail handler
+            LOG_MAIL_ENABLED: false
+            LOG_MAIL_LEVEL: "logging.ERROR"
+            LOG_MAIL_MAILHOST: "mail.example.unlp.edu.ar"
+            LOG_MAIL_PORT: 25
+            LOG_MAIL_FROMADDR: "intelmq@examplefeeds.unlp.edu.ar"
+            LOG_MAIL_TOADDR: "support@example.unlp.edu.ar"
+            LOG_MAIL_SUBJECT: "[INTELMQ] Application Error"
+            LOG_MAIL_CREDENTIALS: None #tuple (username, password)
+            LOG_MAIL_SECURE: None
+            # Start botnet at boot
+            ENABLE_BOTNET_AT_BOOT: true
+        networks:
+            - intelmq-internal
+
+
+networks: 
+    intelmq-internal:
+        driver: bridge

mybots/BOTS

@@ -0,0 +1,17 @@
+{
+        "Collector": {
+        },
+        "Parser": {
+        },
+        "Expert": {
+                  "Example": {
+                    "description": "Example own bot.",
+                    "module": "intelmq.bots.experts.example.expert",
+                    "parameters": {
+                    }
+                }
+        },
+        "Output": {
+
+        }
+}

mybots/bots/experts/example/expert.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+from intelmq.lib.bot import Bot
+
+
+class ExampleExpertBot(Bot):
+
+    def init(self):
+        pass
+
+    def process(self):
+        pass
+
+
+BOT = ExampleExpertBot