From 171a89a1a30612f5107f6b44838bdd047366b2dd Mon Sep 17 00:00:00 2001
From: Einar Lanfranco <einar.lanfranco@gmail.com>
Date: Tue, 23 Mar 2021 10:08:27 -0300
Subject: [PATCH 01/10] Adding Dockerfile to dev version

---
 .docker/intelmq-full-dev/Dockerfile | 14 +++++++++
 .docker/intelmq-full-dev/entrypoint | 48 +++++++++++++++++++++++++++++
 .docker/intelmq-full-dev/update     |  9 ++++++
 3 files changed, 71 insertions(+)
 create mode 100644 .docker/intelmq-full-dev/Dockerfile
 create mode 100644 .docker/intelmq-full-dev/entrypoint
 create mode 100644 .docker/intelmq-full-dev/update

diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile
new file mode 100644
index 0000000..10a3fdf
--- /dev/null
+++ b/.docker/intelmq-full-dev/Dockerfile
@@ -0,0 +1,14 @@
+FROM certat/intelmq-full:1.0
+
+MAINTAINER Einar <elanfranco@cert.unlp.edu.ar>
+MAINTAINER Jeremias <jpretto@cert.unlp.edu.ar>
+
+
+#ADD dev_intelmq /opt/dev_intelmq
+ADD entrypoint /usr/bin/entrypoint
+ADD update /usr/bin/update
+
+RUN chmod +x /usr/bin/entrypoint /usr/bin/update
+
+ENTRYPOINT ["entrypoint"]
+
diff --git a/.docker/intelmq-full-dev/entrypoint b/.docker/intelmq-full-dev/entrypoint
new file mode 100644
index 0000000..0219273
--- /dev/null
+++ b/.docker/intelmq-full-dev/entrypoint
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+function check_config_files {
+
+	echo "Checking for configuration files"
+	for i in $(find /intelmq-bots/etc/ -name "*.conf"); do
+		if [[ ! -f /opt/intelmq/etc/$(basename $i) ]]; then
+			cp $i /opt/intelmq/etc/;
+		fi;
+	done;
+	if [[ ! -f /opt/intelmq/etc/manager/positions.conf ]]; then
+			cp -a /intelmq-bots/etc/manager /opt/intelmq/etc/;
+	fi;
+	chown -R intelmq.www-data /opt/intelmq/etc/;
+
+ }
+
+if [[ ! -z "${DEV}" ]]; then
+  update
+else
+	echo "Mixing bots"
+	if [[ ! -z "${REPO_UPDATE}" ]]; then
+		rm -fr /intelmq-bots
+		git clone ${REPO_UPDATE} /intelmq-bots
+		cp -a intelmq-bots/bots/BOTS /opt/intelmq/etc/
+		check_config_files
+		update
+	else
+		check_config_files
+	fi
+fi
+
+if [ "${LOG_MAIL_ENABLED}" = "true" ]; then
+    sed -i "s/return\ logger/### Code added to fix unexistent mail handler ###\n    mail_handler=logging.handlers.SMTPHandler(mailhost = ('${LOG_MAIL_MAILHOST}', ${LOG_MAIL_PORT}),fromaddr = '${LOG_MAIL_FROMADDR}',toaddrs = ['${LOG_MAIL_TOADDR}'],subject = '${LOG_MAIL_SUBJECT}',credentials = ${LOG_MAIL_CREDENTIALS}, secure = ${LOG_MAIL_SECURE} )\n    mail_handler.setLevel(${LOG_MAIL_LEVEL})\n    mail_handler.setFormatter(logging.Formatter(LOG_FORMAT))\n    logger.addHandler(mail_handler)\n    aux_logger = logger\n    return aux_logger\n    ### End code added to fix unexistent mail handler ###/g" /opt/dev_intelmq/intelmq/lib/utils.py
+fi
+
+
+if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
+	su - intelmq -s /bin/bash -c 'intelmqctl start'
+fi
+
+
+# Requirements TeamCymru Bots
+sed -i 's/MinProtocol = TLSv1.2/MinProtocol = TLSv1.0/' /etc/ssl/openssl.cnf
+
+
+/opt/entrypoint.sh
+
diff --git a/.docker/intelmq-full-dev/update b/.docker/intelmq-full-dev/update
new file mode 100644
index 0000000..144e78d
--- /dev/null
+++ b/.docker/intelmq-full-dev/update
@@ -0,0 +1,9 @@
+#!/bin/bash
+cp -a /mybots/bots/* /opt/intelmq/dev_intelmq/intelmq/bots/
+cd /opt/dev_intelmq && pip3 install -e .
+echo "Install requirements for bots in dev repository"
+for file in $(find /intelmq-bots/ -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
+chown -R intelmq:intelmq /opt/intelmq/
+chown -R intelmq.www-data /opt/intelmq/etc/;
+chmod -R g+w  /opt/intelmq
+su - intelmq -s /bin/bash -c 'intelmqctl upgrade-config'

From 77d05fbc1feb9ff9abf14e1c19f00c43e5888071 Mon Sep 17 00:00:00 2001
From: Jeremias Pretto <jpretto@cert.unlp.edu.ar>
Date: Tue, 23 Mar 2021 11:57:27 -0300
Subject: [PATCH 02/10] Features:  - Support for bot development in intelmq
 2.3.1 version  - Add mail handler  - Add start botnet at boot option

---
 .docker/intelmq-full-dev/Dockerfile        | 19 ++++++--
 .docker/intelmq-full-dev/entrypoint        | 48 ------------------
 .docker/intelmq-full-dev/entrypoint_dev.sh | 14 ++++++
 .docker/intelmq-full-dev/merge_BOTS.py     | 33 +++++++++++++
 .docker/intelmq-full-dev/update            |  9 ----
 .docker/intelmq-full-dev/update.sh         | 16 ++++++
 docker-compose-dev.yml                     | 57 ++++++++++++++++++++++
 mybots/BOTS                                | 17 +++++++
 mybots/bots/experts/example/expert.py      | 14 ++++++
 9 files changed, 165 insertions(+), 62 deletions(-)
 delete mode 100644 .docker/intelmq-full-dev/entrypoint
 create mode 100755 .docker/intelmq-full-dev/entrypoint_dev.sh
 create mode 100644 .docker/intelmq-full-dev/merge_BOTS.py
 delete mode 100644 .docker/intelmq-full-dev/update
 create mode 100755 .docker/intelmq-full-dev/update.sh
 create mode 100644 docker-compose-dev.yml
 create mode 100644 mybots/BOTS
 create mode 100644 mybots/bots/experts/example/expert.py

diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile
index 10a3fdf..8b18930 100644
--- a/.docker/intelmq-full-dev/Dockerfile
+++ b/.docker/intelmq-full-dev/Dockerfile
@@ -3,12 +3,21 @@ FROM certat/intelmq-full:1.0
 MAINTAINER Einar <elanfranco@cert.unlp.edu.ar>
 MAINTAINER Jeremias <jpretto@cert.unlp.edu.ar>
 
+USER root
 
-#ADD dev_intelmq /opt/dev_intelmq
-ADD entrypoint /usr/bin/entrypoint
-ADD update /usr/bin/update
+ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh
+ADD update.sh /opt/dev/update.sh
+ADD merge_BOTS.py /opt/dev/merge_BOTS.py
 
-RUN chmod +x /usr/bin/entrypoint /usr/bin/update
+# Merge bots for merge_BOTS.py
+RUN python3 -m pip install jsonmerge
 
-ENTRYPOINT ["entrypoint"]
+
+# Permission denied when installing new bots
+RUN chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
+
+USER intelmq:intelmq
+ENV PATH="/opt/intelmq/.local/bin:${PATH}"
+
+ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"]
 
diff --git a/.docker/intelmq-full-dev/entrypoint b/.docker/intelmq-full-dev/entrypoint
deleted file mode 100644
index 0219273..0000000
--- a/.docker/intelmq-full-dev/entrypoint
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash
-
-function check_config_files {
-
-	echo "Checking for configuration files"
-	for i in $(find /intelmq-bots/etc/ -name "*.conf"); do
-		if [[ ! -f /opt/intelmq/etc/$(basename $i) ]]; then
-			cp $i /opt/intelmq/etc/;
-		fi;
-	done;
-	if [[ ! -f /opt/intelmq/etc/manager/positions.conf ]]; then
-			cp -a /intelmq-bots/etc/manager /opt/intelmq/etc/;
-	fi;
-	chown -R intelmq.www-data /opt/intelmq/etc/;
-
- }
-
-if [[ ! -z "${DEV}" ]]; then
-  update
-else
-	echo "Mixing bots"
-	if [[ ! -z "${REPO_UPDATE}" ]]; then
-		rm -fr /intelmq-bots
-		git clone ${REPO_UPDATE} /intelmq-bots
-		cp -a intelmq-bots/bots/BOTS /opt/intelmq/etc/
-		check_config_files
-		update
-	else
-		check_config_files
-	fi
-fi
-
-if [ "${LOG_MAIL_ENABLED}" = "true" ]; then
-    sed -i "s/return\ logger/### Code added to fix unexistent mail handler ###\n    mail_handler=logging.handlers.SMTPHandler(mailhost = ('${LOG_MAIL_MAILHOST}', ${LOG_MAIL_PORT}),fromaddr = '${LOG_MAIL_FROMADDR}',toaddrs = ['${LOG_MAIL_TOADDR}'],subject = '${LOG_MAIL_SUBJECT}',credentials = ${LOG_MAIL_CREDENTIALS}, secure = ${LOG_MAIL_SECURE} )\n    mail_handler.setLevel(${LOG_MAIL_LEVEL})\n    mail_handler.setFormatter(logging.Formatter(LOG_FORMAT))\n    logger.addHandler(mail_handler)\n    aux_logger = logger\n    return aux_logger\n    ### End code added to fix unexistent mail handler ###/g" /opt/dev_intelmq/intelmq/lib/utils.py
-fi
-
-
-if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
-	su - intelmq -s /bin/bash -c 'intelmqctl start'
-fi
-
-
-# Requirements TeamCymru Bots
-sed -i 's/MinProtocol = TLSv1.2/MinProtocol = TLSv1.0/' /etc/ssl/openssl.cnf
-
-
-/opt/entrypoint.sh
-
diff --git a/.docker/intelmq-full-dev/entrypoint_dev.sh b/.docker/intelmq-full-dev/entrypoint_dev.sh
new file mode 100755
index 0000000..be94d63
--- /dev/null
+++ b/.docker/intelmq-full-dev/entrypoint_dev.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+/opt/dev/update.sh
+
+if [ "${LOG_MAIL_ENABLED}" = "true" ]; then
+    sed -i "s/return\ logger/### Code added to fix unexistent mail handler ###\n    mail_handler=logging.handlers.SMTPHandler(mailhost = ('${LOG_MAIL_MAILHOST}', ${LOG_MAIL_PORT}),fromaddr = '${LOG_MAIL_FROMADDR}',toaddrs = ['${LOG_MAIL_TOADDR}'],subject = '${LOG_MAIL_SUBJECT}',credentials = ${LOG_MAIL_CREDENTIALS}, secure = ${LOG_MAIL_SECURE} )\n    mail_handler.setLevel(${LOG_MAIL_LEVEL})\n    mail_handler.setFormatter(logging.Formatter(LOG_FORMAT))\n    logger.addHandler(mail_handler)\n    aux_logger = logger\n    return aux_logger\n    ### End code added to fix unexistent mail handler ###/g" /opt/intelmq/intelmq/lib/utils.py
+fi
+
+if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
+	intelmqctl start
+fi
+
+
+/opt/entrypoint.sh
\ No newline at end of file
diff --git a/.docker/intelmq-full-dev/merge_BOTS.py b/.docker/intelmq-full-dev/merge_BOTS.py
new file mode 100644
index 0000000..095d4bd
--- /dev/null
+++ b/.docker/intelmq-full-dev/merge_BOTS.py
@@ -0,0 +1,33 @@
+import json
+from jsonmerge import merge
+from collections import OrderedDict
+
+file1="/opt/dev/mybots/BOTS"
+file2="/opt/intelmq/intelmq/bots/BOTS"
+
+
+with open(file1, 'r') as f:
+    j1 = json.load(f)
+with open(file2, 'r') as f:
+    j2 = json.load(f)
+
+def sortOD(od):
+    res = OrderedDict()
+    for k, v in sorted(od.items()):
+        if isinstance(v, dict):
+            res[k] = sortOD(v)
+        else:
+            res[k] = v
+    return res
+
+
+merged = sortOD(merge(j1,j2))
+
+desired_order_list = ['Collector', 'Parser', 'Expert', 'Output']
+reordered_dict = {k: merged[k] for k in desired_order_list}
+
+reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list})
+
+with open(file2, 'w') as f:
+    json.dump(reordered_dict, f, indent=4)
+
diff --git a/.docker/intelmq-full-dev/update b/.docker/intelmq-full-dev/update
deleted file mode 100644
index 144e78d..0000000
--- a/.docker/intelmq-full-dev/update
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-cp -a /mybots/bots/* /opt/intelmq/dev_intelmq/intelmq/bots/
-cd /opt/dev_intelmq && pip3 install -e .
-echo "Install requirements for bots in dev repository"
-for file in $(find /intelmq-bots/ -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
-chown -R intelmq:intelmq /opt/intelmq/
-chown -R intelmq.www-data /opt/intelmq/etc/;
-chmod -R g+w  /opt/intelmq
-su - intelmq -s /bin/bash -c 'intelmqctl upgrade-config'
diff --git a/.docker/intelmq-full-dev/update.sh b/.docker/intelmq-full-dev/update.sh
new file mode 100755
index 0000000..5ac1252
--- /dev/null
+++ b/.docker/intelmq-full-dev/update.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+echo "Installing requirements for bots in dev repository"
+for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
+
+echo "Merge your BOTS file with BOTS"
+python3 /opt/dev/merge_BOTS.py
+
+echo "Copying BOTS"
+cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
+cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
+
+
+echo "Installing new BOTS"
+cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user
+
+
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
new file mode 100644
index 0000000..21a29cf
--- /dev/null
+++ b/docker-compose-dev.yml
@@ -0,0 +1,57 @@
+version: "3"
+services:
+    redis:
+        image: redis:latest
+        volumes:
+            - ./example_config/redis/redis.conf:/usr/local/etc/redis/redis.conf
+        command:
+            - redis-server
+            - /usr/local/etc/redis/redis.conf
+        restart: always
+        networks: 
+            - intelmq-internal
+    nginx:
+        image: certat/intelmq-nginx:latest
+        restart: always
+        ports: 
+            - 1337:80
+        volumes: 
+            - ./intelmq-manager/html:/www
+        depends_on: 
+            - intelmq
+        networks: 
+            - intelmq-internal
+    intelmq:
+        build: .docker/intelmq-full-dev
+        volumes: 
+            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+            - ./example_config/intelmq-api:/opt/intelmq-api/config
+            - ./intelmq_logs:/opt/intelmq/var/log
+            - ./intelmq_output:/opt/intelmq/var/lib/bots
+            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
+            - ./mybots:/opt/dev/mybots
+        depends_on: 
+            - redis
+        environment: 
+            INTELMQ_PIPELINE_DRIVER: "redis"
+            INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_REDIS_CACHE_HOST: redis
+            # Mail handler
+            LOG_MAIL_ENABLED: false
+            LOG_MAIL_LEVEL: "logging.ERROR"
+            LOG_MAIL_MAILHOST: "mail.example.unlp.edu.ar"
+            LOG_MAIL_PORT: 25
+            LOG_MAIL_FROMADDR: "intelmq@examplefeeds.unlp.edu.ar"
+            LOG_MAIL_TOADDR: "support@example.unlp.edu.ar"
+            LOG_MAIL_SUBJECT: "[INTELMQ] Application Error"
+            LOG_MAIL_CREDENTIALS: None #tuple (username, password)
+            LOG_MAIL_SECURE: None
+            # Start botnet at boot
+            ENABLE_BOTNET_AT_BOOT: true
+        networks:
+            - intelmq-internal
+
+
+networks: 
+    intelmq-internal:
+        driver: bridge
\ No newline at end of file
diff --git a/mybots/BOTS b/mybots/BOTS
new file mode 100644
index 0000000..185295d
--- /dev/null
+++ b/mybots/BOTS
@@ -0,0 +1,17 @@
+{
+        "Collector": {
+        },
+        "Parser": {
+        },
+        "Expert": {
+                  "Example": {
+                    "description": "Example own bot.",
+                    "module": "intelmq.bots.experts.example.expert",
+                    "parameters": {
+                    }
+                }
+        },
+        "Output": {
+
+        }
+}
\ No newline at end of file
diff --git a/mybots/bots/experts/example/expert.py b/mybots/bots/experts/example/expert.py
new file mode 100644
index 0000000..c39a6e2
--- /dev/null
+++ b/mybots/bots/experts/example/expert.py
@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+from intelmq.lib.bot import Bot
+
+
+class ExampleExpertBot(Bot):
+
+    def init(self):
+        pass
+
+    def process(self):
+        pass
+
+
+BOT = ExampleExpertBot

From 612bbb53e683b4646c0501a4e51d65516286c5ca Mon Sep 17 00:00:00 2001
From: Einar Lanfranco <einar.lanfranco@gmail.com>
Date: Tue, 23 Mar 2021 12:30:16 -0300
Subject: [PATCH 03/10] Develop guide md

---
 DEVELOP-GUIDE.md | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 DEVELOP-GUIDE.md

diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md
new file mode 100644
index 0000000..15f7038
--- /dev/null
+++ b/DEVELOP-GUIDE.md
@@ -0,0 +1,35 @@
+# intelmq-docker
+
+## Run & deploy containers in dev mode:
+
+1. `docker-compose -f docker-compose-dev.yml up`
+
+## Docker-compose-dev.yml file
+
+### Volume:  
+
+**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert.
+
+### Add your own bots
+
+Just start coding or pull your bots repository in ,/mybots folder
+
+
+### How to install and look yours bots runnig
+
+
+Just run /opt/dev/update.sh in the container:
+
+1. `docker-compose exec -f docker-compose-dev.yml  intelmq /opt/bin/update.sh`
+
+When you do this:
+
+* Yours BOTS files will be mixed with intelmq original BOTS
+* Yours bots will be installed 
+
+### Additional environment variables
+
+Check options in docker-compose-dev.yml:
+
+* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email)
+* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not.

From 5f1e2c8f1c0ee23374cb70f8ba8b6b5448d3e6d4 Mon Sep 17 00:00:00 2001
From: Einar Lanfranco <einar.lanfranco@gmail.com>
Date: Tue, 23 Mar 2021 13:05:21 -0300
Subject: [PATCH 04/10] Develop guide md update

---
 DEVELOP-GUIDE.md | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md
index 15f7038..1f57e5f 100644
--- a/DEVELOP-GUIDE.md
+++ b/DEVELOP-GUIDE.md
@@ -14,7 +14,6 @@
 
 Just start coding or pull your bots repository in ,/mybots folder
 
-
 ### How to install and look yours bots runnig
 
 
@@ -24,7 +23,7 @@ Just run /opt/dev/update.sh in the container:
 
 When you do this:
 
-* Yours BOTS files will be mixed with intelmq original BOTS
+* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment
 * Yours bots will be installed 
 
 ### Additional environment variables
@@ -33,3 +32,14 @@ Check options in docker-compose-dev.yml:
 
 * LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email)
 * ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not.
+
+
+## For deploy your already developed bots
+
+Just clone your bots git to ./mybots and run the container
+
+For example, using https://github.com/CERTUNLP/intelmq-bots:
+
+1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots`
+0. `docker-compose -f docker-compose-dev.yml up`
+

From 5fba132f9c769e9c7a2bbf40c3dd32b6a8a3fd38 Mon Sep 17 00:00:00 2001
From: Jeremias Pretto <jpretto@cert.unlp.edu.ar>
Date: Tue, 23 Mar 2021 15:19:03 -0300
Subject: [PATCH 05/10] hotfix

---
 .docker/intelmq-full-dev/update.sh               | 1 +
 docker-compose-dev.yml                           | 6 +++---
 mybots/bots/collectors/otherexample/collector.py | 0
 3 files changed, 4 insertions(+), 3 deletions(-)
 create mode 100644 mybots/bots/collectors/otherexample/collector.py

diff --git a/.docker/intelmq-full-dev/update.sh b/.docker/intelmq-full-dev/update.sh
index 5ac1252..a46dc6a 100755
--- a/.docker/intelmq-full-dev/update.sh
+++ b/.docker/intelmq-full-dev/update.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+
 echo "Installing requirements for bots in dev repository"
 for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
 
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 21a29cf..c035be4 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -37,7 +37,7 @@ services:
             INTELMQ_PIPELINE_HOST: redis
             INTELMQ_REDIS_CACHE_HOST: redis
             # Mail handler
-            LOG_MAIL_ENABLED: false
+            LOG_MAIL_ENABLED: "false"
             LOG_MAIL_LEVEL: "logging.ERROR"
             LOG_MAIL_MAILHOST: "mail.example.unlp.edu.ar"
             LOG_MAIL_PORT: 25
@@ -47,11 +47,11 @@ services:
             LOG_MAIL_CREDENTIALS: None #tuple (username, password)
             LOG_MAIL_SECURE: None
             # Start botnet at boot
-            ENABLE_BOTNET_AT_BOOT: true
+            ENABLE_BOTNET_AT_BOOT: "true"
         networks:
             - intelmq-internal
 
 
 networks: 
     intelmq-internal:
-        driver: bridge
\ No newline at end of file
+        driver: bridge
diff --git a/mybots/bots/collectors/otherexample/collector.py b/mybots/bots/collectors/otherexample/collector.py
new file mode 100644
index 0000000..e69de29

From b1c3677f909bb70d9318c6ee99ed1b03f55c9906 Mon Sep 17 00:00:00 2001
From: Jeremias Pretto <jpretto@cert.unlp.edu.ar>
Date: Tue, 23 Mar 2021 15:44:44 -0300
Subject: [PATCH 06/10] 2.3 compatibility

---
 .docker/intelmq-full-dev/merge_BOTS.py | 15 ++++++++++-----
 .docker/intelmq-full-dev/update.sh     | 11 +++++++----
 DEVELOP-GUIDE.md                       |  2 +-
 3 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/.docker/intelmq-full-dev/merge_BOTS.py b/.docker/intelmq-full-dev/merge_BOTS.py
index 095d4bd..f56e6de 100644
--- a/.docker/intelmq-full-dev/merge_BOTS.py
+++ b/.docker/intelmq-full-dev/merge_BOTS.py
@@ -1,14 +1,18 @@
 import json
 from jsonmerge import merge
+import argparse
 from collections import OrderedDict
 
-file1="/opt/dev/mybots/BOTS"
-file2="/opt/intelmq/intelmq/bots/BOTS"
+parser = argparse.ArgumentParser(description='Merge two json.')
+parser.add_argument('input_file_1', type=str, help='input_file_1')
+parser.add_argument('input_file_2', type=str, help='input_file_2')
+parser.add_argument('output_file', type=str, help='output_file')
 
+args = parser.parse_args()
 
-with open(file1, 'r') as f:
+with open(args.input_file_1, 'r') as f:
     j1 = json.load(f)
-with open(file2, 'r') as f:
+with open(args.input_file_2, 'r') as f:
     j2 = json.load(f)
 
 def sortOD(od):
@@ -26,8 +30,9 @@ merged = sortOD(merge(j1,j2))
 desired_order_list = ['Collector', 'Parser', 'Expert', 'Output']
 reordered_dict = {k: merged[k] for k in desired_order_list}
 
+# add other keys
 reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list})
 
-with open(file2, 'w') as f:
+with open(args.output_file, 'w') as f:
     json.dump(reordered_dict, f, indent=4)
 
diff --git a/.docker/intelmq-full-dev/update.sh b/.docker/intelmq-full-dev/update.sh
index a46dc6a..b3a0061 100755
--- a/.docker/intelmq-full-dev/update.sh
+++ b/.docker/intelmq-full-dev/update.sh
@@ -3,15 +3,18 @@
 echo "Installing requirements for bots in dev repository"
 for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
 
+# Backup Original BOTS
+cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk
+
 echo "Merge your BOTS file with BOTS"
-python3 /opt/dev/merge_BOTS.py
+python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS"
+cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
 
 echo "Copying BOTS"
 cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
-cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
-
 
 echo "Installing new BOTS"
 cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user
 
-
+# Restore original BOTS
+mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS
diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md
index 1f57e5f..8c1fd52 100644
--- a/DEVELOP-GUIDE.md
+++ b/DEVELOP-GUIDE.md
@@ -40,6 +40,6 @@ Just clone your bots git to ./mybots and run the container
 
 For example, using https://github.com/CERTUNLP/intelmq-bots:
 
-1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots`
+1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3`
 0. `docker-compose -f docker-compose-dev.yml up`
 

From 9ff46cb1e686824fd7866fe54307e9c6b63558e1 Mon Sep 17 00:00:00 2001
From: Einar Lanfranco <einar.lanfranco@gmail.com>
Date: Thu, 25 Mar 2021 15:41:24 -0300
Subject: [PATCH 07/10] Wagner suggestions

---
 .docker/intelmq-full-dev/Dockerfile    |  7 ++-----
 .docker/intelmq-full-dev/merge_BOTS.py |  2 +-
 .docker/intelmq-full-dev/update.sh     | 28 ++++++++++++++++----------
 mybots/BOTS                            |  2 +-
 4 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile
index 8b18930..f8dc9a0 100644
--- a/.docker/intelmq-full-dev/Dockerfile
+++ b/.docker/intelmq-full-dev/Dockerfile
@@ -3,20 +3,17 @@ FROM certat/intelmq-full:1.0
 MAINTAINER Einar <elanfranco@cert.unlp.edu.ar>
 MAINTAINER Jeremias <jpretto@cert.unlp.edu.ar>
 
-USER root
-
 ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh
 ADD update.sh /opt/dev/update.sh
 ADD merge_BOTS.py /opt/dev/merge_BOTS.py
 
 # Merge bots for merge_BOTS.py
-RUN python3 -m pip install jsonmerge
+RUN sudo python3 -m pip install jsonmerge
 
 
 # Permission denied when installing new bots
-RUN chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
+RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
 
-USER intelmq:intelmq
 ENV PATH="/opt/intelmq/.local/bin:${PATH}"
 
 ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"]
diff --git a/.docker/intelmq-full-dev/merge_BOTS.py b/.docker/intelmq-full-dev/merge_BOTS.py
index f56e6de..3c28494 100644
--- a/.docker/intelmq-full-dev/merge_BOTS.py
+++ b/.docker/intelmq-full-dev/merge_BOTS.py
@@ -35,4 +35,4 @@ reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list}
 
 with open(args.output_file, 'w') as f:
     json.dump(reordered_dict, f, indent=4)
-
+   
\ No newline at end of file
diff --git a/.docker/intelmq-full-dev/update.sh b/.docker/intelmq-full-dev/update.sh
index b3a0061..83b2470 100755
--- a/.docker/intelmq-full-dev/update.sh
+++ b/.docker/intelmq-full-dev/update.sh
@@ -3,18 +3,24 @@
 echo "Installing requirements for bots in dev repository"
 for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
 
-# Backup Original BOTS
-cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk
-
-echo "Merge your BOTS file with BOTS"
-python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS"
-cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
-
-echo "Copying BOTS"
-cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
+if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then 
+    if [ "${AUTO_MIX_BOTS}" = "true" ]; then
+        # Backup Original BOTS
+        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk
+        echo "Merge your BOTS file with BOTS"
+        python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS"
+        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
+        echo "Copying BOTS"
+        cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
+        # Restore original BOTS
+        mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS
+    else
+        cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS
+    fi
+fi
 
 echo "Installing new BOTS"
 cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user
 
-# Restore original BOTS
-mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS
+
+
diff --git a/mybots/BOTS b/mybots/BOTS
index 185295d..feaa198 100644
--- a/mybots/BOTS
+++ b/mybots/BOTS
@@ -14,4 +14,4 @@
         "Output": {
 
         }
-}
\ No newline at end of file
+}

From 9c44bd34c93c79618bbc9c1ea76a70cbc38a3d58 Mon Sep 17 00:00:00 2001
From: Einar Lanfranco <einar.lanfranco@gmail.com>
Date: Thu, 25 Mar 2021 16:23:49 -0300
Subject: [PATCH 08/10] Mail handler requeriment is going to be migrated to
 main intelmq

---
 docker-compose-dev.yml | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index c035be4..e658d2a 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -36,18 +36,10 @@ services:
             INTELMQ_PIPELINE_DRIVER: "redis"
             INTELMQ_PIPELINE_HOST: redis
             INTELMQ_REDIS_CACHE_HOST: redis
-            # Mail handler
-            LOG_MAIL_ENABLED: "false"
-            LOG_MAIL_LEVEL: "logging.ERROR"
-            LOG_MAIL_MAILHOST: "mail.example.unlp.edu.ar"
-            LOG_MAIL_PORT: 25
-            LOG_MAIL_FROMADDR: "intelmq@examplefeeds.unlp.edu.ar"
-            LOG_MAIL_TOADDR: "support@example.unlp.edu.ar"
-            LOG_MAIL_SUBJECT: "[INTELMQ] Application Error"
-            LOG_MAIL_CREDENTIALS: None #tuple (username, password)
-            LOG_MAIL_SECURE: None
             # Start botnet at boot
-            ENABLE_BOTNET_AT_BOOT: "true"
+            ENABLE_BOTNET_AT_BOOT: "false"
+            # Enable this to enable automix of BOTS file
+            AUTO_MIX_BOTS: "false"
         networks:
             - intelmq-internal
 

From 8bfbdcbe44af94590d11fb503d5c2fa54f19c9bc Mon Sep 17 00:00:00 2001
From: Einar Felipe Lanfranco <einar-lanfranco@users.noreply.github.com>
Date: Thu, 25 Mar 2021 16:47:39 -0300
Subject: [PATCH 09/10] Update README.md

Fix for html building in intelmq-manager, otherwise you get 404
---
 README.md | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 2bb3e9b..a8e10d1 100644
--- a/README.md
+++ b/README.md
@@ -14,9 +14,19 @@ If you do have any questions / feedback / questions, please open an issue :)
 0. `sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose`
 0. `git clone https://github.com/certat/intelmq-docker.git --recursive`
 0. `cd intelmq-docker`
-0. `sudo docker-compose pull`
-0. `sudo docker-compose up`
-0. Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+0. `docker-compose pull`
+0. `cd intelmq-manager`
+0. `python3 setup.py`
+0. `cd ..`
+2. `docker-compose up`
+3. Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+
+## For developers 
+
+Please take a look to DEVELOP-GUIDE.md
+
+
+## Build and deploy new images
 
 If you want to build/deploy/test this container run 
 1. `chmod +x build.sh`

From 6ac15032630a4d9f31878c867366fb2664a7229f Mon Sep 17 00:00:00 2001
From: Einar Felipe Lanfranco <einar-lanfranco@users.noreply.github.com>
Date: Thu, 25 Mar 2021 16:52:55 -0300
Subject: [PATCH 10/10] Removing mail handler from update.sh

---
 .docker/intelmq-full-dev/entrypoint_dev.sh | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.docker/intelmq-full-dev/entrypoint_dev.sh b/.docker/intelmq-full-dev/entrypoint_dev.sh
index be94d63..913a56f 100755
--- a/.docker/intelmq-full-dev/entrypoint_dev.sh
+++ b/.docker/intelmq-full-dev/entrypoint_dev.sh
@@ -2,13 +2,9 @@
 
 /opt/dev/update.sh
 
-if [ "${LOG_MAIL_ENABLED}" = "true" ]; then
-    sed -i "s/return\ logger/### Code added to fix unexistent mail handler ###\n    mail_handler=logging.handlers.SMTPHandler(mailhost = ('${LOG_MAIL_MAILHOST}', ${LOG_MAIL_PORT}),fromaddr = '${LOG_MAIL_FROMADDR}',toaddrs = ['${LOG_MAIL_TOADDR}'],subject = '${LOG_MAIL_SUBJECT}',credentials = ${LOG_MAIL_CREDENTIALS}, secure = ${LOG_MAIL_SECURE} )\n    mail_handler.setLevel(${LOG_MAIL_LEVEL})\n    mail_handler.setFormatter(logging.Formatter(LOG_FORMAT))\n    logger.addHandler(mail_handler)\n    aux_logger = logger\n    return aux_logger\n    ### End code added to fix unexistent mail handler ###/g" /opt/intelmq/intelmq/lib/utils.py
-fi
-
 if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
 	intelmqctl start
 fi
 
 
-/opt/entrypoint.sh
\ No newline at end of file
+/opt/entrypoint.sh