Merge pull request #3 from CERTUNLP/main

We developed some adjustments in our repository to integrate with yours in the new version 2.3 adding facilities for bot developers

.docker/intelmq-full-dev/Dockerfile

@@ -0,0 +1,20 @@
+FROM certat/intelmq-full:1.0
+
+MAINTAINER Einar <elanfranco@cert.unlp.edu.ar>
+MAINTAINER Jeremias <jpretto@cert.unlp.edu.ar>
+
+ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh
+ADD update.sh /opt/dev/update.sh
+ADD merge_BOTS.py /opt/dev/merge_BOTS.py
+
+# Merge bots for merge_BOTS.py
+RUN sudo python3 -m pip install jsonmerge
+
+
+# Permission denied when installing new bots
+RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
+
+ENV PATH="/opt/intelmq/.local/bin:${PATH}"
+
+ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"]
+

.docker/intelmq-full-dev/entrypoint_dev.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+/opt/dev/update.sh
+
+if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
+	intelmqctl start
+fi
+
+
+/opt/entrypoint.sh

.docker/intelmq-full-dev/merge_BOTS.py

@@ -0,0 +1,38 @@
+import json
+from jsonmerge import merge
+import argparse
+from collections import OrderedDict
+
+parser = argparse.ArgumentParser(description='Merge two json.')
+parser.add_argument('input_file_1', type=str, help='input_file_1')
+parser.add_argument('input_file_2', type=str, help='input_file_2')
+parser.add_argument('output_file', type=str, help='output_file')
+
+args = parser.parse_args()
+
+with open(args.input_file_1, 'r') as f:
+    j1 = json.load(f)
+with open(args.input_file_2, 'r') as f:
+    j2 = json.load(f)
+
+def sortOD(od):
+    res = OrderedDict()
+    for k, v in sorted(od.items()):
+        if isinstance(v, dict):
+            res[k] = sortOD(v)
+        else:
+            res[k] = v
+    return res
+
+
+merged = sortOD(merge(j1,j2))
+
+desired_order_list = ['Collector', 'Parser', 'Expert', 'Output']
+reordered_dict = {k: merged[k] for k in desired_order_list}
+
+# add other keys
+reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list})
+
+with open(args.output_file, 'w') as f:
+    json.dump(reordered_dict, f, indent=4)
+   

.docker/intelmq-full-dev/update.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+echo "Installing requirements for bots in dev repository"
+for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
+
+if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then 
+    if [ "${AUTO_MIX_BOTS}" = "true" ]; then
+        # Backup Original BOTS
+        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk
+        echo "Merge your BOTS file with BOTS"
+        python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS"
+        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
+        echo "Copying BOTS"
+        cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
+        # Restore original BOTS
+        mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS
+    else
+        cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS
+    fi
+fi
+
+echo "Installing new BOTS"
+cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user
+
+
+

DEVELOP-GUIDE.md

@@ -0,0 +1,45 @@
+# intelmq-docker
+
+## Run & deploy containers in dev mode:
+
+1. `docker-compose -f docker-compose-dev.yml up`
+
+## Docker-compose-dev.yml file
+
+### Volume:  
+
+**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert.
+
+### Add your own bots
+
+Just start coding or pull your bots repository in ,/mybots folder
+
+### How to install and look yours bots runnig
+
+
+Just run /opt/dev/update.sh in the container:
+
+1. `docker-compose exec -f docker-compose-dev.yml  intelmq /opt/bin/update.sh`
+
+When you do this:
+
+* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment
+* Yours bots will be installed 
+
+### Additional environment variables
+
+Check options in docker-compose-dev.yml:
+
+* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email)
+* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not.
+
+
+## For deploy your already developed bots
+
+Just clone your bots git to ./mybots and run the container
+
+For example, using https://github.com/CERTUNLP/intelmq-bots:
+
+1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3`
+0. `docker-compose -f docker-compose-dev.yml up`
+

README.md

@@ -14,9 +14,19 @@ If you do have any questions / feedback / questions, please open an issue :)
 0. `sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose`
 0. `git clone https://github.com/certat/intelmq-docker.git --recursive`
 0. `cd intelmq-docker`
-0. `sudo docker-compose pull`
-0. `sudo docker-compose up`
-0. Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+0. `docker-compose pull`
+0. `cd intelmq-manager`
+0. `python3 setup.py`
+0. `cd ..`
+2. `docker-compose up`
+3. Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+
+## For developers 
+
+Please take a look to DEVELOP-GUIDE.md
+
+
+## Build and deploy new images
 
 If you want to build/deploy/test this container run 
 1. `chmod +x build.sh`

docker-compose-dev.yml

@@ -0,0 +1,49 @@
+version: "3"
+services:
+    redis:
+        image: redis:latest
+        volumes:
+            - ./example_config/redis/redis.conf:/usr/local/etc/redis/redis.conf
+        command:
+            - redis-server
+            - /usr/local/etc/redis/redis.conf
+        restart: always
+        networks: 
+            - intelmq-internal
+    nginx:
+        image: certat/intelmq-nginx:latest
+        restart: always
+        ports: 
+            - 1337:80
+        volumes: 
+            - ./intelmq-manager/html:/www
+        depends_on: 
+            - intelmq
+        networks: 
+            - intelmq-internal
+    intelmq:
+        build: .docker/intelmq-full-dev
+        volumes: 
+            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+            - ./example_config/intelmq-api:/opt/intelmq-api/config
+            - ./intelmq_logs:/opt/intelmq/var/log
+            - ./intelmq_output:/opt/intelmq/var/lib/bots
+            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
+            - ./mybots:/opt/dev/mybots
+        depends_on: 
+            - redis
+        environment: 
+            INTELMQ_PIPELINE_DRIVER: "redis"
+            INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_REDIS_CACHE_HOST: redis
+            # Start botnet at boot
+            ENABLE_BOTNET_AT_BOOT: "false"
+            # Enable this to enable automix of BOTS file
+            AUTO_MIX_BOTS: "false"
+        networks:
+            - intelmq-internal
+
+
+networks: 
+    intelmq-internal:
+        driver: bridge

mybots/BOTS

@@ -0,0 +1,17 @@
+{
+        "Collector": {
+        },
+        "Parser": {
+        },
+        "Expert": {
+                  "Example": {
+                    "description": "Example own bot.",
+                    "module": "intelmq.bots.experts.example.expert",
+                    "parameters": {
+                    }
+                }
+        },
+        "Output": {
+
+        }
+}

mybots/bots/experts/example/expert.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+from intelmq.lib.bot import Bot
+
+
+class ExampleExpertBot(Bot):
+
+    def init(self):
+        pass
+
+    def process(self):
+        pass
+
+
+BOT = ExampleExpertBot