diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile new file mode 100644 index 0000000..f8dc9a0 --- /dev/null +++ b/.docker/intelmq-full-dev/Dockerfile @@ -0,0 +1,20 @@ +FROM certat/intelmq-full:1.0 + +MAINTAINER Einar +MAINTAINER Jeremias + +ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh +ADD update.sh /opt/dev/update.sh +ADD merge_BOTS.py /opt/dev/merge_BOTS.py + +# Merge bots for merge_BOTS.py +RUN sudo python3 -m pip install jsonmerge + + +# Permission denied when installing new bots +RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info + +ENV PATH="/opt/intelmq/.local/bin:${PATH}" + +ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"] + diff --git a/.docker/intelmq-full-dev/entrypoint_dev.sh b/.docker/intelmq-full-dev/entrypoint_dev.sh new file mode 100755 index 0000000..913a56f --- /dev/null +++ b/.docker/intelmq-full-dev/entrypoint_dev.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +/opt/dev/update.sh + +if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then + intelmqctl start +fi + + +/opt/entrypoint.sh diff --git a/.docker/intelmq-full-dev/merge_BOTS.py b/.docker/intelmq-full-dev/merge_BOTS.py new file mode 100644 index 0000000..3c28494 --- /dev/null +++ b/.docker/intelmq-full-dev/merge_BOTS.py @@ -0,0 +1,38 @@ +import json +from jsonmerge import merge +import argparse +from collections import OrderedDict + +parser = argparse.ArgumentParser(description='Merge two json.') +parser.add_argument('input_file_1', type=str, help='input_file_1') +parser.add_argument('input_file_2', type=str, help='input_file_2') +parser.add_argument('output_file', type=str, help='output_file') + +args = parser.parse_args() + +with open(args.input_file_1, 'r') as f: + j1 = json.load(f) +with open(args.input_file_2, 'r') as f: + j2 = json.load(f) + +def sortOD(od): + res = OrderedDict() + for k, v in sorted(od.items()): + if isinstance(v, dict): + res[k] = sortOD(v) + else: + res[k] = v + return res + + +merged = sortOD(merge(j1,j2)) + +desired_order_list = ['Collector', 'Parser', 'Expert', 'Output'] +reordered_dict = {k: merged[k] for k in desired_order_list} + +# add other keys +reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list}) + +with open(args.output_file, 'w') as f: + json.dump(reordered_dict, f, indent=4) + \ No newline at end of file diff --git a/.docker/intelmq-full-dev/update.sh b/.docker/intelmq-full-dev/update.sh new file mode 100755 index 0000000..83b2470 --- /dev/null +++ b/.docker/intelmq-full-dev/update.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +echo "Installing requirements for bots in dev repository" +for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done + +if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then + if [ "${AUTO_MIX_BOTS}" = "true" ]; then + # Backup Original BOTS + cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk + echo "Merge your BOTS file with BOTS" + python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" + cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS + echo "Copying BOTS" + cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/ + # Restore original BOTS + mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS + else + cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS + fi +fi + +echo "Installing new BOTS" +cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user + + + diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md new file mode 100644 index 0000000..8c1fd52 --- /dev/null +++ b/DEVELOP-GUIDE.md @@ -0,0 +1,45 @@ +# intelmq-docker + +## Run & deploy containers in dev mode: + +1. `docker-compose -f docker-compose-dev.yml up` + +## Docker-compose-dev.yml file + +### Volume: + +**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert. + +### Add your own bots + +Just start coding or pull your bots repository in ,/mybots folder + +### How to install and look yours bots runnig + + +Just run /opt/dev/update.sh in the container: + +1. `docker-compose exec -f docker-compose-dev.yml intelmq /opt/bin/update.sh` + +When you do this: + +* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment +* Yours bots will be installed + +### Additional environment variables + +Check options in docker-compose-dev.yml: + +* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email) +* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not. + + +## For deploy your already developed bots + +Just clone your bots git to ./mybots and run the container + +For example, using https://github.com/CERTUNLP/intelmq-bots: + +1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3` +0. `docker-compose -f docker-compose-dev.yml up` + diff --git a/README.md b/README.md index 2bb3e9b..a8e10d1 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,19 @@ If you do have any questions / feedback / questions, please open an issue :) 0. `sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose` 0. `git clone https://github.com/certat/intelmq-docker.git --recursive` 0. `cd intelmq-docker` -0. `sudo docker-compose pull` -0. `sudo docker-compose up` -0. Open your favourite browser -> Go to `http://127.0.0.1:1337/` +0. `docker-compose pull` +0. `cd intelmq-manager` +0. `python3 setup.py` +0. `cd ..` +2. `docker-compose up` +3. Open your favourite browser -> Go to `http://127.0.0.1:1337/` + +## For developers + +Please take a look to DEVELOP-GUIDE.md + + +## Build and deploy new images If you want to build/deploy/test this container run 1. `chmod +x build.sh` diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000..e658d2a --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,49 @@ +version: "3" +services: + redis: + image: redis:latest + volumes: + - ./example_config/redis/redis.conf:/usr/local/etc/redis/redis.conf + command: + - redis-server + - /usr/local/etc/redis/redis.conf + restart: always + networks: + - intelmq-internal + nginx: + image: certat/intelmq-nginx:latest + restart: always + ports: + - 1337:80 + volumes: + - ./intelmq-manager/html:/www + depends_on: + - intelmq + networks: + - intelmq-internal + intelmq: + build: .docker/intelmq-full-dev + volumes: + - ./example_config/intelmq/etc/:/opt/intelmq/etc/ + - ./example_config/intelmq-api:/opt/intelmq-api/config + - ./intelmq_logs:/opt/intelmq/var/log + - ./intelmq_output:/opt/intelmq/var/lib/bots + - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot + - ./mybots:/opt/dev/mybots + depends_on: + - redis + environment: + INTELMQ_PIPELINE_DRIVER: "redis" + INTELMQ_PIPELINE_HOST: redis + INTELMQ_REDIS_CACHE_HOST: redis + # Start botnet at boot + ENABLE_BOTNET_AT_BOOT: "false" + # Enable this to enable automix of BOTS file + AUTO_MIX_BOTS: "false" + networks: + - intelmq-internal + + +networks: + intelmq-internal: + driver: bridge diff --git a/mybots/BOTS b/mybots/BOTS new file mode 100644 index 0000000..feaa198 --- /dev/null +++ b/mybots/BOTS @@ -0,0 +1,17 @@ +{ + "Collector": { + }, + "Parser": { + }, + "Expert": { + "Example": { + "description": "Example own bot.", + "module": "intelmq.bots.experts.example.expert", + "parameters": { + } + } + }, + "Output": { + + } +} diff --git a/mybots/bots/collectors/otherexample/collector.py b/mybots/bots/collectors/otherexample/collector.py new file mode 100644 index 0000000..e69de29 diff --git a/mybots/bots/experts/example/expert.py b/mybots/bots/experts/example/expert.py new file mode 100644 index 0000000..c39a6e2 --- /dev/null +++ b/mybots/bots/experts/example/expert.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +from intelmq.lib.bot import Bot + + +class ExampleExpertBot(Bot): + + def init(self): + pass + + def process(self): + pass + + +BOT = ExampleExpertBot