docker for dev ready

.docker/intelmq-full-dev/Dockerfile

@@ -1,20 +1,18 @@
-FROM certat/intelmq-full:1.0
+FROM certat/intelmq-full:latest
+ENV LANG C.UTF-8
 
-MAINTAINER Einar <elanfranco@cert.unlp.edu.ar>
+LABEL maintainer="Einar <elanfranco@csirtamericas.org>"
-MAINTAINER Jeremias <jpretto@cert.unlp.edu.ar>
+LABEL maintainer="Jeremias <jpretto@cert.unlp.edu.ar>" 
+LABEL maintainer="Mateo <mdurante@cert.unlp.edu.ar>" 
 
-ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh
+WORKDIR /opt
-ADD update.sh /opt/dev/update.sh
+ADD entrypoint-dev.sh /opt/entrypoint-dev.sh
-ADD merge_BOTS.py /opt/dev/merge_BOTS.py
+ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh
+RUN sudo chmod +x /opt/entrypoint-dev.sh \
+    && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh
+RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \
+    && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh
 
-# Merge bots for merge_BOTS.py
+USER intelmq:intelmq
-RUN sudo python3 -m pip install jsonmerge
-
-
-# Permission denied when installing new bots
-RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
-
-ENV PATH="/opt/intelmq/.local/bin:${PATH}"
-
-ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"]
 
+ENTRYPOINT [ "/opt/entrypoint-dev.sh" ]

.docker/intelmq-full-dev/entrypoint-dev.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+export INTELMQ_IS_DOCKER=1
+
+if [[ ${IS_DEV} == "true" ]]
+then
+    cd /etc/intelmq
+    sudo pip3 install hug url-normalize geolib imbox jinja2 pyasn textx tld time-machine
+    sudo pip3 install --force pymisp[fileobjects,openioc,virustotal]
+    /opt/install_reqs_and_deploy_bots.sh
+fi
+
+sudo chown -R intelmq:intelmq /etc/intelmq
+sudo chown -R intelmq:intelmq /opt/intelmq
+
+intelmqctl upgrade-config
+intelmqctl check
+
+intelmq_user="${INTELMQ_API_USER:=intelmq}"
+intelmq_pass="${INTELMQ_API_PASS:=intelmq}"
+
+intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass"
+
+if [[ $1 == "selftest" ]]
+then
+    export INTELMQ_TEST_EXOTIC=1
+    nosetests3 /etc/intelmq/intelmq/tests
+else
+    cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080
+fi

.docker/intelmq-full-dev/entrypoint_dev.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-/opt/dev/update.sh
-
-if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
-	intelmqctl start
-fi
-
-
-/opt/entrypoint.sh

.docker/intelmq-full-dev/install_reqs_and_deploy_bots

@@ -0,0 +1,10 @@
+#!/bin/bash
+for file in $(find /etc/intelmq/intelmq/bots -name "*REQUIREMENTS.txt"); do 
+    cat file >> /tmp/fullrequirements.txt
+done
+cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt
+sudo pip3 install -r /tmp/orderfullrequirements.txt; 
+cd /etc/intelmq
+sudo pip3 install --no-cache-dir -e .
+
+intelmqsetup

.docker/intelmq-full-dev/merge_BOTS.py

@@ -1,38 +0,0 @@
-import json
-from jsonmerge import merge
-import argparse
-from collections import OrderedDict
-
-parser = argparse.ArgumentParser(description='Merge two json.')
-parser.add_argument('input_file_1', type=str, help='input_file_1')
-parser.add_argument('input_file_2', type=str, help='input_file_2')
-parser.add_argument('output_file', type=str, help='output_file')
-
-args = parser.parse_args()
-
-with open(args.input_file_1, 'r') as f:
-    j1 = json.load(f)
-with open(args.input_file_2, 'r') as f:
-    j2 = json.load(f)
-
-def sortOD(od):
-    res = OrderedDict()
-    for k, v in sorted(od.items()):
-        if isinstance(v, dict):
-            res[k] = sortOD(v)
-        else:
-            res[k] = v
-    return res
-
-
-merged = sortOD(merge(j1,j2))
-
-desired_order_list = ['Collector', 'Parser', 'Expert', 'Output']
-reordered_dict = {k: merged[k] for k in desired_order_list}
-
-# add other keys
-reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list})
-
-with open(args.output_file, 'w') as f:
-    json.dump(reordered_dict, f, indent=4)
-   

.docker/intelmq-full-dev/update.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-echo "Installing requirements for bots in dev repository"
-for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
-
-if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then 
-    if [ "${AUTO_MIX_BOTS}" = "true" ]; then
-        # Backup Original BOTS
-        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk
-        echo "Merge your BOTS file with BOTS"
-        python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS"
-        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
-        echo "Copying BOTS"
-        cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
-        # Restore original BOTS
-        mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS
-    else
-        cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS
-    fi
-fi
-
-echo "Installing new BOTS"
-cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user
-
-
-

DEVELOP-GUIDE.md

@@ -2,46 +2,48 @@
 
 ## Run & deploy containers in dev mode:
 
-0. `cd intelmq-manager`
+### Install docker and docker-compose
-0. `python3 setup.py`
+```
-0. `cd ..`
+sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose
-1. `docker-compose -f docker-compose-dev.yml up`
+```
+
+### Clone this repo
+
+```
+git clone https://github.com/certat/intelmq-docker.git --recursive
+cd intelmq-docker
+docker-compose -f docker-compose-dev.yml build
+```
+
+### In next step replace git@github.com:certtools/intelmq.git by your fork of intelmq
+
+```
+git clone git@github.com:certtools/intelmq.git my_fork_of_intelmq/
+docker-compose -f docker-compose-dev.yml up
+```
+
+### Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+
+        Default user/password: intelmq/intelmq
 
 ## Docker-compose-dev.yml file
 
-### Volume:
+### Volumes:
 
-**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert.
+- **./my_fork_of_intelmq/intelmq:/etc/intelmq/intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly.
 
 ### Add your own bots
 
-Just start coding or pull your bots repository in ,/mybots folder
+Just start coding or pull your bots repository in ./my_fork_of_intelmq folder/intelmq/bots
 
-### How to install and look yours bots runnig
+### How to install and look yours bots running
 
+After you change some bot or add someshing new just run command **install_reqs_and_deploy_bots.sh** in the running container
 
-Just run /opt/dev/update.sh in the container:
+```
-
+docker-compose exec -f docker-compose-dev.yml  intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh
-1. `docker-compose exec -f docker-compose-dev.yml  intelmq /opt/bin/update.sh`
+```
 
 When you do this:
 
-* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment
+* Yours bots and REQUERIMENTS will be installed
-* Yours bots will be installed
-
-### Additional environment variables
-
-Check options in docker-compose-dev.yml:
-
-* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email)
-* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not.
-
-
-## For deploy your already developed bots
-
-Just clone your bots git to ./mybots and run the container
-
-For example, using https://github.com/CERTUNLP/intelmq-bots:
-
-1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3`
-0. `docker-compose -f docker-compose-dev.yml up`

docker-compose-dev.yml

@@ -7,16 +7,12 @@ services:
         command:
             - redis-server
             - /usr/local/etc/redis/redis.conf
-        restart: always
         networks: 
             - intelmq-internal
     nginx:
         image: certat/intelmq-nginx:latest
-        restart: always
         ports: 
             - 1337:80
-        volumes: 
-            - ./intelmq-manager/html:/www
         depends_on: 
             - intelmq
         networks: 
@@ -25,25 +21,23 @@ services:
         build: .docker/intelmq-full-dev
         volumes: 
             - ./example_config/intelmq/etc/:/opt/intelmq/etc/
-            - ./example_config/intelmq-api:/opt/intelmq-api/config
+            - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
             - ./intelmq_logs:/opt/intelmq/var/log
             - ./intelmq_output:/opt/intelmq/var/lib/bots
-            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
+            - ./my_fork_of_intelmq/intelmq/:/etc/intelmq/intelmq/
-            - ./mybots:/opt/dev/mybots
         depends_on: 
             - redis
-        environment: 
+        environment:
-            INTELMQ_PIPELINE_DRIVER: "redis"
+            IS_DEV: "true" 
+            INTELMQ_SOURCE_PIPELINE_BROKER: "redis"
+            INTELMQ_PIPELINE_BROKER: "redis"
+            INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis"
             INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_SOURCE_PIPELINE_HOST: redis
+            INTELMQ_DESTINATION_PIPELINE_HOST: redis
             INTELMQ_REDIS_CACHE_HOST: redis
-            # Start botnet at boot
-            ENABLE_BOTNET_AT_BOOT: "false"
-            # Enable this to enable automix of BOTS file
-            AUTO_MIX_BOTS: "false"
         networks:
             - intelmq-internal
-
-
 networks: 
     intelmq-internal:
         driver: bridge