docker for dev ready

2025-12-06 17:22:57 +01:00 · 2021-10-01 17:45:04 -03:00
parent 220c6a7884
commit a84e4edea9
8 changed files with 94 additions and 135 deletions
--- a/.docker/intelmq-full-dev/Dockerfile
+++ b/.docker/intelmq-full-dev/Dockerfile
@@ -1,20 +1,18 @@
-FROM certat/intelmq-full:1.0
+FROM certat/intelmq-full:latest
+ENV LANG C.UTF-8

-MAINTAINER Einar <elanfranco@cert.unlp.edu.ar>
-MAINTAINER Jeremias <jpretto@cert.unlp.edu.ar>
+LABEL maintainer="Einar <elanfranco@csirtamericas.org>"
+LABEL maintainer="Jeremias <jpretto@cert.unlp.edu.ar>" 
+LABEL maintainer="Mateo <mdurante@cert.unlp.edu.ar>" 

-ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh
-ADD update.sh /opt/dev/update.sh
-ADD merge_BOTS.py /opt/dev/merge_BOTS.py
+WORKDIR /opt
+ADD entrypoint-dev.sh /opt/entrypoint-dev.sh
+ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh
+RUN sudo chmod +x /opt/entrypoint-dev.sh \
+    && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh
+RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \
+    && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh

-# Merge bots for merge_BOTS.py
-RUN sudo python3 -m pip install jsonmerge
-
-
-# Permission denied when installing new bots
-RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
-
-ENV PATH="/opt/intelmq/.local/bin:${PATH}"
-
-ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"]
+USER intelmq:intelmq

+ENTRYPOINT [ "/opt/entrypoint-dev.sh" ]
--- a/.docker/intelmq-full-dev/entrypoint-dev.sh
+++ b/.docker/intelmq-full-dev/entrypoint-dev.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+export INTELMQ_IS_DOCKER=1
+
+if [[ ${IS_DEV} == "true" ]]
+then
+    cd /etc/intelmq
+    sudo pip3 install hug url-normalize geolib imbox jinja2 pyasn textx tld time-machine
+    sudo pip3 install --force pymisp[fileobjects,openioc,virustotal]
+    /opt/install_reqs_and_deploy_bots.sh
+fi
+
+sudo chown -R intelmq:intelmq /etc/intelmq
+sudo chown -R intelmq:intelmq /opt/intelmq
+
+intelmqctl upgrade-config
+intelmqctl check
+
+intelmq_user="${INTELMQ_API_USER:=intelmq}"
+intelmq_pass="${INTELMQ_API_PASS:=intelmq}"
+
+intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass"
+
+if [[ $1 == "selftest" ]]
+then
+    export INTELMQ_TEST_EXOTIC=1
+    nosetests3 /etc/intelmq/intelmq/tests
+else
+    cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080
+fi
--- a/.docker/intelmq-full-dev/entrypoint_dev.sh
+++ b/.docker/intelmq-full-dev/entrypoint_dev.sh
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-/opt/dev/update.sh
-
-if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
-	intelmqctl start
-fi
-
-
-/opt/entrypoint.sh
--- a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots
+++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots
@@ -0,0 +1,10 @@
+#!/bin/bash
+for file in $(find /etc/intelmq/intelmq/bots -name "*REQUIREMENTS.txt"); do 
+    cat file >> /tmp/fullrequirements.txt
+done
+cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt
+sudo pip3 install -r /tmp/orderfullrequirements.txt; 
+cd /etc/intelmq
+sudo pip3 install --no-cache-dir -e .
+
+intelmqsetup
--- a/.docker/intelmq-full-dev/merge_BOTS.py
+++ b/.docker/intelmq-full-dev/merge_BOTS.py
@@ -1,38 +0,0 @@
-import json
-from jsonmerge import merge
-import argparse
-from collections import OrderedDict
-
-parser = argparse.ArgumentParser(description='Merge two json.')
-parser.add_argument('input_file_1', type=str, help='input_file_1')
-parser.add_argument('input_file_2', type=str, help='input_file_2')
-parser.add_argument('output_file', type=str, help='output_file')
-
-args = parser.parse_args()
-
-with open(args.input_file_1, 'r') as f:
-    j1 = json.load(f)
-with open(args.input_file_2, 'r') as f:
-    j2 = json.load(f)
-
-def sortOD(od):
-    res = OrderedDict()
-    for k, v in sorted(od.items()):
-        if isinstance(v, dict):
-            res[k] = sortOD(v)
-        else:
-            res[k] = v
-    return res
-
-
-merged = sortOD(merge(j1,j2))
-
-desired_order_list = ['Collector', 'Parser', 'Expert', 'Output']
-reordered_dict = {k: merged[k] for k in desired_order_list}
-
-# add other keys
-reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list})
-
-with open(args.output_file, 'w') as f:
-    json.dump(reordered_dict, f, indent=4)
-   
--- a/.docker/intelmq-full-dev/update.sh
+++ b/.docker/intelmq-full-dev/update.sh
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-echo "Installing requirements for bots in dev repository"
-for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
-
-if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then 
-    if [ "${AUTO_MIX_BOTS}" = "true" ]; then
-        # Backup Original BOTS
-        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk
-        echo "Merge your BOTS file with BOTS"
-        python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS"
-        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
-        echo "Copying BOTS"
-        cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
-        # Restore original BOTS
-        mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS
-    else
-        cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS
-    fi
-fi
-
-echo "Installing new BOTS"
-cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user
-
-
-
--- a/DEVELOP-GUIDE.md
+++ b/DEVELOP-GUIDE.md
@@ -2,46 +2,48 @@

 ## Run & deploy containers in dev mode:

-0. `cd intelmq-manager`
-0. `python3 setup.py`
-0. `cd ..`
-1. `docker-compose -f docker-compose-dev.yml up`
+### Install docker and docker-compose
+```
+sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose
+```
+
+### Clone this repo
+
+```
+git clone https://github.com/certat/intelmq-docker.git --recursive
+cd intelmq-docker
+docker-compose -f docker-compose-dev.yml build
+```
+
+### In next step replace git@github.com:certtools/intelmq.git by your fork of intelmq
+
+```
+git clone git@github.com:certtools/intelmq.git my_fork_of_intelmq/
+docker-compose -f docker-compose-dev.yml up
+```
+
+### Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+
+        Default user/password: intelmq/intelmq

 ## Docker-compose-dev.yml file

-### Volume:
+### Volumes:

-**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert.
+- **./my_fork_of_intelmq/intelmq:/etc/intelmq/intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly.

 ### Add your own bots

-Just start coding or pull your bots repository in ,/mybots folder
+Just start coding or pull your bots repository in ./my_fork_of_intelmq folder/intelmq/bots

-### How to install and look yours bots runnig
+### How to install and look yours bots running

+After you change some bot or add someshing new just run command **install_reqs_and_deploy_bots.sh** in the running container

-Just run /opt/dev/update.sh in the container:
-
-1. `docker-compose exec -f docker-compose-dev.yml  intelmq /opt/bin/update.sh`
+```
+docker-compose exec -f docker-compose-dev.yml  intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh
+```

 When you do this:

-* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment
-* Yours bots will be installed
-
-### Additional environment variables
-
-Check options in docker-compose-dev.yml:
-
-* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email)
-* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not.
-
-
-## For deploy your already developed bots
-
-Just clone your bots git to ./mybots and run the container
-
-For example, using https://github.com/CERTUNLP/intelmq-bots:
-
-1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3`
-0. `docker-compose -f docker-compose-dev.yml up`
+* Yours bots and REQUERIMENTS will be installed
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -7,16 +7,12 @@ services:
        command:
            - redis-server
            - /usr/local/etc/redis/redis.conf
-        restart: always
        networks: 
            - intelmq-internal
    nginx:
        image: certat/intelmq-nginx:latest
-        restart: always
        ports: 
            - 1337:80
-        volumes: 
-            - ./intelmq-manager/html:/www
        depends_on: 
            - intelmq
        networks: 
@@ -25,25 +21,23 @@ services:
        build: .docker/intelmq-full-dev
        volumes: 
            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
-            - ./example_config/intelmq-api:/opt/intelmq-api/config
+            - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
            - ./intelmq_logs:/opt/intelmq/var/log
            - ./intelmq_output:/opt/intelmq/var/lib/bots
-            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
-            - ./mybots:/opt/dev/mybots
+            - ./my_fork_of_intelmq/intelmq/:/etc/intelmq/intelmq/
        depends_on: 
            - redis
-        environment: 
-            INTELMQ_PIPELINE_DRIVER: "redis"
+        environment:
+            IS_DEV: "true" 
+            INTELMQ_SOURCE_PIPELINE_BROKER: "redis"
+            INTELMQ_PIPELINE_BROKER: "redis"
+            INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis"
            INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_SOURCE_PIPELINE_HOST: redis
+            INTELMQ_DESTINATION_PIPELINE_HOST: redis
            INTELMQ_REDIS_CACHE_HOST: redis
-            # Start botnet at boot
-            ENABLE_BOTNET_AT_BOOT: "false"
-            # Enable this to enable automix of BOTS file
-            AUTO_MIX_BOTS: "false"
        networks:
            - intelmq-internal
-
-
 networks: 
    intelmq-internal:
        driver: bridge