From 4ec9c20a813524968396f906944c1090711a29c6 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Fri, 1 Oct 2021 17:45:04 -0300 Subject: [PATCH] docker for dev ready --- .docker/intelmq-full-dev/Dockerfile | 30 +++++---- .docker/intelmq-full-dev/entrypoint-dev.sh | 29 +++++++++ .docker/intelmq-full-dev/entrypoint_dev.sh | 10 --- .../install_reqs_and_deploy_bots | 10 +++ .docker/intelmq-full-dev/merge_BOTS.py | 38 ------------ .docker/intelmq-full-dev/update.sh | 26 -------- DEVELOP-GUIDE.md | 62 ++++++++++--------- docker-compose-dev.yml | 24 +++---- 8 files changed, 94 insertions(+), 135 deletions(-) create mode 100644 .docker/intelmq-full-dev/entrypoint-dev.sh delete mode 100755 .docker/intelmq-full-dev/entrypoint_dev.sh create mode 100755 .docker/intelmq-full-dev/install_reqs_and_deploy_bots delete mode 100644 .docker/intelmq-full-dev/merge_BOTS.py delete mode 100755 .docker/intelmq-full-dev/update.sh diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index f8dc9a0..3b6fa5d 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -1,20 +1,18 @@ -FROM certat/intelmq-full:1.0 +FROM certat/intelmq-full:latest +ENV LANG C.UTF-8 -MAINTAINER Einar -MAINTAINER Jeremias +LABEL maintainer="Einar " +LABEL maintainer="Jeremias " +LABEL maintainer="Mateo " -ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh -ADD update.sh /opt/dev/update.sh -ADD merge_BOTS.py /opt/dev/merge_BOTS.py +WORKDIR /opt +ADD entrypoint-dev.sh /opt/entrypoint-dev.sh +ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh +RUN sudo chmod +x /opt/entrypoint-dev.sh \ + && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh +RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ + && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh -# Merge bots for merge_BOTS.py -RUN sudo python3 -m pip install jsonmerge - - -# Permission denied when installing new bots -RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info - -ENV PATH="/opt/intelmq/.local/bin:${PATH}" - -ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"] +USER intelmq:intelmq +ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh new file mode 100644 index 0000000..67b2d6f --- /dev/null +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -0,0 +1,29 @@ +#!/bin/bash +export INTELMQ_IS_DOCKER=1 + +if [[ ${IS_DEV} == "true" ]] +then + cd /etc/intelmq + sudo pip3 install hug url-normalize geolib imbox jinja2 pyasn textx tld time-machine + sudo pip3 install --force pymisp[fileobjects,openioc,virustotal] + /opt/install_reqs_and_deploy_bots.sh +fi + +sudo chown -R intelmq:intelmq /etc/intelmq +sudo chown -R intelmq:intelmq /opt/intelmq + +intelmqctl upgrade-config +intelmqctl check + +intelmq_user="${INTELMQ_API_USER:=intelmq}" +intelmq_pass="${INTELMQ_API_PASS:=intelmq}" + +intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" + +if [[ $1 == "selftest" ]] +then + export INTELMQ_TEST_EXOTIC=1 + nosetests3 /etc/intelmq/intelmq/tests +else + cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 +fi diff --git a/.docker/intelmq-full-dev/entrypoint_dev.sh b/.docker/intelmq-full-dev/entrypoint_dev.sh deleted file mode 100755 index 913a56f..0000000 --- a/.docker/intelmq-full-dev/entrypoint_dev.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -/opt/dev/update.sh - -if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then - intelmqctl start -fi - - -/opt/entrypoint.sh diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots new file mode 100755 index 0000000..192abd5 --- /dev/null +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -0,0 +1,10 @@ +#!/bin/bash +for file in $(find /etc/intelmq/intelmq/bots -name "*REQUIREMENTS.txt"); do + cat file >> /tmp/fullrequirements.txt +done +cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt +sudo pip3 install -r /tmp/orderfullrequirements.txt; +cd /etc/intelmq +sudo pip3 install --no-cache-dir -e . + +intelmqsetup \ No newline at end of file diff --git a/.docker/intelmq-full-dev/merge_BOTS.py b/.docker/intelmq-full-dev/merge_BOTS.py deleted file mode 100644 index 3c28494..0000000 --- a/.docker/intelmq-full-dev/merge_BOTS.py +++ /dev/null @@ -1,38 +0,0 @@ -import json -from jsonmerge import merge -import argparse -from collections import OrderedDict - -parser = argparse.ArgumentParser(description='Merge two json.') -parser.add_argument('input_file_1', type=str, help='input_file_1') -parser.add_argument('input_file_2', type=str, help='input_file_2') -parser.add_argument('output_file', type=str, help='output_file') - -args = parser.parse_args() - -with open(args.input_file_1, 'r') as f: - j1 = json.load(f) -with open(args.input_file_2, 'r') as f: - j2 = json.load(f) - -def sortOD(od): - res = OrderedDict() - for k, v in sorted(od.items()): - if isinstance(v, dict): - res[k] = sortOD(v) - else: - res[k] = v - return res - - -merged = sortOD(merge(j1,j2)) - -desired_order_list = ['Collector', 'Parser', 'Expert', 'Output'] -reordered_dict = {k: merged[k] for k in desired_order_list} - -# add other keys -reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list}) - -with open(args.output_file, 'w') as f: - json.dump(reordered_dict, f, indent=4) - \ No newline at end of file diff --git a/.docker/intelmq-full-dev/update.sh b/.docker/intelmq-full-dev/update.sh deleted file mode 100755 index 83b2470..0000000 --- a/.docker/intelmq-full-dev/update.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -echo "Installing requirements for bots in dev repository" -for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done - -if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then - if [ "${AUTO_MIX_BOTS}" = "true" ]; then - # Backup Original BOTS - cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk - echo "Merge your BOTS file with BOTS" - python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" - cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS - echo "Copying BOTS" - cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/ - # Restore original BOTS - mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS - else - cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS - fi -fi - -echo "Installing new BOTS" -cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user - - - diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 409dce6..153a547 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -2,46 +2,48 @@ ## Run & deploy containers in dev mode: -0. `cd intelmq-manager` -0. `python3 setup.py` -0. `cd ..` -1. `docker-compose -f docker-compose-dev.yml up` +### Install docker and docker-compose +``` +sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose +``` + +### Clone this repo + +``` +git clone https://github.com/certat/intelmq-docker.git --recursive +cd intelmq-docker +docker-compose -f docker-compose-dev.yml build +``` + +### In next step replace git@github.com:certtools/intelmq.git by your fork of intelmq + +``` +git clone git@github.com:certtools/intelmq.git my_fork_of_intelmq/ +docker-compose -f docker-compose-dev.yml up +``` + +### Open your favourite browser -> Go to `http://127.0.0.1:1337/` + + Default user/password: intelmq/intelmq ## Docker-compose-dev.yml file -### Volume: +### Volumes: -**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert. +- **./my_fork_of_intelmq/intelmq:/etc/intelmq/intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly. ### Add your own bots -Just start coding or pull your bots repository in ,/mybots folder +Just start coding or pull your bots repository in ./my_fork_of_intelmq folder/intelmq/bots -### How to install and look yours bots runnig +### How to install and look yours bots running +After you change some bot or add someshing new just run command **install_reqs_and_deploy_bots.sh** in the running container -Just run /opt/dev/update.sh in the container: - -1. `docker-compose exec -f docker-compose-dev.yml intelmq /opt/bin/update.sh` +``` +docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh +``` When you do this: -* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment -* Yours bots will be installed - -### Additional environment variables - -Check options in docker-compose-dev.yml: - -* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email) -* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not. - - -## For deploy your already developed bots - -Just clone your bots git to ./mybots and run the container - -For example, using https://github.com/CERTUNLP/intelmq-bots: - -1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3` -0. `docker-compose -f docker-compose-dev.yml up` +* Yours bots and REQUERIMENTS will be installed diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index e658d2a..dbbf6ce 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -7,16 +7,12 @@ services: command: - redis-server - /usr/local/etc/redis/redis.conf - restart: always networks: - intelmq-internal nginx: image: certat/intelmq-nginx:latest - restart: always ports: - 1337:80 - volumes: - - ./intelmq-manager/html:/www depends_on: - intelmq networks: @@ -25,25 +21,23 @@ services: build: .docker/intelmq-full-dev volumes: - ./example_config/intelmq/etc/:/opt/intelmq/etc/ - - ./example_config/intelmq-api:/opt/intelmq-api/config + - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot - - ./mybots:/opt/dev/mybots + - ./my_fork_of_intelmq/intelmq/:/etc/intelmq/intelmq/ depends_on: - redis - environment: - INTELMQ_PIPELINE_DRIVER: "redis" + environment: + IS_DEV: "true" + INTELMQ_SOURCE_PIPELINE_BROKER: "redis" + INTELMQ_PIPELINE_BROKER: "redis" + INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis" INTELMQ_PIPELINE_HOST: redis + INTELMQ_SOURCE_PIPELINE_HOST: redis + INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis - # Start botnet at boot - ENABLE_BOTNET_AT_BOOT: "false" - # Enable this to enable automix of BOTS file - AUTO_MIX_BOTS: "false" networks: - intelmq-internal - - networks: intelmq-internal: driver: bridge