From 61129e65fe7ce2c76333b3c9c64925bc8088120d Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 22 Aug 2022 12:51:53 +0200 Subject: [PATCH 01/21] intelmq-full: add ssh, required for rsync https://github.com/certtools/intelmq/discussions/2234 --- .docker/intelmq-full/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.docker/intelmq-full/Dockerfile b/.docker/intelmq-full/Dockerfile index 7983b50..de785b2 100644 --- a/.docker/intelmq-full/Dockerfile +++ b/.docker/intelmq-full/Dockerfile @@ -21,6 +21,7 @@ RUN apt-get update \ sudo \ gcc \ rsync \ + ssh \ python3-pika \ python3-nose \ python3-dev \ From 4ec9c20a813524968396f906944c1090711a29c6 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Fri, 1 Oct 2021 17:45:04 -0300 Subject: [PATCH 02/21] docker for dev ready --- .docker/intelmq-full-dev/Dockerfile | 30 +++++---- .docker/intelmq-full-dev/entrypoint-dev.sh | 29 +++++++++ .docker/intelmq-full-dev/entrypoint_dev.sh | 10 --- .../install_reqs_and_deploy_bots | 10 +++ .docker/intelmq-full-dev/merge_BOTS.py | 38 ------------ .docker/intelmq-full-dev/update.sh | 26 -------- DEVELOP-GUIDE.md | 62 ++++++++++--------- docker-compose-dev.yml | 24 +++---- 8 files changed, 94 insertions(+), 135 deletions(-) create mode 100644 .docker/intelmq-full-dev/entrypoint-dev.sh delete mode 100755 .docker/intelmq-full-dev/entrypoint_dev.sh create mode 100755 .docker/intelmq-full-dev/install_reqs_and_deploy_bots delete mode 100644 .docker/intelmq-full-dev/merge_BOTS.py delete mode 100755 .docker/intelmq-full-dev/update.sh diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index f8dc9a0..3b6fa5d 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -1,20 +1,18 @@ -FROM certat/intelmq-full:1.0 +FROM certat/intelmq-full:latest +ENV LANG C.UTF-8 -MAINTAINER Einar -MAINTAINER Jeremias +LABEL maintainer="Einar " +LABEL maintainer="Jeremias " +LABEL maintainer="Mateo " -ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh -ADD update.sh /opt/dev/update.sh -ADD merge_BOTS.py /opt/dev/merge_BOTS.py +WORKDIR /opt +ADD entrypoint-dev.sh /opt/entrypoint-dev.sh +ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh +RUN sudo chmod +x /opt/entrypoint-dev.sh \ + && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh +RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ + && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh -# Merge bots for merge_BOTS.py -RUN sudo python3 -m pip install jsonmerge - - -# Permission denied when installing new bots -RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info - -ENV PATH="/opt/intelmq/.local/bin:${PATH}" - -ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"] +USER intelmq:intelmq +ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh new file mode 100644 index 0000000..67b2d6f --- /dev/null +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -0,0 +1,29 @@ +#!/bin/bash +export INTELMQ_IS_DOCKER=1 + +if [[ ${IS_DEV} == "true" ]] +then + cd /etc/intelmq + sudo pip3 install hug url-normalize geolib imbox jinja2 pyasn textx tld time-machine + sudo pip3 install --force pymisp[fileobjects,openioc,virustotal] + /opt/install_reqs_and_deploy_bots.sh +fi + +sudo chown -R intelmq:intelmq /etc/intelmq +sudo chown -R intelmq:intelmq /opt/intelmq + +intelmqctl upgrade-config +intelmqctl check + +intelmq_user="${INTELMQ_API_USER:=intelmq}" +intelmq_pass="${INTELMQ_API_PASS:=intelmq}" + +intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" + +if [[ $1 == "selftest" ]] +then + export INTELMQ_TEST_EXOTIC=1 + nosetests3 /etc/intelmq/intelmq/tests +else + cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 +fi diff --git a/.docker/intelmq-full-dev/entrypoint_dev.sh b/.docker/intelmq-full-dev/entrypoint_dev.sh deleted file mode 100755 index 913a56f..0000000 --- a/.docker/intelmq-full-dev/entrypoint_dev.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -/opt/dev/update.sh - -if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then - intelmqctl start -fi - - -/opt/entrypoint.sh diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots new file mode 100755 index 0000000..192abd5 --- /dev/null +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -0,0 +1,10 @@ +#!/bin/bash +for file in $(find /etc/intelmq/intelmq/bots -name "*REQUIREMENTS.txt"); do + cat file >> /tmp/fullrequirements.txt +done +cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt +sudo pip3 install -r /tmp/orderfullrequirements.txt; +cd /etc/intelmq +sudo pip3 install --no-cache-dir -e . + +intelmqsetup \ No newline at end of file diff --git a/.docker/intelmq-full-dev/merge_BOTS.py b/.docker/intelmq-full-dev/merge_BOTS.py deleted file mode 100644 index 3c28494..0000000 --- a/.docker/intelmq-full-dev/merge_BOTS.py +++ /dev/null @@ -1,38 +0,0 @@ -import json -from jsonmerge import merge -import argparse -from collections import OrderedDict - -parser = argparse.ArgumentParser(description='Merge two json.') -parser.add_argument('input_file_1', type=str, help='input_file_1') -parser.add_argument('input_file_2', type=str, help='input_file_2') -parser.add_argument('output_file', type=str, help='output_file') - -args = parser.parse_args() - -with open(args.input_file_1, 'r') as f: - j1 = json.load(f) -with open(args.input_file_2, 'r') as f: - j2 = json.load(f) - -def sortOD(od): - res = OrderedDict() - for k, v in sorted(od.items()): - if isinstance(v, dict): - res[k] = sortOD(v) - else: - res[k] = v - return res - - -merged = sortOD(merge(j1,j2)) - -desired_order_list = ['Collector', 'Parser', 'Expert', 'Output'] -reordered_dict = {k: merged[k] for k in desired_order_list} - -# add other keys -reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list}) - -with open(args.output_file, 'w') as f: - json.dump(reordered_dict, f, indent=4) - \ No newline at end of file diff --git a/.docker/intelmq-full-dev/update.sh b/.docker/intelmq-full-dev/update.sh deleted file mode 100755 index 83b2470..0000000 --- a/.docker/intelmq-full-dev/update.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -echo "Installing requirements for bots in dev repository" -for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done - -if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then - if [ "${AUTO_MIX_BOTS}" = "true" ]; then - # Backup Original BOTS - cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk - echo "Merge your BOTS file with BOTS" - python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" - cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS - echo "Copying BOTS" - cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/ - # Restore original BOTS - mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS - else - cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS - fi -fi - -echo "Installing new BOTS" -cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user - - - diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 409dce6..153a547 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -2,46 +2,48 @@ ## Run & deploy containers in dev mode: -0. `cd intelmq-manager` -0. `python3 setup.py` -0. `cd ..` -1. `docker-compose -f docker-compose-dev.yml up` +### Install docker and docker-compose +``` +sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose +``` + +### Clone this repo + +``` +git clone https://github.com/certat/intelmq-docker.git --recursive +cd intelmq-docker +docker-compose -f docker-compose-dev.yml build +``` + +### In next step replace git@github.com:certtools/intelmq.git by your fork of intelmq + +``` +git clone git@github.com:certtools/intelmq.git my_fork_of_intelmq/ +docker-compose -f docker-compose-dev.yml up +``` + +### Open your favourite browser -> Go to `http://127.0.0.1:1337/` + + Default user/password: intelmq/intelmq ## Docker-compose-dev.yml file -### Volume: +### Volumes: -**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert. +- **./my_fork_of_intelmq/intelmq:/etc/intelmq/intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly. ### Add your own bots -Just start coding or pull your bots repository in ,/mybots folder +Just start coding or pull your bots repository in ./my_fork_of_intelmq folder/intelmq/bots -### How to install and look yours bots runnig +### How to install and look yours bots running +After you change some bot or add someshing new just run command **install_reqs_and_deploy_bots.sh** in the running container -Just run /opt/dev/update.sh in the container: - -1. `docker-compose exec -f docker-compose-dev.yml intelmq /opt/bin/update.sh` +``` +docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh +``` When you do this: -* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment -* Yours bots will be installed - -### Additional environment variables - -Check options in docker-compose-dev.yml: - -* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email) -* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not. - - -## For deploy your already developed bots - -Just clone your bots git to ./mybots and run the container - -For example, using https://github.com/CERTUNLP/intelmq-bots: - -1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3` -0. `docker-compose -f docker-compose-dev.yml up` +* Yours bots and REQUERIMENTS will be installed diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index e658d2a..dbbf6ce 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -7,16 +7,12 @@ services: command: - redis-server - /usr/local/etc/redis/redis.conf - restart: always networks: - intelmq-internal nginx: image: certat/intelmq-nginx:latest - restart: always ports: - 1337:80 - volumes: - - ./intelmq-manager/html:/www depends_on: - intelmq networks: @@ -25,25 +21,23 @@ services: build: .docker/intelmq-full-dev volumes: - ./example_config/intelmq/etc/:/opt/intelmq/etc/ - - ./example_config/intelmq-api:/opt/intelmq-api/config + - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot - - ./mybots:/opt/dev/mybots + - ./my_fork_of_intelmq/intelmq/:/etc/intelmq/intelmq/ depends_on: - redis - environment: - INTELMQ_PIPELINE_DRIVER: "redis" + environment: + IS_DEV: "true" + INTELMQ_SOURCE_PIPELINE_BROKER: "redis" + INTELMQ_PIPELINE_BROKER: "redis" + INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis" INTELMQ_PIPELINE_HOST: redis + INTELMQ_SOURCE_PIPELINE_HOST: redis + INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis - # Start botnet at boot - ENABLE_BOTNET_AT_BOOT: "false" - # Enable this to enable automix of BOTS file - AUTO_MIX_BOTS: "false" networks: - intelmq-internal - - networks: intelmq-internal: driver: bridge From 536fe2cb7b49c6b59c915f0de3c03e3572f67b0c Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Wed, 13 Apr 2022 10:58:55 -0300 Subject: [PATCH 03/21] some issues solved --- .docker/intelmq-full-dev/Dockerfile | 5 +++++ .docker/intelmq-full-dev/install_reqs_and_deploy_bots | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index 3b6fa5d..5726f9d 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -4,6 +4,11 @@ ENV LANG C.UTF-8 LABEL maintainer="Einar " LABEL maintainer="Jeremias " LABEL maintainer="Mateo " +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + git \ + vim \ + && rm -rf /var/lib/apt/lists/* WORKDIR /opt ADD entrypoint-dev.sh /opt/entrypoint-dev.sh diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots index 192abd5..cc75155 100755 --- a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -1,10 +1,11 @@ #!/bin/bash +rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt for file in $(find /etc/intelmq/intelmq/bots -name "*REQUIREMENTS.txt"); do - cat file >> /tmp/fullrequirements.txt + cat $file >> /tmp/fullrequirements.txt done cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt sudo pip3 install -r /tmp/orderfullrequirements.txt; cd /etc/intelmq sudo pip3 install --no-cache-dir -e . -intelmqsetup \ No newline at end of file +sudo intelmqsetup From af3be1c46ccebb94ce06abbf91742e7d4830e7d5 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Fri, 22 Apr 2022 14:48:03 -0300 Subject: [PATCH 04/21] Update Dockerfile --- .docker/intelmq-full-dev/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index 5726f9d..b0e1f30 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -4,11 +4,11 @@ ENV LANG C.UTF-8 LABEL maintainer="Einar " LABEL maintainer="Jeremias " LABEL maintainer="Mateo " -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +RUN sudo apt-get update \ + && sudo apt-get install -y --no-install-recommends \ git \ vim \ - && rm -rf /var/lib/apt/lists/* + && sudo rm -rf /var/lib/apt/lists/* WORKDIR /opt ADD entrypoint-dev.sh /opt/entrypoint-dev.sh From 9d9bef02481b059d1797b996e891cc913c38cd57 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 10 May 2022 12:43:39 -0300 Subject: [PATCH 05/21] Dockerfile --- .docker/intelmq-full-dev/Dockerfile | 5 ---- .docker/intelmq-full-dev/entrypoint-dev.sh | 29 ------------------- .../install_reqs_and_deploy_bots | 14 +++++---- .gitignore | 1 + docker-compose-dev.yml | 4 ++- 5 files changed, 12 insertions(+), 41 deletions(-) delete mode 100644 .docker/intelmq-full-dev/entrypoint-dev.sh diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index b0e1f30..2e3d96a 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -11,13 +11,8 @@ RUN sudo apt-get update \ && sudo rm -rf /var/lib/apt/lists/* WORKDIR /opt -ADD entrypoint-dev.sh /opt/entrypoint-dev.sh ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh -RUN sudo chmod +x /opt/entrypoint-dev.sh \ - && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh USER intelmq:intelmq - -ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh deleted file mode 100644 index 67b2d6f..0000000 --- a/.docker/intelmq-full-dev/entrypoint-dev.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -export INTELMQ_IS_DOCKER=1 - -if [[ ${IS_DEV} == "true" ]] -then - cd /etc/intelmq - sudo pip3 install hug url-normalize geolib imbox jinja2 pyasn textx tld time-machine - sudo pip3 install --force pymisp[fileobjects,openioc,virustotal] - /opt/install_reqs_and_deploy_bots.sh -fi - -sudo chown -R intelmq:intelmq /etc/intelmq -sudo chown -R intelmq:intelmq /opt/intelmq - -intelmqctl upgrade-config -intelmqctl check - -intelmq_user="${INTELMQ_API_USER:=intelmq}" -intelmq_pass="${INTELMQ_API_PASS:=intelmq}" - -intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" - -if [[ $1 == "selftest" ]] -then - export INTELMQ_TEST_EXOTIC=1 - nosetests3 /etc/intelmq/intelmq/tests -else - cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 -fi diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots index cc75155..3bedd27 100755 --- a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -1,11 +1,13 @@ #!/bin/bash -rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt -for file in $(find /etc/intelmq/intelmq/bots -name "*REQUIREMENTS.txt"); do - cat $file >> /tmp/fullrequirements.txt +sudo cp -r $MY_FORK/$MY_BOTS_FOLDER/* /etc/intelmq/intelmq/bots/ +sudo rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt +for req in $(find $MY_FORK/$MY_BOTS_FOLDER -name "*REQUIREMENTS.txt"); do + cat $req >> /tmp/fullrequirements.txt done cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt sudo pip3 install -r /tmp/orderfullrequirements.txt; -cd /etc/intelmq -sudo pip3 install --no-cache-dir -e . -sudo intelmqsetup + +#cd /etc/intelmq +#sudo pip3 install --no-cache-dir -e . +#sudo intelmqsetup diff --git a/.gitignore b/.gitignore index d0f6aba..0142308 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ intelmq_logs/ intelmq_persistence/ +my_fork_of_intelmq/ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index dbbf6ce..0dd8003 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -24,7 +24,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./my_fork_of_intelmq/intelmq/:/etc/intelmq/intelmq/ + - ./my_fork_of_intelmq/:${MY_FORK}/${MY_BOTS_FOLDER} depends_on: - redis environment: @@ -36,6 +36,8 @@ services: INTELMQ_SOURCE_PIPELINE_HOST: redis INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis + MY_FORK: "my_fork_of_intelmq" + MY_BOTS_FOLDER: "intelmq/bots" networks: - intelmq-internal networks: From 08859e2108dd69dfe84470127aa41753add23ee7 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 10 May 2022 18:06:11 -0300 Subject: [PATCH 06/21] Actualizados para los cambios de 3.0.2 que trajeron muchos problemas --- .docker/intelmq-full-dev/Dockerfile | 5 +++++ .docker/intelmq-full-dev/install_reqs_and_deploy_bots | 7 +++---- DEVELOP-GUIDE.md | 8 +++++++- docker-compose-dev.yml | 6 +++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index 2e3d96a..b0e1f30 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -11,8 +11,13 @@ RUN sudo apt-get update \ && sudo rm -rf /var/lib/apt/lists/* WORKDIR /opt +ADD entrypoint-dev.sh /opt/entrypoint-dev.sh ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh +RUN sudo chmod +x /opt/entrypoint-dev.sh \ + && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh USER intelmq:intelmq + +ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots index 3bedd27..6006121 100755 --- a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -7,7 +7,6 @@ done cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt sudo pip3 install -r /tmp/orderfullrequirements.txt; - -#cd /etc/intelmq -#sudo pip3 install --no-cache-dir -e . -#sudo intelmqsetup +cd /etc/intelmq +sudo pip3 install --no-cache-dir -e . +sudo intelmqsetup diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 153a547..53c1e8a 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -30,7 +30,13 @@ docker-compose -f docker-compose-dev.yml up ### Volumes: -- **./my_fork_of_intelmq/intelmq:/etc/intelmq/intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly. +- **./my_fork_of_intelmq/:/my_fork_of_intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly. + +### Environment + #Folder where you clone your repository + MY_FORK: "/my_fork_of_intelmq" + #Foder in your repo where bots are located + MY_BOTS_FOLDER: "intelmq/bots" ### Add your own bots diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0dd8003..9af23da 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -24,7 +24,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./my_fork_of_intelmq/:${MY_FORK}/${MY_BOTS_FOLDER} + - ./mybots:/my_fork_of_intelmq depends_on: - redis environment: @@ -36,8 +36,8 @@ services: INTELMQ_SOURCE_PIPELINE_HOST: redis INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis - MY_FORK: "my_fork_of_intelmq" - MY_BOTS_FOLDER: "intelmq/bots" + MY_FORK: "/my_fork_of_intelmq" + MY_BOTS_FOLDER: "bots" networks: - intelmq-internal networks: From 9e39baeca59f20af810ff1f3d387f665decdb640 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 10 May 2022 18:06:40 -0300 Subject: [PATCH 07/21] Adding new entrypoint --- .docker/intelmq-full-dev/entrypoint-dev.sh | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .docker/intelmq-full-dev/entrypoint-dev.sh diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh new file mode 100644 index 0000000..958f0ce --- /dev/null +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -0,0 +1,26 @@ +#!/bin/bash +export INTELMQ_IS_DOCKER=1 + +if [[ ${IS_DEV} == "true" ]] +then + /opt/install_reqs_and_deploy_bots.sh +fi + +sudo chown -R intelmq:intelmq /etc/intelmq +sudo chown -R intelmq:intelmq /opt/intelmq + +intelmqctl upgrade-config +intelmqctl check + +intelmq_user="${INTELMQ_API_USER:=intelmq}" +intelmq_pass="${INTELMQ_API_PASS:=intelmq}" + +intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" + +if [[ $1 == "selftest" ]] +then + export INTELMQ_TEST_EXOTIC=1 + nosetests3 /etc/intelmq/intelmq/tests +else + cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 +fi \ No newline at end of file From fd95d93d99f27376f8edfb72e90551768c149432 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 10 May 2022 18:10:44 -0300 Subject: [PATCH 08/21] Adding notes to DEVELOP-GUIDE --- DEVELOP-GUIDE.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 53c1e8a..19ad3a8 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -34,13 +34,13 @@ docker-compose -f docker-compose-dev.yml up ### Environment #Folder where you clone your repository - MY_FORK: "/my_fork_of_intelmq" + MY_FORK: "/mybots" #Foder in your repo where bots are located - MY_BOTS_FOLDER: "intelmq/bots" + MY_BOTS_FOLDER: "bots" ### Add your own bots -Just start coding or pull your bots repository in ./my_fork_of_intelmq folder/intelmq/bots +Just start coding or pull your bots repository in ./mybots by default in a subfolder bots, so you need for example mybots/bots/[collectors,parsers,experts,output,parsers] ### How to install and look yours bots running @@ -52,4 +52,4 @@ docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_re When you do this: -* Yours bots and REQUERIMENTS will be installed +* Yours bots REQUERIMENTS.txt and the bots will be installed From d6ed749a273323bf2165c6f456efab60e04b2822 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 14 Jun 2022 17:53:19 -0300 Subject: [PATCH 09/21] Update docker-compose-dev.yml Agregando soporte para emcemder el cluster cuando prende el contenedor --- docker-compose-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 9af23da..4deffbf 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -28,6 +28,7 @@ services: depends_on: - redis environment: + ENABLE_BOTNET_AT_BOOT: "false" IS_DEV: "true" INTELMQ_SOURCE_PIPELINE_BROKER: "redis" INTELMQ_PIPELINE_BROKER: "redis" From 4277635f689df3b2732be60c421e005acc955800 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 14 Jun 2022 17:57:40 -0300 Subject: [PATCH 10/21] Update entrypoint-dev.sh Actualizando el entrypoint para prender del inicio --- .docker/intelmq-full-dev/entrypoint-dev.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh index 958f0ce..4a56ed5 100644 --- a/.docker/intelmq-full-dev/entrypoint-dev.sh +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -23,4 +23,8 @@ then nosetests3 /etc/intelmq/intelmq/tests else cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 -fi \ No newline at end of file +fi + +if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then + intelmqctl start +fi From 821dbbefcca84692554355a4c314e975254be839 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 14 Jun 2022 18:19:48 -0300 Subject: [PATCH 11/21] Update DEVELOP-GUIDE.md --- DEVELOP-GUIDE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 19ad3a8..652d3cd 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -53,3 +53,6 @@ docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_re When you do this: * Yours bots REQUERIMENTS.txt and the bots will be installed + + +* Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true" From 9fb236454614967f47040247ea13bb2d106b4f1e Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 12 Jul 2022 12:24:46 -0300 Subject: [PATCH 12/21] DEV README fix --- DEVELOP-GUIDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 652d3cd..46b843d 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -35,7 +35,7 @@ docker-compose -f docker-compose-dev.yml up ### Environment #Folder where you clone your repository MY_FORK: "/mybots" - #Foder in your repo where bots are located + #Folder in your repo where bots are located MY_BOTS_FOLDER: "bots" ### Add your own bots @@ -44,7 +44,7 @@ Just start coding or pull your bots repository in ./mybots by default in a subfo ### How to install and look yours bots running -After you change some bot or add someshing new just run command **install_reqs_and_deploy_bots.sh** in the running container +After you change some bot or add something new just run command **install_reqs_and_deploy_bots.sh** in the running container ``` docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh From 662f6ab390ce310cf983319604f7c3e72b6293dc Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 12 Jul 2022 14:29:42 -0300 Subject: [PATCH 13/21] fix entrypoint --- .docker/intelmq-full-dev/entrypoint-dev.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh index 4a56ed5..b3f45ff 100644 --- a/.docker/intelmq-full-dev/entrypoint-dev.sh +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -17,6 +17,10 @@ intelmq_pass="${INTELMQ_API_PASS:=intelmq}" intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" +if [[ ${ENABLE_BOTNET_AT_BOOT} == "true" ]]; then + intelmqctl start +fi + if [[ $1 == "selftest" ]] then export INTELMQ_TEST_EXOTIC=1 @@ -25,6 +29,3 @@ else cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 fi -if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then - intelmqctl start -fi From 0e305ca842fec4d4d167d866b47c916bdf94d5e6 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Thu, 11 Aug 2022 12:41:32 -0300 Subject: [PATCH 14/21] Update DEVELOP-GUIDE.md --- DEVELOP-GUIDE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 46b843d..a193fe2 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -56,3 +56,9 @@ When you do this: * Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true" + +### Known isues + +Some dependencies are missing, as far is i know you need to install: + + sudo pip3 install elasticsearch==7.9 geoip2 azure-storage-blob From 9be93b3ddca89f3224b78c150306bac25ae52151 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Thu, 18 Aug 2022 12:13:19 -0300 Subject: [PATCH 15/21] fix to support dependenceies for all intelmq default bots --- .docker/intelmq-full-dev/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index b0e1f30..712aa3b 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -8,6 +8,7 @@ RUN sudo apt-get update \ && sudo apt-get install -y --no-install-recommends \ git \ vim \ + ssh \ && sudo rm -rf /var/lib/apt/lists/* WORKDIR /opt @@ -18,6 +19,11 @@ RUN sudo chmod +x /opt/entrypoint-dev.sh \ RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh +# This section is just for install all the default bots dependencies +RUN for req in $(find /etc/intelmq/intelmq/bots/ -name "*REQUIREMENTS.txt"); do cat $req >> /tmp/fullrequirements.txt; done +RUN cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt +RUN cat /tmp/orderfullrequirements.txt |grep -Eo '(^[^#]+)' | xargs -d "\n" -I {} sh -c 'pip3 install "$1"|| exit 0' sh {} + USER intelmq:intelmq ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] From 1c397f394de1b62113d7bc8c7134fd1b91b840a3 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Thu, 18 Aug 2022 13:04:49 -0300 Subject: [PATCH 16/21] fixing known issues documentation --- DEVELOP-GUIDE.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index a193fe2..510497a 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -57,8 +57,21 @@ When you do this: * Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true" +## Dependencies problems + +Some dependencies from defaults bots are missing in original intelmq image, so we fix it in our Dockerfile build process. Neverles we still facing some issues. + ### Known isues -Some dependencies are missing, as far is i know you need to install: +Blueliv problem: - sudo pip3 install elasticsearch==7.9 geoip2 azure-storage-blob + pip3 install git+git://github.com/Blueliv/api-python-sdk doesn't work because git+git is deprecated, to fix it you need to replace git+git with git+https + + +But you still will have problems because of dependency confict: + + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + pymisp 2.4.148 requires requests<3.0.0,>=2.25.1, but you have requests 2.5.1 which is incompatible. + + + From 4872c7348959add0a55546f62d71d28067590551 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Thu, 18 Aug 2022 13:17:46 -0300 Subject: [PATCH 17/21] fixing known issues documentation --- DEVELOP-GUIDE.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 510497a..2c21cab 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -59,19 +59,24 @@ When you do this: ## Dependencies problems -Some dependencies from defaults bots are missing in original intelmq image, so we fix it in our Dockerfile build process. Neverles we still facing some issues. +Some dependencies from defaults bots are missing in original intelmq image, so we fix it in our Dockerfile build process. Nevertheless, we still facing some issues. ### Known isues -Blueliv problem: +#### Blueliv problem: + +This bot has 2 problems: + +1- It doesn't install: pip3 install git+git://github.com/Blueliv/api-python-sdk doesn't work because git+git is deprecated, to fix it you need to replace git+git with git+https -But you still will have problems because of dependency confict: +2- But if you fix and install it you would cause a dependency conflict with pymisp: - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behavior is the source of the following dependency conflicts. pymisp 2.4.148 requires requests<3.0.0,>=2.25.1, but you have requests 2.5.1 which is incompatible. - +If you don't need blueliv, just don't fix git+git with git+https. + From ef59195dfdacae8bc2ba1d202fca95f302adfb90 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Thu, 18 Aug 2022 13:33:46 -0300 Subject: [PATCH 18/21] fixing volumes in documentation --- DEVELOP-GUIDE.md | 13 ++++++------- docker-compose-dev.yml | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 2c21cab..994ba90 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -30,31 +30,31 @@ docker-compose -f docker-compose-dev.yml up ### Volumes: -- **./my_fork_of_intelmq/:/my_fork_of_intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly. +- **./my_bots:/my_bots** -> this is the folder where your bots source code need to be. ### Environment #Folder where you clone your repository - MY_FORK: "/mybots" + MY_FORK: "/my_bots" #Folder in your repo where bots are located MY_BOTS_FOLDER: "bots" ### Add your own bots -Just start coding or pull your bots repository in ./mybots by default in a subfolder bots, so you need for example mybots/bots/[collectors,parsers,experts,output,parsers] +Just start coding or pull your bots repository in ./my_bots by default in a subfolder bots, so you need for example my_bots/bots/[collectors,parsers,experts,output,parsers] + +You could take a look at the folder and files in https://github.com/certtools/intelmq/tree/develop/intelmq/bots ### How to install and look yours bots running After you change some bot or add something new just run command **install_reqs_and_deploy_bots.sh** in the running container ``` -docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh +docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh ``` When you do this: * Yours bots REQUERIMENTS.txt and the bots will be installed - - * Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true" ## Dependencies problems @@ -79,4 +79,3 @@ This bot has 2 problems: If you don't need blueliv, just don't fix git+git with git+https. - diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 4deffbf..01bd74a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -24,7 +24,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./mybots:/my_fork_of_intelmq + - ./my_bots:/my_bots depends_on: - redis environment: @@ -37,7 +37,7 @@ services: INTELMQ_SOURCE_PIPELINE_HOST: redis INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis - MY_FORK: "/my_fork_of_intelmq" + MY_FORK: "/my_bots" MY_BOTS_FOLDER: "bots" networks: - intelmq-internal From 25166024c84e5e59a2d6d8b780dc33465525ef4e Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 30 Aug 2022 14:59:15 -0300 Subject: [PATCH 19/21] Quick fix in runtime.yaml we need to add destination_pipeline_host: redis, source_pipeline_host: redis bacuase intelmqdump dont care about environment variables --- example_config/intelmq/etc/runtime.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_config/intelmq/etc/runtime.yaml b/example_config/intelmq/etc/runtime.yaml index c289919..0cfad2b 100644 --- a/example_config/intelmq/etc/runtime.yaml +++ b/example_config/intelmq/etc/runtime.yaml @@ -117,7 +117,7 @@ gethostbyname-2-expert: run_mode: continuous global: {destination_pipeline_broker: redis, process_manager: intelmq, source_pipeline_broker: redis, ssl_ca_certificate: null, statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null, - statistics_port: 6379} + statistics_port: 6379, destination_pipeline_host: redis, source_pipeline_host: redis} malc0de-parser: bot_id: malc0de-parser description: Malc0de Parser is the bot responsible to parse the IP Blacklist and From 51a1d7a4bfb94010ccd625bd92ee1860b63fdb0b Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Fri, 11 Nov 2022 13:50:58 -0300 Subject: [PATCH 20/21] Update Dockerfile Good morning, we found an error in sudoers file, it must be /etc/ not /opt/ ! --- .docker/intelmq-full/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/intelmq-full/Dockerfile b/.docker/intelmq-full/Dockerfile index de785b2..08bd9d9 100644 --- a/.docker/intelmq-full/Dockerfile +++ b/.docker/intelmq-full/Dockerfile @@ -47,7 +47,7 @@ COPY ./intelmq-api /opt/intelmq-api RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \ && adduser intelmq sudo \ - && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /opt/sudoers \ + && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \ && sudo chown -R intelmq:intelmq /opt/intelmq \ && mkdir -p /opt/intelmq_persistence \ && sudo chown -R intelmq:intelmq /opt/intelmq_persistence From 7c69331d59408c120b473aab11a324f54d65a2d0 Mon Sep 17 00:00:00 2001 From: Sebastian Waldbauer Date: Tue, 15 Nov 2022 14:36:41 +0100 Subject: [PATCH 21/21] FIX: Fixed some missing dependecies closes #16 closes #15 Signed-off-by: Sebastian Waldbauer --- .docker/intelmq-full/Dockerfile | 2 ++ test.sh | 30 +++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.docker/intelmq-full/Dockerfile b/.docker/intelmq-full/Dockerfile index 08bd9d9..0ba5a94 100644 --- a/.docker/intelmq-full/Dockerfile +++ b/.docker/intelmq-full/Dockerfile @@ -35,6 +35,8 @@ RUN apt-get update \ python3-cerberus \ python3-requests-mock \ python3-shodan \ + python3-elasticsearch \ + python3-pymongo \ libfuzzy-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/test.sh b/test.sh index 60e80bf..7010ee1 100755 --- a/test.sh +++ b/test.sh @@ -1,11 +1,28 @@ #!/bin/bash +echo RUNNING TESTS WITH REDIS echo Setting up redis container redis_id=$(docker run --rm -d -p 6379:6379 -v ~/example_config/redis/redis.conf:/redis.conf redis:latest) +redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id) + +echo Setting up IntelMQ-Container +docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \ + -v $(pwd)/example_config/intelmq-api:/etc/intelmq-api/config \ + -v $(pwd)/intelmq_logs:/etc/intelmq/var/log \ + -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \ + -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \ + -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \ + -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \ + -e "INTELMQ_PIPELINE_HOST=$redis_ip" \ + -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \ + intelmq-full:latest selftest + +echo Removing redis container +docker container kill $redis_id + +echo RUNNING TESTS WITH AMQP echo Setting up AMQP container amq_id=$(docker run --rm -d -p 5672:5672 -p 15672:15672 rabbitmq:latest) - -redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id) amp_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks}}{{ .IPAddress }}{{ end }}' $amq_id) echo Setting up IntelMQ-Container @@ -15,15 +32,10 @@ docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \ -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \ -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \ -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \ - -e "INTELMQ_IS_DOCKER=\"true\"" \ - -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \ - -e "INTELMQ_PIPELINE_HOST=$redis_ip" \ - -e "INTELMQ_PIPELINE_AMQ_HOST=$amp_ip" \ + -e "INTELMQ_PIPELINE_DRIVER=\"amqp\"" \ + -e "INTELMQ_PIPELINE_HOST=$amq_id" \ -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \ intelmq-full:latest selftest -echo Removing redis container -docker container kill $redis_id - echo Removing AMQP container docker container kill $amq_id