From a84e4edea9f9684a840a5298b064c20f02a59396 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Fri, 1 Oct 2021 17:45:04 -0300 Subject: [PATCH 01/30] docker for dev ready --- .docker/intelmq-full-dev/Dockerfile | 30 +++++---- .docker/intelmq-full-dev/entrypoint-dev.sh | 29 +++++++++ .docker/intelmq-full-dev/entrypoint_dev.sh | 10 --- .../install_reqs_and_deploy_bots | 10 +++ .docker/intelmq-full-dev/merge_BOTS.py | 38 ------------ .docker/intelmq-full-dev/update.sh | 26 -------- DEVELOP-GUIDE.md | 62 ++++++++++--------- docker-compose-dev.yml | 24 +++---- 8 files changed, 94 insertions(+), 135 deletions(-) create mode 100644 .docker/intelmq-full-dev/entrypoint-dev.sh delete mode 100755 .docker/intelmq-full-dev/entrypoint_dev.sh create mode 100755 .docker/intelmq-full-dev/install_reqs_and_deploy_bots delete mode 100644 .docker/intelmq-full-dev/merge_BOTS.py delete mode 100755 .docker/intelmq-full-dev/update.sh diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index f8dc9a0..3b6fa5d 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -1,20 +1,18 @@ -FROM certat/intelmq-full:1.0 +FROM certat/intelmq-full:latest +ENV LANG C.UTF-8 -MAINTAINER Einar -MAINTAINER Jeremias +LABEL maintainer="Einar " +LABEL maintainer="Jeremias " +LABEL maintainer="Mateo " -ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh -ADD update.sh /opt/dev/update.sh -ADD merge_BOTS.py /opt/dev/merge_BOTS.py +WORKDIR /opt +ADD entrypoint-dev.sh /opt/entrypoint-dev.sh +ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh +RUN sudo chmod +x /opt/entrypoint-dev.sh \ + && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh +RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ + && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh -# Merge bots for merge_BOTS.py -RUN sudo python3 -m pip install jsonmerge - - -# Permission denied when installing new bots -RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info - -ENV PATH="/opt/intelmq/.local/bin:${PATH}" - -ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"] +USER intelmq:intelmq +ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh new file mode 100644 index 0000000..67b2d6f --- /dev/null +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -0,0 +1,29 @@ +#!/bin/bash +export INTELMQ_IS_DOCKER=1 + +if [[ ${IS_DEV} == "true" ]] +then + cd /etc/intelmq + sudo pip3 install hug url-normalize geolib imbox jinja2 pyasn textx tld time-machine + sudo pip3 install --force pymisp[fileobjects,openioc,virustotal] + /opt/install_reqs_and_deploy_bots.sh +fi + +sudo chown -R intelmq:intelmq /etc/intelmq +sudo chown -R intelmq:intelmq /opt/intelmq + +intelmqctl upgrade-config +intelmqctl check + +intelmq_user="${INTELMQ_API_USER:=intelmq}" +intelmq_pass="${INTELMQ_API_PASS:=intelmq}" + +intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" + +if [[ $1 == "selftest" ]] +then + export INTELMQ_TEST_EXOTIC=1 + nosetests3 /etc/intelmq/intelmq/tests +else + cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 +fi diff --git a/.docker/intelmq-full-dev/entrypoint_dev.sh b/.docker/intelmq-full-dev/entrypoint_dev.sh deleted file mode 100755 index 913a56f..0000000 --- a/.docker/intelmq-full-dev/entrypoint_dev.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -/opt/dev/update.sh - -if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then - intelmqctl start -fi - - -/opt/entrypoint.sh diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots new file mode 100755 index 0000000..192abd5 --- /dev/null +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -0,0 +1,10 @@ +#!/bin/bash +for file in $(find /etc/intelmq/intelmq/bots -name "*REQUIREMENTS.txt"); do + cat file >> /tmp/fullrequirements.txt +done +cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt +sudo pip3 install -r /tmp/orderfullrequirements.txt; +cd /etc/intelmq +sudo pip3 install --no-cache-dir -e . + +intelmqsetup \ No newline at end of file diff --git a/.docker/intelmq-full-dev/merge_BOTS.py b/.docker/intelmq-full-dev/merge_BOTS.py deleted file mode 100644 index 3c28494..0000000 --- a/.docker/intelmq-full-dev/merge_BOTS.py +++ /dev/null @@ -1,38 +0,0 @@ -import json -from jsonmerge import merge -import argparse -from collections import OrderedDict - -parser = argparse.ArgumentParser(description='Merge two json.') -parser.add_argument('input_file_1', type=str, help='input_file_1') -parser.add_argument('input_file_2', type=str, help='input_file_2') -parser.add_argument('output_file', type=str, help='output_file') - -args = parser.parse_args() - -with open(args.input_file_1, 'r') as f: - j1 = json.load(f) -with open(args.input_file_2, 'r') as f: - j2 = json.load(f) - -def sortOD(od): - res = OrderedDict() - for k, v in sorted(od.items()): - if isinstance(v, dict): - res[k] = sortOD(v) - else: - res[k] = v - return res - - -merged = sortOD(merge(j1,j2)) - -desired_order_list = ['Collector', 'Parser', 'Expert', 'Output'] -reordered_dict = {k: merged[k] for k in desired_order_list} - -# add other keys -reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list}) - -with open(args.output_file, 'w') as f: - json.dump(reordered_dict, f, indent=4) - \ No newline at end of file diff --git a/.docker/intelmq-full-dev/update.sh b/.docker/intelmq-full-dev/update.sh deleted file mode 100755 index 83b2470..0000000 --- a/.docker/intelmq-full-dev/update.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -echo "Installing requirements for bots in dev repository" -for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done - -if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then - if [ "${AUTO_MIX_BOTS}" = "true" ]; then - # Backup Original BOTS - cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk - echo "Merge your BOTS file with BOTS" - python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" - cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS - echo "Copying BOTS" - cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/ - # Restore original BOTS - mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS - else - cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS - fi -fi - -echo "Installing new BOTS" -cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user - - - diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 409dce6..153a547 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -2,46 +2,48 @@ ## Run & deploy containers in dev mode: -0. `cd intelmq-manager` -0. `python3 setup.py` -0. `cd ..` -1. `docker-compose -f docker-compose-dev.yml up` +### Install docker and docker-compose +``` +sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose +``` + +### Clone this repo + +``` +git clone https://github.com/certat/intelmq-docker.git --recursive +cd intelmq-docker +docker-compose -f docker-compose-dev.yml build +``` + +### In next step replace git@github.com:certtools/intelmq.git by your fork of intelmq + +``` +git clone git@github.com:certtools/intelmq.git my_fork_of_intelmq/ +docker-compose -f docker-compose-dev.yml up +``` + +### Open your favourite browser -> Go to `http://127.0.0.1:1337/` + + Default user/password: intelmq/intelmq ## Docker-compose-dev.yml file -### Volume: +### Volumes: -**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert. +- **./my_fork_of_intelmq/intelmq:/etc/intelmq/intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly. ### Add your own bots -Just start coding or pull your bots repository in ,/mybots folder +Just start coding or pull your bots repository in ./my_fork_of_intelmq folder/intelmq/bots -### How to install and look yours bots runnig +### How to install and look yours bots running +After you change some bot or add someshing new just run command **install_reqs_and_deploy_bots.sh** in the running container -Just run /opt/dev/update.sh in the container: - -1. `docker-compose exec -f docker-compose-dev.yml intelmq /opt/bin/update.sh` +``` +docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh +``` When you do this: -* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment -* Yours bots will be installed - -### Additional environment variables - -Check options in docker-compose-dev.yml: - -* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email) -* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not. - - -## For deploy your already developed bots - -Just clone your bots git to ./mybots and run the container - -For example, using https://github.com/CERTUNLP/intelmq-bots: - -1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3` -0. `docker-compose -f docker-compose-dev.yml up` +* Yours bots and REQUERIMENTS will be installed diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index e658d2a..dbbf6ce 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -7,16 +7,12 @@ services: command: - redis-server - /usr/local/etc/redis/redis.conf - restart: always networks: - intelmq-internal nginx: image: certat/intelmq-nginx:latest - restart: always ports: - 1337:80 - volumes: - - ./intelmq-manager/html:/www depends_on: - intelmq networks: @@ -25,25 +21,23 @@ services: build: .docker/intelmq-full-dev volumes: - ./example_config/intelmq/etc/:/opt/intelmq/etc/ - - ./example_config/intelmq-api:/opt/intelmq-api/config + - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot - - ./mybots:/opt/dev/mybots + - ./my_fork_of_intelmq/intelmq/:/etc/intelmq/intelmq/ depends_on: - redis - environment: - INTELMQ_PIPELINE_DRIVER: "redis" + environment: + IS_DEV: "true" + INTELMQ_SOURCE_PIPELINE_BROKER: "redis" + INTELMQ_PIPELINE_BROKER: "redis" + INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis" INTELMQ_PIPELINE_HOST: redis + INTELMQ_SOURCE_PIPELINE_HOST: redis + INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis - # Start botnet at boot - ENABLE_BOTNET_AT_BOOT: "false" - # Enable this to enable automix of BOTS file - AUTO_MIX_BOTS: "false" networks: - intelmq-internal - - networks: intelmq-internal: driver: bridge From f533ff5ef64575cd05cc939dfe0928c06b4f5cba Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Wed, 13 Apr 2022 10:58:55 -0300 Subject: [PATCH 02/30] some issues solved --- .docker/intelmq-full-dev/Dockerfile | 5 +++++ .docker/intelmq-full-dev/install_reqs_and_deploy_bots | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index 3b6fa5d..5726f9d 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -4,6 +4,11 @@ ENV LANG C.UTF-8 LABEL maintainer="Einar " LABEL maintainer="Jeremias " LABEL maintainer="Mateo " +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + git \ + vim \ + && rm -rf /var/lib/apt/lists/* WORKDIR /opt ADD entrypoint-dev.sh /opt/entrypoint-dev.sh diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots index 192abd5..cc75155 100755 --- a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -1,10 +1,11 @@ #!/bin/bash +rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt for file in $(find /etc/intelmq/intelmq/bots -name "*REQUIREMENTS.txt"); do - cat file >> /tmp/fullrequirements.txt + cat $file >> /tmp/fullrequirements.txt done cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt sudo pip3 install -r /tmp/orderfullrequirements.txt; cd /etc/intelmq sudo pip3 install --no-cache-dir -e . -intelmqsetup \ No newline at end of file +sudo intelmqsetup From 2ea1b86e76bb76337154d9f2349a44f1b7784edb Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Fri, 22 Apr 2022 14:48:03 -0300 Subject: [PATCH 03/30] Update Dockerfile --- .docker/intelmq-full-dev/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index 5726f9d..b0e1f30 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -4,11 +4,11 @@ ENV LANG C.UTF-8 LABEL maintainer="Einar " LABEL maintainer="Jeremias " LABEL maintainer="Mateo " -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +RUN sudo apt-get update \ + && sudo apt-get install -y --no-install-recommends \ git \ vim \ - && rm -rf /var/lib/apt/lists/* + && sudo rm -rf /var/lib/apt/lists/* WORKDIR /opt ADD entrypoint-dev.sh /opt/entrypoint-dev.sh From 6ea0fd234076eddd3037e83c00f8bb7128ec802b Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 10 May 2022 12:43:39 -0300 Subject: [PATCH 04/30] Dockerfile --- .docker/intelmq-full-dev/Dockerfile | 5 ---- .docker/intelmq-full-dev/entrypoint-dev.sh | 29 ------------------- .../install_reqs_and_deploy_bots | 14 +++++---- .gitignore | 1 + docker-compose-dev.yml | 4 ++- 5 files changed, 12 insertions(+), 41 deletions(-) delete mode 100644 .docker/intelmq-full-dev/entrypoint-dev.sh diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index b0e1f30..2e3d96a 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -11,13 +11,8 @@ RUN sudo apt-get update \ && sudo rm -rf /var/lib/apt/lists/* WORKDIR /opt -ADD entrypoint-dev.sh /opt/entrypoint-dev.sh ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh -RUN sudo chmod +x /opt/entrypoint-dev.sh \ - && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh USER intelmq:intelmq - -ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh deleted file mode 100644 index 67b2d6f..0000000 --- a/.docker/intelmq-full-dev/entrypoint-dev.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -export INTELMQ_IS_DOCKER=1 - -if [[ ${IS_DEV} == "true" ]] -then - cd /etc/intelmq - sudo pip3 install hug url-normalize geolib imbox jinja2 pyasn textx tld time-machine - sudo pip3 install --force pymisp[fileobjects,openioc,virustotal] - /opt/install_reqs_and_deploy_bots.sh -fi - -sudo chown -R intelmq:intelmq /etc/intelmq -sudo chown -R intelmq:intelmq /opt/intelmq - -intelmqctl upgrade-config -intelmqctl check - -intelmq_user="${INTELMQ_API_USER:=intelmq}" -intelmq_pass="${INTELMQ_API_PASS:=intelmq}" - -intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" - -if [[ $1 == "selftest" ]] -then - export INTELMQ_TEST_EXOTIC=1 - nosetests3 /etc/intelmq/intelmq/tests -else - cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 -fi diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots index cc75155..3bedd27 100755 --- a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -1,11 +1,13 @@ #!/bin/bash -rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt -for file in $(find /etc/intelmq/intelmq/bots -name "*REQUIREMENTS.txt"); do - cat $file >> /tmp/fullrequirements.txt +sudo cp -r $MY_FORK/$MY_BOTS_FOLDER/* /etc/intelmq/intelmq/bots/ +sudo rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt +for req in $(find $MY_FORK/$MY_BOTS_FOLDER -name "*REQUIREMENTS.txt"); do + cat $req >> /tmp/fullrequirements.txt done cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt sudo pip3 install -r /tmp/orderfullrequirements.txt; -cd /etc/intelmq -sudo pip3 install --no-cache-dir -e . -sudo intelmqsetup + +#cd /etc/intelmq +#sudo pip3 install --no-cache-dir -e . +#sudo intelmqsetup diff --git a/.gitignore b/.gitignore index d0f6aba..0142308 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ intelmq_logs/ intelmq_persistence/ +my_fork_of_intelmq/ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index dbbf6ce..0dd8003 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -24,7 +24,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./my_fork_of_intelmq/intelmq/:/etc/intelmq/intelmq/ + - ./my_fork_of_intelmq/:${MY_FORK}/${MY_BOTS_FOLDER} depends_on: - redis environment: @@ -36,6 +36,8 @@ services: INTELMQ_SOURCE_PIPELINE_HOST: redis INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis + MY_FORK: "my_fork_of_intelmq" + MY_BOTS_FOLDER: "intelmq/bots" networks: - intelmq-internal networks: From 59c69f03dc828354fa45e13b27ee5d9768c6aa96 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 10 May 2022 18:06:11 -0300 Subject: [PATCH 05/30] Actualizados para los cambios de 3.0.2 que trajeron muchos problemas --- .docker/intelmq-full-dev/Dockerfile | 5 +++++ .docker/intelmq-full-dev/install_reqs_and_deploy_bots | 7 +++---- DEVELOP-GUIDE.md | 8 +++++++- docker-compose-dev.yml | 6 +++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index 2e3d96a..b0e1f30 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -11,8 +11,13 @@ RUN sudo apt-get update \ && sudo rm -rf /var/lib/apt/lists/* WORKDIR /opt +ADD entrypoint-dev.sh /opt/entrypoint-dev.sh ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh +RUN sudo chmod +x /opt/entrypoint-dev.sh \ + && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh USER intelmq:intelmq + +ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots index 3bedd27..6006121 100755 --- a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -7,7 +7,6 @@ done cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt sudo pip3 install -r /tmp/orderfullrequirements.txt; - -#cd /etc/intelmq -#sudo pip3 install --no-cache-dir -e . -#sudo intelmqsetup +cd /etc/intelmq +sudo pip3 install --no-cache-dir -e . +sudo intelmqsetup diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 153a547..53c1e8a 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -30,7 +30,13 @@ docker-compose -f docker-compose-dev.yml up ### Volumes: -- **./my_fork_of_intelmq/intelmq:/etc/intelmq/intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly. +- **./my_fork_of_intelmq/:/my_fork_of_intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly. + +### Environment + #Folder where you clone your repository + MY_FORK: "/my_fork_of_intelmq" + #Foder in your repo where bots are located + MY_BOTS_FOLDER: "intelmq/bots" ### Add your own bots diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0dd8003..9af23da 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -24,7 +24,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./my_fork_of_intelmq/:${MY_FORK}/${MY_BOTS_FOLDER} + - ./mybots:/my_fork_of_intelmq depends_on: - redis environment: @@ -36,8 +36,8 @@ services: INTELMQ_SOURCE_PIPELINE_HOST: redis INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis - MY_FORK: "my_fork_of_intelmq" - MY_BOTS_FOLDER: "intelmq/bots" + MY_FORK: "/my_fork_of_intelmq" + MY_BOTS_FOLDER: "bots" networks: - intelmq-internal networks: From 9b2a3ef4b5949b801eb1d9da5f795b2128e264bd Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 10 May 2022 18:06:40 -0300 Subject: [PATCH 06/30] Adding new entrypoint --- .docker/intelmq-full-dev/entrypoint-dev.sh | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .docker/intelmq-full-dev/entrypoint-dev.sh diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh new file mode 100644 index 0000000..958f0ce --- /dev/null +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -0,0 +1,26 @@ +#!/bin/bash +export INTELMQ_IS_DOCKER=1 + +if [[ ${IS_DEV} == "true" ]] +then + /opt/install_reqs_and_deploy_bots.sh +fi + +sudo chown -R intelmq:intelmq /etc/intelmq +sudo chown -R intelmq:intelmq /opt/intelmq + +intelmqctl upgrade-config +intelmqctl check + +intelmq_user="${INTELMQ_API_USER:=intelmq}" +intelmq_pass="${INTELMQ_API_PASS:=intelmq}" + +intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" + +if [[ $1 == "selftest" ]] +then + export INTELMQ_TEST_EXOTIC=1 + nosetests3 /etc/intelmq/intelmq/tests +else + cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 +fi \ No newline at end of file From 38242fbf5866f180fd44417031b20723208ed7ae Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 10 May 2022 18:10:44 -0300 Subject: [PATCH 07/30] Adding notes to DEVELOP-GUIDE --- DEVELOP-GUIDE.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 53c1e8a..19ad3a8 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -34,13 +34,13 @@ docker-compose -f docker-compose-dev.yml up ### Environment #Folder where you clone your repository - MY_FORK: "/my_fork_of_intelmq" + MY_FORK: "/mybots" #Foder in your repo where bots are located - MY_BOTS_FOLDER: "intelmq/bots" + MY_BOTS_FOLDER: "bots" ### Add your own bots -Just start coding or pull your bots repository in ./my_fork_of_intelmq folder/intelmq/bots +Just start coding or pull your bots repository in ./mybots by default in a subfolder bots, so you need for example mybots/bots/[collectors,parsers,experts,output,parsers] ### How to install and look yours bots running @@ -52,4 +52,4 @@ docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_re When you do this: -* Yours bots and REQUERIMENTS will be installed +* Yours bots REQUERIMENTS.txt and the bots will be installed From 46cd50c7981f06291da6763b21c27a53ffa41b37 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 14 Jun 2022 17:53:19 -0300 Subject: [PATCH 08/30] Update docker-compose-dev.yml Agregando soporte para emcemder el cluster cuando prende el contenedor --- docker-compose-dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 9af23da..4deffbf 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -28,6 +28,7 @@ services: depends_on: - redis environment: + ENABLE_BOTNET_AT_BOOT: "false" IS_DEV: "true" INTELMQ_SOURCE_PIPELINE_BROKER: "redis" INTELMQ_PIPELINE_BROKER: "redis" From 87a9824fef785ea578bfbaae60f0d3e6effaa5fc Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 14 Jun 2022 17:57:40 -0300 Subject: [PATCH 09/30] Update entrypoint-dev.sh Actualizando el entrypoint para prender del inicio --- .docker/intelmq-full-dev/entrypoint-dev.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh index 958f0ce..4a56ed5 100644 --- a/.docker/intelmq-full-dev/entrypoint-dev.sh +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -23,4 +23,8 @@ then nosetests3 /etc/intelmq/intelmq/tests else cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 -fi \ No newline at end of file +fi + +if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then + intelmqctl start +fi From cbdf57a6d4ba45bb99051d098a66a1104b0302d6 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 14 Jun 2022 18:19:48 -0300 Subject: [PATCH 10/30] Update DEVELOP-GUIDE.md --- DEVELOP-GUIDE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 19ad3a8..652d3cd 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -53,3 +53,6 @@ docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_re When you do this: * Yours bots REQUERIMENTS.txt and the bots will be installed + + +* Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true" From 8b6e1b8a649f3fdde5d35f5e793e047f96c4117f Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 12 Jul 2022 12:24:46 -0300 Subject: [PATCH 11/30] DEV README fix --- DEVELOP-GUIDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 652d3cd..46b843d 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -35,7 +35,7 @@ docker-compose -f docker-compose-dev.yml up ### Environment #Folder where you clone your repository MY_FORK: "/mybots" - #Foder in your repo where bots are located + #Folder in your repo where bots are located MY_BOTS_FOLDER: "bots" ### Add your own bots @@ -44,7 +44,7 @@ Just start coding or pull your bots repository in ./mybots by default in a subfo ### How to install and look yours bots running -After you change some bot or add someshing new just run command **install_reqs_and_deploy_bots.sh** in the running container +After you change some bot or add something new just run command **install_reqs_and_deploy_bots.sh** in the running container ``` docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh From f98068693b1dcd61d85b1c22293b4fe05a7d5bf0 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 12 Jul 2022 14:29:42 -0300 Subject: [PATCH 12/30] fix entrypoint --- .docker/intelmq-full-dev/entrypoint-dev.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh index 4a56ed5..b3f45ff 100644 --- a/.docker/intelmq-full-dev/entrypoint-dev.sh +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -17,6 +17,10 @@ intelmq_pass="${INTELMQ_API_PASS:=intelmq}" intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" +if [[ ${ENABLE_BOTNET_AT_BOOT} == "true" ]]; then + intelmqctl start +fi + if [[ $1 == "selftest" ]] then export INTELMQ_TEST_EXOTIC=1 @@ -25,6 +29,3 @@ else cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 fi -if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then - intelmqctl start -fi From af31872816dd9caf6ca2226af21546d52eb1f139 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Thu, 11 Aug 2022 12:41:32 -0300 Subject: [PATCH 13/30] Update DEVELOP-GUIDE.md --- DEVELOP-GUIDE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 46b843d..a193fe2 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -56,3 +56,9 @@ When you do this: * Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true" + +### Known isues + +Some dependencies are missing, as far is i know you need to install: + + sudo pip3 install elasticsearch==7.9 geoip2 azure-storage-blob From d3a1c333415de245ed5ff20446dcb92a3df49261 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Thu, 18 Aug 2022 12:13:19 -0300 Subject: [PATCH 14/30] fix to support dependenceies for all intelmq default bots --- .docker/intelmq-full-dev/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index b0e1f30..712aa3b 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -8,6 +8,7 @@ RUN sudo apt-get update \ && sudo apt-get install -y --no-install-recommends \ git \ vim \ + ssh \ && sudo rm -rf /var/lib/apt/lists/* WORKDIR /opt @@ -18,6 +19,11 @@ RUN sudo chmod +x /opt/entrypoint-dev.sh \ RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh +# This section is just for install all the default bots dependencies +RUN for req in $(find /etc/intelmq/intelmq/bots/ -name "*REQUIREMENTS.txt"); do cat $req >> /tmp/fullrequirements.txt; done +RUN cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt +RUN cat /tmp/orderfullrequirements.txt |grep -Eo '(^[^#]+)' | xargs -d "\n" -I {} sh -c 'pip3 install "$1"|| exit 0' sh {} + USER intelmq:intelmq ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] From 878f5154be394f6321e2029edc22177ed6d43a59 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Thu, 18 Aug 2022 13:04:49 -0300 Subject: [PATCH 15/30] fixing known issues documentation --- DEVELOP-GUIDE.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index a193fe2..510497a 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -57,8 +57,21 @@ When you do this: * Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true" +## Dependencies problems + +Some dependencies from defaults bots are missing in original intelmq image, so we fix it in our Dockerfile build process. Neverles we still facing some issues. + ### Known isues -Some dependencies are missing, as far is i know you need to install: +Blueliv problem: - sudo pip3 install elasticsearch==7.9 geoip2 azure-storage-blob + pip3 install git+git://github.com/Blueliv/api-python-sdk doesn't work because git+git is deprecated, to fix it you need to replace git+git with git+https + + +But you still will have problems because of dependency confict: + + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + pymisp 2.4.148 requires requests<3.0.0,>=2.25.1, but you have requests 2.5.1 which is incompatible. + + + From 40796b596158e6b21589d32127275064ea6e635e Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Thu, 18 Aug 2022 13:17:46 -0300 Subject: [PATCH 16/30] fixing known issues documentation --- DEVELOP-GUIDE.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 510497a..2c21cab 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -59,19 +59,24 @@ When you do this: ## Dependencies problems -Some dependencies from defaults bots are missing in original intelmq image, so we fix it in our Dockerfile build process. Neverles we still facing some issues. +Some dependencies from defaults bots are missing in original intelmq image, so we fix it in our Dockerfile build process. Nevertheless, we still facing some issues. ### Known isues -Blueliv problem: +#### Blueliv problem: + +This bot has 2 problems: + +1- It doesn't install: pip3 install git+git://github.com/Blueliv/api-python-sdk doesn't work because git+git is deprecated, to fix it you need to replace git+git with git+https -But you still will have problems because of dependency confict: +2- But if you fix and install it you would cause a dependency conflict with pymisp: - ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behavior is the source of the following dependency conflicts. pymisp 2.4.148 requires requests<3.0.0,>=2.25.1, but you have requests 2.5.1 which is incompatible. - +If you don't need blueliv, just don't fix git+git with git+https. + From 0aee947169577e7dd611d7e7dabc237fb696dbad Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Thu, 18 Aug 2022 13:33:46 -0300 Subject: [PATCH 17/30] fixing volumes in documentation --- DEVELOP-GUIDE.md | 13 ++++++------- docker-compose-dev.yml | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 2c21cab..994ba90 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -30,31 +30,31 @@ docker-compose -f docker-compose-dev.yml up ### Volumes: -- **./my_fork_of_intelmq/:/my_fork_of_intelmq** -> this is the folder where your source code need to be, we decide to use fork from intelmq so you could inherit intelmq changes and upgrades to your bots code directly. +- **./my_bots:/my_bots** -> this is the folder where your bots source code need to be. ### Environment #Folder where you clone your repository - MY_FORK: "/mybots" + MY_FORK: "/my_bots" #Folder in your repo where bots are located MY_BOTS_FOLDER: "bots" ### Add your own bots -Just start coding or pull your bots repository in ./mybots by default in a subfolder bots, so you need for example mybots/bots/[collectors,parsers,experts,output,parsers] +Just start coding or pull your bots repository in ./my_bots by default in a subfolder bots, so you need for example my_bots/bots/[collectors,parsers,experts,output,parsers] + +You could take a look at the folder and files in https://github.com/certtools/intelmq/tree/develop/intelmq/bots ### How to install and look yours bots running After you change some bot or add something new just run command **install_reqs_and_deploy_bots.sh** in the running container ``` -docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh +docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh ``` When you do this: * Yours bots REQUERIMENTS.txt and the bots will be installed - - * Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true" ## Dependencies problems @@ -79,4 +79,3 @@ This bot has 2 problems: If you don't need blueliv, just don't fix git+git with git+https. - diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 4deffbf..01bd74a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -24,7 +24,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./mybots:/my_fork_of_intelmq + - ./my_bots:/my_bots depends_on: - redis environment: @@ -37,7 +37,7 @@ services: INTELMQ_SOURCE_PIPELINE_HOST: redis INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis - MY_FORK: "/my_fork_of_intelmq" + MY_FORK: "/my_bots" MY_BOTS_FOLDER: "bots" networks: - intelmq-internal From 346c3abec1fa504e56ddb17cee3b5bd729f9bc11 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 30 Aug 2022 14:59:15 -0300 Subject: [PATCH 18/30] Quick fix in runtime.yaml we need to add destination_pipeline_host: redis, source_pipeline_host: redis bacuase intelmqdump dont care about environment variables --- example_config/intelmq/etc/runtime.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_config/intelmq/etc/runtime.yaml b/example_config/intelmq/etc/runtime.yaml index c289919..0cfad2b 100644 --- a/example_config/intelmq/etc/runtime.yaml +++ b/example_config/intelmq/etc/runtime.yaml @@ -117,7 +117,7 @@ gethostbyname-2-expert: run_mode: continuous global: {destination_pipeline_broker: redis, process_manager: intelmq, source_pipeline_broker: redis, ssl_ca_certificate: null, statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null, - statistics_port: 6379} + statistics_port: 6379, destination_pipeline_host: redis, source_pipeline_host: redis} malc0de-parser: bot_id: malc0de-parser description: Malc0de Parser is the bot responsible to parse the IP Blacklist and From f9c139ecc0c9ece9109fdab1a5a53c925c22acb7 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Fri, 18 Nov 2022 13:55:20 -0300 Subject: [PATCH 19/30] Path changes solve --- .docker/intelmq-full-dev/Dockerfile | 6 +----- .docker/intelmq-full-dev/entrypoint-dev.sh | 7 +++---- .docker/intelmq-full/Dockerfile | 5 ++--- docker-compose-dev.yml | 13 +++++++------ 4 files changed, 13 insertions(+), 18 deletions(-) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index 712aa3b..fad5430 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -19,11 +19,7 @@ RUN sudo chmod +x /opt/entrypoint-dev.sh \ RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh -# This section is just for install all the default bots dependencies -RUN for req in $(find /etc/intelmq/intelmq/bots/ -name "*REQUIREMENTS.txt"); do cat $req >> /tmp/fullrequirements.txt; done -RUN cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt -RUN cat /tmp/orderfullrequirements.txt |grep -Eo '(^[^#]+)' | xargs -d "\n" -I {} sh -c 'pip3 install "$1"|| exit 0' sh {} -USER intelmq:intelmq +USER intelmq ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh index b3f45ff..5c031a8 100644 --- a/.docker/intelmq-full-dev/entrypoint-dev.sh +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -24,8 +24,7 @@ fi if [[ $1 == "selftest" ]] then export INTELMQ_TEST_EXOTIC=1 - nosetests3 /etc/intelmq/intelmq/tests + nosetests3 /opt/intelmq/intelmq/tests else - cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 -fi - + cd /opt/intelmq-api && hug -m intelmq_api.serve -p8080 +fi \ No newline at end of file diff --git a/.docker/intelmq-full/Dockerfile b/.docker/intelmq-full/Dockerfile index 7983b50..7c67eec 100644 --- a/.docker/intelmq-full/Dockerfile +++ b/.docker/intelmq-full/Dockerfile @@ -14,8 +14,7 @@ LABEL maintainer="IntelMQ Team " \ org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \ org.opencontainers.image.vendor="intelmq-team" -### -# libfuzzy-dev is used for pydeep +### libfuzzy-dev is used for pydeep RUN apt-get update \ && apt-get install -y --no-install-recommends \ sudo \ @@ -46,7 +45,7 @@ COPY ./intelmq-api /opt/intelmq-api RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \ && adduser intelmq sudo \ - && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /opt/sudoers \ + && echo "intelmq ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/intelmq \ && sudo chown -R intelmq:intelmq /opt/intelmq \ && mkdir -p /opt/intelmq_persistence \ && sudo chown -R intelmq:intelmq /opt/intelmq_persistence diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 01bd74a..40f5a2c 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -18,13 +18,14 @@ services: networks: - intelmq-internal intelmq: - build: .docker/intelmq-full-dev + # build: .docker/intelmq-full-dev + image: certunlp/intelmq3-full-dev:latest volumes: - - ./example_config/intelmq/etc/:/opt/intelmq/etc/ - - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - - ./intelmq_logs:/opt/intelmq/var/log - - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./my_bots:/my_bots + - ./example_config/intelmq/etc/:/opt/intelmq/etc/ + - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json + - ./intelmq_logs:/opt/intelmq/var/log + - ./intelmq_output:/opt/intelmq/var/lib/bots + - ./my_bots:/my_bots depends_on: - redis environment: From c6b4f0ef1ec8447da3dea2e4f998608360db0613 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Fri, 18 Nov 2022 16:00:09 -0300 Subject: [PATCH 20/30] Fixing some more paths, and something in install_reqs_and_deploy_bots --- .docker/intelmq-full-dev/Dockerfile | 35 +++++++++++-------- .../install_reqs_and_deploy_bots | 5 +-- docker-compose-dev.yml | 4 +-- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index fe0f83b..05a40bc 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -1,18 +1,25 @@ -FROM certat/intelmq-full:1.0 +FROM certat/intelmq-full:latest +ENV LANG C.UTF-8 -ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh -ADD update.sh /opt/dev/update.sh -ADD merge_BOTS.py /opt/dev/merge_BOTS.py +LABEL maintainer="Einar " +LABEL maintainer="Jeremias " +LABEL maintainer="Mateo " +RUN sudo apt-get update \ + && sudo apt-get install -y --no-install-recommends \ + git \ + vim \ + ssh \ + && sudo rm -rf /var/lib/apt/lists/* -# Merge bots for merge_BOTS.py -RUN sudo python3 -m pip install jsonmerge +WORKDIR /opt +ADD entrypoint-dev.sh /opt/entrypoint-dev.sh +ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh +RUN sudo chmod +x /opt/entrypoint-dev.sh \ + && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh +RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \ + && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh +RUN usermod -aG intelmq www-data -USER intelmq:intelmq - -# Permission denied when installing new bots -RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info - -ENV PATH="/opt/intelmq/.local/bin:${PATH}" - -ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"] +USER intelmq +ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] \ No newline at end of file diff --git a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots index 6006121..6ba799c 100755 --- a/.docker/intelmq-full-dev/install_reqs_and_deploy_bots +++ b/.docker/intelmq-full-dev/install_reqs_and_deploy_bots @@ -1,12 +1,13 @@ #!/bin/bash -sudo cp -r $MY_FORK/$MY_BOTS_FOLDER/* /etc/intelmq/intelmq/bots/ +sudo cp -r $MY_FORK/$MY_BOTS_FOLDER/* /opt/intelmq/intelmq/bots/ sudo rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt for req in $(find $MY_FORK/$MY_BOTS_FOLDER -name "*REQUIREMENTS.txt"); do cat $req >> /tmp/fullrequirements.txt + echo "" >> /tmp/fullrequirements.txt done cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt sudo pip3 install -r /tmp/orderfullrequirements.txt; -cd /etc/intelmq +cd /opt/intelmq sudo pip3 install --no-cache-dir -e . sudo intelmqsetup diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 038bad1..f6361af 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -22,7 +22,7 @@ services: image: certunlp/intelmq3-full-dev:latest volumes: - ./example_config/intelmq/etc/:/opt/intelmq/etc/ - - ./example_config/intelmq-api/config.json:/opt/intelmq/api-config.json + - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - ./my_bots:/my_bots @@ -38,7 +38,7 @@ services: INTELMQ_SOURCE_PIPELINE_HOST: redis INTELMQ_DESTINATION_PIPELINE_HOST: redis INTELMQ_REDIS_CACHE_HOST: redis - MY_FORK: "/my_bots" + MY_FORK: "/my_bots/" MY_BOTS_FOLDER: "bots" networks: - intelmq-internal From 703b49c581fea55bcf97272a0a8805c37677b1ae Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Fri, 18 Nov 2022 18:05:44 -0300 Subject: [PATCH 21/30] Documentation fix --- DEVELOP-GUIDE.md | 36 +++++++++++-------- docker-compose-dev.yml | 2 +- {mybots => example_bots}/BOTS | 0 .../bots/collectors/otherexample/collector.py | 0 .../bots/experts/example/expert.py | 0 5 files changed, 23 insertions(+), 15 deletions(-) rename {mybots => example_bots}/BOTS (100%) rename {mybots => example_bots}/bots/collectors/otherexample/collector.py (100%) rename {mybots => example_bots}/bots/experts/example/expert.py (100%) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 994ba90..262f494 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -12,13 +12,6 @@ sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker- ``` git clone https://github.com/certat/intelmq-docker.git --recursive cd intelmq-docker -docker-compose -f docker-compose-dev.yml build -``` - -### In next step replace git@github.com:certtools/intelmq.git by your fork of intelmq - -``` -git clone git@github.com:certtools/intelmq.git my_fork_of_intelmq/ docker-compose -f docker-compose-dev.yml up ``` @@ -28,21 +21,36 @@ docker-compose -f docker-compose-dev.yml up ## Docker-compose-dev.yml file +Docker dev shares almost all volumes and environment variables from intelmq-full image. But some are new: + ### Volumes: -- **./my_bots:/my_bots** -> this is the folder where your bots source code need to be. +- **./example_bots:/my_bots** -> this is the folder where your bots source code need to be. ### Environment - #Folder where you clone your repository + +* Two variables to indicate where the source code of your bots is located: + + #Volume in the container where you clone your repository MY_FORK: "/my_bots" - #Folder in your repo where bots are located + #Subfolder in MY_FORK where your where bots are located MY_BOTS_FOLDER: "bots" +* Another thing, you could make your bots to be running when container startup, just setting + ENABLE_BOTNET_AT_BOOT: "true"* + ### Add your own bots -Just start coding or pull your bots repository in ./my_bots by default in a subfolder bots, so you need for example my_bots/bots/[collectors,parsers,experts,output,parsers] +Just start coding or pull your bots repository in some folder like ./my_bots in a subfolder bots, so you need for example my_bots/bots/[collectors,parsers,experts,output,parsers] + +You could take a look at the folder and files in https://github.com/certtools/intelmq/tree/develop/intelmq/bots to start. + +``` +git clone https://github.com/AAAAA/BBBB.git my_bots +``` + +After doing this, you need to change in docker-compose-dev.yml the volume definition from **./example_bots:/my_bots** to **./my_bots:/my_bots** -You could take a look at the folder and files in https://github.com/certtools/intelmq/tree/develop/intelmq/bots ### How to install and look yours bots running @@ -54,8 +62,8 @@ docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_req When you do this: -* Yours bots REQUERIMENTS.txt and the bots will be installed -* Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true" +* Yours bots REQUERIMENTS.txt and yout bots will be installed or updated from **MY_FORK** + ## Dependencies problems diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index f6361af..f1131cb 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -25,7 +25,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./my_bots:/my_bots + - ./mybots:/my_bots depends_on: - redis environment: diff --git a/mybots/BOTS b/example_bots/BOTS similarity index 100% rename from mybots/BOTS rename to example_bots/BOTS diff --git a/mybots/bots/collectors/otherexample/collector.py b/example_bots/bots/collectors/otherexample/collector.py similarity index 100% rename from mybots/bots/collectors/otherexample/collector.py rename to example_bots/bots/collectors/otherexample/collector.py diff --git a/mybots/bots/experts/example/expert.py b/example_bots/bots/experts/example/expert.py similarity index 100% rename from mybots/bots/experts/example/expert.py rename to example_bots/bots/experts/example/expert.py From 7e82a9a6e044b0897b38b5131573a3cee64b2092 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Fri, 18 Nov 2022 18:56:50 -0300 Subject: [PATCH 22/30] volume renamed --- docker-compose-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index f1131cb..b87a7b7 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -25,7 +25,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./mybots:/my_bots + - ./example_bots:/my_bots depends_on: - redis environment: From c4a77efb421722e07b16e3b4f854cb8a483416a1 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Sat, 26 Nov 2022 13:05:15 -0300 Subject: [PATCH 23/30] Update example_bots noop collector --- .../bots/collectors/noop/collector.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 example_bots/bots/collectors/noop/collector.py diff --git a/example_bots/bots/collectors/noop/collector.py b/example_bots/bots/collectors/noop/collector.py new file mode 100644 index 0000000..2aa6a35 --- /dev/null +++ b/example_bots/bots/collectors/noop/collector.py @@ -0,0 +1,33 @@ +""" +SPDX-FileCopyrightText: 2022 Einar Lanfranco +SPDX-License-Identifier: AGPL-3.0-or-later + +Example Collector Bot for Demo purpose only. + +Document possible necessary configurations. +""" +import sys +import time +# imports for additional libraries and intelmq +from intelmq.lib.bot import CollectorBot + + +class NoOpCollectorBot(CollectorBot): + """Este bot no hace nada util""" + paso: str = "step" + cantidad: int = 5 + rate_limit: int = 3600 + + def process(self): + self.logger.info("Comenzando NOOP Collector") + time.sleep(self.cantidad) + for i in range(self.cantidad): + time.sleep(i) + self.logger.info(f'{self.paso}{i}') + report = self.new_report() + report.add("raw","bGEgbmFkYSBtaXNtYQ==") + report.add("feed.url", "http://noop.url") + self.send_message(report) + + +BOT = NoOpCollectorBot \ No newline at end of file From 6947ae3cad4ebf03e314218e0434ab4786bc3211 Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Sat, 26 Nov 2022 14:00:12 -0300 Subject: [PATCH 24/30] Some modifications made to developer guide and a new noop collector bot example added --- DEVELOP-GUIDE.md | 5 +++-- example_bots/BOTS | 17 ----------------- example_bots/bots/collectors/noop/collector.py | 2 +- .../bots/collectors/otherexample/collector.py | 0 example_bots/bots/experts/example/expert.py | 14 -------------- 5 files changed, 4 insertions(+), 34 deletions(-) delete mode 100644 example_bots/BOTS delete mode 100644 example_bots/bots/collectors/otherexample/collector.py delete mode 100644 example_bots/bots/experts/example/expert.py diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 262f494..04b4014 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -41,7 +41,7 @@ Docker dev shares almost all volumes and environment variables from intelmq-full ### Add your own bots -Just start coding or pull your bots repository in some folder like ./my_bots in a subfolder bots, so you need for example my_bots/bots/[collectors,parsers,experts,output,parsers] +Just start coding or pull your bots repository in some folder like, for example, ./my_bots in a subfolder bots, then you have for example my_bots/bots/[collectors,parsers,experts,output,parsers] You could take a look at the folder and files in https://github.com/certtools/intelmq/tree/develop/intelmq/bots to start. @@ -62,7 +62,8 @@ docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_req When you do this: -* Yours bots REQUERIMENTS.txt and yout bots will be installed or updated from **MY_FORK** +* Yours bots REQUERIMENTS.txt and yout bots will be installed or updated from **MY_FORK**. +* Keep in mind that before being installed they will be mixed with the originals of the intelmq project, so it is important not to use the same names, neither for the bot nor for the .py files. ## Dependencies problems diff --git a/example_bots/BOTS b/example_bots/BOTS deleted file mode 100644 index feaa198..0000000 --- a/example_bots/BOTS +++ /dev/null @@ -1,17 +0,0 @@ -{ - "Collector": { - }, - "Parser": { - }, - "Expert": { - "Example": { - "description": "Example own bot.", - "module": "intelmq.bots.experts.example.expert", - "parameters": { - } - } - }, - "Output": { - - } -} diff --git a/example_bots/bots/collectors/noop/collector.py b/example_bots/bots/collectors/noop/collector.py index 2aa6a35..c9b5f22 100644 --- a/example_bots/bots/collectors/noop/collector.py +++ b/example_bots/bots/collectors/noop/collector.py @@ -30,4 +30,4 @@ class NoOpCollectorBot(CollectorBot): self.send_message(report) -BOT = NoOpCollectorBot \ No newline at end of file +BOT = NoOpCollectorBot diff --git a/example_bots/bots/collectors/otherexample/collector.py b/example_bots/bots/collectors/otherexample/collector.py deleted file mode 100644 index e69de29..0000000 diff --git a/example_bots/bots/experts/example/expert.py b/example_bots/bots/experts/example/expert.py deleted file mode 100644 index c39a6e2..0000000 --- a/example_bots/bots/experts/example/expert.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -from intelmq.lib.bot import Bot - - -class ExampleExpertBot(Bot): - - def init(self): - pass - - def process(self): - pass - - -BOT = ExampleExpertBot From 454ca624ff1a97ca5c9c59afc6960d36354944ea Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Mon, 28 Nov 2022 18:24:28 -0300 Subject: [PATCH 25/30] remove example bots --- docker-compose-dev.yml | 2 +- example_bots/.keep | 1 + .../bots/collectors/noop/collector.py | 33 ---- .../intelmq/etc/manager/positions.conf | 54 ++++-- example_config/intelmq/etc/runtime.yaml | 172 +++++++++++++++++- 5 files changed, 203 insertions(+), 59 deletions(-) create mode 100644 example_bots/.keep delete mode 100644 example_bots/bots/collectors/noop/collector.py diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index b87a7b7..0fe852a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -25,7 +25,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./example_bots:/my_bots + - ./intelmq3-bots-example:/my_bots depends_on: - redis environment: diff --git a/example_bots/.keep b/example_bots/.keep new file mode 100644 index 0000000..b3f79f4 --- /dev/null +++ b/example_bots/.keep @@ -0,0 +1 @@ +Link your bots repository here diff --git a/example_bots/bots/collectors/noop/collector.py b/example_bots/bots/collectors/noop/collector.py deleted file mode 100644 index c9b5f22..0000000 --- a/example_bots/bots/collectors/noop/collector.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -SPDX-FileCopyrightText: 2022 Einar Lanfranco -SPDX-License-Identifier: AGPL-3.0-or-later - -Example Collector Bot for Demo purpose only. - -Document possible necessary configurations. -""" -import sys -import time -# imports for additional libraries and intelmq -from intelmq.lib.bot import CollectorBot - - -class NoOpCollectorBot(CollectorBot): - """Este bot no hace nada util""" - paso: str = "step" - cantidad: int = 5 - rate_limit: int = 3600 - - def process(self): - self.logger.info("Comenzando NOOP Collector") - time.sleep(self.cantidad) - for i in range(self.cantidad): - time.sleep(i) - self.logger.info(f'{self.paso}{i}') - report = self.new_report() - report.add("raw","bGEgbmFkYSBtaXNtYQ==") - report.add("feed.url", "http://noop.url") - self.send_message(report) - - -BOT = NoOpCollectorBot diff --git a/example_config/intelmq/etc/manager/positions.conf b/example_config/intelmq/etc/manager/positions.conf index 942a4c7..8d2e0f5 100644 --- a/example_config/intelmq/etc/manager/positions.conf +++ b/example_config/intelmq/etc/manager/positions.conf @@ -1,11 +1,39 @@ { + "Deduplicator-Expert-CISA": { + "x": 653, + "y": 308 + }, + "Filter-Expert-timebased": { + "x": 852, + "y": 380 + }, + "GenericCsv-Parser": { + "x": 579, + "y": 211 + }, + "GenericCsv-Parser-2": { + "x": 456, + "y": 362 + }, + "HTTP-Collector": { + "x": 206, + "y": 361 + }, + "NoOp-Collector": { + "x": 276, + "y": 220 + }, + "Telegram-Output": { + "x": 767, + "y": 510 + }, "cymru-whois-expert": { "x": -280, "y": 414 }, "deduplicator-expert": { - "x": 148, - "y": -107 + "x": 340, + "y": -113 }, "feodo-tracker-browse-collector": { "x": 136, @@ -15,9 +43,9 @@ "x": 97, "y": -287 }, - "file-output": { - "x": -413, - "y": 428 + "file-output-einar": { + "x": 1113, + "y": -93 }, "gethostbyname-1-expert": { "x": -125, @@ -32,16 +60,8 @@ "y": 24 }, "malc0de-windows-format-collector": { - "x": 433, - "y": 121 - }, - "malware-domain-list-collector": { - "x": 465, - "y": -198 - }, - "malware-domain-list-parser": { - "x": 283, - "y": -125 + "x": 697, + "y": -202 }, "spamhaus-drop-collector": { "x": -137, @@ -56,8 +76,8 @@ "y": 30 }, "url2fqdn-expert": { - "x": -75, - "y": 179 + "x": -57, + "y": 39 }, "settings": { "physics": false, diff --git a/example_config/intelmq/etc/runtime.yaml b/example_config/intelmq/etc/runtime.yaml index 0cfad2b..77fc7d1 100644 --- a/example_config/intelmq/etc/runtime.yaml +++ b/example_config/intelmq/etc/runtime.yaml @@ -1,3 +1,152 @@ +Deduplicator-Expert-CISA: + bot_id: Deduplicator-Expert-CISA + description: Detection and drop exact duplicate messages. Message hashes are cached + in the Redis database + enabled: true + group: Expert + groupname: experts + module: intelmq.bots.experts.deduplicator.expert + name: Deduplicator + parameters: + bypass: false + destination_queues: + _default: [Filter-Expert-timebased-queue] + filter_keys: '' + filter_type: blacklist + redis_cache_db: 6 + redis_cache_host: 127.0.0.1 + redis_cache_password: '' + redis_cache_port: 6379 + redis_cache_ttl: 86400 + run_mode: continuous +Filter-Expert-timebased: + bot_id: Filter-Expert-timebased + description: Filter events, supports named paths for splitting the message flow + enabled: true + group: Expert + module: intelmq.bots.experts.filter.expert + name: Filter + parameters: + destination_queues: + _default: [Telegram-Output-queue] + filter_action: '' + filter_key: time.source + filter_regex: '' + filter_value: '' + not_after: '' + not_before: 15 days + run_mode: continuous +GenericCsv-Parser: + bot_id: GenericCsv-Parser + description: 'Parse generic CSV data. Ignoring lines starting with character #. + URLs without protocol can be prefixed with a default value.' + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.generic.parser_csv + name: GenericCsv + parameters: + column_regex_search: '' + columns: '' + columns_required: '' + compose_fields: {} + data_type: '' + default_url_protocol: http:// + delimiter: ',' + destination_queues: {} + filter_text: '' + filter_type: '' + skip_header: false + time_format: '' + type: '' + type_translation: {} + run_mode: continuous +GenericCsv-Parser-2: + bot_id: GenericCsv-Parser-2 + description: 'Parse generic CSV data. Ignoring lines starting with character #. + URLs without protocol can be prefixed with a default value.' + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.generic.parser_csv + name: GenericCsv + parameters: + column_regex_search: '' + columns: extra.cveID,extra.vendorProject,extra.product,extra.vulnerabilityName,time.source,extra.shortDescription,extra.requiredAction,extra.dueDate,extra.notes + compose_fields: {} + data_type: '' + default_url_protocol: http:// + delimiter: ',' + destination_queues: + _default: [Deduplicator-Expert-CISA-queue, Filter-Expert-timebased-queue] + filter_text: '' + filter_type: '' + skip_header: true + time_format: '' + type: '' + type_translation: {} + run_mode: continuous +HTTP-Collector: + bot_id: HTTP-Collector + description: Fetch reports from an URL + enabled: true + group: Collector + groupname: collectors + module: intelmq.bots.collectors.http.collector_http + name: HTTP + parameters: + code: '' + destination_queues: + _default: [GenericCsv-Parser-2-queue] + documentation: '' + extract_files: false + gpg_keyring: '' + http_header: {} + http_password: '' + http_url: https://www.cisa.gov/sites/default/files/csv/known_exploited_vulnerabilities.csv + http_url_formatting: false + http_username: '' + provider: '' + rate_limit: 3600 + signature_url: '' + signature_url_formatting: false + ssl_client_cert: '' + ssl_client_certificate: '' + verify_pgp_signatures: false + run_mode: continuous +NoOp-Collector: + bot_id: NoOp-Collector + description: Este bot no hace nada util + enabled: true + group: Collector + groupname: collectors + module: intelmq.bots.collectors.otherexample.collector + name: NoOp + parameters: + cantidad: 5 + code: '' + destination_queues: + _default: [GenericCsv-Parser-queue] + documentation: '' + paso: step + provider: '' + rate_limit: 3600 + run_mode: continuous +Telegram-Output: + bot_id: Telegram-Output + description: Send events to a REST API listener through HTTP POST + enabled: true + group: Output + groupname: outputs + module: intelmq.bots.outputs.telegram.output + name: Telegram + parameters: + chat_id: 145090811 + destination_queues: {} + message: 'Nuevo CVE siendo explotado: {ev[extra.cveID]}:{ev[extra.vulnerabilityName]}. + Agregado {ev[time.source]}. Detalles: Producto{ev[extra.product]} ' + token: 5985070241:AAFhp3Hrl-_sv67lZoibhW2uBTqpPfBRJiE + run_mode: continuous cymru-whois-expert: bot_id: cymru-whois-expert description: Cymru Whois (IP to ASN) is the bot responsible to add network information @@ -9,7 +158,7 @@ cymru-whois-expert: name: Cymru Whois parameters: destination_queues: - _default: [file-output-queue] + _default: [file-output-einar-queue] overwrite: true redis_cache_db: 5 redis_cache_host: 127.0.0.1 @@ -38,10 +187,12 @@ deduplicator-expert: redis_cache_ttl: 86400 run_mode: continuous feodo-tracker-browse-collector: + bot_id: feodo-tracker-browse-collector description: Generic URL Fetcher is the bot responsible to get the report from an URL. enabled: true group: Collector + groupname: collectors module: intelmq.bots.collectors.http.collector_http name: URL Fetcher parameters: @@ -58,10 +209,12 @@ feodo-tracker-browse-collector: ssl_client_certificate: null run_mode: continuous feodo-tracker-browse-parser: + bot_id: feodo-tracker-browse-parser description: HTML Table Parser is a bot configurable to parse different html table data. enabled: true group: Parser + groupname: parsers module: intelmq.bots.parsers.html_table.parser name: HTML Table parameters: @@ -80,16 +233,19 @@ feodo-tracker-browse-parser: time_format: null type: c2-server run_mode: continuous -file-output: - bot_id: file-output +file-output-einar: + bot_id: file-output-einar description: File is the bot responsible to send events to a file. enabled: true group: Output groupname: outputs module: intelmq.bots.outputs.file.output name: File - parameters: {file: /opt/intelmq/var/lib/bots/file-output/events.txt, hierarchical_output: false, - single_key: null} + parameters: + destination_queues: {} + file: /opt/intelmq/var/lib/bots/file-output/events.txt + hierarchical_output: false + single_key: '' run_mode: continuous gethostbyname-1-expert: bot_id: gethostbyname-1-expert @@ -115,9 +271,9 @@ gethostbyname-2-expert: destination_queues: _default: [cymru-whois-expert-queue] run_mode: continuous -global: {destination_pipeline_broker: redis, process_manager: intelmq, source_pipeline_broker: redis, - ssl_ca_certificate: null, statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null, - statistics_port: 6379, destination_pipeline_host: redis, source_pipeline_host: redis} +global: {destination_pipeline_broker: redis, destination_pipeline_host: redis, process_manager: intelmq, + source_pipeline_broker: redis, source_pipeline_host: redis, ssl_ca_certificate: null, + statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null, statistics_port: 6379} malc0de-parser: bot_id: malc0de-parser description: Malc0de Parser is the bot responsible to parse the IP Blacklist and From 500c4b1483286569f6ebb84501d21fb7843b8de7 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 9 May 2023 14:13:42 -0300 Subject: [PATCH 26/30] Update docker-compose-dev.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tenía mal la referencia a un volumen --- docker-compose-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0fe852a..b87a7b7 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -25,7 +25,7 @@ services: - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json - ./intelmq_logs:/opt/intelmq/var/log - ./intelmq_output:/opt/intelmq/var/lib/bots - - ./intelmq3-bots-example:/my_bots + - ./example_bots:/my_bots depends_on: - redis environment: From c8d43a993bc43b0666e594c69522d28c7890787d Mon Sep 17 00:00:00 2001 From: Einar Lanfranco Date: Tue, 9 May 2023 14:19:07 -0300 Subject: [PATCH 27/30] reset config files --- .../intelmq/etc/manager/positions.conf | 54 ++---- example_config/intelmq/etc/runtime.yaml | 172 +----------------- 2 files changed, 25 insertions(+), 201 deletions(-) diff --git a/example_config/intelmq/etc/manager/positions.conf b/example_config/intelmq/etc/manager/positions.conf index 8d2e0f5..942a4c7 100644 --- a/example_config/intelmq/etc/manager/positions.conf +++ b/example_config/intelmq/etc/manager/positions.conf @@ -1,39 +1,11 @@ { - "Deduplicator-Expert-CISA": { - "x": 653, - "y": 308 - }, - "Filter-Expert-timebased": { - "x": 852, - "y": 380 - }, - "GenericCsv-Parser": { - "x": 579, - "y": 211 - }, - "GenericCsv-Parser-2": { - "x": 456, - "y": 362 - }, - "HTTP-Collector": { - "x": 206, - "y": 361 - }, - "NoOp-Collector": { - "x": 276, - "y": 220 - }, - "Telegram-Output": { - "x": 767, - "y": 510 - }, "cymru-whois-expert": { "x": -280, "y": 414 }, "deduplicator-expert": { - "x": 340, - "y": -113 + "x": 148, + "y": -107 }, "feodo-tracker-browse-collector": { "x": 136, @@ -43,9 +15,9 @@ "x": 97, "y": -287 }, - "file-output-einar": { - "x": 1113, - "y": -93 + "file-output": { + "x": -413, + "y": 428 }, "gethostbyname-1-expert": { "x": -125, @@ -60,8 +32,16 @@ "y": 24 }, "malc0de-windows-format-collector": { - "x": 697, - "y": -202 + "x": 433, + "y": 121 + }, + "malware-domain-list-collector": { + "x": 465, + "y": -198 + }, + "malware-domain-list-parser": { + "x": 283, + "y": -125 }, "spamhaus-drop-collector": { "x": -137, @@ -76,8 +56,8 @@ "y": 30 }, "url2fqdn-expert": { - "x": -57, - "y": 39 + "x": -75, + "y": 179 }, "settings": { "physics": false, diff --git a/example_config/intelmq/etc/runtime.yaml b/example_config/intelmq/etc/runtime.yaml index 77fc7d1..0cfad2b 100644 --- a/example_config/intelmq/etc/runtime.yaml +++ b/example_config/intelmq/etc/runtime.yaml @@ -1,152 +1,3 @@ -Deduplicator-Expert-CISA: - bot_id: Deduplicator-Expert-CISA - description: Detection and drop exact duplicate messages. Message hashes are cached - in the Redis database - enabled: true - group: Expert - groupname: experts - module: intelmq.bots.experts.deduplicator.expert - name: Deduplicator - parameters: - bypass: false - destination_queues: - _default: [Filter-Expert-timebased-queue] - filter_keys: '' - filter_type: blacklist - redis_cache_db: 6 - redis_cache_host: 127.0.0.1 - redis_cache_password: '' - redis_cache_port: 6379 - redis_cache_ttl: 86400 - run_mode: continuous -Filter-Expert-timebased: - bot_id: Filter-Expert-timebased - description: Filter events, supports named paths for splitting the message flow - enabled: true - group: Expert - module: intelmq.bots.experts.filter.expert - name: Filter - parameters: - destination_queues: - _default: [Telegram-Output-queue] - filter_action: '' - filter_key: time.source - filter_regex: '' - filter_value: '' - not_after: '' - not_before: 15 days - run_mode: continuous -GenericCsv-Parser: - bot_id: GenericCsv-Parser - description: 'Parse generic CSV data. Ignoring lines starting with character #. - URLs without protocol can be prefixed with a default value.' - enabled: true - group: Parser - groupname: parsers - module: intelmq.bots.parsers.generic.parser_csv - name: GenericCsv - parameters: - column_regex_search: '' - columns: '' - columns_required: '' - compose_fields: {} - data_type: '' - default_url_protocol: http:// - delimiter: ',' - destination_queues: {} - filter_text: '' - filter_type: '' - skip_header: false - time_format: '' - type: '' - type_translation: {} - run_mode: continuous -GenericCsv-Parser-2: - bot_id: GenericCsv-Parser-2 - description: 'Parse generic CSV data. Ignoring lines starting with character #. - URLs without protocol can be prefixed with a default value.' - enabled: true - group: Parser - groupname: parsers - module: intelmq.bots.parsers.generic.parser_csv - name: GenericCsv - parameters: - column_regex_search: '' - columns: extra.cveID,extra.vendorProject,extra.product,extra.vulnerabilityName,time.source,extra.shortDescription,extra.requiredAction,extra.dueDate,extra.notes - compose_fields: {} - data_type: '' - default_url_protocol: http:// - delimiter: ',' - destination_queues: - _default: [Deduplicator-Expert-CISA-queue, Filter-Expert-timebased-queue] - filter_text: '' - filter_type: '' - skip_header: true - time_format: '' - type: '' - type_translation: {} - run_mode: continuous -HTTP-Collector: - bot_id: HTTP-Collector - description: Fetch reports from an URL - enabled: true - group: Collector - groupname: collectors - module: intelmq.bots.collectors.http.collector_http - name: HTTP - parameters: - code: '' - destination_queues: - _default: [GenericCsv-Parser-2-queue] - documentation: '' - extract_files: false - gpg_keyring: '' - http_header: {} - http_password: '' - http_url: https://www.cisa.gov/sites/default/files/csv/known_exploited_vulnerabilities.csv - http_url_formatting: false - http_username: '' - provider: '' - rate_limit: 3600 - signature_url: '' - signature_url_formatting: false - ssl_client_cert: '' - ssl_client_certificate: '' - verify_pgp_signatures: false - run_mode: continuous -NoOp-Collector: - bot_id: NoOp-Collector - description: Este bot no hace nada util - enabled: true - group: Collector - groupname: collectors - module: intelmq.bots.collectors.otherexample.collector - name: NoOp - parameters: - cantidad: 5 - code: '' - destination_queues: - _default: [GenericCsv-Parser-queue] - documentation: '' - paso: step - provider: '' - rate_limit: 3600 - run_mode: continuous -Telegram-Output: - bot_id: Telegram-Output - description: Send events to a REST API listener through HTTP POST - enabled: true - group: Output - groupname: outputs - module: intelmq.bots.outputs.telegram.output - name: Telegram - parameters: - chat_id: 145090811 - destination_queues: {} - message: 'Nuevo CVE siendo explotado: {ev[extra.cveID]}:{ev[extra.vulnerabilityName]}. - Agregado {ev[time.source]}. Detalles: Producto{ev[extra.product]} ' - token: 5985070241:AAFhp3Hrl-_sv67lZoibhW2uBTqpPfBRJiE - run_mode: continuous cymru-whois-expert: bot_id: cymru-whois-expert description: Cymru Whois (IP to ASN) is the bot responsible to add network information @@ -158,7 +9,7 @@ cymru-whois-expert: name: Cymru Whois parameters: destination_queues: - _default: [file-output-einar-queue] + _default: [file-output-queue] overwrite: true redis_cache_db: 5 redis_cache_host: 127.0.0.1 @@ -187,12 +38,10 @@ deduplicator-expert: redis_cache_ttl: 86400 run_mode: continuous feodo-tracker-browse-collector: - bot_id: feodo-tracker-browse-collector description: Generic URL Fetcher is the bot responsible to get the report from an URL. enabled: true group: Collector - groupname: collectors module: intelmq.bots.collectors.http.collector_http name: URL Fetcher parameters: @@ -209,12 +58,10 @@ feodo-tracker-browse-collector: ssl_client_certificate: null run_mode: continuous feodo-tracker-browse-parser: - bot_id: feodo-tracker-browse-parser description: HTML Table Parser is a bot configurable to parse different html table data. enabled: true group: Parser - groupname: parsers module: intelmq.bots.parsers.html_table.parser name: HTML Table parameters: @@ -233,19 +80,16 @@ feodo-tracker-browse-parser: time_format: null type: c2-server run_mode: continuous -file-output-einar: - bot_id: file-output-einar +file-output: + bot_id: file-output description: File is the bot responsible to send events to a file. enabled: true group: Output groupname: outputs module: intelmq.bots.outputs.file.output name: File - parameters: - destination_queues: {} - file: /opt/intelmq/var/lib/bots/file-output/events.txt - hierarchical_output: false - single_key: '' + parameters: {file: /opt/intelmq/var/lib/bots/file-output/events.txt, hierarchical_output: false, + single_key: null} run_mode: continuous gethostbyname-1-expert: bot_id: gethostbyname-1-expert @@ -271,9 +115,9 @@ gethostbyname-2-expert: destination_queues: _default: [cymru-whois-expert-queue] run_mode: continuous -global: {destination_pipeline_broker: redis, destination_pipeline_host: redis, process_manager: intelmq, - source_pipeline_broker: redis, source_pipeline_host: redis, ssl_ca_certificate: null, - statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null, statistics_port: 6379} +global: {destination_pipeline_broker: redis, process_manager: intelmq, source_pipeline_broker: redis, + ssl_ca_certificate: null, statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null, + statistics_port: 6379, destination_pipeline_host: redis, source_pipeline_host: redis} malc0de-parser: bot_id: malc0de-parser description: Malc0de Parser is the bot responsible to parse the IP Blacklist and From 19cca627556406ba3183e5aff81d03cd42fbca06 Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Tue, 9 May 2023 16:16:19 -0300 Subject: [PATCH 28/30] Update Dockerfile Added new linr --- .docker/intelmq-full-dev/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.docker/intelmq-full-dev/Dockerfile b/.docker/intelmq-full-dev/Dockerfile index 05a40bc..13eb13e 100644 --- a/.docker/intelmq-full-dev/Dockerfile +++ b/.docker/intelmq-full-dev/Dockerfile @@ -22,4 +22,4 @@ RUN usermod -aG intelmq www-data USER intelmq -ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] \ No newline at end of file +ENTRYPOINT [ "/opt/entrypoint-dev.sh" ] From 5f98e21a08473bd62d1231bf3f2ccdae43c510dc Mon Sep 17 00:00:00 2001 From: Einar Felipe Lanfranco Date: Wed, 10 May 2023 14:45:19 -0300 Subject: [PATCH 29/30] Update DEVELOP-GUIDE.md /opt/install_reqs_and_deploy_bots.sh needs to be called without sudo --- DEVELOP-GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOP-GUIDE.md b/DEVELOP-GUIDE.md index 04b4014..ae0a2df 100644 --- a/DEVELOP-GUIDE.md +++ b/DEVELOP-GUIDE.md @@ -57,7 +57,7 @@ After doing this, you need to change in docker-compose-dev.yml the volume defin After you change some bot or add something new just run command **install_reqs_and_deploy_bots.sh** in the running container ``` -docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh +docker-compose exec -f docker-compose-dev.yml intelmq bash /opt/install_reqs_and_deploy_bots.sh ``` When you do this: From 6f9531eeb872165b420aeb09d1fcc0700ef5a41a Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Tue, 22 Aug 2023 15:52:22 +0200 Subject: [PATCH 30/30] Revert to build from local Dockerfile --- docker-compose-dev.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index b87a7b7..39558f2 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -18,8 +18,7 @@ services: networks: - intelmq-internal intelmq: - # build: .docker/intelmq-full-dev - image: certunlp/intelmq3-full-dev:latest + build: .docker/intelmq-full-dev volumes: - ./example_config/intelmq/etc/:/opt/intelmq/etc/ - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json