Update to 3.5.0

Fix wrong command

.docker/intelmq-full-dev/Dockerfile

@@ -0,0 +1,25 @@
+FROM certat/intelmq-full:latest
+ENV LANG C.UTF-8
+
+LABEL maintainer="Einar <elanfranco@csirtamericas.org>"
+LABEL maintainer="Jeremias <jpretto@cert.unlp.edu.ar>" 
+LABEL maintainer="Mateo <mdurante@cert.unlp.edu.ar>" 
+RUN sudo apt-get update \
+    && sudo apt-get install -y --no-install-recommends \
+    git \
+    vim \
+    ssh \
+    && sudo rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt
+ADD entrypoint-dev.sh /opt/entrypoint-dev.sh
+ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh
+RUN sudo chmod +x /opt/entrypoint-dev.sh \
+    && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh
+RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \
+    && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh
+RUN usermod -aG intelmq www-data
+
+USER intelmq
+
+ENTRYPOINT [ "/opt/entrypoint-dev.sh" ]

.docker/intelmq-full-dev/entrypoint-dev.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+export INTELMQ_IS_DOCKER=1
+
+if [[ ${IS_DEV} == "true" ]]
+then
+    /opt/install_reqs_and_deploy_bots.sh
+fi
+
+sudo chown -R intelmq:intelmq /etc/intelmq
+sudo chown -R intelmq:intelmq /opt/intelmq
+
+intelmqctl upgrade-config
+intelmqctl check
+
+intelmq_user="${INTELMQ_API_USER:=intelmq}"
+intelmq_pass="${INTELMQ_API_PASS:=intelmq}"
+
+intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass"
+
+if [[ ${ENABLE_BOTNET_AT_BOOT} == "true" ]]; then
+	intelmqctl start
+fi
+
+if [[ $1 == "selftest" ]]
+then
+    export INTELMQ_TEST_EXOTIC=1
+    pytest-3 /opt/intelmq/intelmq/tests
+else
+    cd /opt/intelmq-api && uvicorn intelmq_api.main:app --port 8080 --host 0.0.0.0
+fi

.docker/intelmq-full-dev/install_reqs_and_deploy_bots

@@ -0,0 +1,13 @@
+#!/bin/bash
+sudo cp -r $MY_FORK/$MY_BOTS_FOLDER/* /opt/intelmq/intelmq/bots/
+sudo rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt
+for req in $(find $MY_FORK/$MY_BOTS_FOLDER -name "*REQUIREMENTS.txt"); do 
+   cat $req >> /tmp/fullrequirements.txt
+   echo "" >> /tmp/fullrequirements.txt
+done
+cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt
+sudo pip3 install -r /tmp/orderfullrequirements.txt; 
+
+cd /opt/intelmq
+sudo pip3 install --no-cache-dir -e .
+sudo intelmqsetup

.docker/intelmq-full/Dockerfile

@@ -0,0 +1,79 @@
+FROM debian:bullseye-slim
+ENV LANG C.UTF-8
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG BUILD_VERSION
+
+LABEL maintainer="IntelMQ Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.authors="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.title="intelmq-full" \
+      org.opencontainers.image.description="IntelMQ with core & api" \
+      org.opencontainers.image.url="https://intelmq.org/" \
+      org.opencontainers.image.source="https://github.com/certtools/intelmq.git" \
+      org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
+      org.opencontainers.image.vendor="intelmq-team"
+
+### libfuzzy-dev is used for pydeep
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+    sudo \
+    gcc \
+    rsync \
+    ssh \
+    python3-pika \
+    python3-dev \
+    python3-setuptools \
+    python3-pip \
+    python3-ruamel.yaml \
+    python3-bs4 \
+    python3-validators \
+    python3-lxml \
+    python3-xmltodict \
+    python3-cerberus \
+    python3-requests-mock \
+    python3-pytest \
+    python3-pytest-cov \
+    python3-shodan \
+    python3-elasticsearch \
+    python3-pymongo \
+    libfuzzy-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+LABEL org.opencontainers.image.created=$BUILD_DATE \
+      org.opencontainers.image.revision=$VCS_REF \
+      org.opencontainers.image.version=$BUILD_VERSION
+
+COPY ./intelmq /opt/intelmq
+COPY ./intelmq-api /opt/intelmq-api
+
+RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \
+    && adduser intelmq sudo \
+    && echo "intelmq ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/intelmq \
+    && sudo chown -R intelmq:intelmq /opt/intelmq \
+    && mkdir -p /opt/intelmq_persistence \
+    && sudo chown -R intelmq:intelmq /opt/intelmq_persistence
+
+### Install IntelMQ
+RUN pip3 install url-normalize geolib imbox jinja2 pyasn textx tld time-machine otxv2 pendulum \
+    && pip3 install --force pymisp[fileobjects,openioc,virustotal]
+
+RUN cd /opt/intelmq \
+    && pip3 install .
+
+RUN cd /opt/intelmq \
+    && intelmqsetup
+
+### Install IntelMQ-API
+RUN cd /opt/intelmq-api \
+    && pip3 install .
+
+ADD entrypoint.sh /opt/entrypoint.sh
+RUN chmod +x /opt/entrypoint.sh \
+    && chown intelmq:intelmq /opt/entrypoint.sh
+
+WORKDIR /opt
+
+#USER intelmq:intelmq
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]

.docker/nginx/Dockerfile

@@ -0,0 +1,25 @@
+FROM nginx:1.13-alpine
+ENV LANG C.UTF-8
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG BUILD_VERSION
+
+LABEL maintainer="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.authors="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.title="intelmq-nginx" \
+      org.opencontainers.image.description="Modified NGINX Server for intelmq" \
+      org.opencontainers.image.url="https://github.com/certtools/intelmq/issues" \
+      org.opencontainers.image.source="https://github.com/certtools/intelmq.git" \
+      org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
+      org.opencontainers.image.vendor="intelmq-team"
+
+LABEL org.opencontainers.image.created=$BUILD_DATE \
+      org.opencontainers.image.revision=$VCS_REF \
+      org.opencontainers.image.version=$BUILD_VERSION
+
+WORKDIR /www
+
+COPY .docker/nginx/config/app.conf /etc/nginx/conf.d/default.conf
+COPY .docker/nginx/config/nginx.conf /etc/nginx/nginx.conf
+COPY intelmq-manager/html/ /www/

.docker/nginx/config/app.conf

@@ -0,0 +1,20 @@
+upstream intelmq_api {
+    server intelmq:8080;
+}
+
+server {
+    listen 80 default_server;
+
+    server_name localhost;
+
+    root /www;
+
+    location / {
+        index index.html;
+        try_files $uri /index.html =404;
+    }
+
+    location /intelmq/ {
+        proxy_pass http://intelmq_api/;
+    }
+}

.docker/nginx/config/nginx.conf

@@ -0,0 +1,27 @@
+user nginx;
+
+worker_processes 1;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+
+    sendfile on;
+
+    keepalive_timeout 65;
+
+    include /etc/nginx/conf.d/*.conf;
+}

.gitignore

@@ -0,0 +1,3 @@
+intelmq_logs/
+intelmq_persistence/
+my_fork_of_intelmq/

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "intelmq"]
+	path = intelmq
+	url = https://github.com/certtools/intelmq.git
+	branch = maintenance
+[submodule "intelmq-manager"]
+	path = intelmq-manager
+	url = https://github.com/certtools/intelmq-manager.git
+	branch = maintenance
+[submodule "intelmq-api"]
+	path = intelmq-api
+	url = https://github.com/certtools/intelmq-api.git
+	branch = maintenance

DEVELOP-GUIDE.md

@@ -0,0 +1,90 @@
+# intelmq-docker
+
+## Run & deploy containers in dev mode:
+
+### Install docker and docker-compose
+```
+sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose
+```
+
+### Clone this repo
+
+```
+git clone https://github.com/certat/intelmq-docker.git --recursive
+cd intelmq-docker
+docker-compose -f docker-compose-dev.yml up
+```
+
+### Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+
+        Default user/password: intelmq/intelmq
+
+## Docker-compose-dev.yml file
+
+Docker dev shares almost all volumes and environment variables from intelmq-full image. But some are new:
+
+### Volumes:
+
+- **./example_bots:/my_bots** -> this is the folder where your bots source code need to be.
+
+### Environment
+
+* Two variables to indicate where the source code of your bots is located:  
+
+            #Volume in the container where you clone your repository
+            MY_FORK: "/my_bots"
+            #Subfolder in MY_FORK where your where bots are located
+            MY_BOTS_FOLDER: "bots"
+
+* Another thing, you could make your bots to be running when container startup, just setting 
+            ENABLE_BOTNET_AT_BOOT: "true"* 
+
+### Add your own bots
+
+Just start coding or pull your bots repository in some folder like, for example, ./my_bots in a subfolder bots, then you have for example my_bots/bots/[collectors,parsers,experts,output,parsers]
+
+You could take a look at the folder and files in https://github.com/certtools/intelmq/tree/develop/intelmq/bots to start.
+
+```
+git clone https://github.com/AAAAA/BBBB.git my_bots
+```
+
+After doing this,  you need to change in docker-compose-dev.yml the volume definition from **./example_bots:/my_bots** to **./my_bots:/my_bots** 
+
+
+### How to install and look yours bots running
+
+After you change some bot or add something new just run command **install_reqs_and_deploy_bots.sh** in the running container
+
+```
+docker-compose -f docker-compose-dev.yml exec intelmq bash /opt/install_reqs_and_deploy_bots.sh
+```
+
+When you do this:
+
+* Yours bots REQUERIMENTS.txt and yout bots will be installed or updated from **MY_FORK**. 
+* Keep in mind that before being installed they will be mixed with the originals of the intelmq project, so it is important not to use the same names, neither for the bot nor for the .py files.
+
+
+## Dependencies problems
+
+Some dependencies from defaults bots are missing in original intelmq image, so we fix it in our Dockerfile build process. Nevertheless, we still facing some issues.
+
+### Known isues
+
+#### Blueliv problem:
+
+This bot has 2 problems: 
+
+1- It doesn't install:
+
+        pip3 install git+git://github.com/Blueliv/api-python-sdk doesn't work because git+git is deprecated, to fix it you need to replace git+git with git+https 
+
+
+2- But if you fix and install it you would cause a dependency conflict with pymisp:
+
+        ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behavior is the source of the following dependency conflicts.
+        pymisp 2.4.148 requires requests<3.0.0,>=2.25.1, but you have requests 2.5.1 which is incompatible.
+
+
+If you don't need blueliv, just don't fix git+git with git+https.

Dockerfile

@@ -1,59 +0,0 @@
-FROM debian:buster
-ENV LANG C.UTF-8
-
-ARG BUILD_DATE
-ARG VCS_REF
-ARG BUILD_VERSION
-
-LABEL maintainer="IntelMQ Team <intelmq-team@cert.at>" \
-      org.label-schema.schema-version="1.0" \
-      org.label-schema.name="certat/intelmq-full" \
-      org.label-schema.description="IntelMQ with core & manager" \
-      org.label-schema.url="https://intelmq.org/" \
-      org.label-schema.vcs-url="https://github.com/certat/intelmq-docker.git" \
-      org.label-schema.vendor="CERT.AT"
-
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-    sudo \
-    gcc \
-    python3-nose \
-    python3-yaml \
-    python3-cerberus \
-    python3-requests-mock \
-    python3-dev \
-    python3-setuptools \
-    python3-pip \
-    && rm -rf /var/lib/apt/lists/*
-
-LABEL org.label-schema.build-date=$BUILD_DATE \
-      org.label-schema.vcs-ref=$VCS_REF \
-      org.label-schema.version=$BUILD_VERSION
-
-
-COPY ./intelmq /opt/intelmq
-COPY ./intelmq-manager /opt/intelmq-manager
-
-WORKDIR /opt
-
-RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \
-    && adduser intelmq sudo \ 
-    && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
-    && sudo chown -R intelmq:intelmq /opt/intelmq
-
-### Install IntelMQ
-RUN cd /opt/intelmq \
-    && pip3 install --no-cache-dir -e . \
-    && intelmqsetup
-
-### Install IntelMQ-Manager (python)
-RUN cd /opt/intelmq-manager \
-    && pip3 install hug mako \
-    && pip3 install --no-cache-dir -e .
-
-ADD entrypoint.sh /opt/entrypoint.sh
-RUN chmod +x /opt/entrypoint.sh
-
-USER intelmq
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]

README.md

@@ -4,37 +4,40 @@
 Do not run this software in production, it might break.
 
 # Information
-This repository is currently maintained by Sebastian Waldbauer (@waldbauer-certat).
+This repository is currently maintained by CERT.at.
 
 If you do have any questions / feedback / questions, please open an issue :)
 
 ## Fastest way to run & deploy
 
 1. `cd ~`
-0. `mkdir intelmq_logs`
 0. `sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose`
-0. `git clone https://github.com/certat/intelmq-docker.git`
+0. `git clone https://github.com/certat/intelmq-docker.git --recursive`
 0. `cd intelmq-docker`
-0. `sudo docker pull certat/intelmq-full:1.0`
-0. `chown -R $USER:$USER example_config`
-0. `sudo docker-compose up`
-0. Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+0. `docker-compose pull`
+2. `docker-compose up`
+3. Open your favourite browser -> Go to `http://127.0.0.1:1337/`
 
-If you want to build/deploy/test this container run 
+## For developers
+
+Please take a look to DEVELOP-GUIDE.md
+
+
+## Build and deploy new images
+
+If you want to build/deploy/test this container run
 1. `chmod +x build.sh`
 0. `chmod +x test.sh`
 0. `chmod +x publish.sh`
 
-**!ATTENTATION!** Only [CERT.AT](https://cert.at/) employee's/maintainer can publish on `cerat/` repository. Change this in `publish.sh`
+**!ATTENTION!** Only [CERT.AT](https://cert.at/) employee's/maintainer can publish on the `certat/` repository. Change this in `publish.sh`
 
 ## How to develop new features & build containers?
 **ATTENTION** Make sure to change `certat/intelmq-full:1.0` to `intelmq-full:1.0` in `docker-compose.yml`
 
-1. `cd ~`
-0. `git clone https://github.com/certtools/intelmq.git`
-0. `git clone https://github.com/certtools/intelmq-manager`
+Start making your changes in `intelmq`, `intelmq-api` or `intelmq-manager`.
 
-Now you can start making changes to source code. If you're finished and ready to test within your docker enviroment
+If you're finished and ready to test within your docker enviroment
 1. `cd ~/intelmq-docker`
 0. `sudo ./build.sh`
 
@@ -42,4 +45,4 @@ Now your docker image should be built successfully. Check for errors :)
 
 Now lets run tests to ensure our image is ready.
 
-1. `sudo ./test.sh`
+1. `sudo ./test.sh`

build.sh

@@ -1,19 +1,27 @@
 #!/bin/bash
 build_date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-git_ref_core=$(cd ../intelmq && git rev-parse --short HEAD)
-git_ref_manager=$(cd ../intelmq-manager && git rev-parse --short HEAD)
-build_version="1.0"
+git_ref_core=$(git -C ./intelmq describe --long --always)
+git_ref_manager=$(git -C ./intelmq-manager describe --long --always)
+git_ref_api=$(git -C ./intelmq-api describe --long --always)
+build_version=$(git -C ./intelmq describe --always)
 
 echo Building new IntelMQ-Image v$build_version
 echo Core      : $git_ref_core
 echo Manager   : $git_ref_manager
+echo Api       : $git_ref_api
 echo Build_date: $build_date
 
-cp -r ../intelmq ./intelmq
-cp -r ../intelmq-manager ./intelmq-manager
+# build static html
+cd ./intelmq-manager && python3 -m pip install . && intelmq-manager-build && cd ..
 
 docker build --build-arg BUILD_DATE=$build_date \
-    --build-arg VCS_REF="IntelMQ=$git_ref_core, IntelMQ-Manager=$git_ref_manager" \
+    --build-arg VCS_REF="IntelMQ-Manager=$git_ref_manager" \
     --build-arg BUILD_VERSION=$build_version \
-    -f Dockerfile \
-    -t intelmq-full:$build_version .
+    -f ./.docker/nginx/Dockerfile \
+    -t intelmq-nginx:latest .
+
+docker build --build-arg BUILD_DATE=$build_date \
+    --build-arg VCS_REF="IntelMQ=$git_ref_core, IntelMQ-API=$git_ref_api, IntelMQ-Manager=$git_ref_manager" \
+    --build-arg BUILD_VERSION=$build_version \
+    -f ./.docker/intelmq-full/Dockerfile \
+    -t intelmq-full:latest .

docker-compose-dev.yml

@@ -0,0 +1,46 @@
+version: "3"
+services:
+    redis:
+        image: redis:latest
+        volumes:
+            - ./example_config/redis/redis.conf:/usr/local/etc/redis/redis.conf
+        command:
+            - redis-server
+            - /usr/local/etc/redis/redis.conf
+        networks: 
+            - intelmq-internal
+    nginx:
+        image: certat/intelmq-nginx:latest
+        ports: 
+            - 1337:80
+        depends_on: 
+            - intelmq
+        networks: 
+            - intelmq-internal
+    intelmq:
+        build: .docker/intelmq-full-dev
+        volumes: 
+           - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+           - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
+           - ./intelmq_logs:/opt/intelmq/var/log
+           - ./intelmq_output:/opt/intelmq/var/lib/bots
+           - ./example_bots:/my_bots
+        depends_on: 
+            - redis
+        environment:
+            ENABLE_BOTNET_AT_BOOT: "false"
+            IS_DEV: "true" 
+            INTELMQ_SOURCE_PIPELINE_BROKER: "redis"
+            INTELMQ_PIPELINE_BROKER: "redis"
+            INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis"
+            INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_SOURCE_PIPELINE_HOST: redis
+            INTELMQ_DESTINATION_PIPELINE_HOST: redis
+            INTELMQ_REDIS_CACHE_HOST: redis
+            MY_FORK: "/my_bots/"
+            MY_BOTS_FOLDER: "bots"
+        networks:
+            - intelmq-internal
+networks: 
+    intelmq-internal:
+        driver: bridge

docker-compose.yml

@@ -9,43 +9,36 @@ services:
             - /usr/local/etc/redis/redis.conf
         restart: always
         networks:
-            - intelmq-network
-    postgres: 
-        image: postgres:latest
+            - intelmq-internal
+    nginx:
+        image: certat/intelmq-nginx:latest
         restart: always
-        environment:
-            POSTGRES_PASSWORD: test
-            POSTGRES_USER: root
-            POSTGRES_DB: test
+        ports:
+            - 1337:80
+        depends_on:
+            - intelmq
         networks:
-            - intelmq-database
-
-# IntelMQ with IntelMQ-Manager!
-    intelmq-full:
-        image: certat/intelmq-full:1.0
-        volumes: 
-            - ./example_config/intelmq/etc:/opt/intelmq/etc
-            - ./example_config/intelmq-manager:/opt/intelmq-manager/config
+            - intelmq-internal
+    intelmq:
+        image: certat/intelmq-full:latest
+        volumes:
+            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+            - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
             - ./intelmq_logs:/opt/intelmq/var/log
-            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
-        ports: 
-            - 127.0.0.1:1337:8080/tcp
-        depends_on: 
+            - ./intelmq_output:/opt/intelmq/var/lib/bots
+        depends_on:
             - redis
-            - postgres
-        environment: 
-            INTELMQ_IS_DOCKER: "true"
-            INTELMQ_PIPELINE_DRIVER: "redis"
+        environment:
+            INTELMQ_SOURCE_PIPELINE_BROKER: "redis"
+            INTELMQ_PIPELINE_BROKER: "redis"
+            INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis"
             INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_SOURCE_PIPELINE_HOST: redis
+            INTELMQ_DESTINATION_PIPELINE_HOST: redis
             INTELMQ_REDIS_CACHE_HOST: redis
-            INTELMQ_MANAGER_CONFIG: "/opt/intelmq-manager/config/config.json"
         networks:
-            - intelmq-network
-            - intelmq-database
+            - intelmq-internal
 
 networks:
-    intelmq-network:
+    intelmq-internal:
         driver: bridge
-    intelmq-database:
-        driver: bridge
-        

entrypoint.sh

@@ -1,7 +1,20 @@
 #!/bin/bash
+export INTELMQ_IS_DOCKER=1
+sudo chown -R intelmq:intelmq /etc/intelmq
+sudo chown -R intelmq:intelmq /opt/intelmq
+
+intelmqctl upgrade-config
+intelmqctl check
+
+intelmq_user="${INTELMQ_API_USER:=intelmq}"
+intelmq_pass="${INTELMQ_API_PASS:=intelmq}"
+
+intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass"
+
 if [[ $1 == "selftest" ]]
 then
-    nosetests3 /opt/intelmq/intelmq/tests
+    export INTELMQ_TEST_EXOTIC=1
+    pytest-3 /opt/intelmq/intelmq/tests
 else
-    hug -f /opt/intelmq-manager/intelmq_manager/serve.py -p8080
-fi
+    cd /opt/intelmq-api && uvicorn intelmq_api.main:app --port 8080 --host 0.0.0.0
+fi

example_bots/.keep

@@ -0,0 +1 @@
+Link your bots repository here

example_config/intelmq-api/config.json

@@ -0,0 +1,8 @@
+{
+    "intelmq_ctl_cmd": ["intelmqctl"],
+    "allowed_path": "/etc/intelmq/var/lib/bots/",
+    "session_store": "/etc/intelmq/api-session.sqlite",
+    "session_duration": 86400,
+    "allow_origins": ["*"],
+    "html_dir": ""
+}

example_config/intelmq-manager/config.json

@@ -1,4 +0,0 @@
-{
-    "intelmq_ctl_cmd": ["/usr/local/bin/intelmqctl"],
-    "allowed_path": "/opt/intelmq/var/lib/"
-}

example_config/intelmq/etc/defaults.conf

@@ -1,39 +0,0 @@
-{
-    "accuracy": 100,
-    "destination_pipeline_broker": "redis",
-    "destination_pipeline_db": 2,
-    "destination_pipeline_host": "127.0.0.1",
-    "destination_pipeline_password": null,
-    "destination_pipeline_port": 6379,
-    "error_dump_message": true,
-    "error_log_exception": true,
-    "error_log_message": false,
-    "error_max_retries": 3,
-    "error_procedure": "pass",
-    "error_retry_delay": 15,
-    "http_proxy": null,
-    "http_timeout_max_tries": 3,
-    "http_timeout_sec": 30,
-    "http_user_agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
-    "http_verify_cert": true,
-    "https_proxy": null,
-    "load_balance": false,
-    "log_processed_messages_count": 500,
-    "log_processed_messages_seconds": 900,
-    "logging_handler": "file",
-    "logging_level": "INFO",
-    "logging_path": "/opt/intelmq/var/log/",
-    "logging_syslog": "/dev/log",
-    "process_manager": "intelmq",
-    "rate_limit": 0,
-    "source_pipeline_broker": "redis",
-    "source_pipeline_db": 2,
-    "source_pipeline_host": "127.0.0.1",
-    "source_pipeline_password": null,
-    "source_pipeline_port": 6379,
-    "ssl_ca_certificate": null,
-    "statistics_database": 3,
-    "statistics_host": "127.0.0.1",
-    "statistics_password": null,
-    "statistics_port": 6379
-}

example_config/intelmq/etc/feeds.yaml

@@ -287,7 +287,7 @@ providers:
             http_url: https://urlhaus.abuse.ch/feeds/tld/<TLD>/,
               https://urlhaus.abuse.ch/feeds/country/<CC>/, or
               https://urlhaus.abuse.ch/feeds/asn/<ASN>/
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -296,7 +296,7 @@ providers:
             skip_header: false
             default_url_protocol: http://
             type_translation: '{"malware_download": "malware-distribution"}'
-            delimeter: ","
+            delimiter: ","
             columns:
               - time.source
               - source.url
@@ -406,7 +406,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -429,7 +429,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://www.team-cymru.org/Services/Bogons/fullbogons-ipv6.txt
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -542,11 +542,11 @@ providers:
       public: yes
   Turris:
     Greylist:
-      description: The data are processed and clasified every week and behaviour of
+      description: The data are processed and classified every week and behaviour of
         IP addresses that accessed a larger number of Turris routers is evaluated.
         The result is a list of addresses that have tried to obtain information about
-        services on the router or tried to gain access to them. We publish this so
-        called "greylist" that also contains a list of tags for each address which
+        services on the router or tried to gain access to them. The list also
+        contains a list of tags for each address which
         indicate what behaviour of the address was observed.
       additional_information:
       bots:
@@ -561,61 +561,72 @@ providers:
           module: intelmq.bots.parsers.turris.parser
           parameters:
       revision: 2018-01-20
-      documentation: https://project.turris.cz/greylist-data/legend.txt
+      documentation: https://project.turris.cz/en/greylist
       public: yes
-  Malc0de:
-    Bind Format:
-      description: This feed includes FQDN's of malicious hosts, the file format is
-        in Bind file format.
+    Greylist with PGP signature verification:
+      description: |
+        The data are processed and classified every week and behaviour of
+        IP addresses that accessed a larger number of Turris routers is evaluated.
+        The result is a list of addresses that have tried to obtain information about
+        services on the router or tried to gain access to them. The list also
+        contains a list of tags for each address which
+        indicate what behaviour of the address was observed.
+
+        The Turris Greylist feed provides PGP signatures for the provided files.
+        You will need to import the public PGP key from the linked documentation
+        page, currently available at
+        https://pgp.mit.edu/pks/lookup?op=vindex&search=0x10876666
+        or from below.
+        See the URL Fetcher Collector documentation for more information on
+        PGP signature verification.
+
+        PGP Public key:
+        ```
+        -----BEGIN PGP PUBLIC KEY BLOCK-----
+        Version: SKS 1.1.6
+        Comment: Hostname: pgp.mit.edu
+
+        mQINBFRl7D8BEADaRFoDa/+r27Gtqrdn8sZL4aSYTU4Q3gDr3TfigK8H26Un/Y79a/DUL1o0
+        o8SRae3uwVcjJDHZ6KDnxThbqF7URfpuCcCYxOs8p/eu3dSueqEGTODHWF4ChIh2japJDc4t
+        3FQHbIh2e3GHotVqJGhvxMmWqBFoZ/mlWvhjs99FFBZ87qbUNk7l1UAGEXeWeECgz9nGox40
+        3YpCgEsnJJsKC53y5LD/wBf4z+z0GsLg2GMRejmPRgrkSE/d9VjF/+niifAj2ZVFoINSVjjI
+        8wQFc8qLiExdzwLdgc+ggdzk5scY3ugI5IBt1zflxMIOG4BxKj/5IWsnhKMG2NLVGUYOODoG
+        pKhcY0gCHypw1bmkp2m+BDVyg4KM2fFPgQ554DAX3xdukMCzzZyBxR3UdT4dN7xRVhpph3Y2
+        Amh1E/dpde9uwKFk1oRHkRZ3UT1XtpbXtFNY0wCiGXPt6KznJAJcomYFkeLHjJo3nMK0hISV
+        GSNetVLfNWlTkeo93E1innbSaDEN70H4jPivjdVjSrLtIGfr2IudUJI84dGmvMxssWuM2qdg
+        FSzoTHw9UE9KT3SltKPS+F7u9x3h1J492YaVDncATRjPZUBDhbvo6Pcezhup7XTnI3gbRQc2
+        oEUDb933nwuobHm3VsUcf9686v6j8TYehsbjk+zdA4BoS/IdCwARAQABtC5UdXJyaXMgR3Jl
+        eWxpc3QgR2VuZXJhdG9yIDxncmV5bGlzdEB0dXJyaXMuY3o+iQI4BBMBAgAiBQJUZew/AhsD
+        BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRDAQrU3EIdmZoH4D/9Jo6j9RZxCAPTaQ9WZ
+        WOdb1Eqd/206bObEX+xJAago+8vuy+waatHYBM9/+yxh0SIg2g5whd6J7A++7ePpt5XzX6hq
+        bzdG8qGtsCRu+CpDJ40UwHep79Ck6O/A9KbZcZW1z/DhbYT3z/ZVWALy4RtgmyC67Vr+j/C7
+        KNQ529bs3kP9AzvEIeBC4wdKl8dUSuZIPFbgf565zRNKLtHVgVhiuDPcxKmBEl4/PLYF30a9
+        5Tgp8/PNa2qp1DV/EZjcsxvSRIZB3InGBvdKdSzvs4N/wLnKWedj1GGm7tJhSkJa4MLBSOIx
+        yamhTS/3A5Cd1qoDhLkp7DGVXSdgEtpoZDC0jR7nTS6pXojcgQaF7SfJ3cjZaLI5rjsx0YLk
+        G4PzonQKCAAQG1G9haCDniD8NrrkZ3eFiafoKEECRFETIG0BJHjPdSWcK9jtNCupBYb7JCiz
+        Q0hwLh2wrw/wCutQezD8XfsBFFIQC18TsJAVgdHLZnGYkd5dIbV/1scOcm52w6EGIeMBBYlB
+        J2+JNukH5sJDA6zAXNl2I1H1eZsP4+FSNIfB6LdovHVPAjn7qXCw3+IonnQK8+g8YJkbbhKJ
+        sPejfg+ndpe5u0zX+GvQCFBFu03muANA0Y/OOeGIQwU93d/akN0P1SRfq+bDXnkRIJQOD6XV
+        0ZPKVXlNOjy/z2iN2A==
+        =wjkM
+        -----END PGP PUBLIC KEY BLOCK-----
+        ```
       additional_information:
       bots:
         collector:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
-            http_url: https://malc0de.com/bl/ZONES
-            rate_limit: 10800
-            name: __FEED__
+            http_url: https://www.turris.cz/greylist-data/greylist-latest.csv
+            name: Greylist
             provider: __PROVIDER__
+            rate_limit: 43200
+            signature_url: https://www.turris.cz/greylist-data/greylist-latest.csv.asc
+            verify_pgp_signatures: false
         parser:
-          module: intelmq.bots.parsers.malc0de.parser
+          module: intelmq.bots.parsers.turris.parser
           parameters:
       revision: 2018-01-20
-      documentation: http://malc0de.com/dashboard/
-      public: yes
-    Windows Format:
-      description: This feed includes FQDN's of malicious hosts, the file format is
-        in Windows Hosts file format.
-      additional_information:
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: https://malc0de.com/bl/BOOT
-            rate_limit: 10800
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.malc0de.parser
-          parameters:
-      revision: 2018-01-20
-      documentation: http://malc0de.com/dashboard/
-      public: yes
-    IP Blacklist:
-      description: This feed includes IP Addresses of malicious hosts.
-      additional_information:
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: https://malc0de.com/bl/IP_Blacklist.txt
-            rate_limit: 10800
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.malc0de.parser
-          parameters:
-      revision: 2018-01-20
-      documentation: http://malc0de.com/dashboard/
+      documentation: https://project.turris.cz/en/greylist
       public: yes
   University of Toulouse:
     Blacklist:
@@ -1008,6 +1019,50 @@ providers:
       revision: 2018-01-20
       documentation: http://www.blocklist.de/en/export.html
       public: yes
+  CERT-Bund:
+    CB-Report Malware infections via IMAP:
+      description: CERT-Bund sends reports for the malware-infected hosts.
+      additional_information: Traffic from malware related hosts contacting
+        command-and-control servers is caught and sent to national CERT teams.
+        There are two e-mail feeds with identical CSV structure -- one reports on
+        general malware infections, the other on the Avalanche botnet.
+      bots:
+        collector:
+          module: intelmq.bots.collectors.mail.collector_mail_attach
+          parameters:
+            mail_host: __HOST__
+            mail_password: __PASSWORD__
+            mail_ssl: true
+            mail_user: __USERNAME__
+            attach_regex: events.csv
+            extract_files: false
+            rate_limit: 86400
+            subject_regex: ^\\[CB-Report#.* Malware infections (\\(Avalanche\\) )?in country
+            folder: INBOX
+            name: __FEED__
+            provider: __PROVIDER__
+        parser:
+          module: intelmq.bots.parsers.generic.parser_csv
+          parameters:
+            skip_header: true
+            default_url_protocol: http://
+            time_format: from_format|%Y-%m-%d %H:%M:%S
+            delimiter: ","
+            columns:
+              - source.asn
+              - source.ip
+              - time.source
+              - classification.type
+              - malware.name
+              - source.port
+              - destination.ip
+              - destination.port
+              - destination.fqdn
+              - protocol.transport
+            type: infected-system
+      revision: 2020-08-20
+      documentation:
+      public: no
   CERT.PL:
     N6 Stomp Stream:
       description: N6 Collector - CERT.pl's N6 Collector - N6 feed via STOMP interface.
@@ -1081,7 +1136,7 @@ providers:
             http_url: http://support.clean-mx.de/clean-mx/xmlviruses?response=alive&domain=
             http_timeout_sec: 120
             http_user_agent: "{{ your user agent }}"
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1101,7 +1156,7 @@ providers:
             http_url: http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&domain=
             http_timeout_sec: 120
             http_user_agent: "{{ your user agent }}"
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1110,24 +1165,6 @@ providers:
       revision: 2018-01-20
       documentation: http://clean-mx.de/
       public: no
-  Malware Domain List:
-    Blacklist:
-      description: No description provided by feed provider.
-      additional_information:
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: http://www.malwaredomainlist.com/updatescsv.php
-            rate_limit: 3600
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.malwaredomainlist.parser
-          parameters:
-      revision: 2018-01-20
-      documentation: http://www.malwaredomainlist.com/
-      public: yes
   AnubisNetworks:
     Cyberfeed Stream:
       description: Fetches and parsers the Cyberfeed data stream.
@@ -1205,10 +1242,12 @@ providers:
       revision: 2018-01-20
       documentation: https://osint.bambenekconsulting.com/feeds/
       public: yes
-  DynDNS:
-    Infected Domains:
-      description: DynDNS ponmocup. List of ponmocup malware redirection domains and
-        infected web-servers. See also http://security-research.dyndns.org/pub/botnet-links.html
+  cAPTure:
+    Ponmocup Domains CIF Format:
+      description: List of ponmocup malware redirection domains and infected web-servers from cAPTure.
+        See also http://security-research.dyndns.org/pub/botnet-links.htm
+        and http://c-apt-ure.blogspot.com/search/label/ponmocup
+        The data in the CIF format is not equal to the Shadowserver CSV format. Reasons are unknown.
       additional_information:
       bots:
         collector:
@@ -1216,7 +1255,7 @@ providers:
           parameters:
             http_url: http://security-research.dyndns.org/pub/malware-feeds/ponmocup-infected-domains-CIF-latest.txt
             rate_limit: 10800
-            name: __FEED__
+            name: Infected Domains
             provider: __PROVIDER__
         parser:
           module: intelmq.bots.parsers.dyn.parser
@@ -1224,6 +1263,40 @@ providers:
       revision: 2018-01-20
       documentation: http://security-research.dyndns.org/pub/malware-feeds/
       public: yes
+    Ponmocup Domains Shadowserver Format:
+      description: List of ponmocup malware redirection domains and infected web-servers from cAPTure.
+        See also http://security-research.dyndns.org/pub/botnet-links.htm
+        and http://c-apt-ure.blogspot.com/search/label/ponmocup
+        The data in the Shadowserver CSV is not equal to the CIF format format. Reasons are unknown.
+      additional_information:
+      bots:
+        collector:
+          module: intelmq.bots.collectors.http.collector_http
+          parameters:
+            http_url: http://security-research.dyndns.org/pub/malware-feeds/ponmocup-infected-domains-shadowserver.csv
+            rate_limit: 10800
+            name: Infected Domains
+            provider: __PROVIDER__
+        parser:
+          module: intelmq.bots.parsers.generic.parser_csv
+          parameters:
+            columns:
+              - time.source
+              - source.ip
+              - source.fqdn
+              - source.urlpath
+              - source.port
+              - protocol.application
+              - extra.tag
+              - extra.redirect_target
+              - extra.category
+            compose_fields: {"source.url": "http://{0}{1}"}
+            skip_header: true
+            delimiter: ","
+            type: malware-distribution
+      revision: 2020-07-08
+      documentation: http://security-research.dyndns.org/pub/malware-feeds/
+      public: yes
   DShield:
     Suspicious Domains:
       description: There are many suspicious domains on the internet. In an effort
@@ -1236,7 +1309,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://www.dshield.org/feeds/suspiciousdomains_High.txt
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1255,7 +1328,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://www.dshield.org/block.txt
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1272,7 +1345,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://dshield.org/asdetailsascii.html?as={{ AS Number }}
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1299,7 +1372,7 @@ providers:
       revision: 2018-01-20
       documentation: http://vxvault.net/ViriList.php
       public: yes
-  ShadowServer:
+  Shadowserver:
     Via IMAP:
       description: Shadowserver sends out a variety of reports (see https://www.shadowserver.org/wiki/pmwiki.php/Services/Reports).
       additional_information: The configuration retrieves the data from a e-mails via IMAP from the attachments.
@@ -1356,6 +1429,28 @@ providers:
       revision: 2018-01-20
       documentation: https://www.shadowserver.org/what-we-do/network-reporting/
       public: no
+    Via API:
+      description: Shadowserver sends out a variety of reports to subscribers, see documentation.
+      additional_information: This configuration fetches user-configurable reports from the Shadowserver Reports API. For a list of reports, have a look at the Shadowserver collector and parser documentation.
+      bots:
+        collector:
+          module: intelmq.bots.collectors.shadowserver.collector_reports_api
+          parameters:
+            country: <CC>
+            api_key: <API key>
+            secret: <API secret>
+            types: <single report or list of reports>
+            rate_limit: 86400
+            redis_cache_db: 12
+            redis_cache_host: 127.0.0.1
+            redis_cache_port: 6379
+            redis_cache_ttl: 864000
+        parser:
+          module: intelmq.bots.parsers.shadowserver.parser_json
+          parameters:
+      revision: 2020-01-08
+      documentation: https://www.shadowserver.org/what-we-do/network-reporting/api-documentation/
+      public: no
   Fraunhofer:
     DGA Archive:
       description: Fraunhofer DGA collector fetches data from Fraunhofer's domain
@@ -1417,7 +1512,7 @@ providers:
       documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange
       public: no
     CTIP via Interflow:
-      description: Collects CTIP (Sinkhole data) files from the Interflow API.The feed is available via Microsoft’s Government Security Program (GSP).
+      description: Collects the CTIP Infected feed (Sinkhole data for your country) files from the Interflow API.The feed is available via Microsoft’s Government Security Program (GSP).
       additional_information: Depending on the file sizes you may need to increase the parameter 'http_timeout_sec' of the collector. As many IPs occur very often in the data, you may want to use a deduplicator specifically for the feed.
       bots:
         collector:
@@ -1436,8 +1531,8 @@ providers:
       revision: 2018-03-06
       documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange
       public: no
-    CTIP via Azure:
-      description: Collects CTIP (Sinkhole data) files from a shared Azure Storage. The feed is available via Microsoft’s Government Security Program (GSP).
+    CTIP Infected via Azure:
+      description: Collects the CTIP (Sinkhole data) from a shared Azure Storage. The feed is available via Microsoft’s Government Security Program (GSP).
       additional_information: The cache is needed for memorizing which files have already been processed, the TTL should be higher than the oldest file available in the storage (currently the last three days are available). The connection string contains endpoint as well as authentication information.
       bots:
         collector:
@@ -1458,6 +1553,28 @@ providers:
       revision: 2020-05-29
       documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange
       public: no
+    CTIP C2 via Azure:
+      description: Collects the CTIP C2 feed from a shared Azure Storage. The feed is available via Microsoft’s Government Security Program (GSP).
+      additional_information: The cache is needed for memorizing which files have already been processed, the TTL should be higher than the oldest file available in the storage (currently the last three days are available). The connection string contains endpoint as well as authentication information.
+      bots:
+        collector:
+          module: intelmq.bots.collectors.microsoft.collector_azure
+          parameters:
+            connection_string: "{{your connection string}}"
+            container_name: "ctip-c2"
+            name: __FEED__
+            provider: __PROVIDER__
+            rate_limit: 3600
+            redis_cache_db: 5
+            redis_cache_host: 127.0.0.1
+            redis_cache_port: 6379
+            redis_cache_ttl: 864000
+        parser:
+          module: intelmq.bots.parsers.microsoft.parser_ctip
+          parameters:
+      revision: 2020-05-29
+      documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange
+      public: no
   Threatminer:
     Recent domains:
       description: Latest malicious domains.
@@ -1563,10 +1680,10 @@ providers:
             listen 443 ssl http2;
             server_name [your host name];
             client_max_body_size 50M;
-            
+
             ssl_certificate [path to your key];
             ssl_certificate_key [path to your certificate];
-            
+
             location /[your private url] {
                  if ($http_authorization != '[your private password]') {
                      return 403;
@@ -1595,7 +1712,7 @@ providers:
     DailyIOC:
       description: Daily IOC from tweets and articles
       additional_information: |
-        collector's `extra_fields` parameter may be any of fields from the github [content API response](https://developer.github.com/v3/repos/contents/)
+        collector's `extra_fields` parameter may be any of fields from the github `content API response <https://developer.github.com/v3/repos/contents/>`_
       bots:
         collector:
           module: intelmq.bots.collectors.github_api.collector_github_contents_api
@@ -1612,7 +1729,7 @@ providers:
       public: yes
   CZ.NIC:
     HaaS:
-      description: SSH attackers against HaaS (Honeypot as a Sevice) provided by CZ.NIC, z.s.p.o. The dump is published once a day.
+      description: SSH attackers against HaaS (Honeypot as a Service) provided by CZ.NIC, z.s.p.o. The dump is published once a day.
       bots:
         collector:
           module: intelmq.bots.collectors.http.collector_http
@@ -1628,6 +1745,24 @@ providers:
       revision: 2020-07-22
       documentation: https://haas.nic.cz/
       public: yes
+    Proki:
+      description: Aggregation of various sources on malicious IP addresses (malware spreaders or C&C servers).
+      bots:
+        collector:
+          module: intelmq.bots.collectors.http.collector_http
+          parameters:
+            http_url: https://proki.csirt.cz/api/1/__APIKEY__/data/day/{time[%Y/%m/%d]}
+            http_url_formatting:
+              days: -1
+            rate_limit: 86400
+            name: __FEED__
+            provider: __PROVIDER__
+        parser:
+          module: intelmq.bots.parsers.cznic.parser_proki
+          parameters:
+      revision: 2020-08-17
+      documentation: https://csirt.cz/en/proki/
+      public: no
   ESET:
     ETI Domains:
       description: Domain data from ESET's TAXII API.
@@ -1665,3 +1800,25 @@ providers:
       revision: 2020-06-30
       documentation: https://www.eset.com/int/business/services/threat-intelligence/
       public: no
+  Shodan:
+    Country Stream:
+      description: Collects the Shodan stream for one or multiple countries from the Shodan API.
+      additional_information: A Shodan account with streaming permissions is needed.
+      bots:
+        collector:
+          module: intelmq.bots.collectors.shodan.collector_stream
+          parameters:
+            api_key: <API key>
+            countries: <comma-separated list of country codes>
+            error_retry_delay: 0
+            name: __FEED__
+            provider: __PROVIDER__
+        parser:
+          module: intelmq.bots.parsers.shodan.parser
+          parameters:
+            ignore_errors: false
+            error_retry_delay: 0
+            minimal_mode: false
+      revision: 2021-03-22
+      documentation: https://developer.shodan.io/api/stream
+      public: no

example_config/intelmq/etc/harmonization.conf

@@ -1,16 +1,16 @@
 {
     "event": {
         "classification.identifier": {
-            "description": "The lowercase identifier defines the actual software or service (e.g. 'heartbleed' or 'ntp_version') or standardized malware name (e.g. 'zeus'). Note that you MAY overwrite this field during processing for your individual setup. This field is not standardized across IntelMQ setups/users.",
+            "description": "The lowercase identifier defines the actual software or service (e.g. ``heartbleed`` or ``ntp_version``) or standardized malware name (e.g. ``zeus``). Note that you MAY overwrite this field during processing for your individual setup. This field is not standardized across IntelMQ setups/users.",
             "type": "String"
         },
         "classification.taxonomy": {
-            "description": "We recognize the need for the CSIRT teams to apply a static (incident) taxonomy to abuse data. With this goal in mind the type IOC will serve as a basis for this activity. Each value of the dynamic type mapping translates to a an element in the static taxonomy. The European CSIRT teams for example have decided to apply the eCSIRT.net incident classification. The value of the taxonomy key is thus a derivative of the dynamic type above. For more information about check [ENISA taxonomies](http://www.enisa.europa.eu/activities/cert/support/incident-management/browsable/incident-handling-process/incident-taxonomy/existing-taxonomies).",
+            "description": "We recognize the need for the CSIRT teams to apply a static (incident) taxonomy to abuse data. With this goal in mind the type IOC will serve as a basis for this activity. Each value of the dynamic type mapping translates to a an element in the static taxonomy. The European CSIRT teams for example have decided to apply the eCSIRT.net incident classification. The value of the taxonomy key is thus a derivative of the dynamic type above. For more information about check `ENISA taxonomies <http://www.enisa.europa.eu/activities/cert/support/incident-management/browsable/incident-handling-process/incident-taxonomy/existing-taxonomies>`_.",
             "length": 100,
-            "type": "LowercaseString"
+            "type": "ClassificationTaxonomy"
         },
         "classification.type": {
-            "description": "The abuse type IOC is one of the most crucial pieces of information for any given abuse event. The main idea of dynamic typing is to keep our ontology flexible, since we need to evolve with the evolving threatscape of abuse data. In contrast with the static taxonomy below, the dynamic typing is used to perform business decisions in the abuse handling pipeline. Furthermore, the value data set should be kept as minimal as possible to avoid \u201ctype explosion\u201d, which in turn dilutes the business value of the dynamic typing. In general, we normally have two types of abuse type IOC: ones referring to a compromised resource or ones referring to pieces of the criminal infrastructure, such as a command and control servers for example.",
+            "description": "The abuse type IOC is one of the most crucial pieces of information for any given abuse event. The main idea of dynamic typing is to keep our ontology flexible, since we need to evolve with the evolving threatscape of abuse data. In contrast with the static taxonomy below, the dynamic typing is used to perform business decisions in the abuse handling pipeline. Furthermore, the value data set should be kept as minimal as possible to avoid *type explosion*, which in turn dilutes the business value of the dynamic typing. In general, we normally have two types of abuse type IOC: ones referring to a compromised resource or ones referring to pieces of the criminal infrastructure, such as a command and control servers for example.",
             "type": "ClassificationType"
         },
         "comment": {
@@ -356,7 +356,7 @@
             "type": "DateTime"
         },
         "time.source": {
-            "description": "The time of occurence of the event as reported the feed (source).",
+            "description": "The time of occurrence of the event as reported the feed (source).",
             "type": "DateTime"
         },
         "tlp": {

example_config/intelmq/etc/harmonization.conf.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2016 Sebastian Wagner
+SPDX-License-Identifier: AGPL-3.0-or-later

example_config/intelmq/etc/manager/positions.conf

@@ -27,14 +27,6 @@
         "x": -252,
         "y": 243
     },
-    "malc0de-parser": {
-        "x": 297,
-        "y": 24
-    },
-    "malc0de-windows-format-collector": {
-        "x": 433,
-        "y": 121
-    },
     "malware-domain-list-collector": {
         "x": 465,
         "y": -198

example_config/intelmq/etc/pipeline.conf

@@ -1,86 +0,0 @@
-{
-    "cymru-whois-expert": {
-        "source-queue": "cymru-whois-expert-queue",
-        "destination-queues": [
-            "file-output-queue"
-        ]
-    },
-    "deduplicator-expert": {
-        "source-queue": "deduplicator-expert-queue",
-        "destination-queues": [
-            "taxonomy-expert-queue"
-        ]
-    },
-    "feodo-tracker-browse-collector": {
-        "destination-queues": [
-            "feodo-tracker-browse-parser-queue"
-        ]
-    },
-    "feodo-tracker-browse-parser": {
-        "source-queue": "feodo-tracker-browse-parser-queue",
-        "destination-queues": [
-            "deduplicator-expert-queue"
-        ]
-    },
-    "file-output": {
-        "source-queue": "file-output-queue"
-    },
-    "gethostbyname-1-expert": {
-        "source-queue": "gethostbyname-1-expert-queue",
-        "destination-queues": [
-            "cymru-whois-expert-queue"
-        ]
-    },
-    "gethostbyname-2-expert": {
-        "source-queue": "gethostbyname-2-expert-queue",
-        "destination-queues": [
-            "cymru-whois-expert-queue"
-        ]
-    },
-    "malc0de-parser": {
-        "source-queue": "malc0de-parser-queue",
-        "destination-queues": [
-            "deduplicator-expert-queue"
-        ]
-    },
-    "malc0de-windows-format-collector": {
-        "destination-queues": [
-            "malc0de-parser-queue"
-        ]
-    },
-    "malware-domain-list-collector": {
-        "destination-queues": [
-            "malware-domain-list-parser-queue"
-        ]
-    },
-    "malware-domain-list-parser": {
-        "source-queue": "malware-domain-list-parser-queue",
-        "destination-queues": [
-            "deduplicator-expert-queue"
-        ]
-    },
-    "spamhaus-drop-collector": {
-        "destination-queues": [
-            "spamhaus-drop-parser-queue"
-        ]
-    },
-    "spamhaus-drop-parser": {
-        "source-queue": "spamhaus-drop-parser-queue",
-        "destination-queues": [
-            "deduplicator-expert-queue"
-        ]
-    },
-    "taxonomy-expert": {
-        "source-queue": "taxonomy-expert-queue",
-        "destination-queues": [
-            "url2fqdn-expert-queue"
-        ]
-    },
-    "url2fqdn-expert": {
-        "source-queue": "url2fqdn-expert-queue",
-        "destination-queues": [
-            "gethostbyname-1-expert-queue",
-            "gethostbyname-2-expert-queue"
-        ]
-    }
-}

example_config/intelmq/etc/runtime.conf

@@ -1,230 +0,0 @@
-{
-    "cymru-whois-expert": {
-        "bot_id": "cymru-whois-expert",
-        "description": "Cymru Whois (IP to ASN) is the bot responsible to add network information to the events (BGP, ASN, AS Name, Country, etc..).",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.cymru_whois.expert",
-        "name": "Cymru Whois",
-        "parameters": {
-            "overwrite": true,
-            "redis_cache_db": 5,
-            "redis_cache_password": null,
-            "redis_cache_port": 6379,
-            "redis_cache_ttl": 86400
-        },
-        "run_mode": "continuous"
-    },
-    "deduplicator-expert": {
-        "bot_id": "deduplicator-expert",
-        "description": "Deduplicator is the bot responsible for detection and removal of duplicate messages. Messages get cached for <redis_cache_ttl> seconds. If found in the cache, it is assumed to be a duplicate.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.deduplicator.expert",
-        "name": "Deduplicator",
-        "parameters": {
-            "filter_keys": "raw,time.observation",
-            "filter_type": "blacklist",
-            "redis_cache_db": 6,
-            "redis_cache_port": 6379,
-            "redis_cache_ttl": 86400
-        },
-        "run_mode": "continuous"
-    },
-    "feodo-tracker-browse-collector": {
-        "description": "Generic URL Fetcher is the bot responsible to get the report from an URL.",
-        "enabled": true,
-        "group": "Collector",
-        "module": "intelmq.bots.collectors.http.collector_http",
-        "name": "URL Fetcher",
-        "parameters": {
-            "extract_files": false,
-            "http_password": null,
-            "http_url": "https://feodotracker.abuse.ch/browse",
-            "http_url_formatting": false,
-            "http_username": null,
-            "name": "Feodo Tracker Browse",
-            "provider": "Abuse.ch",
-            "rate_limit": 86400,
-            "ssl_client_certificate": null
-        },
-        "run_mode": "continuous",
-        "groupname": "collectors",
-        "bot_id": "feodo-tracker-browse-collector"
-    },
-    "feodo-tracker-browse-parser": {
-        "description": "HTML Table Parser is a bot configurable to parse different html table data.",
-        "enabled": true,
-        "group": "Parser",
-        "module": "intelmq.bots.parsers.html_table.parser",
-        "name": "HTML Table",
-        "parameters": {
-            "attribute_name": "",
-            "attribute_value": "",
-            "columns": "time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc",
-            "default_url_protocol": "http://",
-            "ignore_values": ",,,,Not listed,,",
-            "skip_table_head": true,
-            "split_column": "",
-            "split_index": 0,
-            "split_separator": "",
-            "table_index": 0,
-            "time_format": null,
-            "type": "c2server"
-        },
-        "run_mode": "continuous",
-        "groupname": "parsers",
-        "bot_id": "feodo-tracker-browse-parser"
-    },
-    "file-output": {
-        "bot_id": "file-output",
-        "description": "File is the bot responsible to send events to a file.",
-        "enabled": true,
-        "group": "Output",
-        "groupname": "outputs",
-        "module": "intelmq.bots.outputs.file.output",
-        "name": "File",
-        "parameters": {
-            "file": "/opt/intelmq/var/lib/bots/file-output/events.txt",
-            "hierarchical_output": false,
-            "single_key": null
-        },
-        "run_mode": "continuous"
-    },
-    "gethostbyname-1-expert": {
-        "bot_id": "gethostbyname-1-expert",
-        "description": "fqdn2ip is the bot responsible to parsing the ip from the fqdn.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.gethostbyname.expert",
-        "name": "Gethostbyname",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "gethostbyname-2-expert": {
-        "bot_id": "gethostbyname-2-expert",
-        "description": "fqdn2ip is the bot responsible to parsing the ip from the fqdn.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.gethostbyname.expert",
-        "name": "Gethostbyname",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "malc0de-parser": {
-        "bot_id": "malc0de-parser",
-        "description": "Malc0de Parser is the bot responsible to parse the IP Blacklist and either Windows Format or Bind Format reports and sanitize the information.",
-        "enabled": true,
-        "group": "Parser",
-        "groupname": "parsers",
-        "module": "intelmq.bots.parsers.malc0de.parser",
-        "name": "Malc0de",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "malc0de-windows-format-collector": {
-        "bot_id": "malc0de-windows-format-collector",
-        "description": "",
-        "enabled": true,
-        "group": "Collector",
-        "groupname": "collectors",
-        "module": "intelmq.bots.collectors.http.collector_http",
-        "name": "Malc0de Windows Format",
-        "parameters": {
-            "http_password": null,
-            "http_url": "https://malc0de.com/bl/BOOT",
-            "http_username": null,
-            "name": "Windows Format",
-            "provider": "Malc0de",
-            "rate_limit": 10800,
-            "ssl_client_certificate": null
-        },
-        "run_mode": "continuous"
-    },
-    "malware-domain-list-collector": {
-        "bot_id": "malware-domain-list-collector",
-        "description": "Malware Domain List Collector is the bot responsible to get the report from source of information.",
-        "enabled": true,
-        "group": "Collector",
-        "groupname": "collectors",
-        "module": "intelmq.bots.collectors.http.collector_http",
-        "name": "Malware Domain List",
-        "parameters": {
-            "http_url": "http://www.malwaredomainlist.com/updatescsv.php",
-            "name": "Malware Domain List",
-            "provider": "Malware Domain List",
-            "rate_limit": 3600
-        },
-        "run_mode": "continuous"
-    },
-    "malware-domain-list-parser": {
-        "bot_id": "malware-domain-list-parser",
-        "description": "Malware Domain List Parser is the bot responsible to parse the report and sanitize the information.",
-        "enabled": true,
-        "group": "Parser",
-        "groupname": "parsers",
-        "module": "intelmq.bots.parsers.malwaredomainlist.parser",
-        "name": "Malware Domain List",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "spamhaus-drop-collector": {
-        "bot_id": "spamhaus-drop-collector",
-        "description": "",
-        "enabled": true,
-        "group": "Collector",
-        "groupname": "collectors",
-        "module": "intelmq.bots.collectors.http.collector_http",
-        "name": "Spamhaus Drop",
-        "parameters": {
-            "http_password": null,
-            "http_url": "https://www.spamhaus.org/drop/drop.txt",
-            "http_username": null,
-            "name": "Drop",
-            "provider": "Spamhaus",
-            "rate_limit": 3600,
-            "ssl_client_certificate": null
-        },
-        "run_mode": "continuous"
-    },
-    "spamhaus-drop-parser": {
-        "bot_id": "spamhaus-drop-parser",
-        "description": "Spamhaus Drop Parser is the bot responsible to parse the DROP, EDROP, DROPv6, and ASN-DROP reports and sanitize the information.",
-        "enabled": true,
-        "group": "Parser",
-        "groupname": "parsers",
-        "module": "intelmq.bots.parsers.spamhaus.parser_drop",
-        "name": "Spamhaus Drop",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "taxonomy-expert": {
-        "bot_id": "taxonomy-expert",
-        "description": "Taxonomy is the bot responsible to apply the eCSIRT Taxonomy to all events.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.taxonomy.expert",
-        "name": "Taxonomy",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "url2fqdn-expert": {
-        "bot_id": "url2fqdn-expert",
-        "description": "url2fqdn is the bot responsible to parsing the fqdn from the url.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.url2fqdn.expert",
-        "name": "URL2FQDN",
-        "parameters": {
-            "load_balance": true,
-            "overwrite": false
-        },
-        "run_mode": "continuous"
-    }
-}

example_config/intelmq/etc/runtime.yaml

@@ -0,0 +1,179 @@
+cymru-whois-expert:
+  bot_id: cymru-whois-expert
+  description: Cymru Whois (IP to ASN) is the bot responsible to add network information
+    to the events (BGP, ASN, AS Name, Country, etc..).
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.cymru_whois.expert
+  name: Cymru Whois
+  parameters:
+    destination_queues:
+      _default: [file-output-queue]
+    overwrite: true
+    redis_cache_db: 5
+    redis_cache_host: 127.0.0.1
+    redis_cache_password: null
+    redis_cache_port: 6379
+    redis_cache_ttl: 86400
+  run_mode: continuous
+deduplicator-expert:
+  bot_id: deduplicator-expert
+  description: Deduplicator is the bot responsible for detection and removal of duplicate
+    messages. Messages get cached for <redis_cache_ttl> seconds. If found in the cache,
+    it is assumed to be a duplicate.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.deduplicator.expert
+  name: Deduplicator
+  parameters:
+    destination_queues:
+      _default: [taxonomy-expert-queue]
+    filter_keys: raw,time.observation
+    filter_type: blacklist
+    redis_cache_db: 6
+    redis_cache_host: 127.0.0.1
+    redis_cache_port: 6379
+    redis_cache_ttl: 86400
+  run_mode: continuous
+feodo-tracker-browse-collector:
+  description: Generic URL Fetcher is the bot responsible to get the report from an
+    URL.
+  enabled: true
+  group: Collector
+  module: intelmq.bots.collectors.http.collector_http
+  name: URL Fetcher
+  parameters:
+    destination_queues:
+      _default: [feodo-tracker-browse-parser-queue]
+    extract_files: false
+    http_password: null
+    http_url: https://feodotracker.abuse.ch/browse
+    http_url_formatting: false
+    http_username: null
+    name: Feodo Tracker Browse
+    provider: Abuse.ch
+    rate_limit: 86400
+    ssl_client_certificate: null
+  run_mode: continuous
+feodo-tracker-browse-parser:
+  description: HTML Table Parser is a bot configurable to parse different html table
+    data.
+  enabled: true
+  group: Parser
+  module: intelmq.bots.parsers.html_table.parser
+  name: HTML Table
+  parameters:
+    attribute_name: ''
+    attribute_value: ''
+    columns: time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc
+    default_url_protocol: http://
+    destination_queues:
+      _default: [deduplicator-expert-queue]
+    ignore_values: ',,,,,'
+    skip_table_head: true
+    split_column: ''
+    split_index: 0
+    split_separator: ''
+    table_index: 0
+    time_format: null
+    type: c2-server
+  run_mode: continuous
+file-output:
+  bot_id: file-output
+  description: File is the bot responsible to send events to a file.
+  enabled: true
+  group: Output
+  groupname: outputs
+  module: intelmq.bots.outputs.file.output
+  name: File
+  parameters: {file: /opt/intelmq/var/lib/bots/file-output/events.txt, hierarchical_output: false,
+    single_key: null}
+  run_mode: continuous
+gethostbyname-1-expert:
+  bot_id: gethostbyname-1-expert
+  description: fqdn2ip is the bot responsible to parsing the ip from the fqdn.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.gethostbyname.expert
+  name: Gethostbyname
+  parameters:
+    destination_queues:
+      _default: [cymru-whois-expert-queue]
+  run_mode: continuous
+gethostbyname-2-expert:
+  bot_id: gethostbyname-2-expert
+  description: fqdn2ip is the bot responsible to parsing the ip from the fqdn.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.gethostbyname.expert
+  name: Gethostbyname
+  parameters:
+    destination_queues:
+      _default: [cymru-whois-expert-queue]
+  run_mode: continuous
+global: {destination_pipeline_broker: redis, process_manager: intelmq, source_pipeline_broker: redis,
+  ssl_ca_certificate: null, statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null,
+  statistics_port: 6379, destination_pipeline_host: redis, source_pipeline_host: redis}
+spamhaus-drop-collector:
+  bot_id: spamhaus-drop-collector
+  description: ''
+  enabled: true
+  group: Collector
+  groupname: collectors
+  module: intelmq.bots.collectors.http.collector_http
+  name: Spamhaus Drop
+  parameters:
+    destination_queues:
+      _default: [spamhaus-drop-parser-queue]
+    http_password: null
+    http_url: https://www.spamhaus.org/drop/drop.txt
+    http_username: null
+    name: Drop
+    provider: Spamhaus
+    rate_limit: 3600
+    ssl_client_certificate: null
+  run_mode: continuous
+spamhaus-drop-parser:
+  bot_id: spamhaus-drop-parser
+  description: Spamhaus Drop Parser is the bot responsible to parse the DROP, EDROP,
+    DROPv6, and ASN-DROP reports and sanitize the information.
+  enabled: true
+  group: Parser
+  groupname: parsers
+  module: intelmq.bots.parsers.spamhaus.parser_drop
+  name: Spamhaus Drop
+  parameters:
+    destination_queues:
+      _default: [deduplicator-expert-queue]
+  run_mode: continuous
+taxonomy-expert:
+  bot_id: taxonomy-expert
+  description: Taxonomy is the bot responsible to apply the eCSIRT Taxonomy to all
+    events.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.taxonomy.expert
+  name: Taxonomy
+  parameters:
+    destination_queues:
+      _default: [url2fqdn-expert-queue]
+  run_mode: continuous
+url2fqdn-expert:
+  bot_id: url2fqdn-expert
+  description: url2fqdn is the bot responsible to parsing the fqdn from the url.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.url2fqdn.expert
+  name: URL2FQDN
+  parameters:
+    destination_queues:
+      _default: [gethostbyname-1-expert-queue, gethostbyname-2-expert-queue]
+    load_balance: true
+    overwrite: false
+  run_mode: continuous

example_config/intelmq/etc/runtime.yaml.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2021 Birger Schacht
+SPDX-License-Identifier: AGPL-3.0-or-later

inspect-container.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+base_path=$(pwd)
+echo $base_path
+
+echo [START] Creating new network
+network_id=$(docker network create -d bridge intelmq-testing-network)
+echo [DONE ] Network created
+
+echo [START] Setting up redis container
+redis_id=$(docker run --rm -d --network=intelmq-testing-network -p 6379:6379 -v $base_path/intelmq_docker/example_config/redis/redis.conf:/redis.conf redis:latest)
+redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
+echo [DONE ] Redis container running $redis_ip:6379
+
+echo [START] IntelMQ
+intelmq_id=$(docker run --network=intelmq-testing-network --cap-add=SYS_PTRACE -p 8080:8080 --rm -d -v $base_path/intelmq_persistence:/opt/intelmq_persistence -v $base_path/example_config/intelmq/etc:/opt/intelmq/etc -v $base_path/example_config/intelmq-api/config.json:/etc/intelmq/api-config.json:ro -v $base_path/intelmq_logs:/opt/intelmq/var/log -v $base_path/example_config/intelmq/var/lib:/opt/intelmq/var/lib -e "INTELMQ_IS_DOCKER=true" -e "INTELMQ_SOURCE_PIPELINE_BROKER=redis" -e "INTELMQ_PIPELINE_BROKER=redis" -e "INTELMQ_DESTIONATION_PIPELINE_BROKER=redis" -e "INTELMQ_PIPELINE_HOST=$redis_ip" -e "INTELMQ_SOURCE_PIPELINE_HOST=$redis_ip" -e "INTELMQ_DESTINATION_PIPELINE_HOST=$redis_ip" -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" intelmq-full:latest)
+intelmq_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $intelmq_id)
+echo [DONE ] IntelMQ running
+
+echo [START] IntelMQ-Manager
+intelmq_manager_id=$(docker run --rm -d -p 1337:80 --network=intelmq-testing-network --add-host intelmq:$intelmq_ip intelmq-nginx:latest)
+echo [DONE ] IntelMQ-Manager running
+
+echo [START] Preparing profiling
+docker exec -it $intelmq_id bash -c 'sudo apt update && sudo apt install -y htop && sudo pip3 install py-spy shodan'
+echo [DONE ] All profiling installed
+
+echo Execing into intelmq instance
+docker exec -it $intelmq_id /bin/bash
+
+echo Killing all containers
+docker container kill $redis_id $intelmq_id $intelmq_manager_id
+
+echo Removing network
+docker network rm intelmq-testing-network

intelmq_output/.gitignore

@@ -0,0 +1 @@
+*

publish.sh

@@ -1,8 +1,17 @@
 #!/bin/bash
-build_version="1.0"
+build_version="3.5.0"
+namespace="certat"
 
 docker login
 
-docker tag intelmq-full:$build_version certat/intelmq-full:$build_version
+docker tag intelmq-nginx:latest $namespace/intelmq-nginx:latest
 
-docker push certat/intelmq-full:$build_version
+docker push $namespace/intelmq-nginx:latest
+
+docker tag intelmq-full:latest $namespace/intelmq-full:latest
+docker tag intelmq-full:latest $namespace/intelmq-full:1.0
+docker tag intelmq-full:latest $namespace/intelmq-full:$build_version
+
+docker push $namespace/intelmq-full:latest
+docker push $namespace/intelmq-full:1.0
+docker push $namespace/intelmq-full:$build_version

test.sh

@@ -1,16 +1,53 @@
 #!/bin/bash
-redis_id=$(sudo docker run --rm -d -p 6379:6379 -v ~/intelmq-docker/example_config/redis/redis.conf:/redis.conf redis:latest)
+echo RUNNING TESTS WITH REDIS
+echo Setting up redis container
+redis_id=$(docker run --rm -d -p 6379:6379 -v ~/example_config/redis/redis.conf:/redis.conf redis:latest)
+redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
 
-redis_ip=$(sudo docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
-
-sudo docker run --rm -v ~/intelmq-docker/example_config/intelmq/etc:/opt/intelmq/etc \
-    -v ~/intelmq-docker/example_config/intelmq-manager:/opt/intelmq-manager/config \
-    -v ~/intelmq-docker/intelmq_logs:/opt/intelmq/var/log \
-    -v ~/intelmq-docker/example_config/intelmq/var/lib:/opt/intelmq/var/lib \
-    -e "INTELMQ_IS_DOCKER=\"true\"" \
-    -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \
+echo Setting up IntelMQ-Container
+docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \
+    -v $(pwd)/example_config/intelmq-api:/etc/intelmq-api/config \
+    -v $(pwd)/intelmq_logs:/etc/intelmq/var/log \
+    -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \
+    -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \
+    -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \
+    -e "INTELMQ_PIPELINE_DRIVER=redis" \
     -e "INTELMQ_PIPELINE_HOST=$redis_ip" \
+    -e "INTELMQ_SOURCE_PIPELINE_HOST=$redis_ip" \
+    -e "INTELMQ_DESTINATION_PIPELINE_HOST=$redis_ip" \
     -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \
-    -e "INTELMQ_MANAGER_CONFIG=\"/opt/intelmq-manager/config/config.json\"" \
-    intelmq-full:1.0 selftest
-sudo docker container stop $redis_id
+    intelmq-full:latest selftest
+
+echo Removing redis container
+docker container kill $redis_id
+
+echo RUNNING TESTS WITH AMQP
+
+echo Setting up AMQP container
+amq_id=$(docker run --rm -d -p 5672:5672 -p 15672:15672 rabbitmq:latest)
+amp_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks}}{{ .IPAddress }}{{ end }}' $amq_id)
+
+echo Setting up IntelMQ-Container
+docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \
+    -v $(pwd)/example_config/intelmq-api:/etc/intelmq-api/config \
+    -v $(pwd)/intelmq_logs:/etc/intelmq/var/log \
+    -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \
+    -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \
+    -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \
+    -e "INTELMQ_PIPELINE_DRIVER=\"amqp\"" \
+    -e "INTELMQ_PIPELINE_HOST=$amq_id" \
+    -e "INTELMQ_SOURCE_PIPELINE_HOST=$amq_ip" \
+    -e "INTELMQ_DESTINATION_PIPELINE_HOST=$amq_ip" \
+    -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \
+    intelmq-full:latest selftest
+
+echo Removing AMQP container
+docker container kill $amq_id
+
+# restore broke priviliges
+
+for mounted_dir in example_config intelmq_logs intelmq_output intelmq_persistence;
+do
+    echo "Restoring broken privelages to `whoami` for directory $mounted_dir"
+    sudo chown -R `whoami`:`whoami` $(pwd)/$mounted_dir
+done

update_submodules.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+git submodule update --remote
+git pull --recurse-submodules

versions.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+intelmq_full_built=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.created" }}' intelmq-full:latest)
+intelmq_full_vers=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' intelmq-full:latest)
+intelmq_full_rev=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' intelmq-full:latest)
+
+echo IntelMQ built at \"$intelmq_full_built\" \(Version $intelmq_full_vers\)
+revisions=$(echo $intelmq_full_rev | tr "," "\n")
+for rev in $revisions
+do
+    echo "> $rev"
+done