FIX: Remove server_tokens from nginx

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

.docker/intelmq-full-dev/Dockerfile

@@ -0,0 +1,20 @@
+FROM certat/intelmq-full:1.0
+
+MAINTAINER Einar <elanfranco@cert.unlp.edu.ar>
+MAINTAINER Jeremias <jpretto@cert.unlp.edu.ar>
+
+ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh
+ADD update.sh /opt/dev/update.sh
+ADD merge_BOTS.py /opt/dev/merge_BOTS.py
+
+# Merge bots for merge_BOTS.py
+RUN sudo python3 -m pip install jsonmerge
+
+
+# Permission denied when installing new bots
+RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
+
+ENV PATH="/opt/intelmq/.local/bin:${PATH}"
+
+ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"]
+

.docker/intelmq-full-dev/entrypoint_dev.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+/opt/dev/update.sh
+
+if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
+	intelmqctl start
+fi
+
+
+/opt/entrypoint.sh

.docker/intelmq-full-dev/merge_BOTS.py

@@ -0,0 +1,38 @@
+import json
+from jsonmerge import merge
+import argparse
+from collections import OrderedDict
+
+parser = argparse.ArgumentParser(description='Merge two json.')
+parser.add_argument('input_file_1', type=str, help='input_file_1')
+parser.add_argument('input_file_2', type=str, help='input_file_2')
+parser.add_argument('output_file', type=str, help='output_file')
+
+args = parser.parse_args()
+
+with open(args.input_file_1, 'r') as f:
+    j1 = json.load(f)
+with open(args.input_file_2, 'r') as f:
+    j2 = json.load(f)
+
+def sortOD(od):
+    res = OrderedDict()
+    for k, v in sorted(od.items()):
+        if isinstance(v, dict):
+            res[k] = sortOD(v)
+        else:
+            res[k] = v
+    return res
+
+
+merged = sortOD(merge(j1,j2))
+
+desired_order_list = ['Collector', 'Parser', 'Expert', 'Output']
+reordered_dict = {k: merged[k] for k in desired_order_list}
+
+# add other keys
+reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list})
+
+with open(args.output_file, 'w') as f:
+    json.dump(reordered_dict, f, indent=4)
+   

.docker/intelmq-full-dev/update.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+echo "Installing requirements for bots in dev repository"
+for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
+
+if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then 
+    if [ "${AUTO_MIX_BOTS}" = "true" ]; then
+        # Backup Original BOTS
+        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk
+        echo "Merge your BOTS file with BOTS"
+        python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS"
+        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
+        echo "Copying BOTS"
+        cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
+        # Restore original BOTS
+        mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS
+    else
+        cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS
+    fi
+fi
+
+echo "Installing new BOTS"
+cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user
+
+
+

.docker/intelmq-full/Dockerfile

@@ -0,0 +1,69 @@
+FROM debian:buster
+ENV LANG C.UTF-8
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG BUILD_VERSION
+
+LABEL maintainer="IntelMQ Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.authors="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.title="intelmq-full" \
+      org.opencontainers.image.description="IntelMQ with core & api" \
+      org.opencontainers.image.url="https://intelmq.org/" \
+      org.opencontainers.image.source="https://github.com/certtools/intelmq.git" \
+      org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
+      org.opencontainers.image.vendor="intelmq-team"
+
+
+###
+# libfuzzy-dev is used for pydeep
+#
+#
+#
+#
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+    sudo \
+    gcc \
+    rsync \
+    python3-nose \
+    python3-dev \
+    python3-setuptools \
+    python3-pip \
+    libfuzzy-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+LABEL org.opencontainers.image.created=$BUILD_DATE \
+      org.opencontainers.image.revision=$VCS_REF \
+      org.opencontainers.image.version=$BUILD_VERSION
+
+WORKDIR /opt
+
+COPY ./intelmq /etc/intelmq
+COPY ./intelmq-api /etc/intelmq-api
+
+RUN useradd -d /etc/intelmq -U -s /bin/bash intelmq \
+    && adduser intelmq sudo \
+    && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
+    && sudo chown -R intelmq:intelmq /etc/intelmq \
+    && mkdir -p /opt/intelmq_persistence \
+    && sudo chown -R intelmq:intelmq /opt/intelmq_persistence
+
+### Install IntelMQ
+RUN cd /etc/intelmq \
+    && pip3 install hug bs4 pika validators textx lxml url-normalize geolib pyasn pyyaml requests-mock cerberus imbox tld xmltodict jinja2 \
+    && pip3 install --force pymisp[fileobjects,openioc,virustotal] \
+    && pip3 install --no-cache-dir -e . \
+    && intelmqsetup
+
+### Install IntelMQ-API
+RUN cd /etc/intelmq-api \
+    && python3 setup.py install
+
+ADD entrypoint.sh /opt/entrypoint.sh
+RUN chmod +x /opt/entrypoint.sh \
+    && chown intelmq:intelmq /opt/entrypoint.sh
+
+USER intelmq:intelmq
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]

.docker/nginx/Dockerfile

@@ -0,0 +1,25 @@
+FROM nginx:1.13-alpine
+ENV LANG C.UTF-8
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG BUILD_VERSION
+
+LABEL maintainer="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.authors="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.title="intelmq-nginx" \
+      org.opencontainers.image.description="Modified NGINX Server for intelmq" \
+      org.opencontainers.image.url="https://github.com/certtools/intelmq/issues" \
+      org.opencontainers.image.source="https://github.com/certtools/intelmq.git" \
+      org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
+      org.opencontainers.image.vendor="intelmq-team"
+
+LABEL org.opencontainers.image.created=$BUILD_DATE \
+      org.opencontainers.image.revision=$VCS_REF \
+      org.opencontainers.image.version=$BUILD_VERSION
+
+WORKDIR /www
+
+COPY .docker/nginx/config/app.conf /etc/nginx/conf.d/default.conf
+COPY .docker/nginx/config/nginx.conf /etc/nginx/nginx.conf
+COPY intelmq-manager/html/ /www/

.docker/nginx/config/app.conf

@@ -0,0 +1,21 @@
+upstream intelmq_api {
+    server intelmq:8080;
+}
+
+server {
+    server_tokens off;
+    listen 80 default_server;
+
+    server_name localhost;
+
+    root /www;
+
+    location / {
+        index index.html;
+        try_files $uri /index.html =404;
+    }
+
+    location /intelmq/ {
+        proxy_pass http://intelmq_api/;
+    }
+}

.docker/nginx/config/nginx.conf

@@ -0,0 +1,27 @@
+user nginx;
+
+worker_processes 1;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+
+    sendfile on;
+
+    keepalive_timeout 65;
+
+    include /etc/nginx/conf.d/*.conf;
+}

.gitignore

@@ -0,0 +1,2 @@
+intelmq_logs/
+intelmq_persistence/

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "intelmq"]
+	path = intelmq
+	url = https://github.com/certtools/intelmq.git
+[submodule "intelmq-manager"]
+	path = intelmq-manager
+	url = https://github.com/certtools/intelmq-manager.git
+[submodule "intelmq-api"]
+	path = intelmq-api
+	url = https://github.com/certtools/intelmq-api.git

DEVELOP-GUIDE.md

@@ -0,0 +1,47 @@
+# intelmq-docker
+
+## Run & deploy containers in dev mode:
+
+0. `cd intelmq-manager`
+0. `python3 setup.py`
+0. `cd ..`
+1. `docker-compose -f docker-compose-dev.yml up`
+
+## Docker-compose-dev.yml file
+
+### Volume:
+
+**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert.
+
+### Add your own bots
+
+Just start coding or pull your bots repository in ,/mybots folder
+
+### How to install and look yours bots runnig
+
+
+Just run /opt/dev/update.sh in the container:
+
+1. `docker-compose exec -f docker-compose-dev.yml  intelmq /opt/bin/update.sh`
+
+When you do this:
+
+* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment
+* Yours bots will be installed
+
+### Additional environment variables
+
+Check options in docker-compose-dev.yml:
+
+* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email)
+* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not.
+
+
+## For deploy your already developed bots
+
+Just clone your bots git to ./mybots and run the container
+
+For example, using https://github.com/CERTUNLP/intelmq-bots:
+
+1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3`
+0. `docker-compose -f docker-compose-dev.yml up`

Dockerfile

@@ -1,59 +0,0 @@
-FROM debian:buster
-ENV LANG C.UTF-8
-
-ARG BUILD_DATE
-ARG VCS_REF
-ARG BUILD_VERSION
-
-LABEL maintainer="IntelMQ Team <intelmq-team@cert.at>" \
-      org.label-schema.schema-version="1.0" \
-      org.label-schema.name="certat/intelmq-full" \
-      org.label-schema.description="IntelMQ with core & manager" \
-      org.label-schema.url="https://intelmq.org/" \
-      org.label-schema.vcs-url="https://github.com/certat/intelmq-docker.git" \
-      org.label-schema.vendor="CERT.AT"
-
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-    sudo \
-    gcc \
-    python3-nose \
-    python3-yaml \
-    python3-cerberus \
-    python3-requests-mock \
-    python3-dev \
-    python3-setuptools \
-    python3-pip \
-    && rm -rf /var/lib/apt/lists/*
-
-LABEL org.label-schema.build-date=$BUILD_DATE \
-      org.label-schema.vcs-ref=$VCS_REF \
-      org.label-schema.version=$BUILD_VERSION
-
-
-COPY ./intelmq /opt/intelmq
-COPY ./intelmq-manager /opt/intelmq-manager
-
-WORKDIR /opt
-
-RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \
-    && adduser intelmq sudo \ 
-    && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
-    && sudo chown -R intelmq:intelmq /opt/intelmq
-
-### Install IntelMQ
-RUN cd /opt/intelmq \
-    && pip3 install --no-cache-dir -e . \
-    && intelmqsetup
-
-### Install IntelMQ-Manager (python)
-RUN cd /opt/intelmq-manager \
-    && pip3 install hug mako \
-    && pip3 install --no-cache-dir -e .
-
-ADD entrypoint.sh /opt/entrypoint.sh
-RUN chmod +x /opt/entrypoint.sh
-
-USER intelmq
-
-ENTRYPOINT [ "/opt/entrypoint.sh" ]

README.md

@@ -11,30 +11,33 @@ If you do have any questions / feedback / questions, please open an issue :)
 ## Fastest way to run & deploy
 
 1. `cd ~`
-0. `mkdir intelmq_logs`
 0. `sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose`
-0. `git clone https://github.com/certat/intelmq-docker.git`
+0. `git clone https://github.com/certat/intelmq-docker.git --recursive`
 0. `cd intelmq-docker`
-0. `sudo docker pull certat/intelmq-full:1.0`
-0. `chown -R $USER:$USER example_config`
-0. `sudo docker-compose up`
-0. Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+0. `docker-compose pull`
+2. `docker-compose up`
+3. Open your favourite browser -> Go to `http://127.0.0.1:1337/`
 
-If you want to build/deploy/test this container run 
+## For developers
+
+Please take a look to DEVELOP-GUIDE.md
+
+
+## Build and deploy new images
+
+If you want to build/deploy/test this container run
 1. `chmod +x build.sh`
 0. `chmod +x test.sh`
 0. `chmod +x publish.sh`
 
-**!ATTENTATION!** Only [CERT.AT](https://cert.at/) employee's/maintainer can publish on `cerat/` repository. Change this in `publish.sh`
+**!ATTENTION!** Only [CERT.AT](https://cert.at/) employee's/maintainer can publish on the `certat/` repository. Change this in `publish.sh`
 
 ## How to develop new features & build containers?
 **ATTENTION** Make sure to change `certat/intelmq-full:1.0` to `intelmq-full:1.0` in `docker-compose.yml`
 
-1. `cd ~`
-0. `git clone https://github.com/certtools/intelmq.git`
-0. `git clone https://github.com/certtools/intelmq-manager`
+Start making your changes in `intelmq`, `intelmq-api` or `intelmq-manager`.
 
-Now you can start making changes to source code. If you're finished and ready to test within your docker enviroment
+If you're finished and ready to test within your docker enviroment
 1. `cd ~/intelmq-docker`
 0. `sudo ./build.sh`
 
@@ -42,4 +45,4 @@ Now your docker image should be built successfully. Check for errors :)
 
 Now lets run tests to ensure our image is ready.
 
-1. `sudo ./test.sh`
+1. `sudo ./test.sh`

build.sh

@@ -1,19 +1,33 @@
 #!/bin/bash
 build_date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-git_ref_core=$(cd ../intelmq && git rev-parse --short HEAD)
-git_ref_manager=$(cd ../intelmq-manager && git rev-parse --short HEAD)
-build_version="1.0"
+git_ref_core=$(cd ./intelmq && git describe --long)
+git_ref_manager=$(cd ./intelmq-manager && git describe --long)
+git_ref_api=$(cd ./intelmq-api && git describe --long)
+build_version=$(cd ./intelmq && git describe)
 
 echo Building new IntelMQ-Image v$build_version
 echo Core      : $git_ref_core
 echo Manager   : $git_ref_manager
+echo Api       : $git_ref_api
 echo Build_date: $build_date
 
-cp -r ../intelmq ./intelmq
-cp -r ../intelmq-manager ./intelmq-manager
+# build static html
+cd ./intelmq-manager \
+    && python3 setup.py build \
+    && cd ..
 
 docker build --build-arg BUILD_DATE=$build_date \
-    --build-arg VCS_REF="IntelMQ=$git_ref_core, IntelMQ-Manager=$git_ref_manager" \
+    --build-arg VCS_REF="IntelMQ-Manager=$git_ref_manager" \
     --build-arg BUILD_VERSION=$build_version \
-    -f Dockerfile \
-    -t intelmq-full:$build_version .
+    -f ./.docker/nginx/Dockerfile \
+    -t intelmq-nginx:latest .
+
+docker build --build-arg BUILD_DATE=$build_date \
+    --build-arg VCS_REF="IntelMQ=$git_ref_core, IntelMQ-API=$git_ref_api, IntelMQ-Manager=$git_ref_manager" \
+    --build-arg BUILD_VERSION=$build_version \
+    -f ./.docker/intelmq-full/Dockerfile \
+    -t intelmq-full:latest .
+
+cd ./intelmq-manager \
+    && rm -r html \
+    && cd ..

custom_bots/ecs/expert.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: 2021 Birger Schacht
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import json
+from intelmq.lib.bot import Bot
+
+
+class ECSExpertBot(Bot):
+    """Write some fields to the output field in ECS format"""
+
+    def process(self):
+        msg = self.receive_message()
+
+        ecs = {}
+
+        # If the event source has no original timestamp, this value is
+        # typically populated by the first time the event was received by the
+        # pipeline.
+        # (https://www.elastic.co/guide/en/ecs/current/ecs-base.html)
+        ecs['@timestamp'] = msg['time.source'] if 'time.source' in msg else msg['time.observation']
+
+        if 'feed.provider' in msg:
+            ecs['event.provider'] = msg['feed.provider']
+        if 'source.ip' in msg:
+            ecs['server.ip'] = msg['source.ip']
+        if 'source.fqdn' in msg:
+            ecs['server.domain'] = msg['source.fqdn']
+        if 'feed.name' in msg:
+            ecs['event.dataset'] = msg['feed.name']
+
+        msg.add("output", json.dumps(ecs))
+
+        self.send_message(msg)
+        self.acknowledge_message()
+
+
+BOT = ECSExpertBot

docker-compose-dev.yml

@@ -0,0 +1,49 @@
+version: "3"
+services:
+    redis:
+        image: redis:latest
+        volumes:
+            - ./example_config/redis/redis.conf:/usr/local/etc/redis/redis.conf
+        command:
+            - redis-server
+            - /usr/local/etc/redis/redis.conf
+        restart: always
+        networks: 
+            - intelmq-internal
+    nginx:
+        image: certat/intelmq-nginx:latest
+        restart: always
+        ports: 
+            - 1337:80
+        volumes: 
+            - ./intelmq-manager/html:/www
+        depends_on: 
+            - intelmq
+        networks: 
+            - intelmq-internal
+    intelmq:
+        build: .docker/intelmq-full-dev
+        volumes: 
+            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+            - ./example_config/intelmq-api:/opt/intelmq-api/config
+            - ./intelmq_logs:/opt/intelmq/var/log
+            - ./intelmq_output:/opt/intelmq/var/lib/bots
+            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
+            - ./mybots:/opt/dev/mybots
+        depends_on: 
+            - redis
+        environment: 
+            INTELMQ_PIPELINE_DRIVER: "redis"
+            INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_REDIS_CACHE_HOST: redis
+            # Start botnet at boot
+            ENABLE_BOTNET_AT_BOOT: "false"
+            # Enable this to enable automix of BOTS file
+            AUTO_MIX_BOTS: "false"
+        networks:
+            - intelmq-internal
+
+
+networks: 
+    intelmq-internal:
+        driver: bridge

docker-compose.yml

@@ -9,43 +9,39 @@ services:
             - /usr/local/etc/redis/redis.conf
         restart: always
         networks:
-            - intelmq-network
-    postgres: 
-        image: postgres:latest
+            - intelmq-internal
+    nginx:
+        image: certat/intelmq-nginx:latest
         restart: always
-        environment:
-            POSTGRES_PASSWORD: test
-            POSTGRES_USER: root
-            POSTGRES_DB: test
+        ports:
+            - 1337:80
+        depends_on:
+            - intelmq
         networks:
-            - intelmq-database
-
-# IntelMQ with IntelMQ-Manager!
-    intelmq-full:
-        image: certat/intelmq-full:1.0
-        volumes: 
-            - ./example_config/intelmq/etc:/opt/intelmq/etc
-            - ./example_config/intelmq-manager:/opt/intelmq-manager/config
+            - intelmq-internal
+    intelmq:
+        image: certat/intelmq-full:latest
+        volumes:
+            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+            - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
             - ./intelmq_logs:/opt/intelmq/var/log
-            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
-        ports: 
-            - 127.0.0.1:1337:8080/tcp
-        depends_on: 
+            - ./intelmq_output:/opt/intelmq/var/lib/bots
+
+            - ./custom_bots/ecs/__init__.py:/etc/intelmq/intelmq/bots/experts/ecs/__init__.py:ro
+            - ./custom_bots/ecs/expert.py:/etc/intelmq/intelmq/bots/experts/ecs/expert.py:ro
+        depends_on:
             - redis
-            - postgres
-        environment: 
-            INTELMQ_IS_DOCKER: "true"
-            INTELMQ_PIPELINE_DRIVER: "redis"
+        environment:
+            INTELMQ_SOURCE_PIPELINE_BROKER: "redis"
+            INTELMQ_PIPELINE_BROKER: "redis"
+            INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis"
             INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_SOURCE_PIPELINE_HOST: redis
+            INTELMQ_DESTINATION_PIPELINE_HOST: redis
             INTELMQ_REDIS_CACHE_HOST: redis
-            INTELMQ_MANAGER_CONFIG: "/opt/intelmq-manager/config/config.json"
         networks:
-            - intelmq-network
-            - intelmq-database
+            - intelmq-internal
 
 networks:
-    intelmq-network:
+    intelmq-internal:
         driver: bridge
-    intelmq-database:
-        driver: bridge
-        

entrypoint.sh

@@ -1,7 +1,20 @@
 #!/bin/bash
+export INTELMQ_IS_DOCKER=1
+sudo chown -R intelmq:intelmq /etc/intelmq
+sudo chown -R intelmq:intelmq /opt/intelmq
+
+intelmqctl upgrade-config
+intelmqctl check
+
+intelmq_user="${INTELMQ_API_USER:=intelmq}"
+intelmq_pass="${INTELMQ_API_PASS:=intelmq}"
+
+intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass"
+
 if [[ $1 == "selftest" ]]
 then
-    nosetests3 /opt/intelmq/intelmq/tests
+    export INTELMQ_TEST_EXOTIC=1
+    nosetests3 /etc/intelmq/intelmq/tests
 else
-    hug -f /opt/intelmq-manager/intelmq_manager/serve.py -p8080
-fi
+    cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080
+fi

example_config/intelmq-api/config.json

@@ -0,0 +1,8 @@
+{
+    "intelmq_ctl_cmd": ["intelmqctl"],
+    "allowed_path": "/etc/intelmq/var/lib/bots/",
+    "session_store": "/etc/intelmq/api-session.sqlite",
+    "session_duration": 86400,
+    "allow_origins": ["*"],
+    "html_dir": ""
+}

example_config/intelmq-manager/config.json

@@ -1,4 +0,0 @@
-{
-    "intelmq_ctl_cmd": ["/usr/local/bin/intelmqctl"],
-    "allowed_path": "/opt/intelmq/var/lib/"
-}

example_config/intelmq/etc/defaults.conf

@@ -1,39 +0,0 @@
-{
-    "accuracy": 100,
-    "destination_pipeline_broker": "redis",
-    "destination_pipeline_db": 2,
-    "destination_pipeline_host": "127.0.0.1",
-    "destination_pipeline_password": null,
-    "destination_pipeline_port": 6379,
-    "error_dump_message": true,
-    "error_log_exception": true,
-    "error_log_message": false,
-    "error_max_retries": 3,
-    "error_procedure": "pass",
-    "error_retry_delay": 15,
-    "http_proxy": null,
-    "http_timeout_max_tries": 3,
-    "http_timeout_sec": 30,
-    "http_user_agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
-    "http_verify_cert": true,
-    "https_proxy": null,
-    "load_balance": false,
-    "log_processed_messages_count": 500,
-    "log_processed_messages_seconds": 900,
-    "logging_handler": "file",
-    "logging_level": "INFO",
-    "logging_path": "/opt/intelmq/var/log/",
-    "logging_syslog": "/dev/log",
-    "process_manager": "intelmq",
-    "rate_limit": 0,
-    "source_pipeline_broker": "redis",
-    "source_pipeline_db": 2,
-    "source_pipeline_host": "127.0.0.1",
-    "source_pipeline_password": null,
-    "source_pipeline_port": 6379,
-    "ssl_ca_certificate": null,
-    "statistics_database": 3,
-    "statistics_host": "127.0.0.1",
-    "statistics_password": null,
-    "statistics_port": 6379
-}

example_config/intelmq/etc/feeds.yaml

@@ -287,7 +287,7 @@ providers:
             http_url: https://urlhaus.abuse.ch/feeds/tld/<TLD>/,
               https://urlhaus.abuse.ch/feeds/country/<CC>/, or
               https://urlhaus.abuse.ch/feeds/asn/<ASN>/
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -296,7 +296,7 @@ providers:
             skip_header: false
             default_url_protocol: http://
             type_translation: '{"malware_download": "malware-distribution"}'
-            delimeter: ","
+            delimiter: ","
             columns:
               - time.source
               - source.url
@@ -406,7 +406,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -429,7 +429,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://www.team-cymru.org/Services/Bogons/fullbogons-ipv6.txt
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -542,11 +542,11 @@ providers:
       public: yes
   Turris:
     Greylist:
-      description: The data are processed and clasified every week and behaviour of
+      description: The data are processed and classified every week and behaviour of
         IP addresses that accessed a larger number of Turris routers is evaluated.
         The result is a list of addresses that have tried to obtain information about
-        services on the router or tried to gain access to them. We publish this so
-        called "greylist" that also contains a list of tags for each address which
+        services on the router or tried to gain access to them. The list also
+        contains a list of tags for each address which
         indicate what behaviour of the address was observed.
       additional_information:
       bots:
@@ -561,7 +561,72 @@ providers:
           module: intelmq.bots.parsers.turris.parser
           parameters:
       revision: 2018-01-20
-      documentation: https://project.turris.cz/greylist-data/legend.txt
+      documentation: https://project.turris.cz/en/greylist
+      public: yes
+    Greylist with PGP signature verification:
+      description: |
+        The data are processed and classified every week and behaviour of
+        IP addresses that accessed a larger number of Turris routers is evaluated.
+        The result is a list of addresses that have tried to obtain information about
+        services on the router or tried to gain access to them. The list also
+        contains a list of tags for each address which
+        indicate what behaviour of the address was observed.
+
+        The Turris Greylist feed provides PGP signatures for the provided files.
+        You will need to import the public PGP key from the linked documentation
+        page, currently available at
+        https://pgp.mit.edu/pks/lookup?op=vindex&search=0x10876666
+        or from below.
+        See the URL Fetcher Collector documentation for more information on
+        PGP signature verification.
+
+        PGP Public key:
+        ```
+        -----BEGIN PGP PUBLIC KEY BLOCK-----
+        Version: SKS 1.1.6
+        Comment: Hostname: pgp.mit.edu
+
+        mQINBFRl7D8BEADaRFoDa/+r27Gtqrdn8sZL4aSYTU4Q3gDr3TfigK8H26Un/Y79a/DUL1o0
+        o8SRae3uwVcjJDHZ6KDnxThbqF7URfpuCcCYxOs8p/eu3dSueqEGTODHWF4ChIh2japJDc4t
+        3FQHbIh2e3GHotVqJGhvxMmWqBFoZ/mlWvhjs99FFBZ87qbUNk7l1UAGEXeWeECgz9nGox40
+        3YpCgEsnJJsKC53y5LD/wBf4z+z0GsLg2GMRejmPRgrkSE/d9VjF/+niifAj2ZVFoINSVjjI
+        8wQFc8qLiExdzwLdgc+ggdzk5scY3ugI5IBt1zflxMIOG4BxKj/5IWsnhKMG2NLVGUYOODoG
+        pKhcY0gCHypw1bmkp2m+BDVyg4KM2fFPgQ554DAX3xdukMCzzZyBxR3UdT4dN7xRVhpph3Y2
+        Amh1E/dpde9uwKFk1oRHkRZ3UT1XtpbXtFNY0wCiGXPt6KznJAJcomYFkeLHjJo3nMK0hISV
+        GSNetVLfNWlTkeo93E1innbSaDEN70H4jPivjdVjSrLtIGfr2IudUJI84dGmvMxssWuM2qdg
+        FSzoTHw9UE9KT3SltKPS+F7u9x3h1J492YaVDncATRjPZUBDhbvo6Pcezhup7XTnI3gbRQc2
+        oEUDb933nwuobHm3VsUcf9686v6j8TYehsbjk+zdA4BoS/IdCwARAQABtC5UdXJyaXMgR3Jl
+        eWxpc3QgR2VuZXJhdG9yIDxncmV5bGlzdEB0dXJyaXMuY3o+iQI4BBMBAgAiBQJUZew/AhsD
+        BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRDAQrU3EIdmZoH4D/9Jo6j9RZxCAPTaQ9WZ
+        WOdb1Eqd/206bObEX+xJAago+8vuy+waatHYBM9/+yxh0SIg2g5whd6J7A++7ePpt5XzX6hq
+        bzdG8qGtsCRu+CpDJ40UwHep79Ck6O/A9KbZcZW1z/DhbYT3z/ZVWALy4RtgmyC67Vr+j/C7
+        KNQ529bs3kP9AzvEIeBC4wdKl8dUSuZIPFbgf565zRNKLtHVgVhiuDPcxKmBEl4/PLYF30a9
+        5Tgp8/PNa2qp1DV/EZjcsxvSRIZB3InGBvdKdSzvs4N/wLnKWedj1GGm7tJhSkJa4MLBSOIx
+        yamhTS/3A5Cd1qoDhLkp7DGVXSdgEtpoZDC0jR7nTS6pXojcgQaF7SfJ3cjZaLI5rjsx0YLk
+        G4PzonQKCAAQG1G9haCDniD8NrrkZ3eFiafoKEECRFETIG0BJHjPdSWcK9jtNCupBYb7JCiz
+        Q0hwLh2wrw/wCutQezD8XfsBFFIQC18TsJAVgdHLZnGYkd5dIbV/1scOcm52w6EGIeMBBYlB
+        J2+JNukH5sJDA6zAXNl2I1H1eZsP4+FSNIfB6LdovHVPAjn7qXCw3+IonnQK8+g8YJkbbhKJ
+        sPejfg+ndpe5u0zX+GvQCFBFu03muANA0Y/OOeGIQwU93d/akN0P1SRfq+bDXnkRIJQOD6XV
+        0ZPKVXlNOjy/z2iN2A==
+        =wjkM
+        -----END PGP PUBLIC KEY BLOCK-----
+        ```
+      additional_information:
+      bots:
+        collector:
+          module: intelmq.bots.collectors.http.collector_http
+          parameters:
+            http_url: https://www.turris.cz/greylist-data/greylist-latest.csv
+            name: Greylist
+            provider: __PROVIDER__
+            rate_limit: 43200
+            signature_url: https://www.turris.cz/greylist-data/greylist-latest.csv.asc
+            verify_pgp_signatures: false
+        parser:
+          module: intelmq.bots.parsers.turris.parser
+          parameters:
+      revision: 2018-01-20
+      documentation: https://project.turris.cz/en/greylist
       public: yes
   Malc0de:
     Bind Format:
@@ -1008,6 +1073,50 @@ providers:
       revision: 2018-01-20
       documentation: http://www.blocklist.de/en/export.html
       public: yes
+  CERT-Bund:
+    CB-Report Malware infections via IMAP:
+      description: CERT-Bund sends reports for the malware-infected hosts.
+      additional_information: Traffic from malware related hosts contacting
+        command-and-control servers is caught and sent to national CERT teams.
+        There are two e-mail feeds with identical CSV structure -- one reports on
+        general malware infections, the other on the Avalanche botnet.
+      bots:
+        collector:
+          module: intelmq.bots.collectors.mail.collector_mail_attach
+          parameters:
+            mail_host: __HOST__
+            mail_password: __PASSWORD__
+            mail_ssl: true
+            mail_user: __USERNAME__
+            attach_regex: events.csv
+            extract_files: false
+            rate_limit: 86400
+            subject_regex: ^\\[CB-Report#.* Malware infections (\\(Avalanche\\) )?in country
+            folder: INBOX
+            name: __FEED__
+            provider: __PROVIDER__
+        parser:
+          module: intelmq.bots.parsers.generic.parser_csv
+          parameters:
+            skip_header: true
+            default_url_protocol: http://
+            time_format: from_format|%Y-%m-%d %H:%M:%S
+            delimiter: ","
+            columns:
+              - source.asn
+              - source.ip
+              - time.source
+              - classification.type
+              - malware.name
+              - source.port
+              - destination.ip
+              - destination.port
+              - destination.fqdn
+              - protocol.transport
+            type: infected-system
+      revision: 2020-08-20
+      documentation:
+      public: no
   CERT.PL:
     N6 Stomp Stream:
       description: N6 Collector - CERT.pl's N6 Collector - N6 feed via STOMP interface.
@@ -1081,7 +1190,7 @@ providers:
             http_url: http://support.clean-mx.de/clean-mx/xmlviruses?response=alive&domain=
             http_timeout_sec: 120
             http_user_agent: "{{ your user agent }}"
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1101,7 +1210,7 @@ providers:
             http_url: http://support.clean-mx.de/clean-mx/xmlphishing?response=alive&domain=
             http_timeout_sec: 120
             http_user_agent: "{{ your user agent }}"
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1110,24 +1219,6 @@ providers:
       revision: 2018-01-20
       documentation: http://clean-mx.de/
       public: no
-  Malware Domain List:
-    Blacklist:
-      description: No description provided by feed provider.
-      additional_information:
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: http://www.malwaredomainlist.com/updatescsv.php
-            rate_limit: 3600
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.malwaredomainlist.parser
-          parameters:
-      revision: 2018-01-20
-      documentation: http://www.malwaredomainlist.com/
-      public: yes
   AnubisNetworks:
     Cyberfeed Stream:
       description: Fetches and parsers the Cyberfeed data stream.
@@ -1205,10 +1296,12 @@ providers:
       revision: 2018-01-20
       documentation: https://osint.bambenekconsulting.com/feeds/
       public: yes
-  DynDNS:
-    Infected Domains:
-      description: DynDNS ponmocup. List of ponmocup malware redirection domains and
-        infected web-servers. See also http://security-research.dyndns.org/pub/botnet-links.html
+  cAPTure:
+    Ponmocup Domains CIF Format:
+      description: List of ponmocup malware redirection domains and infected web-servers from cAPTure.
+        See also http://security-research.dyndns.org/pub/botnet-links.htm
+        and http://c-apt-ure.blogspot.com/search/label/ponmocup
+        The data in the CIF format is not equal to the Shadowserver CSV format. Reasons are unknown.
       additional_information:
       bots:
         collector:
@@ -1216,7 +1309,7 @@ providers:
           parameters:
             http_url: http://security-research.dyndns.org/pub/malware-feeds/ponmocup-infected-domains-CIF-latest.txt
             rate_limit: 10800
-            name: __FEED__
+            name: Infected Domains
             provider: __PROVIDER__
         parser:
           module: intelmq.bots.parsers.dyn.parser
@@ -1224,6 +1317,40 @@ providers:
       revision: 2018-01-20
       documentation: http://security-research.dyndns.org/pub/malware-feeds/
       public: yes
+    Ponmocup Domains Shadowserver Format:
+      description: List of ponmocup malware redirection domains and infected web-servers from cAPTure.
+        See also http://security-research.dyndns.org/pub/botnet-links.htm
+        and http://c-apt-ure.blogspot.com/search/label/ponmocup
+        The data in the Shadowserver CSV is not equal to the CIF format format. Reasons are unknown.
+      additional_information:
+      bots:
+        collector:
+          module: intelmq.bots.collectors.http.collector_http
+          parameters:
+            http_url: http://security-research.dyndns.org/pub/malware-feeds/ponmocup-infected-domains-shadowserver.csv
+            rate_limit: 10800
+            name: Infected Domains
+            provider: __PROVIDER__
+        parser:
+          module: intelmq.bots.parsers.generic.parser_csv
+          parameters:
+            columns:
+              - time.source
+              - source.ip
+              - source.fqdn
+              - source.urlpath
+              - source.port
+              - protocol.application
+              - extra.tag
+              - extra.redirect_target
+              - extra.category
+            compose_fields: {"source.url": "http://{0}{1}"}
+            skip_header: true
+            delimiter: ","
+            type: malware-distribution
+      revision: 2020-07-08
+      documentation: http://security-research.dyndns.org/pub/malware-feeds/
+      public: yes
   DShield:
     Suspicious Domains:
       description: There are many suspicious domains on the internet. In an effort
@@ -1236,7 +1363,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://www.dshield.org/feeds/suspiciousdomains_High.txt
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1255,7 +1382,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://www.dshield.org/block.txt
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1272,7 +1399,7 @@ providers:
           module: intelmq.bots.collectors.http.collector_http
           parameters:
             http_url: https://dshield.org/asdetailsascii.html?as={{ AS Number }}
-            rate_limit: 129600
+            rate_limit: 86400
             name: __FEED__
             provider: __PROVIDER__
         parser:
@@ -1299,7 +1426,7 @@ providers:
       revision: 2018-01-20
       documentation: http://vxvault.net/ViriList.php
       public: yes
-  ShadowServer:
+  Shadowserver:
     Via IMAP:
       description: Shadowserver sends out a variety of reports (see https://www.shadowserver.org/wiki/pmwiki.php/Services/Reports).
       additional_information: The configuration retrieves the data from a e-mails via IMAP from the attachments.
@@ -1356,6 +1483,28 @@ providers:
       revision: 2018-01-20
       documentation: https://www.shadowserver.org/what-we-do/network-reporting/
       public: no
+    Via API:
+      description: Shadowserver sends out a variety of reports to subscribers, see documentation.
+      additional_information: This configuration fetches user-configurable reports from the Shadowserver Reports API. For a list of reports, have a look at the Shadowserver collector and parser documentation.
+      bots:
+        collector:
+          module: intelmq.bots.collectors.shadowserver.collector_reports_api
+          parameters:
+            country: <CC>
+            api_key: <API key>
+            secret: <API secret>
+            types: <single report or list of reports>
+            rate_limit: 86400
+            redis_cache_db: 12
+            redis_cache_host: 127.0.0.1
+            redis_cache_port: 6379
+            redis_cache_ttl: 864000
+        parser:
+          module: intelmq.bots.parsers.shadowserver.parser_json
+          parameters:
+      revision: 2020-01-08
+      documentation: https://www.shadowserver.org/what-we-do/network-reporting/api-documentation/
+      public: no
   Fraunhofer:
     DGA Archive:
       description: Fraunhofer DGA collector fetches data from Fraunhofer's domain
@@ -1417,7 +1566,7 @@ providers:
       documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange
       public: no
     CTIP via Interflow:
-      description: Collects CTIP (Sinkhole data) files from the Interflow API.The feed is available via Microsoft’s Government Security Program (GSP).
+      description: Collects the CTIP Infected feed (Sinkhole data for your country) files from the Interflow API.The feed is available via Microsoft’s Government Security Program (GSP).
       additional_information: Depending on the file sizes you may need to increase the parameter 'http_timeout_sec' of the collector. As many IPs occur very often in the data, you may want to use a deduplicator specifically for the feed.
       bots:
         collector:
@@ -1436,8 +1585,8 @@ providers:
       revision: 2018-03-06
       documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange
       public: no
-    CTIP via Azure:
-      description: Collects CTIP (Sinkhole data) files from a shared Azure Storage. The feed is available via Microsoft’s Government Security Program (GSP).
+    CTIP Infected via Azure:
+      description: Collects the CTIP (Sinkhole data) from a shared Azure Storage. The feed is available via Microsoft’s Government Security Program (GSP).
       additional_information: The cache is needed for memorizing which files have already been processed, the TTL should be higher than the oldest file available in the storage (currently the last three days are available). The connection string contains endpoint as well as authentication information.
       bots:
         collector:
@@ -1458,6 +1607,28 @@ providers:
       revision: 2020-05-29
       documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange
       public: no
+    CTIP C2 via Azure:
+      description: Collects the CTIP C2 feed from a shared Azure Storage. The feed is available via Microsoft’s Government Security Program (GSP).
+      additional_information: The cache is needed for memorizing which files have already been processed, the TTL should be higher than the oldest file available in the storage (currently the last three days are available). The connection string contains endpoint as well as authentication information.
+      bots:
+        collector:
+          module: intelmq.bots.collectors.microsoft.collector_azure
+          parameters:
+            connection_string: "{{your connection string}}"
+            container_name: "ctip-c2"
+            name: __FEED__
+            provider: __PROVIDER__
+            rate_limit: 3600
+            redis_cache_db: 5
+            redis_cache_host: 127.0.0.1
+            redis_cache_port: 6379
+            redis_cache_ttl: 864000
+        parser:
+          module: intelmq.bots.parsers.microsoft.parser_ctip
+          parameters:
+      revision: 2020-05-29
+      documentation: https://docs.microsoft.com/en-us/security/gsp/informationsharingandexchange
+      public: no
   Threatminer:
     Recent domains:
       description: Latest malicious domains.
@@ -1563,10 +1734,10 @@ providers:
             listen 443 ssl http2;
             server_name [your host name];
             client_max_body_size 50M;
-            
+
             ssl_certificate [path to your key];
             ssl_certificate_key [path to your certificate];
-            
+
             location /[your private url] {
                  if ($http_authorization != '[your private password]') {
                      return 403;
@@ -1595,7 +1766,7 @@ providers:
     DailyIOC:
       description: Daily IOC from tweets and articles
       additional_information: |
-        collector's `extra_fields` parameter may be any of fields from the github [content API response](https://developer.github.com/v3/repos/contents/)
+        collector's `extra_fields` parameter may be any of fields from the github `content API response <https://developer.github.com/v3/repos/contents/>`_
       bots:
         collector:
           module: intelmq.bots.collectors.github_api.collector_github_contents_api
@@ -1612,7 +1783,7 @@ providers:
       public: yes
   CZ.NIC:
     HaaS:
-      description: SSH attackers against HaaS (Honeypot as a Sevice) provided by CZ.NIC, z.s.p.o. The dump is published once a day.
+      description: SSH attackers against HaaS (Honeypot as a Service) provided by CZ.NIC, z.s.p.o. The dump is published once a day.
       bots:
         collector:
           module: intelmq.bots.collectors.http.collector_http
@@ -1628,6 +1799,24 @@ providers:
       revision: 2020-07-22
       documentation: https://haas.nic.cz/
       public: yes
+    Proki:
+      description: Aggregation of various sources on malicious IP addresses (malware spreaders or C&C servers).
+      bots:
+        collector:
+          module: intelmq.bots.collectors.http.collector_http
+          parameters:
+            http_url: https://proki.csirt.cz/api/1/__APIKEY__/data/day/{time[%Y/%m/%d]}
+            http_url_formatting:
+              days: -1
+            rate_limit: 86400
+            name: __FEED__
+            provider: __PROVIDER__
+        parser:
+          module: intelmq.bots.parsers.cznic.parser_proki
+          parameters:
+      revision: 2020-08-17
+      documentation: https://csirt.cz/en/proki/
+      public: no
   ESET:
     ETI Domains:
       description: Domain data from ESET's TAXII API.
@@ -1665,3 +1854,25 @@ providers:
       revision: 2020-06-30
       documentation: https://www.eset.com/int/business/services/threat-intelligence/
       public: no
+  Shodan:
+    Country Stream:
+      description: Collects the Shodan stream for one or multiple countries from the Shodan API.
+      additional_information: A Shodan account with streaming permissions is needed.
+      bots:
+        collector:
+          module: intelmq.bots.collectors.shodan.collector_stream
+          parameters:
+            api_key: <API key>
+            countries: <comma-separated list of country codes>
+            error_retry_delay: 0
+            name: __FEED__
+            provider: __PROVIDER__
+        parser:
+          module: intelmq.bots.parsers.shodan.parser
+          parameters:
+            ignore_errors: false
+            error_retry_delay: 0
+            minimal_mode: false
+      revision: 2021-03-22
+      documentation: https://developer.shodan.io/api/stream
+      public: no

example_config/intelmq/etc/harmonization.conf

@@ -1,16 +1,16 @@
 {
     "event": {
         "classification.identifier": {
-            "description": "The lowercase identifier defines the actual software or service (e.g. 'heartbleed' or 'ntp_version') or standardized malware name (e.g. 'zeus'). Note that you MAY overwrite this field during processing for your individual setup. This field is not standardized across IntelMQ setups/users.",
+            "description": "The lowercase identifier defines the actual software or service (e.g. ``heartbleed`` or ``ntp_version``) or standardized malware name (e.g. ``zeus``). Note that you MAY overwrite this field during processing for your individual setup. This field is not standardized across IntelMQ setups/users.",
             "type": "String"
         },
         "classification.taxonomy": {
-            "description": "We recognize the need for the CSIRT teams to apply a static (incident) taxonomy to abuse data. With this goal in mind the type IOC will serve as a basis for this activity. Each value of the dynamic type mapping translates to a an element in the static taxonomy. The European CSIRT teams for example have decided to apply the eCSIRT.net incident classification. The value of the taxonomy key is thus a derivative of the dynamic type above. For more information about check [ENISA taxonomies](http://www.enisa.europa.eu/activities/cert/support/incident-management/browsable/incident-handling-process/incident-taxonomy/existing-taxonomies).",
+            "description": "We recognize the need for the CSIRT teams to apply a static (incident) taxonomy to abuse data. With this goal in mind the type IOC will serve as a basis for this activity. Each value of the dynamic type mapping translates to a an element in the static taxonomy. The European CSIRT teams for example have decided to apply the eCSIRT.net incident classification. The value of the taxonomy key is thus a derivative of the dynamic type above. For more information about check `ENISA taxonomies <http://www.enisa.europa.eu/activities/cert/support/incident-management/browsable/incident-handling-process/incident-taxonomy/existing-taxonomies>`_.",
             "length": 100,
-            "type": "LowercaseString"
+            "type": "ClassificationTaxonomy"
         },
         "classification.type": {
-            "description": "The abuse type IOC is one of the most crucial pieces of information for any given abuse event. The main idea of dynamic typing is to keep our ontology flexible, since we need to evolve with the evolving threatscape of abuse data. In contrast with the static taxonomy below, the dynamic typing is used to perform business decisions in the abuse handling pipeline. Furthermore, the value data set should be kept as minimal as possible to avoid \u201ctype explosion\u201d, which in turn dilutes the business value of the dynamic typing. In general, we normally have two types of abuse type IOC: ones referring to a compromised resource or ones referring to pieces of the criminal infrastructure, such as a command and control servers for example.",
+            "description": "The abuse type IOC is one of the most crucial pieces of information for any given abuse event. The main idea of dynamic typing is to keep our ontology flexible, since we need to evolve with the evolving threatscape of abuse data. In contrast with the static taxonomy below, the dynamic typing is used to perform business decisions in the abuse handling pipeline. Furthermore, the value data set should be kept as minimal as possible to avoid *type explosion*, which in turn dilutes the business value of the dynamic typing. In general, we normally have two types of abuse type IOC: ones referring to a compromised resource or ones referring to pieces of the criminal infrastructure, such as a command and control servers for example.",
             "type": "ClassificationType"
         },
         "comment": {
@@ -356,7 +356,7 @@
             "type": "DateTime"
         },
         "time.source": {
-            "description": "The time of occurence of the event as reported the feed (source).",
+            "description": "The time of occurrence of the event as reported the feed (source).",
             "type": "DateTime"
         },
         "tlp": {

example_config/intelmq/etc/harmonization.conf.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2016 Sebastian Wagner
+SPDX-License-Identifier: AGPL-3.0-or-later

example_config/intelmq/etc/pipeline.conf

@@ -1,86 +0,0 @@
-{
-    "cymru-whois-expert": {
-        "source-queue": "cymru-whois-expert-queue",
-        "destination-queues": [
-            "file-output-queue"
-        ]
-    },
-    "deduplicator-expert": {
-        "source-queue": "deduplicator-expert-queue",
-        "destination-queues": [
-            "taxonomy-expert-queue"
-        ]
-    },
-    "feodo-tracker-browse-collector": {
-        "destination-queues": [
-            "feodo-tracker-browse-parser-queue"
-        ]
-    },
-    "feodo-tracker-browse-parser": {
-        "source-queue": "feodo-tracker-browse-parser-queue",
-        "destination-queues": [
-            "deduplicator-expert-queue"
-        ]
-    },
-    "file-output": {
-        "source-queue": "file-output-queue"
-    },
-    "gethostbyname-1-expert": {
-        "source-queue": "gethostbyname-1-expert-queue",
-        "destination-queues": [
-            "cymru-whois-expert-queue"
-        ]
-    },
-    "gethostbyname-2-expert": {
-        "source-queue": "gethostbyname-2-expert-queue",
-        "destination-queues": [
-            "cymru-whois-expert-queue"
-        ]
-    },
-    "malc0de-parser": {
-        "source-queue": "malc0de-parser-queue",
-        "destination-queues": [
-            "deduplicator-expert-queue"
-        ]
-    },
-    "malc0de-windows-format-collector": {
-        "destination-queues": [
-            "malc0de-parser-queue"
-        ]
-    },
-    "malware-domain-list-collector": {
-        "destination-queues": [
-            "malware-domain-list-parser-queue"
-        ]
-    },
-    "malware-domain-list-parser": {
-        "source-queue": "malware-domain-list-parser-queue",
-        "destination-queues": [
-            "deduplicator-expert-queue"
-        ]
-    },
-    "spamhaus-drop-collector": {
-        "destination-queues": [
-            "spamhaus-drop-parser-queue"
-        ]
-    },
-    "spamhaus-drop-parser": {
-        "source-queue": "spamhaus-drop-parser-queue",
-        "destination-queues": [
-            "deduplicator-expert-queue"
-        ]
-    },
-    "taxonomy-expert": {
-        "source-queue": "taxonomy-expert-queue",
-        "destination-queues": [
-            "url2fqdn-expert-queue"
-        ]
-    },
-    "url2fqdn-expert": {
-        "source-queue": "url2fqdn-expert-queue",
-        "destination-queues": [
-            "gethostbyname-1-expert-queue",
-            "gethostbyname-2-expert-queue"
-        ]
-    }
-}

example_config/intelmq/etc/runtime.conf

@@ -1,230 +0,0 @@
-{
-    "cymru-whois-expert": {
-        "bot_id": "cymru-whois-expert",
-        "description": "Cymru Whois (IP to ASN) is the bot responsible to add network information to the events (BGP, ASN, AS Name, Country, etc..).",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.cymru_whois.expert",
-        "name": "Cymru Whois",
-        "parameters": {
-            "overwrite": true,
-            "redis_cache_db": 5,
-            "redis_cache_password": null,
-            "redis_cache_port": 6379,
-            "redis_cache_ttl": 86400
-        },
-        "run_mode": "continuous"
-    },
-    "deduplicator-expert": {
-        "bot_id": "deduplicator-expert",
-        "description": "Deduplicator is the bot responsible for detection and removal of duplicate messages. Messages get cached for <redis_cache_ttl> seconds. If found in the cache, it is assumed to be a duplicate.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.deduplicator.expert",
-        "name": "Deduplicator",
-        "parameters": {
-            "filter_keys": "raw,time.observation",
-            "filter_type": "blacklist",
-            "redis_cache_db": 6,
-            "redis_cache_port": 6379,
-            "redis_cache_ttl": 86400
-        },
-        "run_mode": "continuous"
-    },
-    "feodo-tracker-browse-collector": {
-        "description": "Generic URL Fetcher is the bot responsible to get the report from an URL.",
-        "enabled": true,
-        "group": "Collector",
-        "module": "intelmq.bots.collectors.http.collector_http",
-        "name": "URL Fetcher",
-        "parameters": {
-            "extract_files": false,
-            "http_password": null,
-            "http_url": "https://feodotracker.abuse.ch/browse",
-            "http_url_formatting": false,
-            "http_username": null,
-            "name": "Feodo Tracker Browse",
-            "provider": "Abuse.ch",
-            "rate_limit": 86400,
-            "ssl_client_certificate": null
-        },
-        "run_mode": "continuous",
-        "groupname": "collectors",
-        "bot_id": "feodo-tracker-browse-collector"
-    },
-    "feodo-tracker-browse-parser": {
-        "description": "HTML Table Parser is a bot configurable to parse different html table data.",
-        "enabled": true,
-        "group": "Parser",
-        "module": "intelmq.bots.parsers.html_table.parser",
-        "name": "HTML Table",
-        "parameters": {
-            "attribute_name": "",
-            "attribute_value": "",
-            "columns": "time.source,source.ip,malware.name,status,extra.SBL,source.as_name,source.geolocation.cc",
-            "default_url_protocol": "http://",
-            "ignore_values": ",,,,Not listed,,",
-            "skip_table_head": true,
-            "split_column": "",
-            "split_index": 0,
-            "split_separator": "",
-            "table_index": 0,
-            "time_format": null,
-            "type": "c2server"
-        },
-        "run_mode": "continuous",
-        "groupname": "parsers",
-        "bot_id": "feodo-tracker-browse-parser"
-    },
-    "file-output": {
-        "bot_id": "file-output",
-        "description": "File is the bot responsible to send events to a file.",
-        "enabled": true,
-        "group": "Output",
-        "groupname": "outputs",
-        "module": "intelmq.bots.outputs.file.output",
-        "name": "File",
-        "parameters": {
-            "file": "/opt/intelmq/var/lib/bots/file-output/events.txt",
-            "hierarchical_output": false,
-            "single_key": null
-        },
-        "run_mode": "continuous"
-    },
-    "gethostbyname-1-expert": {
-        "bot_id": "gethostbyname-1-expert",
-        "description": "fqdn2ip is the bot responsible to parsing the ip from the fqdn.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.gethostbyname.expert",
-        "name": "Gethostbyname",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "gethostbyname-2-expert": {
-        "bot_id": "gethostbyname-2-expert",
-        "description": "fqdn2ip is the bot responsible to parsing the ip from the fqdn.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.gethostbyname.expert",
-        "name": "Gethostbyname",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "malc0de-parser": {
-        "bot_id": "malc0de-parser",
-        "description": "Malc0de Parser is the bot responsible to parse the IP Blacklist and either Windows Format or Bind Format reports and sanitize the information.",
-        "enabled": true,
-        "group": "Parser",
-        "groupname": "parsers",
-        "module": "intelmq.bots.parsers.malc0de.parser",
-        "name": "Malc0de",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "malc0de-windows-format-collector": {
-        "bot_id": "malc0de-windows-format-collector",
-        "description": "",
-        "enabled": true,
-        "group": "Collector",
-        "groupname": "collectors",
-        "module": "intelmq.bots.collectors.http.collector_http",
-        "name": "Malc0de Windows Format",
-        "parameters": {
-            "http_password": null,
-            "http_url": "https://malc0de.com/bl/BOOT",
-            "http_username": null,
-            "name": "Windows Format",
-            "provider": "Malc0de",
-            "rate_limit": 10800,
-            "ssl_client_certificate": null
-        },
-        "run_mode": "continuous"
-    },
-    "malware-domain-list-collector": {
-        "bot_id": "malware-domain-list-collector",
-        "description": "Malware Domain List Collector is the bot responsible to get the report from source of information.",
-        "enabled": true,
-        "group": "Collector",
-        "groupname": "collectors",
-        "module": "intelmq.bots.collectors.http.collector_http",
-        "name": "Malware Domain List",
-        "parameters": {
-            "http_url": "http://www.malwaredomainlist.com/updatescsv.php",
-            "name": "Malware Domain List",
-            "provider": "Malware Domain List",
-            "rate_limit": 3600
-        },
-        "run_mode": "continuous"
-    },
-    "malware-domain-list-parser": {
-        "bot_id": "malware-domain-list-parser",
-        "description": "Malware Domain List Parser is the bot responsible to parse the report and sanitize the information.",
-        "enabled": true,
-        "group": "Parser",
-        "groupname": "parsers",
-        "module": "intelmq.bots.parsers.malwaredomainlist.parser",
-        "name": "Malware Domain List",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "spamhaus-drop-collector": {
-        "bot_id": "spamhaus-drop-collector",
-        "description": "",
-        "enabled": true,
-        "group": "Collector",
-        "groupname": "collectors",
-        "module": "intelmq.bots.collectors.http.collector_http",
-        "name": "Spamhaus Drop",
-        "parameters": {
-            "http_password": null,
-            "http_url": "https://www.spamhaus.org/drop/drop.txt",
-            "http_username": null,
-            "name": "Drop",
-            "provider": "Spamhaus",
-            "rate_limit": 3600,
-            "ssl_client_certificate": null
-        },
-        "run_mode": "continuous"
-    },
-    "spamhaus-drop-parser": {
-        "bot_id": "spamhaus-drop-parser",
-        "description": "Spamhaus Drop Parser is the bot responsible to parse the DROP, EDROP, DROPv6, and ASN-DROP reports and sanitize the information.",
-        "enabled": true,
-        "group": "Parser",
-        "groupname": "parsers",
-        "module": "intelmq.bots.parsers.spamhaus.parser_drop",
-        "name": "Spamhaus Drop",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "taxonomy-expert": {
-        "bot_id": "taxonomy-expert",
-        "description": "Taxonomy is the bot responsible to apply the eCSIRT Taxonomy to all events.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.taxonomy.expert",
-        "name": "Taxonomy",
-        "parameters": {},
-        "run_mode": "continuous"
-    },
-    "url2fqdn-expert": {
-        "bot_id": "url2fqdn-expert",
-        "description": "url2fqdn is the bot responsible to parsing the fqdn from the url.",
-        "enabled": true,
-        "group": "Expert",
-        "groupname": "experts",
-        "module": "intelmq.bots.experts.url2fqdn.expert",
-        "name": "URL2FQDN",
-        "parameters": {
-            "load_balance": true,
-            "overwrite": false
-        },
-        "run_mode": "continuous"
-    }
-}

example_config/intelmq/etc/runtime.yaml

@@ -0,0 +1,211 @@
+cymru-whois-expert:
+  bot_id: cymru-whois-expert
+  description: Cymru Whois (IP to ASN) is the bot responsible to add network information
+    to the events (BGP, ASN, AS Name, Country, etc..).
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.cymru_whois.expert
+  name: Cymru Whois
+  parameters:
+    destination_queues:
+      _default: [file-output-queue]
+    overwrite: true
+    redis_cache_db: 5
+    redis_cache_host: 127.0.0.1
+    redis_cache_password: null
+    redis_cache_port: 6379
+    redis_cache_ttl: 86400
+  run_mode: continuous
+deduplicator-expert:
+  bot_id: deduplicator-expert
+  description: Deduplicator is the bot responsible for detection and removal of duplicate
+    messages. Messages get cached for <redis_cache_ttl> seconds. If found in the cache,
+    it is assumed to be a duplicate.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.deduplicator.expert
+  name: Deduplicator
+  parameters:
+    destination_queues:
+      _default: [taxonomy-expert-queue]
+    filter_keys: raw,time.observation
+    filter_type: blacklist
+    redis_cache_db: 6
+    redis_cache_host: 127.0.0.1
+    redis_cache_port: 6379
+    redis_cache_ttl: 86400
+  run_mode: continuous
+feodo-tracker-browse-collector:
+  description: Generic URL Fetcher is the bot responsible to get the report from an
+    URL.
+  enabled: true
+  group: Collector
+  module: intelmq.bots.collectors.http.collector_http
+  name: URL Fetcher
+  parameters:
+    destination_queues:
+      _default: [feodo-tracker-browse-parser-queue]
+    extract_files: false
+    http_password: null
+    http_url: https://feodotracker.abuse.ch/browse
+    http_url_formatting: false
+    http_username: null
+    name: Feodo Tracker Browse
+    provider: Abuse.ch
+    rate_limit: 86400
+    ssl_client_certificate: null
+  run_mode: continuous
+feodo-tracker-browse-parser:
+  description: HTML Table Parser is a bot configurable to parse different html table
+    data.
+  enabled: true
+  group: Parser
+  module: intelmq.bots.parsers.html_table.parser
+  name: HTML Table
+  parameters:
+    attribute_name: ''
+    attribute_value: ''
+    columns: time.source,source.ip,malware.name,status,source.as_name,source.geolocation.cc
+    default_url_protocol: http://
+    destination_queues:
+      _default: [deduplicator-expert-queue]
+    ignore_values: ',,,,,'
+    skip_table_head: true
+    split_column: ''
+    split_index: 0
+    split_separator: ''
+    table_index: 0
+    time_format: null
+    type: c2-server
+  run_mode: continuous
+file-output:
+  bot_id: file-output
+  description: File is the bot responsible to send events to a file.
+  enabled: true
+  group: Output
+  groupname: outputs
+  module: intelmq.bots.outputs.file.output
+  name: File
+  parameters: {file: /opt/intelmq/var/lib/bots/file-output/events.txt, hierarchical_output: false,
+    single_key: null}
+  run_mode: continuous
+gethostbyname-1-expert:
+  bot_id: gethostbyname-1-expert
+  description: fqdn2ip is the bot responsible to parsing the ip from the fqdn.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.gethostbyname.expert
+  name: Gethostbyname
+  parameters:
+    destination_queues:
+      _default: [cymru-whois-expert-queue]
+  run_mode: continuous
+gethostbyname-2-expert:
+  bot_id: gethostbyname-2-expert
+  description: fqdn2ip is the bot responsible to parsing the ip from the fqdn.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.gethostbyname.expert
+  name: Gethostbyname
+  parameters:
+    destination_queues:
+      _default: [cymru-whois-expert-queue]
+  run_mode: continuous
+global: {destination_pipeline_broker: redis, process_manager: intelmq, source_pipeline_broker: redis,
+  ssl_ca_certificate: null, statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null,
+  statistics_port: 6379}
+malc0de-parser:
+  bot_id: malc0de-parser
+  description: Malc0de Parser is the bot responsible to parse the IP Blacklist and
+    either Windows Format or Bind Format reports and sanitize the information.
+  enabled: true
+  group: Parser
+  groupname: parsers
+  module: intelmq.bots.parsers.malc0de.parser
+  name: Malc0de
+  parameters:
+    destination_queues:
+      _default: [deduplicator-expert-queue]
+  run_mode: continuous
+malc0de-windows-format-collector:
+  bot_id: malc0de-windows-format-collector
+  description: ''
+  enabled: true
+  group: Collector
+  groupname: collectors
+  module: intelmq.bots.collectors.http.collector_http
+  name: Malc0de Windows Format
+  parameters:
+    destination_queues:
+      _default: [malc0de-parser-queue]
+    http_password: null
+    http_url: https://malc0de.com/bl/BOOT
+    http_username: null
+    name: Windows Format
+    provider: Malc0de
+    rate_limit: 10800
+    ssl_client_certificate: null
+  run_mode: continuous
+spamhaus-drop-collector:
+  bot_id: spamhaus-drop-collector
+  description: ''
+  enabled: true
+  group: Collector
+  groupname: collectors
+  module: intelmq.bots.collectors.http.collector_http
+  name: Spamhaus Drop
+  parameters:
+    destination_queues:
+      _default: [spamhaus-drop-parser-queue]
+    http_password: null
+    http_url: https://www.spamhaus.org/drop/drop.txt
+    http_username: null
+    name: Drop
+    provider: Spamhaus
+    rate_limit: 3600
+    ssl_client_certificate: null
+  run_mode: continuous
+spamhaus-drop-parser:
+  bot_id: spamhaus-drop-parser
+  description: Spamhaus Drop Parser is the bot responsible to parse the DROP, EDROP,
+    DROPv6, and ASN-DROP reports and sanitize the information.
+  enabled: true
+  group: Parser
+  groupname: parsers
+  module: intelmq.bots.parsers.spamhaus.parser_drop
+  name: Spamhaus Drop
+  parameters:
+    destination_queues:
+      _default: [deduplicator-expert-queue]
+  run_mode: continuous
+taxonomy-expert:
+  bot_id: taxonomy-expert
+  description: Taxonomy is the bot responsible to apply the eCSIRT Taxonomy to all
+    events.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.taxonomy.expert
+  name: Taxonomy
+  parameters:
+    destination_queues:
+      _default: [url2fqdn-expert-queue]
+  run_mode: continuous
+url2fqdn-expert:
+  bot_id: url2fqdn-expert
+  description: url2fqdn is the bot responsible to parsing the fqdn from the url.
+  enabled: true
+  group: Expert
+  groupname: experts
+  module: intelmq.bots.experts.url2fqdn.expert
+  name: URL2FQDN
+  parameters:
+    destination_queues:
+      _default: [gethostbyname-1-expert-queue, gethostbyname-2-expert-queue]
+    load_balance: true
+    overwrite: false
+  run_mode: continuous

example_config/intelmq/etc/runtime.yaml.license

@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2021 Birger Schacht
+SPDX-License-Identifier: AGPL-3.0-or-later

inspect-container.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+docker run --rm -it --entrypoint /bin/bash \
+    -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \
+    -v $(pwd)/example_config/intelmq/etc:/etc/intelmq/etc \
+    -v $(pwd)/example_config/intelmq-api/config.json:/etc/intelmq/api-config.json:ro \
+    -v $(pwd)/intelmq_logs:/etc/intelmq/var/log \
+    -v $(pwd)/example_config/intelmq/var/lib:/etc/intelmq/var/lib \
+    -e "INTELMQ_IS_DOCKER=\"true\"" \
+    intelmq-full:latest

intelmq_output/.gitignore

@@ -0,0 +1 @@
+*

mybots/BOTS

@@ -0,0 +1,17 @@
+{
+        "Collector": {
+        },
+        "Parser": {
+        },
+        "Expert": {
+                  "Example": {
+                    "description": "Example own bot.",
+                    "module": "intelmq.bots.experts.example.expert",
+                    "parameters": {
+                    }
+                }
+        },
+        "Output": {
+
+        }
+}

mybots/bots/experts/example/expert.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+from intelmq.lib.bot import Bot
+
+
+class ExampleExpertBot(Bot):
+
+    def init(self):
+        pass
+
+    def process(self):
+        pass
+
+
+BOT = ExampleExpertBot

publish.sh

@@ -1,8 +1,17 @@
 #!/bin/bash
-build_version="1.0"
+build_version="3.0.0"
+namespace="certat"
 
 docker login
 
-docker tag intelmq-full:$build_version certat/intelmq-full:$build_version
+docker tag intelmq-nginx:latest $namespace/intelmq-nginx:latest
 
-docker push certat/intelmq-full:$build_version
+docker push $namespace/intelmq-nginx:latest
+
+docker tag intelmq-full:latest $namespace/intelmq-full:latest
+docker tag intelmq-full:latest $namespace/intelmq-full:1.0
+docker tag intelmq-full:latest $namespace/intelmq-full:$build_version
+
+docker push $namespace/intelmq-full:latest
+docker push $namespace/intelmq-full:1.0
+docker push $namespace/intelmq-full:$build_version

test.sh

@@ -1,16 +1,29 @@
 #!/bin/bash
-redis_id=$(sudo docker run --rm -d -p 6379:6379 -v ~/intelmq-docker/example_config/redis/redis.conf:/redis.conf redis:latest)
+echo Setting up redis container
+redis_id=$(docker run --rm -d -p 6379:6379 -v ~/example_config/redis/redis.conf:/redis.conf redis:latest)
 
-redis_ip=$(sudo docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
+echo Setting up AMQP container
+amq_id=$(docker run --rm -d -p 5672:5672 -p 15672:15672 rabbitmq:latest)
 
-sudo docker run --rm -v ~/intelmq-docker/example_config/intelmq/etc:/opt/intelmq/etc \
-    -v ~/intelmq-docker/example_config/intelmq-manager:/opt/intelmq-manager/config \
-    -v ~/intelmq-docker/intelmq_logs:/opt/intelmq/var/log \
-    -v ~/intelmq-docker/example_config/intelmq/var/lib:/opt/intelmq/var/lib \
+redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
+amp_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks}}{{ .IPAddress }}{{ end }}' $amq_id)
+
+echo Setting up IntelMQ-Container
+docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \
+    -v $(pwd)/example_config/intelmq-api:/etc/intelmq-api/config \
+    -v $(pwd)/intelmq_logs:/etc/intelmq/var/log \
+    -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \
+    -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \
+    -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \
     -e "INTELMQ_IS_DOCKER=\"true\"" \
     -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \
     -e "INTELMQ_PIPELINE_HOST=$redis_ip" \
+    -e "INTELMQ_PIPELINE_AMQ_HOST=$amp_ip" \
     -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \
-    -e "INTELMQ_MANAGER_CONFIG=\"/opt/intelmq-manager/config/config.json\"" \
-    intelmq-full:1.0 selftest
-sudo docker container stop $redis_id
+    intelmq-full:latest selftest
+
+echo Removing redis container
+docker container kill $redis_id
+
+echo Removing AMQP container
+docker container kill $amq_id

versions.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+intelmq_full_built=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.created" }}' intelmq-full:latest)
+intelmq_full_vers=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' intelmq-full:latest)
+intelmq_full_rev=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' intelmq-full:latest)
+
+echo IntelMQ built at \"$intelmq_full_built\" \(Version $intelmq_full_vers\)
+revisions=$(echo $intelmq_full_rev | tr "," "\n")
+for rev in $revisions
+do
+    echo "> $rev"
+done