Update to 3.5.0

Fix wrong command

.docker/intelmq-full-dev/Dockerfile

@@ -1,20 +1,25 @@
-FROM certat/intelmq-full:1.0
+FROM certat/intelmq-full:latest
+ENV LANG C.UTF-8
 
-MAINTAINER Einar <elanfranco@cert.unlp.edu.ar>
-MAINTAINER Jeremias <jpretto@cert.unlp.edu.ar>
+LABEL maintainer="Einar <elanfranco@csirtamericas.org>"
+LABEL maintainer="Jeremias <jpretto@cert.unlp.edu.ar>" 
+LABEL maintainer="Mateo <mdurante@cert.unlp.edu.ar>" 
+RUN sudo apt-get update \
+    && sudo apt-get install -y --no-install-recommends \
+    git \
+    vim \
+    ssh \
+    && sudo rm -rf /var/lib/apt/lists/*
 
-ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh
-ADD update.sh /opt/dev/update.sh
-ADD merge_BOTS.py /opt/dev/merge_BOTS.py
+WORKDIR /opt
+ADD entrypoint-dev.sh /opt/entrypoint-dev.sh
+ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh
+RUN sudo chmod +x /opt/entrypoint-dev.sh \
+    && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh
+RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \
+    && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh
+RUN usermod -aG intelmq www-data
 
-# Merge bots for merge_BOTS.py
-RUN sudo python3 -m pip install jsonmerge
-
-
-# Permission denied when installing new bots
-RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
-
-ENV PATH="/opt/intelmq/.local/bin:${PATH}"
-
-ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"]
+USER intelmq
 
+ENTRYPOINT [ "/opt/entrypoint-dev.sh" ]

.docker/intelmq-full-dev/entrypoint-dev.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+export INTELMQ_IS_DOCKER=1
+
+if [[ ${IS_DEV} == "true" ]]
+then
+    /opt/install_reqs_and_deploy_bots.sh
+fi
+
+sudo chown -R intelmq:intelmq /etc/intelmq
+sudo chown -R intelmq:intelmq /opt/intelmq
+
+intelmqctl upgrade-config
+intelmqctl check
+
+intelmq_user="${INTELMQ_API_USER:=intelmq}"
+intelmq_pass="${INTELMQ_API_PASS:=intelmq}"
+
+intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass"
+
+if [[ ${ENABLE_BOTNET_AT_BOOT} == "true" ]]; then
+	intelmqctl start
+fi
+
+if [[ $1 == "selftest" ]]
+then
+    export INTELMQ_TEST_EXOTIC=1
+    pytest-3 /opt/intelmq/intelmq/tests
+else
+    cd /opt/intelmq-api && uvicorn intelmq_api.main:app --port 8080 --host 0.0.0.0
+fi

.docker/intelmq-full-dev/entrypoint_dev.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-/opt/dev/update.sh
-
-if [ "${ENABLE_BOTNET_AT_BOOT}" = "true" ]; then
-	intelmqctl start
-fi
-
-
-/opt/entrypoint.sh

.docker/intelmq-full-dev/install_reqs_and_deploy_bots

@@ -0,0 +1,13 @@
+#!/bin/bash
+sudo cp -r $MY_FORK/$MY_BOTS_FOLDER/* /opt/intelmq/intelmq/bots/
+sudo rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt
+for req in $(find $MY_FORK/$MY_BOTS_FOLDER -name "*REQUIREMENTS.txt"); do 
+   cat $req >> /tmp/fullrequirements.txt
+   echo "" >> /tmp/fullrequirements.txt
+done
+cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt
+sudo pip3 install -r /tmp/orderfullrequirements.txt; 
+
+cd /opt/intelmq
+sudo pip3 install --no-cache-dir -e .
+sudo intelmqsetup

.docker/intelmq-full-dev/merge_BOTS.py

@@ -1,38 +0,0 @@
-import json
-from jsonmerge import merge
-import argparse
-from collections import OrderedDict
-
-parser = argparse.ArgumentParser(description='Merge two json.')
-parser.add_argument('input_file_1', type=str, help='input_file_1')
-parser.add_argument('input_file_2', type=str, help='input_file_2')
-parser.add_argument('output_file', type=str, help='output_file')
-
-args = parser.parse_args()
-
-with open(args.input_file_1, 'r') as f:
-    j1 = json.load(f)
-with open(args.input_file_2, 'r') as f:
-    j2 = json.load(f)
-
-def sortOD(od):
-    res = OrderedDict()
-    for k, v in sorted(od.items()):
-        if isinstance(v, dict):
-            res[k] = sortOD(v)
-        else:
-            res[k] = v
-    return res
-
-
-merged = sortOD(merge(j1,j2))
-
-desired_order_list = ['Collector', 'Parser', 'Expert', 'Output']
-reordered_dict = {k: merged[k] for k in desired_order_list}
-
-# add other keys
-reordered_dict.update({k: merged[k] for k in merged.keys() - desired_order_list})
-
-with open(args.output_file, 'w') as f:
-    json.dump(reordered_dict, f, indent=4)
-   

.docker/intelmq-full-dev/update.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-echo "Installing requirements for bots in dev repository"
-for file in $(find /opt/dev/mybots -name "*REQUIREMENTS.txt"); do pip3 install -r $file; done
-
-if [ test -f /opt/intelmq/intelmq/bots/BOTS ]; then 
-    if [ "${AUTO_MIX_BOTS}" = "true" ]; then
-        # Backup Original BOTS
-        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/intelmq/bots/BOTS.bk
-        echo "Merge your BOTS file with BOTS"
-        python3 /opt/dev/merge_BOTS.py "/opt/dev/mybots/BOTS" "/opt/intelmq/intelmq/bots/BOTS" "/opt/intelmq/intelmq/bots/BOTS"
-        cp /opt/intelmq/intelmq/bots/BOTS /opt/intelmq/etc/BOTS
-        echo "Copying BOTS"
-        cp -a /opt/dev/mybots/bots/* /opt/intelmq/intelmq/bots/
-        # Restore original BOTS
-        mv /opt/intelmq/intelmq/bots/BOTS.bk /opt/intelmq/intelmq/bots/BOTS
-    else
-        cp /opt/intelmq/etc/BOTS /opt/intelmq/intelmq/bots/BOTS
-    fi
-fi
-
-echo "Installing new BOTS"
-cd /opt/intelmq && pip3 install -e . --user && python3 setup.py install --user
-
-
-

.docker/intelmq-full/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:buster
+FROM debian:bullseye-slim
 ENV LANG C.UTF-8
 
 ARG BUILD_DATE
@@ -14,22 +14,29 @@ LABEL maintainer="IntelMQ Team <intelmq-team@cert.at>" \
       org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
       org.opencontainers.image.vendor="intelmq-team"
 
-
-###
-# libfuzzy-dev is used for pydeep
-#
-#
-#
-#
+### libfuzzy-dev is used for pydeep
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
     sudo \
     gcc \
     rsync \
-    python3-nose \
+    ssh \
+    python3-pika \
     python3-dev \
     python3-setuptools \
     python3-pip \
+    python3-ruamel.yaml \
+    python3-bs4 \
+    python3-validators \
+    python3-lxml \
+    python3-xmltodict \
+    python3-cerberus \
+    python3-requests-mock \
+    python3-pytest \
+    python3-pytest-cov \
+    python3-shodan \
+    python3-elasticsearch \
+    python3-pymongo \
     libfuzzy-dev \
     && rm -rf /var/lib/apt/lists/*
 
@@ -37,33 +44,36 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
       org.opencontainers.image.revision=$VCS_REF \
       org.opencontainers.image.version=$BUILD_VERSION
 
-WORKDIR /opt
+COPY ./intelmq /opt/intelmq
+COPY ./intelmq-api /opt/intelmq-api
 
-COPY ./intelmq /etc/intelmq
-COPY ./intelmq-api /etc/intelmq-api
-
-RUN useradd -d /etc/intelmq -U -s /bin/bash intelmq \
+RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \
     && adduser intelmq sudo \
-    && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
-    && sudo chown -R intelmq:intelmq /etc/intelmq \
+    && echo "intelmq ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/intelmq \
+    && sudo chown -R intelmq:intelmq /opt/intelmq \
     && mkdir -p /opt/intelmq_persistence \
     && sudo chown -R intelmq:intelmq /opt/intelmq_persistence
 
 ### Install IntelMQ
-RUN cd /etc/intelmq \
-    && pip3 install hug bs4 pika validators textx lxml url-normalize geolib pyasn pyyaml requests-mock cerberus imbox tld xmltodict jinja2 \
-    && pip3 install --force pymisp[fileobjects,openioc,virustotal] \
-    && pip3 install --no-cache-dir -e . \
+RUN pip3 install url-normalize geolib imbox jinja2 pyasn textx tld time-machine otxv2 pendulum \
+    && pip3 install --force pymisp[fileobjects,openioc,virustotal]
+
+RUN cd /opt/intelmq \
+    && pip3 install .
+
+RUN cd /opt/intelmq \
     && intelmqsetup
 
 ### Install IntelMQ-API
-RUN cd /etc/intelmq-api \
-    && python3 setup.py install
+RUN cd /opt/intelmq-api \
+    && pip3 install .
 
 ADD entrypoint.sh /opt/entrypoint.sh
 RUN chmod +x /opt/entrypoint.sh \
     && chown intelmq:intelmq /opt/entrypoint.sh
 
-USER intelmq:intelmq
+WORKDIR /opt
+
+#USER intelmq:intelmq
 
 ENTRYPOINT [ "/opt/entrypoint.sh" ]

.docker/nginx/config/app.conf

@@ -3,7 +3,6 @@ upstream intelmq_api {
 }
 
 server {
-    server_tokens off;
     listen 80 default_server;
 
     server_name localhost;

.gitignore

@@ -1,2 +1,3 @@
 intelmq_logs/
 intelmq_persistence/
+my_fork_of_intelmq/

.gitmodules

@@ -1,9 +1,12 @@
 [submodule "intelmq"]
 	path = intelmq
 	url = https://github.com/certtools/intelmq.git
+	branch = maintenance
 [submodule "intelmq-manager"]
 	path = intelmq-manager
 	url = https://github.com/certtools/intelmq-manager.git
+	branch = maintenance
 [submodule "intelmq-api"]
 	path = intelmq-api
 	url = https://github.com/certtools/intelmq-api.git
+	branch = maintenance

DEVELOP-GUIDE.md

@@ -2,46 +2,89 @@
 
 ## Run & deploy containers in dev mode:
 
-0. `cd intelmq-manager`
-0. `python3 setup.py`
-0. `cd ..`
-1. `docker-compose -f docker-compose-dev.yml up`
+### Install docker and docker-compose
+```
+sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-compose
+```
+
+### Clone this repo
+
+```
+git clone https://github.com/certat/intelmq-docker.git --recursive
+cd intelmq-docker
+docker-compose -f docker-compose-dev.yml up
+```
+
+### Open your favourite browser -> Go to `http://127.0.0.1:1337/`
+
+        Default user/password: intelmq/intelmq
 
 ## Docker-compose-dev.yml file
 
-### Volume:
+Docker dev shares almost all volumes and environment variables from intelmq-full image. But some are new:
 
-**./mybots:/opt/dev/mybots** -> this is the folder where your source code need to be, you could see one expert example in mybots/bots/experts/example and a BOTS json definition file containing the default configuration for example expert.
+### Volumes:
+
+- **./example_bots:/my_bots** -> this is the folder where your bots source code need to be.
+
+### Environment
+
+* Two variables to indicate where the source code of your bots is located:  
+
+            #Volume in the container where you clone your repository
+            MY_FORK: "/my_bots"
+            #Subfolder in MY_FORK where your where bots are located
+            MY_BOTS_FOLDER: "bots"
+
+* Another thing, you could make your bots to be running when container startup, just setting 
+            ENABLE_BOTNET_AT_BOOT: "true"* 
 
 ### Add your own bots
 
-Just start coding or pull your bots repository in ,/mybots folder
+Just start coding or pull your bots repository in some folder like, for example, ./my_bots in a subfolder bots, then you have for example my_bots/bots/[collectors,parsers,experts,output,parsers]
 
-### How to install and look yours bots runnig
+You could take a look at the folder and files in https://github.com/certtools/intelmq/tree/develop/intelmq/bots to start.
+
+```
+git clone https://github.com/AAAAA/BBBB.git my_bots
+```
+
+After doing this,  you need to change in docker-compose-dev.yml the volume definition from **./example_bots:/my_bots** to **./my_bots:/my_bots** 
 
 
-Just run /opt/dev/update.sh in the container:
+### How to install and look yours bots running
 
-1. `docker-compose exec -f docker-compose-dev.yml  intelmq /opt/bin/update.sh`
+After you change some bot or add something new just run command **install_reqs_and_deploy_bots.sh** in the running container
+
+```
+docker-compose -f docker-compose-dev.yml exec intelmq bash /opt/install_reqs_and_deploy_bots.sh
+```
 
 When you do this:
 
-* Yours BOTS files will be mixed with intelmq original BOTS and the copied to runtime environment
-* Yours bots will be installed
-
-### Additional environment variables
-
-Check options in docker-compose-dev.yml:
-
-* LOG_MAIL_* -> these variables add support for mail handler (to tell intelmq to notificate you errors using email)
-* ENABLE_BOTNET_AT_BOOT: true/false, to configure if bot has to start at docker boot or not.
+* Yours bots REQUERIMENTS.txt and yout bots will be installed or updated from **MY_FORK**. 
+* Keep in mind that before being installed they will be mixed with the originals of the intelmq project, so it is important not to use the same names, neither for the bot nor for the .py files.
 
 
-## For deploy your already developed bots
+## Dependencies problems
 
-Just clone your bots git to ./mybots and run the container
+Some dependencies from defaults bots are missing in original intelmq image, so we fix it in our Dockerfile build process. Nevertheless, we still facing some issues.
 
-For example, using https://github.com/CERTUNLP/intelmq-bots:
+### Known isues
 
-1. `git clone https://github.com/CERTUNLP/intelmq-bots mybots -b 2.3`
-0. `docker-compose -f docker-compose-dev.yml up`
+#### Blueliv problem:
+
+This bot has 2 problems: 
+
+1- It doesn't install:
+
+        pip3 install git+git://github.com/Blueliv/api-python-sdk doesn't work because git+git is deprecated, to fix it you need to replace git+git with git+https 
+
+
+2- But if you fix and install it you would cause a dependency conflict with pymisp:
+
+        ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behavior is the source of the following dependency conflicts.
+        pymisp 2.4.148 requires requests<3.0.0,>=2.25.1, but you have requests 2.5.1 which is incompatible.
+
+
+If you don't need blueliv, just don't fix git+git with git+https.

README.md

@@ -4,7 +4,7 @@
 Do not run this software in production, it might break.
 
 # Information
-This repository is currently maintained by Sebastian Waldbauer (@waldbauer-certat).
+This repository is currently maintained by CERT.at.
 
 If you do have any questions / feedback / questions, please open an issue :)
 

build.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 build_date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-git_ref_core=$(cd ./intelmq && git describe --long)
-git_ref_manager=$(cd ./intelmq-manager && git describe --long)
-git_ref_api=$(cd ./intelmq-api && git describe --long)
-build_version=$(cd ./intelmq && git describe)
+git_ref_core=$(git -C ./intelmq describe --long --always)
+git_ref_manager=$(git -C ./intelmq-manager describe --long --always)
+git_ref_api=$(git -C ./intelmq-api describe --long --always)
+build_version=$(git -C ./intelmq describe --always)
 
 echo Building new IntelMQ-Image v$build_version
 echo Core      : $git_ref_core
@@ -12,9 +12,7 @@ echo Api       : $git_ref_api
 echo Build_date: $build_date
 
 # build static html
-cd ./intelmq-manager \
-    && python3 setup.py build \
-    && cd ..
+cd ./intelmq-manager && python3 -m pip install . && intelmq-manager-build && cd ..
 
 docker build --build-arg BUILD_DATE=$build_date \
     --build-arg VCS_REF="IntelMQ-Manager=$git_ref_manager" \
@@ -27,7 +25,3 @@ docker build --build-arg BUILD_DATE=$build_date \
     --build-arg BUILD_VERSION=$build_version \
     -f ./.docker/intelmq-full/Dockerfile \
     -t intelmq-full:latest .
-
-cd ./intelmq-manager \
-    && rm -r html \
-    && cd ..

custom_bots/ecs/expert.py

@@ -1,38 +0,0 @@
-# SPDX-FileCopyrightText: 2021 Birger Schacht
-#
-# SPDX-License-Identifier: AGPL-3.0-or-later
-
-import json
-from intelmq.lib.bot import Bot
-
-
-class ECSExpertBot(Bot):
-    """Write some fields to the output field in ECS format"""
-
-    def process(self):
-        msg = self.receive_message()
-
-        ecs = {}
-
-        # If the event source has no original timestamp, this value is
-        # typically populated by the first time the event was received by the
-        # pipeline.
-        # (https://www.elastic.co/guide/en/ecs/current/ecs-base.html)
-        ecs['@timestamp'] = msg['time.source'] if 'time.source' in msg else msg['time.observation']
-
-        if 'feed.provider' in msg:
-            ecs['event.provider'] = msg['feed.provider']
-        if 'source.ip' in msg:
-            ecs['server.ip'] = msg['source.ip']
-        if 'source.fqdn' in msg:
-            ecs['server.domain'] = msg['source.fqdn']
-        if 'feed.name' in msg:
-            ecs['event.dataset'] = msg['feed.name']
-
-        msg.add("output", json.dumps(ecs))
-
-        self.send_message(msg)
-        self.acknowledge_message()
-
-
-BOT = ECSExpertBot

docker-compose-dev.yml

@@ -7,16 +7,12 @@ services:
         command:
             - redis-server
             - /usr/local/etc/redis/redis.conf
-        restart: always
         networks: 
             - intelmq-internal
     nginx:
         image: certat/intelmq-nginx:latest
-        restart: always
         ports: 
             - 1337:80
-        volumes: 
-            - ./intelmq-manager/html:/www
         depends_on: 
             - intelmq
         networks: 
@@ -24,26 +20,27 @@ services:
     intelmq:
         build: .docker/intelmq-full-dev
         volumes: 
-            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
-            - ./example_config/intelmq-api:/opt/intelmq-api/config
-            - ./intelmq_logs:/opt/intelmq/var/log
-            - ./intelmq_output:/opt/intelmq/var/lib/bots
-            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
-            - ./mybots:/opt/dev/mybots
+           - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+           - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
+           - ./intelmq_logs:/opt/intelmq/var/log
+           - ./intelmq_output:/opt/intelmq/var/lib/bots
+           - ./example_bots:/my_bots
         depends_on: 
             - redis
-        environment: 
-            INTELMQ_PIPELINE_DRIVER: "redis"
-            INTELMQ_PIPELINE_HOST: redis
-            INTELMQ_REDIS_CACHE_HOST: redis
-            # Start botnet at boot
+        environment:
             ENABLE_BOTNET_AT_BOOT: "false"
-            # Enable this to enable automix of BOTS file
-            AUTO_MIX_BOTS: "false"
+            IS_DEV: "true" 
+            INTELMQ_SOURCE_PIPELINE_BROKER: "redis"
+            INTELMQ_PIPELINE_BROKER: "redis"
+            INTELMQ_DESTIONATION_PIPELINE_BROKER: "redis"
+            INTELMQ_PIPELINE_HOST: redis
+            INTELMQ_SOURCE_PIPELINE_HOST: redis
+            INTELMQ_DESTINATION_PIPELINE_HOST: redis
+            INTELMQ_REDIS_CACHE_HOST: redis
+            MY_FORK: "/my_bots/"
+            MY_BOTS_FOLDER: "bots"
         networks:
             - intelmq-internal
-
-
 networks: 
     intelmq-internal:
         driver: bridge

docker-compose.yml

@@ -26,9 +26,6 @@ services:
             - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
             - ./intelmq_logs:/opt/intelmq/var/log
             - ./intelmq_output:/opt/intelmq/var/lib/bots
-
-            - ./custom_bots/ecs/__init__.py:/etc/intelmq/intelmq/bots/experts/ecs/__init__.py:ro
-            - ./custom_bots/ecs/expert.py:/etc/intelmq/intelmq/bots/experts/ecs/expert.py:ro
         depends_on:
             - redis
         environment:

entrypoint.sh

@@ -14,7 +14,7 @@ intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass"
 if [[ $1 == "selftest" ]]
 then
     export INTELMQ_TEST_EXOTIC=1
-    nosetests3 /etc/intelmq/intelmq/tests
+    pytest-3 /opt/intelmq/intelmq/tests
 else
-    cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080
+    cd /opt/intelmq-api && uvicorn intelmq_api.main:app --port 8080 --host 0.0.0.0
 fi

example_bots/.keep

@@ -0,0 +1 @@
+Link your bots repository here

example_config/intelmq/etc/feeds.yaml

@@ -628,60 +628,6 @@ providers:
       revision: 2018-01-20
       documentation: https://project.turris.cz/en/greylist
       public: yes
-  Malc0de:
-    Bind Format:
-      description: This feed includes FQDN's of malicious hosts, the file format is
-        in Bind file format.
-      additional_information:
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: https://malc0de.com/bl/ZONES
-            rate_limit: 10800
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.malc0de.parser
-          parameters:
-      revision: 2018-01-20
-      documentation: http://malc0de.com/dashboard/
-      public: yes
-    Windows Format:
-      description: This feed includes FQDN's of malicious hosts, the file format is
-        in Windows Hosts file format.
-      additional_information:
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: https://malc0de.com/bl/BOOT
-            rate_limit: 10800
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.malc0de.parser
-          parameters:
-      revision: 2018-01-20
-      documentation: http://malc0de.com/dashboard/
-      public: yes
-    IP Blacklist:
-      description: This feed includes IP Addresses of malicious hosts.
-      additional_information:
-      bots:
-        collector:
-          module: intelmq.bots.collectors.http.collector_http
-          parameters:
-            http_url: https://malc0de.com/bl/IP_Blacklist.txt
-            rate_limit: 10800
-            name: __FEED__
-            provider: __PROVIDER__
-        parser:
-          module: intelmq.bots.parsers.malc0de.parser
-          parameters:
-      revision: 2018-01-20
-      documentation: http://malc0de.com/dashboard/
-      public: yes
   University of Toulouse:
     Blacklist:
       description: Various blacklist feeds

example_config/intelmq/etc/manager/positions.conf

@@ -27,14 +27,6 @@
         "x": -252,
         "y": 243
     },
-    "malc0de-parser": {
-        "x": 297,
-        "y": 24
-    },
-    "malc0de-windows-format-collector": {
-        "x": 433,
-        "y": 121
-    },
     "malware-domain-list-collector": {
         "x": 465,
         "y": -198

example_config/intelmq/etc/runtime.yaml

@@ -117,39 +117,7 @@ gethostbyname-2-expert:
   run_mode: continuous
 global: {destination_pipeline_broker: redis, process_manager: intelmq, source_pipeline_broker: redis,
   ssl_ca_certificate: null, statistics_database: 3, statistics_host: 127.0.0.1, statistics_password: null,
-  statistics_port: 6379}
-malc0de-parser:
-  bot_id: malc0de-parser
-  description: Malc0de Parser is the bot responsible to parse the IP Blacklist and
-    either Windows Format or Bind Format reports and sanitize the information.
-  enabled: true
-  group: Parser
-  groupname: parsers
-  module: intelmq.bots.parsers.malc0de.parser
-  name: Malc0de
-  parameters:
-    destination_queues:
-      _default: [deduplicator-expert-queue]
-  run_mode: continuous
-malc0de-windows-format-collector:
-  bot_id: malc0de-windows-format-collector
-  description: ''
-  enabled: true
-  group: Collector
-  groupname: collectors
-  module: intelmq.bots.collectors.http.collector_http
-  name: Malc0de Windows Format
-  parameters:
-    destination_queues:
-      _default: [malc0de-parser-queue]
-    http_password: null
-    http_url: https://malc0de.com/bl/BOOT
-    http_username: null
-    name: Windows Format
-    provider: Malc0de
-    rate_limit: 10800
-    ssl_client_certificate: null
-  run_mode: continuous
+  statistics_port: 6379, destination_pipeline_host: redis, source_pipeline_host: redis}
 spamhaus-drop-collector:
   bot_id: spamhaus-drop-collector
   description: ''

inspect-container.sh

@@ -1,9 +1,34 @@
 #!/bin/bash
-docker run --rm -it --entrypoint /bin/bash \
-    -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \
-    -v $(pwd)/example_config/intelmq/etc:/etc/intelmq/etc \
-    -v $(pwd)/example_config/intelmq-api/config.json:/etc/intelmq/api-config.json:ro \
-    -v $(pwd)/intelmq_logs:/etc/intelmq/var/log \
-    -v $(pwd)/example_config/intelmq/var/lib:/etc/intelmq/var/lib \
-    -e "INTELMQ_IS_DOCKER=\"true\"" \
-    intelmq-full:latest
+base_path=$(pwd)
+echo $base_path
+
+echo [START] Creating new network
+network_id=$(docker network create -d bridge intelmq-testing-network)
+echo [DONE ] Network created
+
+echo [START] Setting up redis container
+redis_id=$(docker run --rm -d --network=intelmq-testing-network -p 6379:6379 -v $base_path/intelmq_docker/example_config/redis/redis.conf:/redis.conf redis:latest)
+redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
+echo [DONE ] Redis container running $redis_ip:6379
+
+echo [START] IntelMQ
+intelmq_id=$(docker run --network=intelmq-testing-network --cap-add=SYS_PTRACE -p 8080:8080 --rm -d -v $base_path/intelmq_persistence:/opt/intelmq_persistence -v $base_path/example_config/intelmq/etc:/opt/intelmq/etc -v $base_path/example_config/intelmq-api/config.json:/etc/intelmq/api-config.json:ro -v $base_path/intelmq_logs:/opt/intelmq/var/log -v $base_path/example_config/intelmq/var/lib:/opt/intelmq/var/lib -e "INTELMQ_IS_DOCKER=true" -e "INTELMQ_SOURCE_PIPELINE_BROKER=redis" -e "INTELMQ_PIPELINE_BROKER=redis" -e "INTELMQ_DESTIONATION_PIPELINE_BROKER=redis" -e "INTELMQ_PIPELINE_HOST=$redis_ip" -e "INTELMQ_SOURCE_PIPELINE_HOST=$redis_ip" -e "INTELMQ_DESTINATION_PIPELINE_HOST=$redis_ip" -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" intelmq-full:latest)
+intelmq_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $intelmq_id)
+echo [DONE ] IntelMQ running
+
+echo [START] IntelMQ-Manager
+intelmq_manager_id=$(docker run --rm -d -p 1337:80 --network=intelmq-testing-network --add-host intelmq:$intelmq_ip intelmq-nginx:latest)
+echo [DONE ] IntelMQ-Manager running
+
+echo [START] Preparing profiling
+docker exec -it $intelmq_id bash -c 'sudo apt update && sudo apt install -y htop && sudo pip3 install py-spy shodan'
+echo [DONE ] All profiling installed
+
+echo Execing into intelmq instance
+docker exec -it $intelmq_id /bin/bash
+
+echo Killing all containers
+docker container kill $redis_id $intelmq_id $intelmq_manager_id
+
+echo Removing network
+docker network rm intelmq-testing-network

mybots/BOTS

@@ -1,17 +0,0 @@
-{
-        "Collector": {
-        },
-        "Parser": {
-        },
-        "Expert": {
-                  "Example": {
-                    "description": "Example own bot.",
-                    "module": "intelmq.bots.experts.example.expert",
-                    "parameters": {
-                    }
-                }
-        },
-        "Output": {
-
-        }
-}

mybots/bots/experts/example/expert.py

@@ -1,14 +0,0 @@
-# -*- coding: utf-8 -*-
-from intelmq.lib.bot import Bot
-
-
-class ExampleExpertBot(Bot):
-
-    def init(self):
-        pass
-
-    def process(self):
-        pass
-
-
-BOT = ExampleExpertBot

publish.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-build_version="3.0.0"
+build_version="3.5.0"
 namespace="certat"
 
 docker login

test.sh

@@ -1,11 +1,30 @@
 #!/bin/bash
+echo RUNNING TESTS WITH REDIS
 echo Setting up redis container
 redis_id=$(docker run --rm -d -p 6379:6379 -v ~/example_config/redis/redis.conf:/redis.conf redis:latest)
+redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
+
+echo Setting up IntelMQ-Container
+docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \
+    -v $(pwd)/example_config/intelmq-api:/etc/intelmq-api/config \
+    -v $(pwd)/intelmq_logs:/etc/intelmq/var/log \
+    -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \
+    -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \
+    -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \
+    -e "INTELMQ_PIPELINE_DRIVER=redis" \
+    -e "INTELMQ_PIPELINE_HOST=$redis_ip" \
+    -e "INTELMQ_SOURCE_PIPELINE_HOST=$redis_ip" \
+    -e "INTELMQ_DESTINATION_PIPELINE_HOST=$redis_ip" \
+    -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \
+    intelmq-full:latest selftest
+
+echo Removing redis container
+docker container kill $redis_id
+
+echo RUNNING TESTS WITH AMQP
 
 echo Setting up AMQP container
 amq_id=$(docker run --rm -d -p 5672:5672 -p 15672:15672 rabbitmq:latest)
-
-redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
 amp_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks}}{{ .IPAddress }}{{ end }}' $amq_id)
 
 echo Setting up IntelMQ-Container
@@ -15,15 +34,20 @@ docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \
     -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \
     -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \
     -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \
-    -e "INTELMQ_IS_DOCKER=\"true\"" \
-    -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \
-    -e "INTELMQ_PIPELINE_HOST=$redis_ip" \
-    -e "INTELMQ_PIPELINE_AMQ_HOST=$amp_ip" \
+    -e "INTELMQ_PIPELINE_DRIVER=\"amqp\"" \
+    -e "INTELMQ_PIPELINE_HOST=$amq_id" \
+    -e "INTELMQ_SOURCE_PIPELINE_HOST=$amq_ip" \
+    -e "INTELMQ_DESTINATION_PIPELINE_HOST=$amq_ip" \
     -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \
     intelmq-full:latest selftest
 
-echo Removing redis container
-docker container kill $redis_id
-
 echo Removing AMQP container
 docker container kill $amq_id
+
+# restore broke priviliges
+
+for mounted_dir in example_config intelmq_logs intelmq_output intelmq_persistence;
+do
+    echo "Restoring broken privelages to `whoami` for directory $mounted_dir"
+    sudo chown -R `whoami`:`whoami` $(pwd)/$mounted_dir
+done

update_submodules.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+git submodule update --remote
+git pull --recurse-submodules