Merge branch 'pr23-certnupl'

.docker/intelmq-full-dev/Dockerfile

@@ -18,12 +18,8 @@ RUN sudo chmod +x /opt/entrypoint-dev.sh \
     && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh
 RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \
     && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh
+RUN usermod -aG intelmq www-data
 
-# This section is just for install all the default bots dependencies
+USER intelmq
-RUN for req in $(find /etc/intelmq/intelmq/bots/ -name "*REQUIREMENTS.txt"); do cat $req >> /tmp/fullrequirements.txt; done
-RUN cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt
-RUN cat /tmp/orderfullrequirements.txt |grep -Eo '(^[^#]+)' | xargs  -d "\n" -I {} sh -c 'pip3 install "$1"|| exit 0' sh {}
-
-USER intelmq:intelmq
 
 ENTRYPOINT [ "/opt/entrypoint-dev.sh" ]

.docker/intelmq-full-dev/entrypoint-dev.sh

@@ -28,4 +28,3 @@ then
 else
     cd /etc/intelmq-api && uvicorn intelmq_api.main:app --port 8080 --host 0.0.0.0
 fi
-

.docker/intelmq-full-dev/install_reqs_and_deploy_bots

@@ -1,12 +1,13 @@
 #!/bin/bash
-sudo cp -r $MY_FORK/$MY_BOTS_FOLDER/* /etc/intelmq/intelmq/bots/
+sudo cp -r $MY_FORK/$MY_BOTS_FOLDER/* /opt/intelmq/intelmq/bots/
 sudo rm -f /tmp/orderfullrequirements.txt /tmp/fullrequirements.txt
 for req in $(find $MY_FORK/$MY_BOTS_FOLDER -name "*REQUIREMENTS.txt"); do 
    cat $req >> /tmp/fullrequirements.txt
+   echo "" >> /tmp/fullrequirements.txt
 done
 cat /tmp/fullrequirements.txt | sort | uniq > /tmp/orderfullrequirements.txt
 sudo pip3 install -r /tmp/orderfullrequirements.txt; 
 
-cd /etc/intelmq
+cd /opt/intelmq
 sudo pip3 install --no-cache-dir -e .
 sudo intelmqsetup

.docker/intelmq-full/Dockerfile

@@ -14,8 +14,7 @@ LABEL maintainer="IntelMQ Team <intelmq-team@cert.at>" \
       org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
       org.opencontainers.image.vendor="intelmq-team"
 
-###
+### libfuzzy-dev is used for pydeep
-# libfuzzy-dev is used for pydeep
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
     sudo \
@@ -50,7 +49,7 @@ COPY ./intelmq-api /opt/intelmq-api
 
 RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \
     && adduser intelmq sudo \
-    && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
+    && echo "intelmq ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/intelmq \
     && sudo chown -R intelmq:intelmq /opt/intelmq \
     && mkdir -p /opt/intelmq_persistence \
     && sudo chown -R intelmq:intelmq /opt/intelmq_persistence

DEVELOP-GUIDE.md

@@ -12,13 +12,6 @@ sudo apt update && sudo apt upgrade -y && sudo apt install docker.io git docker-
 ```
 git clone https://github.com/certat/intelmq-docker.git --recursive
 cd intelmq-docker
-docker-compose -f docker-compose-dev.yml build
-```
-
-### In next step replace git@github.com:certtools/intelmq.git by your fork of intelmq
-
-```
-git clone git@github.com:certtools/intelmq.git my_fork_of_intelmq/
 docker-compose -f docker-compose-dev.yml up
 ```
 
@@ -28,34 +21,50 @@ docker-compose -f docker-compose-dev.yml up
 
 ## Docker-compose-dev.yml file
 
+Docker dev shares almost all volumes and environment variables from intelmq-full image. But some are new:
+
 ### Volumes:
 
-- **./my_bots:/my_bots** -> this is the folder where your bots source code need to be.
+- **./example_bots:/my_bots** -> this is the folder where your bots source code need to be.
 
 ### Environment
-            #Folder where you clone your repository
+
+* Two variables to indicate where the source code of your bots is located:  
+
+            #Volume in the container where you clone your repository
             MY_FORK: "/my_bots"
-            #Folder in your repo where bots are located
+            #Subfolder in MY_FORK where your where bots are located
             MY_BOTS_FOLDER: "bots"
 
+* Another thing, you could make your bots to be running when container startup, just setting 
+            ENABLE_BOTNET_AT_BOOT: "true"* 
+
 ### Add your own bots
 
-Just start coding or pull your bots repository in ./my_bots by default in a subfolder bots, so you need for example my_bots/bots/[collectors,parsers,experts,output,parsers]
+Just start coding or pull your bots repository in some folder like, for example, ./my_bots in a subfolder bots, then you have for example my_bots/bots/[collectors,parsers,experts,output,parsers]
+
+You could take a look at the folder and files in https://github.com/certtools/intelmq/tree/develop/intelmq/bots to start.
+
+```
+git clone https://github.com/AAAAA/BBBB.git my_bots
+```
+
+After doing this,  you need to change in docker-compose-dev.yml the volume definition from **./example_bots:/my_bots** to **./my_bots:/my_bots** 
 
-You could take a look at the folder and files in https://github.com/certtools/intelmq/tree/develop/intelmq/bots
 
 ### How to install and look yours bots running
 
 After you change some bot or add something new just run command **install_reqs_and_deploy_bots.sh** in the running container
 
 ```
-docker-compose exec -f docker-compose-dev.yml intelmq sudo bash /opt/install_reqs_and_deploy_bots.sh
+docker-compose exec -f docker-compose-dev.yml intelmq bash /opt/install_reqs_and_deploy_bots.sh
 ```
 
 When you do this:
 
-* Yours bots REQUERIMENTS.txt and the bots will be installed
+* Yours bots REQUERIMENTS.txt and yout bots will be installed or updated from **MY_FORK**. 
-* Another thing, you could make your bots to be running when container startup, just setting ENABLE_BOTNET_AT_BOOT: "true"
+* Keep in mind that before being installed they will be mixed with the originals of the intelmq project, so it is important not to use the same names, neither for the bot nor for the .py files.
+
 
 ## Dependencies problems
 

docker-compose-dev.yml

@@ -20,11 +20,11 @@ services:
     intelmq:
         build: .docker/intelmq-full-dev
         volumes: 
-            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+           - ./example_config/intelmq/etc/:/opt/intelmq/etc/
-            - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
+           - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
-            - ./intelmq_logs:/opt/intelmq/var/log
+           - ./intelmq_logs:/opt/intelmq/var/log
-            - ./intelmq_output:/opt/intelmq/var/lib/bots
+           - ./intelmq_output:/opt/intelmq/var/lib/bots
-            - ./my_bots:/my_bots
+           - ./example_bots:/my_bots
         depends_on: 
             - redis
         environment:
@@ -37,7 +37,7 @@ services:
             INTELMQ_SOURCE_PIPELINE_HOST: redis
             INTELMQ_DESTINATION_PIPELINE_HOST: redis
             INTELMQ_REDIS_CACHE_HOST: redis
-            MY_FORK: "/my_bots"
+            MY_FORK: "/my_bots/"
             MY_BOTS_FOLDER: "bots"
         networks:
             - intelmq-internal

example_bots/.keep

@@ -0,0 +1 @@
+Link your bots repository here

mybots/BOTS

@@ -1,17 +0,0 @@
-{
-        "Collector": {
-        },
-        "Parser": {
-        },
-        "Expert": {
-                  "Example": {
-                    "description": "Example own bot.",
-                    "module": "intelmq.bots.experts.example.expert",
-                    "parameters": {
-                    }
-                }
-        },
-        "Output": {
-
-        }
-}

mybots/bots/experts/example/expert.py

@@ -1,14 +0,0 @@
-# -*- coding: utf-8 -*-
-from intelmq.lib.bot import Bot
-
-
-class ExampleExpertBot(Bot):
-
-    def init(self):
-        pass
-
-    def process(self):
-        pass
-
-
-BOT = ExampleExpertBot