Merge branch 'main' of https://github.com/certat/intelmq-docker into main

.docker/intelmq-full-dev/Dockerfile

@@ -1,25 +1,18 @@
-FROM certat/intelmq-full:latest
+FROM certat/intelmq-full:1.0
-ENV LANG C.UTF-8
 
-LABEL maintainer="Einar <elanfranco@csirtamericas.org>"
+ADD entrypoint_dev.sh /opt/dev/entrypoint_dev.sh
-LABEL maintainer="Jeremias <jpretto@cert.unlp.edu.ar>" 
+ADD update.sh /opt/dev/update.sh
-LABEL maintainer="Mateo <mdurante@cert.unlp.edu.ar>" 
+ADD merge_BOTS.py /opt/dev/merge_BOTS.py
-RUN sudo apt-get update \
-    && sudo apt-get install -y --no-install-recommends \
-    git \
-    vim \
-    ssh \
-    && sudo rm -rf /var/lib/apt/lists/*
 
-WORKDIR /opt
+# Merge bots for merge_BOTS.py
-ADD entrypoint-dev.sh /opt/entrypoint-dev.sh
+RUN sudo python3 -m pip install jsonmerge
-ADD install_reqs_and_deploy_bots /opt/install_reqs_and_deploy_bots.sh
-RUN sudo chmod +x /opt/entrypoint-dev.sh \
-    && sudo chown intelmq:intelmq /opt/entrypoint-dev.sh
-RUN sudo chmod +x /opt/install_reqs_and_deploy_bots.sh \
-    && sudo chown intelmq:intelmq /opt/install_reqs_and_deploy_bots.sh
 
+USER intelmq:intelmq
 
-USER intelmq
+# Permission denied when installing new bots
+RUN sudo chown -R intelmq:intelmq /opt/intelmq/intelmq.egg-info
+
+ENV PATH="/opt/intelmq/.local/bin:${PATH}"
+
+ENTRYPOINT ["/opt/dev/entrypoint_dev.sh"]
 
-ENTRYPOINT [ "/opt/entrypoint-dev.sh" ]

.docker/intelmq-full/Dockerfile

@@ -20,6 +20,7 @@ RUN apt-get update \
     sudo \
     gcc \
     rsync \
+    ssh \
     python3-pika \
     python3-nose \
     python3-dev \
@@ -33,6 +34,8 @@ RUN apt-get update \
     python3-cerberus \
     python3-requests-mock \
     python3-shodan \
+    python3-elasticsearch \
+    python3-pymongo \
     libfuzzy-dev \
     && rm -rf /var/lib/apt/lists/*
 

docker-compose-dev.yml

@@ -22,7 +22,7 @@ services:
         image: certunlp/intelmq3-full-dev:latest
         volumes: 
            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
-           - ./example_config/intelmq-api/config.json:/etc/intelmq/api-config.json
+           - ./example_config/intelmq-api/config.json:/opt/intelmq/api-config.json
            - ./intelmq_logs:/opt/intelmq/var/log
            - ./intelmq_output:/opt/intelmq/var/lib/bots
            - ./my_bots:/my_bots

test.sh

@@ -1,11 +1,28 @@
 #!/bin/bash
+echo RUNNING TESTS WITH REDIS
 echo Setting up redis container
 redis_id=$(docker run --rm -d -p 6379:6379 -v ~/example_config/redis/redis.conf:/redis.conf redis:latest)
+redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
+
+echo Setting up IntelMQ-Container
+docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \
+    -v $(pwd)/example_config/intelmq-api:/etc/intelmq-api/config \
+    -v $(pwd)/intelmq_logs:/etc/intelmq/var/log \
+    -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \
+    -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \
+    -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \
+    -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \
+    -e "INTELMQ_PIPELINE_HOST=$redis_ip" \
+    -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \
+    intelmq-full:latest selftest
+
+echo Removing redis container
+docker container kill $redis_id
+
+echo RUNNING TESTS WITH AMQP
 
 echo Setting up AMQP container
 amq_id=$(docker run --rm -d -p 5672:5672 -p 15672:15672 rabbitmq:latest)
-
-redis_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
 amp_ip=$(docker inspect -f '{{ range.NetworkSettings.Networks}}{{ .IPAddress }}{{ end }}' $amq_id)
 
 echo Setting up IntelMQ-Container
@@ -15,15 +32,10 @@ docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \
     -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \
     -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \
     -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \
-    -e "INTELMQ_IS_DOCKER=\"true\"" \
+    -e "INTELMQ_PIPELINE_DRIVER=\"amqp\"" \
-    -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \
+    -e "INTELMQ_PIPELINE_HOST=$amq_id" \
-    -e "INTELMQ_PIPELINE_HOST=$redis_ip" \
-    -e "INTELMQ_PIPELINE_AMQ_HOST=$amp_ip" \
     -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \
     intelmq-full:latest selftest
 
-echo Removing redis container
-docker container kill $redis_id
-
 echo Removing AMQP container
 docker container kill $amq_id