[ENH] Updated IntelMQ-Full

Now we're using the api instead of the manager.
This is future proof & will be updated constantly.

Removed manager config & added api config

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

[NGINX] Added default webserver (nginx)

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

Updated intelmq-full docker image

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

Changed build process

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

Updated utils

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

Fixed entrypoint

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

Updated docker-compose

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

Misc

Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>

.docker/intelmq-full/Dockerfile

@@ -6,12 +6,13 @@ ARG VCS_REF
 ARG BUILD_VERSION
 
 LABEL maintainer="IntelMQ Team <intelmq-team@cert.at>" \
-      org.label-schema.schema-version="1.0" \
+      org.opencontainers.image.authors="IntelMQ-Team <intelmq-team@cert.at>" \
-      org.label-schema.name="certat/intelmq-full" \
+      org.opencontainers.image.title="intelmq-full" \
-      org.label-schema.description="IntelMQ with core & manager" \
+      org.opencontainers.image.description="IntelMQ with core & api" \
-      org.label-schema.url="https://intelmq.org/" \
+      org.opencontainers.image.url="https://intelmq.org/" \
-      org.label-schema.vcs-url="https://github.com/certat/intelmq-docker.git" \
+      org.opencontainers.image.source="https://github.com/certtools/intelmq.git" \
-      org.label-schema.vendor="CERT.AT"
+      org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
+      org.opencontainers.image.vendor="intelmq-team"
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
@@ -26,16 +27,15 @@ RUN apt-get update \
     python3-pip \
     && rm -rf /var/lib/apt/lists/*
 
-LABEL org.label-schema.build-date=$BUILD_DATE \
+LABEL org.opencontainers.image.created=$BUILD_DATE \
-      org.label-schema.vcs-ref=$VCS_REF \
+      org.opencontainers.image.revision=$VCS_REF \
-      org.label-schema.version=$BUILD_VERSION
+      org.opencontainers.image.version=$BUILD_VERSION
-
-
-COPY ./intelmq /opt/intelmq
-COPY ./intelmq-manager /opt/intelmq-manager
 
 WORKDIR /opt
 
+COPY ./intelmq /opt/intelmq
+COPY ./intelmq-api /opt/intelmq-api
+
 RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \
     && adduser intelmq sudo \ 
     && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
@@ -43,17 +43,14 @@ RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \
 
 ### Install IntelMQ
 RUN cd /opt/intelmq \
+    && pip3 install hug \
     && pip3 install --no-cache-dir -e . \
     && intelmqsetup
 
-### Install IntelMQ-Manager (python)
-RUN cd /opt/intelmq-manager \
-    && pip3 install hug mako \
-    && pip3 install --no-cache-dir -e .
-
 ADD entrypoint.sh /opt/entrypoint.sh
-RUN chmod +x /opt/entrypoint.sh
+RUN chmod +x /opt/entrypoint.sh \
+    && chown intelmq:intelmq /opt/entrypoint.sh
 
-USER intelmq
+USER intelmq:intelmq
 
 ENTRYPOINT [ "/opt/entrypoint.sh" ]

.docker/nginx/Dockerfile

@@ -0,0 +1,24 @@
+FROM nginx:1.13-alpine
+ENV LANG C.UTF-8
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG BUILD_VERSION
+
+LABEL maintainer="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.authors="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.title="intelmq-nginx" \
+      org.opencontainers.image.description="Modified NGINX Server for intelmq" \
+      org.opencontainers.image.url="https://github.com/certtools/intelmq/issues" \
+      org.opencontainers.image.source="https://github.com/certtools/intelmq.git" \
+      org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
+      org.opencontainers.image.vendor="intelmq-team"
+
+LABEL org.opencontainers.image.created=$BUILD_DATE \
+      org.opencontainers.image.revision=$VCS_REF \
+      org.opencontainers.image.version=$BUILD_VERSION
+
+WORKDIR /www
+
+COPY .docker/nginx/config/app.conf /etc/nginx/conf.d/default.conf
+COPY .docker/nginx/config/nginx.conf /etc/nginx/nginx.conf

.docker/nginx/config/app.conf

@@ -0,0 +1,20 @@
+upstream intelmq_api {
+    server intelmq:8080;
+}
+
+server {
+    listen 80 default_server;
+
+    server_name localhost;
+    
+    root /www;
+
+    location / {
+        index index.html;
+        try_files $uri /index.html =404;
+    }
+
+    location /intelmq/ {
+        proxy_pass http://intelmq_api/;
+    }
+}

.docker/nginx/config/nginx.conf

@@ -0,0 +1,27 @@
+user nginx;
+
+worker_processes 1;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+
+    sendfile on;
+
+    keepalive_timeout 65;
+
+    include /etc/nginx/conf.d/*.conf;
+}

.gitignore

@@ -0,0 +1,2 @@
+example_config/nginx/html
+intelmq_logs/

README.md

@@ -16,7 +16,7 @@ If you do have any questions / feedback / questions, please open an issue :)
 0. `git clone https://github.com/certat/intelmq-docker.git`
 0. `cd intelmq-docker`
 0. `sudo docker pull certat/intelmq-full:1.0`
-0. `chown -R $USER:$USER example_config`
+0. `sudo docker pull certat/intelmq-nginx:latest`
 0. `sudo docker-compose up`
 0. Open your favourite browser -> Go to `http://127.0.0.1:1337/`
 

build.sh

@@ -2,18 +2,34 @@
 build_date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 git_ref_core=$(cd ../intelmq && git rev-parse --short HEAD)
 git_ref_manager=$(cd ../intelmq-manager && git rev-parse --short HEAD)
+git_ref_api=$(cd ../intelmq-api && git rev-parse --short HEAD)
 build_version="1.0"
 
 echo Building new IntelMQ-Image v$build_version
 echo Core      : $git_ref_core
 echo Manager   : $git_ref_manager
+echo Api       : $git_ref_api
 echo Build_date: $build_date
 
-cp -r ../intelmq ./intelmq
+cp -R ../intelmq ./intelmq
-cp -r ../intelmq-manager ./intelmq-manager
+
+# build static html
+cp -R ../intelmq-manager ./intelmq-manager
+cd ./intelmq-manager \
+    && python3 setup.py build \
+    && cp -R ./html ../example_config/nginx/html \
+    && cd ..
+
+cp -R ../intelmq-api ./intelmq-api
 
 docker build --build-arg BUILD_DATE=$build_date \
-    --build-arg VCS_REF="IntelMQ=$git_ref_core, IntelMQ-Manager=$git_ref_manager" \
+    --build-arg VCS_REF="IntelMQ-Manager=$git_ref_manager" \
     --build-arg BUILD_VERSION=$build_version \
-    -f Dockerfile \
+    -f ./.docker/nginx/Dockerfile \
+    -t intelmq-nginx:latest .
+
+docker build --build-arg BUILD_DATE=$build_date \
+    --build-arg VCS_REF="IntelMQ=$git_ref_core, IntelMQ-API=$git_ref_api" \
+    --build-arg BUILD_VERSION=$build_version \
+    -f ./.docker/intelmq-full/Dockerfile \
     -t intelmq-full:$build_version .

docker-compose.yml

@@ -10,26 +10,26 @@ services:
         restart: always
         networks:
             - intelmq-network
-    postgres: 
+    nginx:
-        image: postgres:latest
+        image: intelmq-nginx:latest
         restart: always
-        environment:
-            POSTGRES_PASSWORD: test
-            POSTGRES_USER: root
-            POSTGRES_DB: test
         networks: 
-            - intelmq-database
+            - intelmq-network
-
+        ports: 
-# IntelMQ with IntelMQ-Manager!
+            - 1337:80
-    intelmq-full:
-        image: certat/intelmq-full:1.0
         volumes: 
-            - ./example_config/intelmq/etc:/opt/intelmq/etc
+            - ./example_config/nginx/html:/www
-            - ./example_config/intelmq-manager:/opt/intelmq-manager/config
+        depends_on: 
+            - intelmq
+        links: 
+            - intelmq
+    intelmq:
+        image: intelmq-full:1.0
+        volumes: 
+            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+            - ./example_config/intelmq-api:/opt/intelmq-api/config
             - ./intelmq_logs:/opt/intelmq/var/log
             - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
-        ports: 
-            - 127.0.0.1:1337:8080/tcp
         depends_on: 
             - redis
             - postgres
@@ -38,14 +38,9 @@ services:
             INTELMQ_PIPELINE_DRIVER: "redis"
             INTELMQ_PIPELINE_HOST: redis
             INTELMQ_REDIS_CACHE_HOST: redis
-            INTELMQ_MANAGER_CONFIG: "/opt/intelmq-manager/config/config.json"
         networks:
             - intelmq-network
-            - intelmq-database
 
 networks:
     intelmq-network:
         driver: bridge
-    intelmq-database:
-        driver: bridge
-        

entrypoint.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 if [[ $1 == "selftest" ]]
 then
+    INTELMQ_TEST_EXOTIC=1
     nosetests3 /opt/intelmq/intelmq/tests
 else
-    hug -f /opt/intelmq-manager/intelmq_manager/serve.py -p8080
+    INTELMQ_API_CONFIG=/opt/intelmq-api/config/config.json
+    cd intelmq-api && hug -m intelmq_api.serve -p8080
 fi

example_config/intelmq-api/config.json

@@ -0,0 +1,7 @@
+{
+    "intelmq_ctl_cmd": ["intelmqctl"],
+    "allowed_path": "/opt/intelmq/var/lib/bots/",
+    "session_duration": 86400,
+    "allow_origins": ["*"],
+    "html_dir": ""
+}

example_config/intelmq-manager/config.json

@@ -1,4 +0,0 @@
-{
-    "intelmq_ctl_cmd": ["/usr/local/bin/intelmqctl"],
-    "allowed_path": "/opt/intelmq/var/lib/"
-}

example_config/intelmq/etc/defaults.conf

@@ -2,7 +2,7 @@
     "accuracy": 100,
     "destination_pipeline_broker": "redis",
     "destination_pipeline_db": 2,
-    "destination_pipeline_host": "127.0.0.1",
+    "destination_pipeline_host": "redis",
     "destination_pipeline_password": null,
     "destination_pipeline_port": 6379,
     "error_dump_message": true,
@@ -28,12 +28,12 @@
     "rate_limit": 0,
     "source_pipeline_broker": "redis",
     "source_pipeline_db": 2,
-    "source_pipeline_host": "127.0.0.1",
+    "source_pipeline_host": "redis",
     "source_pipeline_password": null,
     "source_pipeline_port": 6379,
     "ssl_ca_certificate": null,
     "statistics_database": 3,
-    "statistics_host": "127.0.0.1",
+    "statistics_host": "redis",
     "statistics_password": null,
     "statistics_port": 6379
 }

publish.sh

@@ -3,6 +3,10 @@ build_version="1.0"
 
 docker login
 
+docker tag intelmq-nginx:latest certat/intelmq-nginx:latest
+
+docker push certat/intelmq-nginx:latest
+
 docker tag intelmq-full:$build_version certat/intelmq-full:$build_version
 
 docker push certat/intelmq-full:$build_version

test.sh

@@ -4,13 +4,12 @@ redis_id=$(sudo docker run --rm -d -p 6379:6379 -v ~/intelmq-docker/example_conf
 redis_ip=$(sudo docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)
 
 sudo docker run --rm -v ~/intelmq-docker/example_config/intelmq/etc:/opt/intelmq/etc \
-    -v ~/intelmq-docker/example_config/intelmq-manager:/opt/intelmq-manager/config \
+    -v ~/intelmq-docker/example_config/intelmq_api:/opt/intelmq_api/config \
     -v ~/intelmq-docker/intelmq_logs:/opt/intelmq/var/log \
     -v ~/intelmq-docker/example_config/intelmq/var/lib:/opt/intelmq/var/lib \
     -e "INTELMQ_IS_DOCKER=\"true\"" \
     -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \
     -e "INTELMQ_PIPELINE_HOST=$redis_ip" \
     -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \
-    -e "INTELMQ_MANAGER_CONFIG=\"/opt/intelmq-manager/config/config.json\"" \
     intelmq-full:1.0 selftest
 sudo docker container stop $redis_id

versions.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+intelmq_full_built=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.created" }}' intelmq-full:1.0)
+intelmq_full_vers=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' intelmq-full:1.0)
+intelmq_full_rev=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' intelmq-full:1.0)
+
+echo IntelMQ built at \"$intelmq_full_built\" \(Version $intelmq_full_vers\)
+revisions=$(echo $intelmq_full_rev | tr "," "\n")
+for rev in $revisions
+do
+    echo "> $rev"
+done