[ENH] Updated IntelMQ-Full

Now we're using the api instead of the manager. This is future proof & will be updated constantly. Removed manager config & added api config Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at> [NGINX] Added default webserver (nginx) Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at> Updated intelmq-full docker image Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at> Changed build process Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at> Updated utils Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at> Fixed entrypoint Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at> Updated docker-compose Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at> Misc Signed-off-by: Sebastian Waldbauer <waldbauer@cert.at>
2025-12-06 17:22:57 +01:00 · 2021-01-24 15:14:10 +01:00
parent 3a105d97bc
commit 0082f38a2f
15 changed files with 156 additions and 56 deletions
--- a/.docker/intelmq-full/Dockerfile
+++ b/.docker/intelmq-full/Dockerfile
@@ -6,12 +6,13 @@ ARG VCS_REF
 ARG BUILD_VERSION

 LABEL maintainer="IntelMQ Team <intelmq-team@cert.at>" \
-      org.label-schema.schema-version="1.0" \
-      org.label-schema.name="certat/intelmq-full" \
-      org.label-schema.description="IntelMQ with core & manager" \
-      org.label-schema.url="https://intelmq.org/" \
-      org.label-schema.vcs-url="https://github.com/certat/intelmq-docker.git" \
-      org.label-schema.vendor="CERT.AT"
+      org.opencontainers.image.authors="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.title="intelmq-full" \
+      org.opencontainers.image.description="IntelMQ with core & api" \
+      org.opencontainers.image.url="https://intelmq.org/" \
+      org.opencontainers.image.source="https://github.com/certtools/intelmq.git" \
+      org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
+      org.opencontainers.image.vendor="intelmq-team"

 RUN apt-get update \
    && apt-get install -y --no-install-recommends \
@@ -26,16 +27,15 @@ RUN apt-get update \
    python3-pip \
    && rm -rf /var/lib/apt/lists/*

-LABEL org.label-schema.build-date=$BUILD_DATE \
-      org.label-schema.vcs-ref=$VCS_REF \
-      org.label-schema.version=$BUILD_VERSION
-
-
-COPY ./intelmq /opt/intelmq
-COPY ./intelmq-manager /opt/intelmq-manager
+LABEL org.opencontainers.image.created=$BUILD_DATE \
+      org.opencontainers.image.revision=$VCS_REF \
+      org.opencontainers.image.version=$BUILD_VERSION

 WORKDIR /opt

+COPY ./intelmq /opt/intelmq
+COPY ./intelmq-api /opt/intelmq-api
+
 RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \
    && adduser intelmq sudo \ 
    && echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
@@ -43,17 +43,14 @@ RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \

 ### Install IntelMQ
 RUN cd /opt/intelmq \
+    && pip3 install hug \
    && pip3 install --no-cache-dir -e . \
    && intelmqsetup

-### Install IntelMQ-Manager (python)
-RUN cd /opt/intelmq-manager \
-    && pip3 install hug mako \
-    && pip3 install --no-cache-dir -e .
-
 ADD entrypoint.sh /opt/entrypoint.sh
-RUN chmod +x /opt/entrypoint.sh
+RUN chmod +x /opt/entrypoint.sh \
+    && chown intelmq:intelmq /opt/entrypoint.sh

-USER intelmq
+USER intelmq:intelmq

 ENTRYPOINT [ "/opt/entrypoint.sh" ]
--- a/.docker/nginx/Dockerfile
+++ b/.docker/nginx/Dockerfile
@@ -0,0 +1,24 @@
+FROM nginx:1.13-alpine
+ENV LANG C.UTF-8
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG BUILD_VERSION
+
+LABEL maintainer="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.authors="IntelMQ-Team <intelmq-team@cert.at>" \
+      org.opencontainers.image.title="intelmq-nginx" \
+      org.opencontainers.image.description="Modified NGINX Server for intelmq" \
+      org.opencontainers.image.url="https://github.com/certtools/intelmq/issues" \
+      org.opencontainers.image.source="https://github.com/certtools/intelmq.git" \
+      org.opencontainers.image.documentation="https://intelmq.readthedocs.io/en/latest/" \
+      org.opencontainers.image.vendor="intelmq-team"
+
+LABEL org.opencontainers.image.created=$BUILD_DATE \
+      org.opencontainers.image.revision=$VCS_REF \
+      org.opencontainers.image.version=$BUILD_VERSION
+
+WORKDIR /www
+
+COPY .docker/nginx/config/app.conf /etc/nginx/conf.d/default.conf
+COPY .docker/nginx/config/nginx.conf /etc/nginx/nginx.conf
--- a/.docker/nginx/config/app.conf
+++ b/.docker/nginx/config/app.conf
@@ -0,0 +1,20 @@
+upstream intelmq_api {
+    server intelmq:8080;
+}
+
+server {
+    listen 80 default_server;
+
+    server_name localhost;
+    
+    root /www;
+
+    location / {
+        index index.html;
+        try_files $uri /index.html =404;
+    }
+
+    location /intelmq/ {
+        proxy_pass http://intelmq_api/;
+    }
+}
--- a/.docker/nginx/config/nginx.conf
+++ b/.docker/nginx/config/nginx.conf
@@ -0,0 +1,27 @@
+user nginx;
+
+worker_processes 1;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+
+    sendfile on;
+
+    keepalive_timeout 65;
+
+    include /etc/nginx/conf.d/*.conf;
+}
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,2 @@
+example_config/nginx/html
+intelmq_logs/
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ If you do have any questions / feedback / questions, please open an issue :)
 0. `git clone https://github.com/certat/intelmq-docker.git`
 0. `cd intelmq-docker`
 0. `sudo docker pull certat/intelmq-full:1.0`
-0. `chown -R $USER:$USER example_config`
+0. `sudo docker pull certat/intelmq-nginx:latest`
 0. `sudo docker-compose up`
 0. Open your favourite browser -> Go to `http://127.0.0.1:1337/`

--- a/build.sh
+++ b/build.sh
@@ -2,18 +2,34 @@
 build_date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 git_ref_core=$(cd ../intelmq && git rev-parse --short HEAD)
 git_ref_manager=$(cd ../intelmq-manager && git rev-parse --short HEAD)
+git_ref_api=$(cd ../intelmq-api && git rev-parse --short HEAD)
 build_version="1.0"

 echo Building new IntelMQ-Image v$build_version
 echo Core      : $git_ref_core
 echo Manager   : $git_ref_manager
+echo Api       : $git_ref_api
 echo Build_date: $build_date

-cp -r ../intelmq ./intelmq
-cp -r ../intelmq-manager ./intelmq-manager
+cp -R ../intelmq ./intelmq
+
+# build static html
+cp -R ../intelmq-manager ./intelmq-manager
+cd ./intelmq-manager \
+    && python3 setup.py build \
+    && cp -R ./html ../example_config/nginx/html \
+    && cd ..
+
+cp -R ../intelmq-api ./intelmq-api

 docker build --build-arg BUILD_DATE=$build_date \
-    --build-arg VCS_REF="IntelMQ=$git_ref_core, IntelMQ-Manager=$git_ref_manager" \
+    --build-arg VCS_REF="IntelMQ-Manager=$git_ref_manager" \
    --build-arg BUILD_VERSION=$build_version \
-    -f Dockerfile \
+    -f ./.docker/nginx/Dockerfile \
+    -t intelmq-nginx:latest .
+
+docker build --build-arg BUILD_DATE=$build_date \
+    --build-arg VCS_REF="IntelMQ=$git_ref_core, IntelMQ-API=$git_ref_api" \
+    --build-arg BUILD_VERSION=$build_version \
+    -f ./.docker/intelmq-full/Dockerfile \
    -t intelmq-full:$build_version .
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,26 +10,26 @@ services:
        restart: always
        networks:
            - intelmq-network
-    postgres: 
-        image: postgres:latest
+    nginx:
+        image: intelmq-nginx:latest
        restart: always
-        environment:
-            POSTGRES_PASSWORD: test
-            POSTGRES_USER: root
-            POSTGRES_DB: test
-        networks:
-            - intelmq-database
-
-# IntelMQ with IntelMQ-Manager!
-    intelmq-full:
-        image: certat/intelmq-full:1.0
+        networks: 
+            - intelmq-network
+        ports: 
+            - 1337:80
        volumes: 
-            - ./example_config/intelmq/etc:/opt/intelmq/etc
-            - ./example_config/intelmq-manager:/opt/intelmq-manager/config
+            - ./example_config/nginx/html:/www
+        depends_on: 
+            - intelmq
+        links: 
+            - intelmq
+    intelmq:
+        image: intelmq-full:1.0
+        volumes: 
+            - ./example_config/intelmq/etc/:/opt/intelmq/etc/
+            - ./example_config/intelmq-api:/opt/intelmq-api/config
            - ./intelmq_logs:/opt/intelmq/var/log
            - ./example_config/intelmq/var/lib/bot:/opt/intelmq/var/lib/bot
-        ports: 
-            - 127.0.0.1:1337:8080/tcp
        depends_on: 
            - redis
            - postgres
@@ -38,14 +38,9 @@ services:
            INTELMQ_PIPELINE_DRIVER: "redis"
            INTELMQ_PIPELINE_HOST: redis
            INTELMQ_REDIS_CACHE_HOST: redis
-            INTELMQ_MANAGER_CONFIG: "/opt/intelmq-manager/config/config.json"
        networks:
            - intelmq-network
-            - intelmq-database

 networks:
    intelmq-network:
        driver: bridge
-    intelmq-database:
-        driver: bridge
-        
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -1,7 +1,9 @@
 #!/bin/bash
 if [[ $1 == "selftest" ]]
 then
+    INTELMQ_TEST_EXOTIC=1
    nosetests3 /opt/intelmq/intelmq/tests
 else
-    hug -f /opt/intelmq-manager/intelmq_manager/serve.py -p8080
+    INTELMQ_API_CONFIG=/opt/intelmq-api/config/config.json
+    cd intelmq-api && hug -m intelmq_api.serve -p8080
 fi
--- a/example_config/intelmq-api/config.json
+++ b/example_config/intelmq-api/config.json
@@ -0,0 +1,7 @@
+{
+    "intelmq_ctl_cmd": ["intelmqctl"],
+    "allowed_path": "/opt/intelmq/var/lib/bots/",
+    "session_duration": 86400,
+    "allow_origins": ["*"],
+    "html_dir": ""
+}
--- a/example_config/intelmq-manager/config.json
+++ b/example_config/intelmq-manager/config.json
@@ -1,4 +0,0 @@
-{
-    "intelmq_ctl_cmd": ["/usr/local/bin/intelmqctl"],
-    "allowed_path": "/opt/intelmq/var/lib/"
-}
--- a/example_config/intelmq/etc/defaults.conf
+++ b/example_config/intelmq/etc/defaults.conf
@@ -2,7 +2,7 @@
    "accuracy": 100,
    "destination_pipeline_broker": "redis",
    "destination_pipeline_db": 2,
-    "destination_pipeline_host": "127.0.0.1",
+    "destination_pipeline_host": "redis",
    "destination_pipeline_password": null,
    "destination_pipeline_port": 6379,
    "error_dump_message": true,
@@ -28,12 +28,12 @@
    "rate_limit": 0,
    "source_pipeline_broker": "redis",
    "source_pipeline_db": 2,
-    "source_pipeline_host": "127.0.0.1",
+    "source_pipeline_host": "redis",
    "source_pipeline_password": null,
    "source_pipeline_port": 6379,
    "ssl_ca_certificate": null,
    "statistics_database": 3,
-    "statistics_host": "127.0.0.1",
+    "statistics_host": "redis",
    "statistics_password": null,
    "statistics_port": 6379
 }
--- a/publish.sh
+++ b/publish.sh
@@ -3,6 +3,10 @@ build_version="1.0"

 docker login

+docker tag intelmq-nginx:latest certat/intelmq-nginx:latest
+
+docker push certat/intelmq-nginx:latest
+
 docker tag intelmq-full:$build_version certat/intelmq-full:$build_version

 docker push certat/intelmq-full:$build_version
--- a/test.sh
+++ b/test.sh
@@ -4,13 +4,12 @@ redis_id=$(sudo docker run --rm -d -p 6379:6379 -v ~/intelmq-docker/example_conf
 redis_ip=$(sudo docker inspect -f '{{ range.NetworkSettings.Networks }}{{ .IPAddress }}{{ end }}' $redis_id)

 sudo docker run --rm -v ~/intelmq-docker/example_config/intelmq/etc:/opt/intelmq/etc \
-    -v ~/intelmq-docker/example_config/intelmq-manager:/opt/intelmq-manager/config \
+    -v ~/intelmq-docker/example_config/intelmq_api:/opt/intelmq_api/config \
    -v ~/intelmq-docker/intelmq_logs:/opt/intelmq/var/log \
    -v ~/intelmq-docker/example_config/intelmq/var/lib:/opt/intelmq/var/lib \
    -e "INTELMQ_IS_DOCKER=\"true\"" \
    -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \
    -e "INTELMQ_PIPELINE_HOST=$redis_ip" \
    -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \
-    -e "INTELMQ_MANAGER_CONFIG=\"/opt/intelmq-manager/config/config.json\"" \
    intelmq-full:1.0 selftest
 sudo docker container stop $redis_id
--- a/versions.sh
+++ b/versions.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+intelmq_full_built=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.created" }}' intelmq-full:1.0)
+intelmq_full_vers=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.version" }}' intelmq-full:1.0)
+intelmq_full_rev=$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' intelmq-full:1.0)
+
+echo IntelMQ built at \"$intelmq_full_built\" \(Version $intelmq_full_vers\)
+revisions=$(echo $intelmq_full_rev | tr "," "\n")
+for rev in $revisions
+do
+    echo "> $rev"
+done