From 90e696543f975ae972de9e066a151b0b92c02e63 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 20 Jul 2023 16:53:21 +0200 Subject: [PATCH 1/2] Update submodules and Docker for IntelMQ 3.2 --- .docker/intelmq-full/Dockerfile | 4 ++-- build.sh | 2 +- entrypoint.sh | 2 +- intelmq | 2 +- intelmq-api | 2 +- intelmq-manager | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.docker/intelmq-full/Dockerfile b/.docker/intelmq-full/Dockerfile index 0ba5a94..849cf26 100644 --- a/.docker/intelmq-full/Dockerfile +++ b/.docker/intelmq-full/Dockerfile @@ -55,7 +55,7 @@ RUN useradd -d /opt/intelmq -U -s /bin/bash intelmq \ && sudo chown -R intelmq:intelmq /opt/intelmq_persistence ### Install IntelMQ -RUN pip3 install hug url-normalize geolib imbox jinja2 pyasn textx tld time-machine otxv2 \ +RUN pip3 install url-normalize geolib imbox jinja2 pyasn textx tld time-machine otxv2 \ && pip3 install --force pymisp[fileobjects,openioc,virustotal] RUN cd /opt/intelmq \ @@ -66,7 +66,7 @@ RUN cd /opt/intelmq \ ### Install IntelMQ-API RUN cd /opt/intelmq-api \ - && python3 setup.py install + && pip3 install . ADD entrypoint.sh /opt/entrypoint.sh RUN chmod +x /opt/entrypoint.sh \ diff --git a/build.sh b/build.sh index b9eb3d3..81e6711 100755 --- a/build.sh +++ b/build.sh @@ -12,7 +12,7 @@ echo Api : $git_ref_api echo Build_date: $build_date # build static html -cd ./intelmq-manager && python3 setup.py build && cd .. +cd ./intelmq-manager && python3 -m pip install . && intelmq-manager-build && cd .. docker build --build-arg BUILD_DATE=$build_date \ --build-arg VCS_REF="IntelMQ-Manager=$git_ref_manager" \ diff --git a/entrypoint.sh b/entrypoint.sh index 1652e84..caa66f1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -16,5 +16,5 @@ then export INTELMQ_TEST_EXOTIC=1 nosetests3 /opt/intelmq/intelmq/tests else - cd /opt/intelmq-api && hug -m intelmq_api.serve -p8080 + cd /opt/intelmq-api && uvicorn intelmq_api.main:app -p8080 fi diff --git a/intelmq b/intelmq index 38c6e97..3a32cbe 160000 --- a/intelmq +++ b/intelmq @@ -1 +1 @@ -Subproject commit 38c6e971f9fd0d5fdbb89a4589e1b1167097a4a1 +Subproject commit 3a32cbe062e0d0499e18730339250a9f774f22ff diff --git a/intelmq-api b/intelmq-api index 87e82ff..6d4f34a 160000 --- a/intelmq-api +++ b/intelmq-api @@ -1 +1 @@ -Subproject commit 87e82ff3b0eec404804dd2a83d87a7b16d2b1d6b +Subproject commit 6d4f34a37a63c195baf728bcaef53b7a643122cf diff --git a/intelmq-manager b/intelmq-manager index 6649d99..1367ea8 160000 --- a/intelmq-manager +++ b/intelmq-manager @@ -1 +1 @@ -Subproject commit 6649d99a757dac8fbf28192e453e25fb4e0b0056 +Subproject commit 1367ea86ba43cd4bbcfe6479f01cbe897e6a3698 From a597fec2a9a6d7add8e791e661cac833330a9cf4 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Tue, 25 Jul 2023 16:01:46 +0200 Subject: [PATCH 2/2] Update to IntelMQ 3.2 Move from nosetests to pytest. Move from hug to FastAPI --- .docker/intelmq-full-dev/entrypoint-dev.sh | 4 ++-- .docker/intelmq-full/Dockerfile | 3 ++- entrypoint.sh | 2 +- publish.sh | 2 +- test.sh | 6 +++++- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.docker/intelmq-full-dev/entrypoint-dev.sh b/.docker/intelmq-full-dev/entrypoint-dev.sh index b3f45ff..6553d14 100644 --- a/.docker/intelmq-full-dev/entrypoint-dev.sh +++ b/.docker/intelmq-full-dev/entrypoint-dev.sh @@ -24,8 +24,8 @@ fi if [[ $1 == "selftest" ]] then export INTELMQ_TEST_EXOTIC=1 - nosetests3 /etc/intelmq/intelmq/tests + pytest-3 /etc/intelmq/intelmq/tests else - cd /etc/intelmq-api && hug -m intelmq_api.serve -p8080 + cd /etc/intelmq-api && uvicorn intelmq_api.main:app -p8080 fi diff --git a/.docker/intelmq-full/Dockerfile b/.docker/intelmq-full/Dockerfile index 849cf26..b93e277 100644 --- a/.docker/intelmq-full/Dockerfile +++ b/.docker/intelmq-full/Dockerfile @@ -23,7 +23,6 @@ RUN apt-get update \ rsync \ ssh \ python3-pika \ - python3-nose \ python3-dev \ python3-setuptools \ python3-pip \ @@ -34,6 +33,8 @@ RUN apt-get update \ python3-xmltodict \ python3-cerberus \ python3-requests-mock \ + python3-pytest \ + python3-pytest-cov \ python3-shodan \ python3-elasticsearch \ python3-pymongo \ diff --git a/entrypoint.sh b/entrypoint.sh index caa66f1..e59fc49 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -14,7 +14,7 @@ intelmq-api-adduser --user "$intelmq_user" --password "$intelmq_pass" if [[ $1 == "selftest" ]] then export INTELMQ_TEST_EXOTIC=1 - nosetests3 /opt/intelmq/intelmq/tests + pytest-3 /opt/intelmq/intelmq/tests else cd /opt/intelmq-api && uvicorn intelmq_api.main:app -p8080 fi diff --git a/publish.sh b/publish.sh index 7cc9fd2..288e428 100755 --- a/publish.sh +++ b/publish.sh @@ -1,5 +1,5 @@ #!/bin/bash -build_version="3.0.2" +build_version="3.2.0" namespace="certat" docker login diff --git a/test.sh b/test.sh index 7010ee1..018426e 100755 --- a/test.sh +++ b/test.sh @@ -11,8 +11,10 @@ docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \ -v $(pwd)/intelmq_output:/etc/intelmq/var/lib/bots \ -v $(pwd)/example_config/intelmq/var/lib/bot:/etc/intelmq/var/lib/bot \ -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \ - -e "INTELMQ_PIPELINE_DRIVER=\"redis\"" \ + -e "INTELMQ_PIPELINE_DRIVER=redis" \ -e "INTELMQ_PIPELINE_HOST=$redis_ip" \ + -e "INTELMQ_SOURCE_PIPELINE_HOST=$redis_ip" \ + -e "INTELMQ_DESTINATION_PIPELINE_HOST=$redis_ip" \ -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \ intelmq-full:latest selftest @@ -34,6 +36,8 @@ docker run --rm -v $(pwd)/example_config/intelmq/etc/:/etc/intelmq/etc/ \ -v $(pwd)/intelmq_persistence:/opt/intelmq_persistence \ -e "INTELMQ_PIPELINE_DRIVER=\"amqp\"" \ -e "INTELMQ_PIPELINE_HOST=$amq_id" \ + -e "INTELMQ_SOURCE_PIPELINE_HOST=$amq_ip" \ + -e "INTELMQ_DESTINATION_PIPELINE_HOST=$amq_ip" \ -e "INTELMQ_REDIS_CACHE_HOST=$redis_ip" \ intelmq-full:latest selftest