From f1d0db81714941b6652337638291cac55124e8e8 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 11 Sep 2023 13:30:11 +0000 Subject: [PATCH 01/11] /app to /kibana/app --- salt/nginx/etc/nginx.conf | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 05da0b5d8..b2616e946 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -230,7 +230,19 @@ http { proxy_cookie_path /api/ /influxdb/api/; } - location /kibana/ { + location /app/ { + auth_request /auth/sessions/whoami; + rewrite /app/(.*) /app/$1 break; + proxy_pass http://{{ GLOBALS.manager }}:5601/app/; + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + + location /kibana/ { auth_request /auth/sessions/whoami; rewrite /kibana/(.*) /$1 break; proxy_pass http://{{ GLOBALS.manager }}:5601/; From 5bac1e4d15f65ba01e3337acf7be0921e6a6fa99 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 18 Sep 2023 21:31:15 +0000 Subject: [PATCH 02/11] Show correct dates and Kibana URL for already processed EVTX files --- salt/common/tools/sbin_jinja/so-import-evtx | 51 +++++++++++---------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index 59a13612c..f48f935bc 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -80,8 +80,8 @@ function evtx2es() { -e "SHIFTTS=$SHIFTDATE" \ -v "$EVTX:/tmp/data.evtx" \ -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \ - -v "/nsm/import/evtx-end_newest:/tmp/newest" \ - -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \ + -v "/nsm/import/$HASH/evtx-end_newest:/tmp/newest" \ + -v "/nsm/import/$HASH/evtx-start_oldest:/tmp/oldest" \ --entrypoint "/evtx_calc_timestamps.sh" \ {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} >> $LOG_FILE 2>&1 } @@ -111,12 +111,6 @@ INVALID_EVTXS_COUNT=0 VALID_EVTXS_COUNT=0 SKIPPED_EVTXS_COUNT=0 -touch /nsm/import/evtx-start_oldest -touch /nsm/import/evtx-end_newest - -echo $START_OLDEST > /nsm/import/evtx-start_oldest -echo $END_NEWEST > /nsm/import/evtx-end_newest - # paths must be quoted in case they include spaces for EVTX in $INPUT_FILES; do EVTX=$(/usr/bin/realpath "$EVTX") @@ -141,8 +135,15 @@ for EVTX in $INPUT_FILES; do status "- this EVTX has already been imported; skipping" SKIPPED_EVTXS_COUNT=$((SKIPPED_EVTXS_COUNT + 1)) else + # create EVTX directory EVTX_DIR=$HASH_DIR/evtx mkdir -p $EVTX_DIR + # create import timestamp files + for i in evtx-start_oldest evtx-end_newest; do + if ! [ -f "$i" ]; then + touch /nsm/import/$HASH/$i + fi + done # import evtx and write them to import ingest pipeline status "- importing logs to Elasticsearch..." @@ -154,28 +155,28 @@ for EVTX in $INPUT_FILES; do VALID_EVTXS_COUNT=$((VALID_EVTXS_COUNT + 1)) fi - # compare $START to $START_OLDEST - START=$(cat /nsm/import/evtx-start_oldest) - START_COMPARE=$(date -d $START +%s) - START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s) - if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then - START_OLDEST=$START - fi - - # compare $ENDNEXT to $END_NEWEST - END=$(cat /nsm/import/evtx-end_newest) - ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"` - ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s) - END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s) - if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then - END_NEWEST=$ENDNEXT - fi - cp -f "${EVTX}" "${EVTX_DIR}"/data.evtx chmod 644 "${EVTX_DIR}"/data.evtx fi # end of valid evtx + # compare $START to $START_OLDEST + START=$(cat /nsm/import/$HASH/evtx-start_oldest) + START_COMPARE=$(date -d $START +%s) + START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s) + if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then + START_OLDEST=$START + fi + + # compare $ENDNEXT to $END_NEWEST + END=$(cat /nsm/import/$HASH/evtx-end_newest) + ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"` + ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s) + END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s) + if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then + END_NEWEST=$ENDNEXT + fi + status done # end of for-loop processing evtx files From a1e963f834918a909245c74551f98bb37933b7a8 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 19 Sep 2023 13:28:20 +0000 Subject: [PATCH 03/11] Reverse timestamps where necessary --- salt/common/tools/sbin_jinja/so-import-evtx | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index f48f935bc..d12f34593 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -160,8 +160,18 @@ for EVTX in $INPUT_FILES; do fi # end of valid evtx - # compare $START to $START_OLDEST + # determine start and end and make sure they aren't reversed START=$(cat /nsm/import/$HASH/evtx-start_oldest) + END=$(cat /nsm/import/$HASH/evtx-end_newest) + START_EPOCH=`date -d "$START" +"%s"` + END_EPOCH=`date -d "$END" +"%s"` + if [ "$START_EPOCH" -gt "$END_EPOCH" ]; then + TEMP=$START + START=$END + END=$TEMP + fi + + # compare $START to $START_OLDEST START_COMPARE=$(date -d $START +%s) START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s) if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then @@ -169,7 +179,6 @@ for EVTX in $INPUT_FILES; do fi # compare $ENDNEXT to $END_NEWEST - END=$(cat /nsm/import/$HASH/evtx-end_newest) ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"` ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s) END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s) From 508260bd468bbeafaa86f0b05b879df75a32ec70 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 19 Sep 2023 13:32:03 +0000 Subject: [PATCH 04/11] Use event.created for timestamp --- salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 index 688000fb7..52b6bae7a 100644 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 @@ -80,6 +80,7 @@ { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, + { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } ], From 2e0ea3f37412b766773a7725ddd19e4d15df0590 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 19 Sep 2023 13:33:12 +0000 Subject: [PATCH 05/11] Set final pipeline --- salt/elasticsearch/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index cc2f5e1cd..91e5191f6 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3689,6 +3689,7 @@ elasticsearch: refresh_interval: 30s number_of_shards: 1 number_of_replicas: 0 + final_pipeline: ".fleet_final_pipeline-1" composed_of: - agent-mappings - dtc-agent-mappings From 3fa3f83007e216f29579778595a66654d24518d4 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 20 Sep 2023 08:22:52 -0400 Subject: [PATCH 06/11] Update soc_sensoroni.yaml --- salt/sensoroni/soc_sensoroni.yaml | 54 +++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index eb63dbe25..db51da358 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -2,53 +2,53 @@ sensoroni: enabled: description: Enable or disable Sensoroni. advanced: True - helpLink: sensoroni.html + helpLink: grid.html config: analyze: enabled: description: Enable or disable the analyzer. advanced: True - helpLink: sensoroni.html + helpLink: cases.html timeout_ms: description: Timeout period for the analyzer. advanced: True - helpLink: sensoroni.html + helpLink: cases.html parallel_limit: description: Parallel limit for the analyzer. advanced: True - helpLink: sensoroni.html + helpLink: cases.html node_checkin_interval_ms: description: Interval in ms to checkin to the soc_host. advanced: True - helpLink: sensoroni.html + helpLink: grid.html node_description: description: Description of the specific node. - helpLink: sensoroni.html + helpLink: grid.html node: True forcedType: string sensoronikey: description: Shared key for sensoroni authentication. - helpLink: sensoroni.html + helpLink: grid.html global: True sensitive: True advanced: True soc_host: description: Host for sensoroni agents to connect to. - helpLink: sensoroni.html + helpLink: grid.html global: True advanced: True analyzers: emailrep: api_key: description: API key for the EmailRep analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the EmailRep analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -56,21 +56,21 @@ sensoroni: greynoise: api_key: description: API key for the GreyNoise analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string api_version: description: API version for the GreyNoise analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: string base_url: description: Base URL for the GreyNoise analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -78,7 +78,7 @@ sensoroni: localfile: file_path: description: File path for the LocalFile analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -86,14 +86,14 @@ sensoroni: otx: api_key: description: API key for the OTX analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the OTX analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -101,14 +101,14 @@ sensoroni: pulsedive: api_key: description: API key for the Pulsedive analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the Pulsedive analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -116,14 +116,14 @@ sensoroni: spamhaus: lookup_host: description: Host to use for lookups. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: string nameservers: description: Nameservers used for queries. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -131,35 +131,35 @@ sensoroni: urlscan: api_key: description: API key for the Urlscan analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the Urlscan analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: string enabled: description: Analyzer enabled - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: bool timeout: description: Timeout for the Urlscan analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: int visibility: description: Type of visibility. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -167,14 +167,14 @@ sensoroni: virustotal: api_key: description: API key for the VirusTotal analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the VirusTotal analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True From fa3a79a7875e03dc115cd8d93fb98c93764d532f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 21 Sep 2023 09:41:44 -0400 Subject: [PATCH 07/11] Update soup to prune in background --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 1251f9a57..c3f9f29d4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -434,8 +434,8 @@ post_to_2.4.10() { } post_to_2.4.20() { - echo "Pruning unused volumes" - docker volume prune -f + echo "Pruning unused docker volumes on all nodes - This process will run in the background." + salt --async \* cmd.run "docker volume prune -f" POSTVERSION=2.4.20 } From eeeae08ec885ae8be7d56b96f1d52ca3530688e5 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 21 Sep 2023 18:39:06 +0000 Subject: [PATCH 08/11] /app/ to /app/dashboards/ --- salt/nginx/etc/nginx.conf | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index b2616e946..795663384 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -230,18 +230,19 @@ http { proxy_cookie_path /api/ /influxdb/api/; } - location /app/ { - auth_request /auth/sessions/whoami; - rewrite /app/(.*) /app/$1 break; - proxy_pass http://{{ GLOBALS.manager }}:5601/app/; - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header X-Forwarded-Proto $scheme; - + location /app/dashboards/ { + auth_request /auth/sessions/whoami; + rewrite /app/dashboards/(.*) /app/dashboards/$1 break; + proxy_pass http://{{ GLOBALS.manager }}:5601/app/; + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + location /kibana/ { auth_request /auth/sessions/whoami; rewrite /kibana/(.*) /$1 break; From c95af6b9922d7b8cbf13d9b2b951243594ca7c2d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 25 Sep 2023 14:39:33 -0400 Subject: [PATCH 09/11] Add a note about testing analyzers outside of the Sensoroni Docker container --- salt/sensoroni/files/analyzers/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index 8b1f44f29..a75799558 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -141,7 +141,6 @@ Additionally, to support airgapped users, the dependency packages themselves, an pip download -r /requirements.txt -d /source-packages ``` - ### Analyzer Architecture The Sensoroni Docker container is responsible for executing analyzers. Only the manager's Sensoroni container will process analyzer jobs. Other nodes in the grid, such as sensors and search nodes, will not be assigned analyzer jobs. @@ -154,6 +153,12 @@ The analyzer itself will only run when a user in SOC enqueues an analyzer job, s python -m urlhaus '{"artifactType":"url","value":"https://bigbadbotnet.invalid",...}' ``` +To manually test an analyzer outside of the Sensoroni Docker container, use a command similar to the following: + +```bash +PYTHONPATH=. python urlhaus/urlhaus.py '{"artifactType":"url","value":"https://bigbadbotnet.invalid",...}' +``` + It is up to each analyzer to determine whether the provided input is compatible with that analyzer. This is assisted by the analyzer metadata, as described earlier in this document, with the use of the `supportedTypes` list. Once the analyzer completes its functionality, it must terminate promptly. See the following sections for more details on expected internal behavior of the analyzer. From 7cb9b5f2577b92cbcd8d908050eaffb40812807d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 25 Sep 2023 14:41:20 -0400 Subject: [PATCH 10/11] Add the blank line that was removed from the previous commit --- salt/sensoroni/files/analyzers/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index a75799558..19335a545 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -141,6 +141,7 @@ Additionally, to support airgapped users, the dependency packages themselves, an pip download -r /requirements.txt -d /source-packages ``` + ### Analyzer Architecture The Sensoroni Docker container is responsible for executing analyzers. Only the manager's Sensoroni container will process analyzer jobs. Other nodes in the grid, such as sensors and search nodes, will not be assigned analyzer jobs. From 0bba68769bf6602dfeb8a1484adfa246c17d1fd0 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 26 Sep 2023 14:05:12 +0000 Subject: [PATCH 11/11] Make scan.pe.image_version type of 'float' --- .../templates/component/so/so-scan-mappings.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/so/so-scan-mappings.json b/salt/elasticsearch/templates/component/so/so-scan-mappings.json index 8ddbe6077..008a6ab10 100644 --- a/salt/elasticsearch/templates/component/so/so-scan-mappings.json +++ b/salt/elasticsearch/templates/component/so/so-scan-mappings.json @@ -20,7 +20,10 @@ "type": "float" } } - } + }, + "image_version": { + "type": "float" + } } }, "elf": {