From fe4129c8e04c23611d9bf75e133f5a0ceb0f7701 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 29 Jan 2025 09:11:52 -0500 Subject: [PATCH 1/8] env discovery.type single-node change only managers and heavynodes are eligible for discovery.type=single-node --- salt/elasticsearch/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 48280c506..4699dfb1f 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -38,7 +38,7 @@ so-elasticsearch: {% endfor %} {% endif %} - environment: - {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %} + {% if (GLOBALS.role in GLOBALS.manager_roles and ELASTICSEARCH_SEED_HOSTS | length == 1) or GLOBALS.role == 'so-heavynode' %} - discovery.type=single-node {% endif %} - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true From 27e977378283a757adcd8e131062ac5da608637a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 29 Jan 2025 10:07:52 -0500 Subject: [PATCH 2/8] Update so-functions --- setup/so-functions | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 94b6aab21..fa7e8a043 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2291,6 +2291,9 @@ update_packages() { retry 150 10 "apt-get -y update" "" "Err:" >> "$setup_log" 2>&1 || fail_setup info "Running apt-get upgrade" retry 150 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || fail_setup + else + info "Updating packages" + logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*" fi } From b874619f0dbc5234730770ab99d48f1aa5d84395 Mon Sep 17 00:00:00 2001 From: Joshua Brower Date: Mon, 3 Feb 2025 09:31:08 -0500 Subject: [PATCH 3/8] Fix ip-mappings ILM --- salt/elasticsearch/defaults.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ffb302977..9c3d11c75 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -638,12 +638,19 @@ elasticsearch: mapping: total_fields: limit: 1500 + lifecycle: + name: so-ip-mappings-logs number_of_replicas: 0 number_of_shards: 1 refresh_interval: 30s sort: field: '@timestamp' order: desc + policy: + phases: + hot: + actions: {} + min_age: 0ms so-items: index_sorting: false index_template: From d0fa6eaf83f8fe5b0d704d3d851886eeaed879eb Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 29 Jan 2025 15:48:48 -0700 Subject: [PATCH 4/8] New Limit on Bulk Creating Related Events Used by the UI and API to hint at a user that not every event will be attached to a case. Supports values up to 10,000 (the default limit on the number of documents returned by a single ES search). --- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 97a24ead7..b97ba11e6 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2147,6 +2147,7 @@ soc: eventFetchLimit: 500 relativeTimeValue: 24 relativeTimeUnit: 30 + maxBulkEscalateEvents: 100 mostRecentlyUsedLimit: 5 ackEnabled: true escalateEnabled: true diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index a667d1f7a..ea651cb47 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -496,7 +496,11 @@ soc: global: True advanced: True forcedType: "[]{}" - alerts: *appSettings + alerts: + <<: *appSettings + maxBulkEscalateEvents: + description: Maximum number of events to escalate in a single bulk escalation. + global: True cases: *appSettings dashboards: *appSettings detections: From 23ebe966e0f73b9d4a41ba3187ef7f2816a29bfa Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Tue, 4 Feb 2025 10:33:04 -0700 Subject: [PATCH 5/8] Added Large Values Warning maxBulkEscalateEvents now has a warning that large values may run into other limits. --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index ea651cb47..8d6bab06b 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -499,7 +499,7 @@ soc: alerts: <<: *appSettings maxBulkEscalateEvents: - description: Maximum number of events to escalate in a single bulk escalation. + description: Maximum number of events to escalate in a single bulk escalation. Large values may run into other limits. global: True cases: *appSettings dashboards: *appSettings From 95fe212202c790c13eb98c13456e29c68299b024 Mon Sep 17 00:00:00 2001 From: Joshua Brower Date: Wed, 5 Feb 2025 09:29:45 -0500 Subject: [PATCH 6/8] Rework for MSI --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 850669ba0..673fe6f2e 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -77,9 +77,10 @@ do done printf "\n\n### Generating MSI...\n" +cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe docker run \ ---mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ -{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o /output/so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs +--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \ +{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs printf "\n### MSI Generated...\n" printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n" From 4636a8d9b189b9b420100dd878470f8097df9378 Mon Sep 17 00:00:00 2001 From: Joshua Brower Date: Wed, 5 Feb 2025 09:38:33 -0500 Subject: [PATCH 7/8] Refresh Agent installers --- salt/manager/tools/sbin/soup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 88dd5c9d8..259ef9abd 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -511,8 +511,7 @@ post_to_2.4.90() { } post_to_2.4.100() { - echo "Regenerating Elastic Agent Installers" - /sbin/so-elastic-agent-gen-installers + echo "Nothing to apply" POSTVERSION=2.4.100 } @@ -532,6 +531,9 @@ post_to_2.4.120() { # Manually rollover suricata alerts index to ensure data_stream.dataset expected mapping is set to 'suricata' rollover_index "logs-suricata.alerts-so" + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers + POSTVERSION=2.4.120 } From bf19c6e7303e4ba993a914d249fda1f9e410e91a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 5 Feb 2025 15:04:04 -0500 Subject: [PATCH 8/8] ca download; ignore shard errors on startup; clarify oidc id --- .github/workflows/contrib.yml | 2 +- salt/common/tools/sbin/so-log-check | 1 + salt/kratos/soc_kratos.yaml | 2 +- salt/soc/enabled.sls | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/contrib.yml b/.github/workflows/contrib.yml index 395675b43..2cbdb2788 100644 --- a/.github/workflows/contrib.yml +++ b/.github/workflows/contrib.yml @@ -18,7 +18,7 @@ jobs: with: path-to-signatures: 'signatures_v1.json' path-to-document: 'https://securityonionsolutions.com/cla' - allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,weslambert,defensivedepth,m0duspwnens + allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens remote-organization-name: Security-Onion-Solutions remote-repository-name: licensing diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 9b54cf586..91417171c 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -125,6 +125,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tls handshake error" # Docker registry container when new node comes onlines EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to get license information" # Logstash trying to contact ES before it's ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process already finished" # Telegraf script finished just as the auto kill timeout kicked in + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No shard available" # Typical error when making a query before ES has finished loading all indices fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then diff --git a/salt/kratos/soc_kratos.yaml b/salt/kratos/soc_kratos.yaml index 82490184b..bc95d9b03 100644 --- a/salt/kratos/soc_kratos.yaml +++ b/salt/kratos/soc_kratos.yaml @@ -11,7 +11,7 @@ kratos: helpLink: oidc.html config: id: - description: Customize the OIDC provider name. This name appears on the login page. Required. + description: Customize the OIDC provider name. This name appears on the login page. Required. It is strongly recommended to leave this to the default value, unless you are aware of the other configuration pieces that will be affected by changing it. global: True forcedType: string helpLink: oidc.html diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index 59f8f8e82..d687289b5 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -51,6 +51,7 @@ so-soc: - /opt/so/conf/soc/migrations:/opt/so/conf/soc/migrations:rw - /nsm/backup/detections-migration:/nsm/backup/detections-migration:ro - /opt/so/state:/opt/so/state:rw + - /etc/pki/ca.crt:/opt/sensoroni/html/so-ca.crt:ro - extra_hosts: {% for node in DOCKER_EXTRA_HOSTS %} {% for hostname, ip in node.items() %}