From 98a67530f5d01f3b665bbccf48fc4565fb2397ef Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 16 Jul 2025 12:14:25 -0400 Subject: [PATCH 01/13] update qcow2 hosted location --- salt/_runners/setup_hypervisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/_runners/setup_hypervisor.py b/salt/_runners/setup_hypervisor.py index 6ddd571c9..9d7116d59 100644 --- a/salt/_runners/setup_hypervisor.py +++ b/salt/_runners/setup_hypervisor.py @@ -165,7 +165,7 @@ def _validate_image_checksum(path, expected_sha256): return True # Constants -IMAGE_URL = "https://yum.oracle.com/templates/OracleLinux/OL9/u5/x86_64/OL9U5_x86_64-kvm-b253.qcow2" +IMAGE_URL = "https://download.securityonion.net/file/securityonion/OL9U5_x86_64-kvm-b253.qcow2" IMAGE_SHA256 = "3b00bbbefc8e78dd28d9f538834fb9e2a03d5ccdc2cadf2ffd0036c0a8f02021" IMAGE_PATH = "/nsm/libvirt/boot/OL9U5_x86_64-kvm-b253.qcow2" MANAGER_HOSTNAME = socket.gethostname() From cd5de5cd0516799d310569c70cda3d348cb52aea Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 16 Jul 2025 12:14:54 -0400 Subject: [PATCH 02/13] add sos hw models --- salt/hypervisor/defaults.yaml | 122 +++++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 30 deletions(-) diff --git a/salt/hypervisor/defaults.yaml b/salt/hypervisor/defaults.yaml index 06509828c..8cf754193 100644 --- a/salt/hypervisor/defaults.yaml +++ b/salt/hypervisor/defaults.yaml @@ -17,42 +17,104 @@ hypervisor: 6: pci_0000_02_00_1 7: pci_0000_41_00_0 8: pci_0000_41_00_1 - model1: + SOSSNNV: hardware: cpu: 128 - memory: 128 + memory: 256 disk: - 1: pci_0000_c7_00_0 - 2: pci_0000_c8_00_0 + 1: pci_0000_42_00_0 + 2: pci_0000_43_00_0 + 3: pci_0000_44_00_0 + 4: pci_0000_45_00_0 copper: - 1: pci_0000_c4_00_0 - 2: pci_0000_c4_00_1 - 3: pci_0000_c4_00_2 - 4: pci_0000_c4_00_3 + sfp: + 1: pci_0000_02_00_0 + 2: pci_0000_02_00_1 + 3: pci_0000_41_00_0 + 4: pci_0000_41_00_1 + SOSSNNV-DE02: + cpu: 128 + memory: 384 + disk: + 1: pci_0000_41_00_0 + 2: pci_0000_42_00_0 + 3: pci_0000_81_00_0 + 4: pci_0000_82_00_0 + 5: pci_0000_83_00_0 + 6: pci_0000_84_00_0 + copper: + 1: pci_0000_85_00_0 + 2: pci_0000_85_00_1 + 3: pci_0000_85_00_2 + 4: pci_0000_85_00_3 + sfp: + 5: pci_0000_c4_00_0 + 6: pci_0000_c4_00_1 + 7: pci_0000_c5_00_0 + 8: pci_0000_c5_00_1 + 9: pci_0000_c5_00_2 + 10: pci_0000_c5_00_3 + SOSSN7200: + cpu: 128 + memory: 256 + copper: + 1: pci_0000_03_00_0 + 2: pci_0000_03_00_1 + 3: pci_0000_03_00_2 + 4: pci_0000_03_00_3 sfp: 5: pci_0000_02_00_0 6: pci_0000_02_00_1 - 7: pci_0000_41_00_0 - 8: pci_0000_41_00_1 - model2: - cpu: 256 - memory: 256 - disk: - 1: pci_0000_c7_00_0 - 2: pci_0000_c8_00_0 - 3: pci_0000_c9_00_0 - 4: pci_0000_c10_00_0 + 7: pci_0000_81_00_0 + 8: pci_0000_81_00_1 + 9: pci_0000_81_00_2 + 10: pci_0000_81_00_3 + SOSSN7200-DE02: + cpu: 128 + memory: 384 copper: - 1: pci_0000_c4_00_0 - 2: pci_0000_c4_00_1 - 3: pci_0000_c4_00_2 - 4: pci_0000_c4_00_3 - 5: pci_0000_c5_00_0 - 6: pci_0000_c5_00_1 - 7: pci_0000_c5_00_2 - 8: pci_0000_c5_00_3 + 1: pci_0000_82_00_0 + 2: pci_0000_82_00_1 + 3: pci_0000_82_00_2 + 4: pci_0000_82_00_3 sfp: - 9: pci_0000_02_00_0 - 10: pci_0000_02_00_1 - 11: pci_0000_41_00_0 - 12: pci_0000_41_00_1 \ No newline at end of file + 5: pci_0000_c4_00_0 + 6: pci_0000_c4_00_1 + 7: pci_0000_c5_00_0 + 8: pci_0000_c5_00_1 + 9: pci_0000_c6_00_0 + 10: pci_0000_c6_00_1 + 11: pci_0000_c6_00_2 + 12: pci_0000_c6_00_3 + SOS4000: + cpu: 128 + memory: 256 + copper: + 1: pci_0000_03_00_0 + 2: pci_0000_03_00_1 + 3: pci_0000_03_00_2 + 4: pci_0000_03_00_3 + sfp: + 5: pci_0000_02_00_0 + 6: pci_0000_02_00_1 + 7: pci_0000_81_00_0 + 8: pci_0000_81_00_1 + 9: pci_0000_81_00_2 + 10: pci_0000_81_00_3 + SOS5000-DE02: + cpu: 128 + memory: 384 + copper: + 1: pci_0000_82_00_0 + 2: pci_0000_82_00_1 + 3: pci_0000_82_00_2 + 4: pci_0000_82_00_3 + sfp: + 5: pci_0000_c4_00_0 + 6: pci_0000_c4_00_1 + 7: pci_0000_c5_00_0 + 8: pci_0000_c5_00_1 + 9: pci_0000_c6_00_0 + 10: pci_0000_c6_00_1 + 11: pci_0000_c6_00_2 + 12: pci_0000_c6_00_3 From a3fb2f13be3848c906f53511c2c73fc41c4a8ef7 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 16 Jul 2025 14:14:16 -0400 Subject: [PATCH 03/13] dont show state changes for user-data --- salt/libvirt/images/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/libvirt/images/init.sls b/salt/libvirt/images/init.sls index b6a5baf04..21ab0553d 100644 --- a/salt/libvirt/images/init.sls +++ b/salt/libvirt/images/init.sls @@ -48,6 +48,7 @@ manage_userdata_sool9: file.managed: - name: /nsm/libvirt/images/sool9/user-data - source: salt://libvirt/images/sool9/user-data + - show_changes: False # Manage qcow2 image manage_qcow2_sool9: From 1aa876f4ebe601790acebc75d22e38b64e19bb4b Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 16 Jul 2025 14:20:55 -0400 Subject: [PATCH 04/13] add missing hardware key --- salt/hypervisor/defaults.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/hypervisor/defaults.yaml b/salt/hypervisor/defaults.yaml index 8cf754193..938fa8bb2 100644 --- a/salt/hypervisor/defaults.yaml +++ b/salt/hypervisor/defaults.yaml @@ -33,6 +33,7 @@ hypervisor: 3: pci_0000_41_00_0 4: pci_0000_41_00_1 SOSSNNV-DE02: + hardware: cpu: 128 memory: 384 disk: @@ -55,6 +56,7 @@ hypervisor: 9: pci_0000_c5_00_2 10: pci_0000_c5_00_3 SOSSN7200: + hardware: cpu: 128 memory: 256 copper: @@ -70,6 +72,7 @@ hypervisor: 9: pci_0000_81_00_2 10: pci_0000_81_00_3 SOSSN7200-DE02: + hardware: cpu: 128 memory: 384 copper: @@ -87,6 +90,7 @@ hypervisor: 11: pci_0000_c6_00_2 12: pci_0000_c6_00_3 SOS4000: + hardware: cpu: 128 memory: 256 copper: @@ -102,6 +106,7 @@ hypervisor: 9: pci_0000_81_00_2 10: pci_0000_81_00_3 SOS5000-DE02: + hardware: cpu: 128 memory: 384 copper: From e61e2f04b3e645645f05ce2b6c3ff1a5a5374479 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 16 Jul 2025 15:24:43 -0400 Subject: [PATCH 05/13] handle hw not having sfp,disk or copper. show none for total if that is the case --- .../hypervisor/soc_hypervisor.yaml.jinja | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja index 4222ff6b2..07577a1fb 100644 --- a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja +++ b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja @@ -22,7 +22,7 @@ | | CPU Cores | Memory (GB) | Disk | Copper | SFP | |-----------|-----------|-------------|-------------|-------------|-------------| | Available | {{ cpu_free }} | {{ mem_free }} | {{ disk_free | replace('\n', ',') if disk_free else 'None' }} | {{ copper_free | replace('\n', ',') if copper_free else 'None' }} | {{ sfp_free | replace('\n', ',') if sfp_free else 'None' }} | -| Total | {{ cpu_total }} | {{ mem_total }} | {{ disk_total | replace('\n', ',') }} | {{ copper_total | replace('\n', ',') }} | {{ sfp_total | replace('\n', ',') }} | +| Total | {{ cpu_total }} | {{ mem_total }} | {{ disk_total | replace('\n', ',') if disk_total else 'None' }} | {{ copper_total | replace('\n', ',') if copper_total else 'None' }} | {{ sfp_total | replace('\n', ',') if sfp_total else 'None' }} | {%- if baseDomainStatus == 'Initialized' %} {%- if vm_list %} @@ -60,7 +60,8 @@ Base domain has not been initialized. {%- macro get_available_pci(hw_config, device_type, used_indices) -%} {%- set available = [] -%} -{%- for idx in hw_config.get(device_type, {}).keys() -%} +{%- set device_config = hw_config.get(device_type, {}) or {} -%} +{%- for idx in device_config.keys() -%} {%- if idx | string not in used_indices -%} {%- do available.append(idx) -%} {%- endif -%} @@ -155,9 +156,9 @@ Base domain has not been initialized. {# Get total resources #} {%- set cpu_total = hw_config.cpu -%} {%- set mem_total = hw_config.memory -%} -{%- set disk_total = hw_config.disk.keys() | join('\n') -%} -{%- set copper_total = hw_config.copper.keys() | join('\n') -%} -{%- set sfp_total = hw_config.sfp.keys() | join('\n') -%} +{%- set disk_total = (hw_config.get('disk', {}) or {}).keys() | join('\n') if hw_config.get('disk', {}) else '' -%} +{%- set copper_total = (hw_config.get('copper', {}) or {}).keys() | join('\n') if hw_config.get('copper', {}) else '' -%} +{%- set sfp_total = (hw_config.get('sfp', {}) or {}).keys() | join('\n') if hw_config.get('sfp', {}) else '' -%} {# Update field labels with total and free values #} {%- set updated_template = TEMPLATE.copy() -%} @@ -170,20 +171,26 @@ Base domain has not been initialized. {%- do update_resource_field(updated_field, mem_free, mem_total, 'GB') -%} {%- elif field.field == 'disk' -%} {%- set disk_free_list = disk_free.split(',') if disk_free else [] -%} +{%- set disk_free_safe = disk_free if disk_free is defined else '' -%} +{%- set disk_total_safe = disk_total if disk_total is defined else '' -%} {%- do updated_field.update({ - 'label': field.label | replace('FREE', disk_free) | replace('TOTAL', disk_total | replace('\n', ',')), + 'label': field.label | replace('FREE', disk_free_safe) | replace('TOTAL', disk_total_safe | replace('\n', ',')), 'options': disk_free_list }) -%} {%- elif field.field == 'copper' -%} {%- set copper_free_list = copper_free.split(',') if copper_free else [] -%} +{%- set copper_free_safe = copper_free if copper_free is defined else '' -%} +{%- set copper_total_safe = copper_total if copper_total is defined else '' -%} {%- do updated_field.update({ - 'label': field.label | replace('FREE', copper_free) | replace('TOTAL', copper_total | replace('\n', ',')), + 'label': field.label | replace('FREE', copper_free_safe) | replace('TOTAL', copper_total_safe | replace('\n', ',')), 'options': copper_free_list }) -%} {%- elif field.field == 'sfp' -%} {%- set sfp_free_list = sfp_free.split(',') if sfp_free else [] -%} +{%- set sfp_free_safe = sfp_free if sfp_free is defined else '' -%} +{%- set sfp_total_safe = sfp_total if sfp_total is defined else '' -%} {%- do updated_field.update({ - 'label': field.label | replace('FREE', sfp_free) | replace('TOTAL', sfp_total | replace('\n', ',')), + 'label': field.label | replace('FREE', sfp_free_safe) | replace('TOTAL', sfp_total_safe | replace('\n', ',')), 'options': sfp_free_list }) -%} {%- endif -%} From b3eb06f53e839ac9792d842836a21706e8e42ff8 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 16 Jul 2025 15:56:34 -0500 Subject: [PATCH 06/13] ja4 Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/defaults.yaml | 5 ++ salt/elasticsearch/files/ingest/zeek.conn | 4 ++ salt/elasticsearch/files/ingest/zeek.http | 1 + salt/elasticsearch/files/ingest/zeek.http2 | 1 + salt/elasticsearch/files/ingest/zeek.ja4ssh | 10 +++ salt/elasticsearch/files/ingest/zeek.ssl | 2 + salt/elasticsearch/files/ingest/zeek.x509 | 1 + .../templates/component/ecs/hash.json | 69 +++++++++++++++++++ 8 files changed, 93 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/zeek.ja4ssh create mode 100644 salt/elasticsearch/templates/component/ecs/hash.json diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 73c29c8c6..4046cf146 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -567,6 +567,7 @@ elasticsearch: - common-settings - common-dynamic-mappings - winlog-mappings + - hash-mappings data_stream: {} ignore_missing_component_templates: [] index_patterns: @@ -3874,6 +3875,7 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + - hash-mappings data_stream: {} ignore_missing_component_templates: [] index_patterns: @@ -3987,6 +3989,7 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + - hash-mappings data_stream: {} ignore_missing_component_templates: [] index_patterns: @@ -4100,6 +4103,7 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + - hash-mappings data_stream: {} ignore_missing_component_templates: [] index_patterns: @@ -4329,6 +4333,7 @@ elasticsearch: - zeek-mappings - common-settings - common-dynamic-mappings + - hash-mappings data_stream: {} ignore_missing_component_templates: [] index_patterns: diff --git a/salt/elasticsearch/files/ingest/zeek.conn b/salt/elasticsearch/files/ingest/zeek.conn index 6051d93a1..fe3d3b832 100644 --- a/salt/elasticsearch/files/ingest/zeek.conn +++ b/salt/elasticsearch/files/ingest/zeek.conn @@ -24,6 +24,10 @@ { "rename": { "field": "message2.resp_cc", "target_field": "server.country_code", "ignore_missing": true } }, { "rename": { "field": "message2.sensorname", "target_field": "observer.name", "ignore_missing": true } }, { "rename": { "field": "message2.vlan", "target_field": "network.vlan.id", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4l", "target_field": "hash.ja4l", "ignore_missing" : true }}, + { "rename": { "field": "message2.ja4ls", "target_field": "hash.ja4ls", "ignore_missing" : true }}, + { "rename": { "field": "message2.ja4t", "target_field": "hash.ja4t", "ignore_missing" : true }}, + { "rename": { "field": "message2.ja4ts", "target_field": "hash.ja4ts", "ignore_missing" : true }}, { "script": { "lang": "painless", "source": "ctx.network.bytes = (ctx.client.bytes + ctx.server.bytes)", "ignore_failure": true } }, { "set": { "if": "ctx.connection?.state == 'S0'", "field": "connection.state_description", "value": "Connection attempt seen, no reply" } }, { "set": { "if": "ctx.connection?.state == 'S1'", "field": "connection.state_description", "value": "Connection established, not terminated" } }, diff --git a/salt/elasticsearch/files/ingest/zeek.http b/salt/elasticsearch/files/ingest/zeek.http index 2224da2f7..8b2a320f9 100644 --- a/salt/elasticsearch/files/ingest/zeek.http +++ b/salt/elasticsearch/files/ingest/zeek.http @@ -27,6 +27,7 @@ { "rename": { "field": "message2.resp_fuids", "target_field": "log.id.resp_fuids", "ignore_missing": true } }, { "rename": { "field": "message2.resp_filenames", "target_field": "file.resp_filenames", "ignore_missing": true } }, { "rename": { "field": "message2.resp_mime_types", "target_field": "file.resp_mime_types", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4h", "target_field": "hash.ja4h", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", "ignore_failure": true } }, { "script": { "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()", "ignore_failure": true } }, { "script": { "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.http2 b/salt/elasticsearch/files/ingest/zeek.http2 index eeeecef8c..e72259dda 100644 --- a/salt/elasticsearch/files/ingest/zeek.http2 +++ b/salt/elasticsearch/files/ingest/zeek.http2 @@ -27,6 +27,7 @@ { "rename": { "field": "message2.resp_filenames", "target_field": "file.resp_filenames", "ignore_missing": true } }, { "rename": { "field": "message2.resp_mime_types", "target_field": "file.resp_mime_types", "ignore_missing": true } }, { "rename": { "field": "message2.stream_id", "target_field": "http2.stream_id", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4h", "target_field": "hash.ja4h", "ignore_missing": true } }, { "remove": { "field": "message2.tags", "ignore_failure": true } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "script": { "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ja4ssh b/salt/elasticsearch/files/ingest/zeek.ja4ssh new file mode 100644 index 000000000..45a569792 --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.ja4ssh @@ -0,0 +1,10 @@ +{ + "description": "zeek.ja4ssh", + "processors": [ + {"set": {"field": "event.dataset","value": "ja4ssh"}}, + {"remove": {"field": "host","ignore_missing": true,"ignore_failure": true}}, + {"json": {"field": "message","target_field": "message2","ignore_failure": true}}, + {"rename": {"field": "message2.ja4ssh", "target_field": "ja4.ja4ssh", "ignore_missing": true}}, + {"pipeline": {"name": "zeek.common"}} + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/zeek.ssl b/salt/elasticsearch/files/ingest/zeek.ssl index 87174d3d2..c694ab333 100644 --- a/salt/elasticsearch/files/ingest/zeek.ssl +++ b/salt/elasticsearch/files/ingest/zeek.ssl @@ -23,6 +23,8 @@ { "rename": { "field": "message2.validation_status","target_field": "ssl.validation_status", "ignore_missing": true } }, { "rename": { "field": "message2.ja3", "target_field": "hash.ja3", "ignore_missing": true } }, { "rename": { "field": "message2.ja3s", "target_field": "hash.ja3s", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4", "target_field": "hash.ja4", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4s", "target_field": "hash.ja4s", "ignore_missing": true } }, { "foreach": { "if": "ctx?.tls?.client?.hash?.sha256 !=null", diff --git a/salt/elasticsearch/files/ingest/zeek.x509 b/salt/elasticsearch/files/ingest/zeek.x509 index 64d06131a..daec96245 100644 --- a/salt/elasticsearch/files/ingest/zeek.x509 +++ b/salt/elasticsearch/files/ingest/zeek.x509 @@ -42,6 +42,7 @@ { "dot_expander": { "field": "basic_constraints.path_length", "path": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.basic_constraints.path_length", "target_field": "x509.basic_constraints.path_length", "ignore_missing": true } }, { "rename": { "field": "message2.fingerprint", "target_field": "hash.sha256", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4x", "target_field": "hash.ja4x", "ignore_missing": true } }, { "pipeline": { "name": "zeek.common_ssl" } } ] } diff --git a/salt/elasticsearch/templates/component/ecs/hash.json b/salt/elasticsearch/templates/component/ecs/hash.json new file mode 100644 index 000000000..c9d1f5a5d --- /dev/null +++ b/salt/elasticsearch/templates/component/ecs/hash.json @@ -0,0 +1,69 @@ +{ + "template": { + "mappings": { + "properties": { + "hash": { + "type": "object", + "properties": { + "ja3": { + "type": "keyword", + "ignore_above": 1024 + }, + "ja3s": { + "type": "keyword", + "ignore_above": 1024 + }, + "hassh": { + "type": "keyword", + "ignore_above": 1024 + }, + "md5": { + "type": "keyword", + "ignore_above": 1024 + }, + "sha1": { + "type": "keyword", + "ignore_above": 1024 + }, + "sha256": { + "type": "keyword", + "ignore_above": 1024 + }, + "ja4": { + "type": "keyword", + "ignore_above": 1024 + }, + "ja4l": { + "type": "keyword", + "ignore_above": 1024 + }, + "ja4ls": { + "type": "keyword", + "ignore_above": 1024 + }, + "ja4t": { + "type": "keyword", + "ignore_above": 1024 + }, + "ja4ts": { + "type": "keyword", + "ignore_above": 1024 + }, + "ja4ssh": { + "type": "keyword", + "ignore_above": 1024 + }, + "ja4h": { + "type": "keyword", + "ignore_above": 1024 + }, + "ja4x": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + } + } +} \ No newline at end of file From c29f11863e96b86540fae76386c5e5ec3b8fe7d4 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 17 Jul 2025 10:47:00 -0500 Subject: [PATCH 07/13] ja4 ignore empty strings --- salt/elasticsearch/files/ingest/zeek.conn | 8 ++++---- salt/elasticsearch/files/ingest/zeek.http | 2 +- salt/elasticsearch/files/ingest/zeek.http2 | 2 +- salt/elasticsearch/files/ingest/zeek.ja4ssh | 2 +- salt/elasticsearch/files/ingest/zeek.ssl | 4 ++-- salt/elasticsearch/files/ingest/zeek.x509 | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/salt/elasticsearch/files/ingest/zeek.conn b/salt/elasticsearch/files/ingest/zeek.conn index fe3d3b832..57558e863 100644 --- a/salt/elasticsearch/files/ingest/zeek.conn +++ b/salt/elasticsearch/files/ingest/zeek.conn @@ -24,10 +24,10 @@ { "rename": { "field": "message2.resp_cc", "target_field": "server.country_code", "ignore_missing": true } }, { "rename": { "field": "message2.sensorname", "target_field": "observer.name", "ignore_missing": true } }, { "rename": { "field": "message2.vlan", "target_field": "network.vlan.id", "ignore_missing": true } }, - { "rename": { "field": "message2.ja4l", "target_field": "hash.ja4l", "ignore_missing" : true }}, - { "rename": { "field": "message2.ja4ls", "target_field": "hash.ja4ls", "ignore_missing" : true }}, - { "rename": { "field": "message2.ja4t", "target_field": "hash.ja4t", "ignore_missing" : true }}, - { "rename": { "field": "message2.ja4ts", "target_field": "hash.ja4ts", "ignore_missing" : true }}, + { "rename": { "field": "message2.ja4l", "target_field": "hash.ja4l", "ignore_missing" : true, "if": "ctx.message2?.ja4l != null && ctx.message2.ja4l.length() > 0" }}, + { "rename": { "field": "message2.ja4ls", "target_field": "hash.ja4ls", "ignore_missing" : true, "if": "ctx.message2?.ja4ls != null && ctx.message2.ja4ls.length() > 0" }}, + { "rename": { "field": "message2.ja4t", "target_field": "hash.ja4t", "ignore_missing" : true, "if": "ctx.message2?.ja4t != null && ctx.message2.ja4t.length() > 0" }}, + { "rename": { "field": "message2.ja4ts", "target_field": "hash.ja4ts", "ignore_missing" : true, "if": "ctx.message2?.ja4ts != null && ctx.message2.ja4ts.length() > 0" }}, { "script": { "lang": "painless", "source": "ctx.network.bytes = (ctx.client.bytes + ctx.server.bytes)", "ignore_failure": true } }, { "set": { "if": "ctx.connection?.state == 'S0'", "field": "connection.state_description", "value": "Connection attempt seen, no reply" } }, { "set": { "if": "ctx.connection?.state == 'S1'", "field": "connection.state_description", "value": "Connection established, not terminated" } }, diff --git a/salt/elasticsearch/files/ingest/zeek.http b/salt/elasticsearch/files/ingest/zeek.http index 8b2a320f9..40642052a 100644 --- a/salt/elasticsearch/files/ingest/zeek.http +++ b/salt/elasticsearch/files/ingest/zeek.http @@ -27,7 +27,7 @@ { "rename": { "field": "message2.resp_fuids", "target_field": "log.id.resp_fuids", "ignore_missing": true } }, { "rename": { "field": "message2.resp_filenames", "target_field": "file.resp_filenames", "ignore_missing": true } }, { "rename": { "field": "message2.resp_mime_types", "target_field": "file.resp_mime_types", "ignore_missing": true } }, - { "rename": { "field": "message2.ja4h", "target_field": "hash.ja4h", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4h", "target_field": "hash.ja4h", "ignore_missing": true, "if": "ctx?.message2?.ja4h != null && ctx.message2.ja4h.length() > 0" } }, { "script": { "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", "ignore_failure": true } }, { "script": { "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()", "ignore_failure": true } }, { "script": { "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.http2 b/salt/elasticsearch/files/ingest/zeek.http2 index e72259dda..34cce0f7b 100644 --- a/salt/elasticsearch/files/ingest/zeek.http2 +++ b/salt/elasticsearch/files/ingest/zeek.http2 @@ -27,7 +27,7 @@ { "rename": { "field": "message2.resp_filenames", "target_field": "file.resp_filenames", "ignore_missing": true } }, { "rename": { "field": "message2.resp_mime_types", "target_field": "file.resp_mime_types", "ignore_missing": true } }, { "rename": { "field": "message2.stream_id", "target_field": "http2.stream_id", "ignore_missing": true } }, - { "rename": { "field": "message2.ja4h", "target_field": "hash.ja4h", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4h", "target_field": "hash.ja4h", "ignore_missing": true, "if": "ctx?.message2?.ja4h != null && ctx.message2.ja4h.length() > 0" } }, { "remove": { "field": "message2.tags", "ignore_failure": true } }, { "remove": { "field": ["host"], "ignore_failure": true } }, { "script": { "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", "ignore_failure": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.ja4ssh b/salt/elasticsearch/files/ingest/zeek.ja4ssh index 45a569792..5901e65f1 100644 --- a/salt/elasticsearch/files/ingest/zeek.ja4ssh +++ b/salt/elasticsearch/files/ingest/zeek.ja4ssh @@ -4,7 +4,7 @@ {"set": {"field": "event.dataset","value": "ja4ssh"}}, {"remove": {"field": "host","ignore_missing": true,"ignore_failure": true}}, {"json": {"field": "message","target_field": "message2","ignore_failure": true}}, - {"rename": {"field": "message2.ja4ssh", "target_field": "ja4.ja4ssh", "ignore_missing": true}}, + {"rename": {"field": "message2.ja4ssh", "target_field": "hash.ja4ssh", "ignore_missing": true, "if": "ctx?.message2?.ja4ssh != null && ctx.message2.ja4ssh.length() > 0" }}, {"pipeline": {"name": "zeek.common"}} ] } \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/zeek.ssl b/salt/elasticsearch/files/ingest/zeek.ssl index c694ab333..0bd6fedb2 100644 --- a/salt/elasticsearch/files/ingest/zeek.ssl +++ b/salt/elasticsearch/files/ingest/zeek.ssl @@ -23,8 +23,8 @@ { "rename": { "field": "message2.validation_status","target_field": "ssl.validation_status", "ignore_missing": true } }, { "rename": { "field": "message2.ja3", "target_field": "hash.ja3", "ignore_missing": true } }, { "rename": { "field": "message2.ja3s", "target_field": "hash.ja3s", "ignore_missing": true } }, - { "rename": { "field": "message2.ja4", "target_field": "hash.ja4", "ignore_missing": true } }, - { "rename": { "field": "message2.ja4s", "target_field": "hash.ja4s", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4", "target_field": "hash.ja4", "ignore_missing": true, "if": "ctx?.message2?.ja4 != null && ctx.message2.ja4.length() > 0" } }, + { "rename": { "field": "message2.ja4s", "target_field": "hash.ja4s", "ignore_missing": true, "if": "ctx?.message2?.ja4s != null && ctx.message2.ja4s.length() > 0" } }, { "foreach": { "if": "ctx?.tls?.client?.hash?.sha256 !=null", diff --git a/salt/elasticsearch/files/ingest/zeek.x509 b/salt/elasticsearch/files/ingest/zeek.x509 index daec96245..b639cb417 100644 --- a/salt/elasticsearch/files/ingest/zeek.x509 +++ b/salt/elasticsearch/files/ingest/zeek.x509 @@ -42,7 +42,7 @@ { "dot_expander": { "field": "basic_constraints.path_length", "path": "message2", "ignore_failure": true } }, { "rename": { "field": "message2.basic_constraints.path_length", "target_field": "x509.basic_constraints.path_length", "ignore_missing": true } }, { "rename": { "field": "message2.fingerprint", "target_field": "hash.sha256", "ignore_missing": true } }, - { "rename": { "field": "message2.ja4x", "target_field": "hash.ja4x", "ignore_missing": true } }, + { "rename": { "field": "message2.ja4x", "target_field": "hash.ja4x", "ignore_missing": true, "if": "ctx?.message2?.ja4x != null && ctx.message2.ja4x.length() > 0" } }, { "pipeline": { "name": "zeek.common_ssl" } } ] } From 0b1f2252ee6c07964c9321ebf5379d641d292d8b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 17 Jul 2025 13:27:54 -0500 Subject: [PATCH 08/13] elasticsearch troubleshoot script --- .../tools/sbin/so-elasticsearch-troubleshoot | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 salt/elasticsearch/tools/sbin/so-elasticsearch-troubleshoot diff --git a/salt/elasticsearch/tools/sbin/so-elasticsearch-troubleshoot b/salt/elasticsearch/tools/sbin/so-elasticsearch-troubleshoot new file mode 100644 index 000000000..b6b201c3c --- /dev/null +++ b/salt/elasticsearch/tools/sbin/so-elasticsearch-troubleshoot @@ -0,0 +1,194 @@ +#!/bin/bash + +. /usr/sbin/so-common + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +BOLD='\033[1;37m' +NC='\033[0m' + +log_title() { + if [ $1 == "LOG" ]; then + echo -e "\n${BOLD}================ $2 ================${NC}\n" + elif [ $1 == "OK" ]; then + echo -e "${GREEN} $2 ${NC}" + elif [ $1 == "WARN" ]; then + echo -e "${YELLOW} $2 ${NC}" + elif [ $1 == "ERROR" ]; then + echo -e "${RED} $2 ${NC}" + fi +} + +health_report() { + if ! health_report_output=$(so-elasticsearch-query _health_report?format=json --fail 2>/dev/null); then + log_title "ERROR" "Failed to retrieve health report from Elasticsearch" + return 1 + fi + non_green_count=$(echo "$health_report_output" | jq '[.indicators | to_entries[] | select(.value.status != "green")] | length') + + if [ "$non_green_count" -gt 0 ]; then + echo "$health_report_output" | jq -r '.indicators | to_entries[] | select(.value.status != "green") | .key' | while read -r indicator_name; do + indicator=$(echo "$health_report_output" | jq -r ".indicators.\"$indicator_name\"") + status=$(echo "$indicator" | jq -r '.status') + symptom=$(echo "$indicator" | jq -r '.symptom // "No symptom available"') + + # reormat indicator name + display_name=$(echo "$indicator_name" | tr '_' ' ' | sed 's/\b\(.\)/\u\1/g') + + if [ "$status" = "yellow" ]; then + log_title "WARN" "$display_name: $symptom" + else + log_title "ERROR" "$display_name: $symptom" + fi + + # diagnosis if available + echo "$indicator" | jq -c '.diagnosis[]? // empty' | while read -r diagnosis; do + cause=$(echo "$diagnosis" | jq -r '.cause // "Unknown"') + action=$(echo "$diagnosis" | jq -r '.action // "No action specified"') + + echo -e " ${BOLD}Cause:${NC} $cause\n" + echo -e " ${BOLD}Action:${NC} $action\n" + + # Check for affected indices + affected_indices=$(echo "$diagnosis" | jq -r '.affected_resources.indices[]? // empty') + if [ -n "$affected_indices" ]; then + echo -e " ${BOLD}Affected indices:${NC}" + total_indices=$(echo "$affected_indices" | wc -l) + echo "$affected_indices" | head -10 | while read -r index; do + echo " - $index" + done + if [ "$total_indices" -gt 10 ]; then + remaining=$((total_indices - 10)) + echo " ... and $remaining more indices (truncated for readability)" + fi + fi + echo + done + done + else + log_title "OK" "All health indicators are green" + fi +} + +elasticsearch_status() { + log_title "LOG" "Elasticsearch Status" + if so-elasticsearch-query / --fail --output /dev/null; then + health_report + else + log_title "ERROR" "Elasticsearch API is not accessible" + so-status + log_title "ERROR" "Make sure Elasticsearch is running. Addtionally, check for startup errors in /opt/so/log/elasticsearch/securityonion.log${NC}\n" + + exit 1 + fi + +} + +indices_by_age() { + log_title "LOG" "Indices by Creation Date - Size > 1KB" + log_title "WARN" "Since high/flood watermark has been reached consider updating ILM policies.\n" + if ! indices_output=$(so-elasticsearch-query '_cat/indices?v&s=creation.date:asc&h=creation.date.string,index,status,health,docs.count,pri.store.size&bytes=b&format=json' --fail 2>/dev/null); then + log_title "ERROR" "Failed to retrieve indices list from Elasticsearch" + return 1 + fi + + # Filter for indices with size > 1KB (1024 bytes) and format output + echo -e "${BOLD}Creation Date Name Size${NC}" + echo -e "${BOLD}--------------------------------------------------------------------------------------------------------------${NC}" + + # Create list of indices excluding .internal, so-detection*, so-case* + echo "$indices_output" | jq -r '.[] | select((."pri.store.size" | tonumber) > 1024) | select(.index | (startswith(".internal") or startswith("so-detection") or startswith("so-case")) | not ) | "\(."creation.date.string") | \(.index) | \(."pri.store.size")"' | while IFS='|' read -r creation_date index_name size_bytes; do + # Convert bytes to GB / MB + if [ "$size_bytes" -gt 1073741824 ]; then + size_human=$(echo "scale=2; $size_bytes / 1073741824" | bc)GB + else + size_human=$(echo "scale=2; $size_bytes / 1048576" | bc)MB + fi + + creation_date=$(date -d "$creation_date" '+%Y-%m-%dT%H:%MZ' ) + + # Format output with spacing + printf "%-19s %-76s %10s\n" "$creation_date" "$index_name" "$size_human" + done +} + +watermark_settings() { + watermark_path=".defaults.cluster.routing.allocation.disk.watermark" + if ! watermark_output=$(so-elasticsearch-query _cluster/settings?include_defaults=true\&filter_path=*.cluster.routing.allocation.disk.* --fail 2>/dev/null); then + log_title "ERROR" "Failed to retrieve watermark settings from Elasticsearch" + return 1 + fi + + if ! disk_allocation_output=$(so-elasticsearch-query _cat/nodes?v\&h=name,ip,disk.used_percent,disk.avail,disk.total\&format=json --fail 2>/dev/null); then + log_title "ERROR" "Failed to retrieve disk allocation data from Elasticsearch" + return 1 + fi + + flood=$(echo $watermark_output | jq -r "$watermark_path.flood_stage" ) + high=$(echo $watermark_output | jq -r "$watermark_path.high" ) + low=$(echo $watermark_output | jq -r "$watermark_path.low" ) + + # Strip percentage signs for comparison + flood_num=${flood%\%} + high_num=${high%\%} + low_num=${low%\%} + + # Check each nodes disk usage + log_title "LOG" "Disk Usage Check" + echo -e "${BOLD}LOW:${GREEN}$low${NC}${BOLD} HIGH:${YELLOW}${high}${NC}${BOLD} FLOOD:${RED}${flood}${NC}\n" + + echo "$disk_allocation_output" | jq -r '.[] | "\(.name)|\(.["disk.used_percent"])"' | while IFS='|' read -r node_name disk_used; do + disk_used_num=$(echo $disk_used | bc) + + if (( $(echo "$disk_used_num >= $flood_num" | bc -l) )); then + log_title "ERROR" "$node_name is at or above the flood watermark ($flood)! Disk usage: ${disk_used}%" + touch /tmp/watermark_reached + elif (( $(echo "$disk_used_num >= $high_num" | bc -l) )); then + log_title "ERROR" "$node_name is at or above the high watermark ($high)! Disk usage: ${disk_used}%" + touch /tmp/watermark_reached + else + log_title "OK" "$node_name disk usage: ${disk_used}%" + fi + done + + # Check if we need to show indices by age + if [ -f /tmp/watermark_reached ]; then + indices_by_age + rm -f /tmp/watermark_reached + fi + +} + +unassigned_shards() { + + if ! unassigned_shards_output=$(so-elasticsearch-query _cat/shards?v\&h=index,shard,prirep,state,unassigned.reason,unassigned.details\&s=state\&format=json --fail 2>/dev/null); then + log_title "ERROR" "Failed to retrieve shard data from Elasticsearch" + return 1 + fi + + log_title "LOG" "Unassigned Shards Check" + # Check if there are any UNASSIGNED shards + unassigned_count=$(echo "$unassigned_shards_output" | jq '[.[] | select(.state == "UNASSIGNED")] | length') + + if [ "$unassigned_count" -gt 0 ]; then + echo "$unassigned_shards_output" | jq -r '.[] | select(.state == "UNASSIGNED") | "\(.index)|\(.shard)|\(.prirep)|\(."unassigned.reason")"' | while IFS='|' read -r index shard prirep reason; do + if [ "$prirep" = "r" ]; then + log_title "WARN" "Replica shard for index $index is unassigned. Reason: $reason" + elif [ "$prirep" = "p" ]; then + log_title "ERROR" "Primary shard for index $index is unassigned. Reason: $reason" + fi + done + else + log_title "OK" "All shards are assigned" + fi +} + +main() { + elasticsearch_status + watermark_settings + unassigned_shards +} + +main From f27714890a08a9a16ad24deedab6945c157571a3 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 18 Jul 2025 09:35:51 -0500 Subject: [PATCH 09/13] update file ownership to socore --- salt/idh/config.sls | 2 +- salt/idstools/config.sls | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/idh/config.sls b/salt/idh/config.sls index 91f809f9e..2e6315007 100644 --- a/salt/idh/config.sls +++ b/salt/idh/config.sls @@ -86,7 +86,7 @@ idh_sbin: file.recurse: - name: /usr/sbin - source: salt://idh/tools/sbin - - user: 934 + - user: 939 - group: 939 - file_mode: 755 diff --git a/salt/idstools/config.sls b/salt/idstools/config.sls index a44b02807..cea75ab9a 100644 --- a/salt/idstools/config.sls +++ b/salt/idstools/config.sls @@ -20,7 +20,7 @@ idstools_sbin: file.recurse: - name: /usr/sbin - source: salt://idstools/tools/sbin - - user: 934 + - user: 939 - group: 939 - file_mode: 755 @@ -29,7 +29,7 @@ idstools_sbin: # file.recurse: # - name: /usr/sbin # - source: salt://idstools/tools/sbin_jinja -# - user: 934 +# - user: 939 # - group: 939 # - file_mode: 755 # - template: jinja @@ -38,7 +38,7 @@ idstools_so-rule-update: file.managed: - name: /usr/sbin/so-rule-update - source: salt://idstools/tools/sbin_jinja/so-rule-update - - user: 934 + - user: 939 - group: 939 - mode: 755 - template: jinja From 05a368681a6d76f085357fe37de3a5635d4a8e06 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 21 Jul 2025 10:53:54 -0400 Subject: [PATCH 10/13] Create config.zeek.ja4 --- salt/zeek/files/config.zeek.ja4 | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 salt/zeek/files/config.zeek.ja4 diff --git a/salt/zeek/files/config.zeek.ja4 b/salt/zeek/files/config.zeek.ja4 new file mode 100644 index 000000000..e3dd08a48 --- /dev/null +++ b/salt/zeek/files/config.zeek.ja4 @@ -0,0 +1,25 @@ +module FINGERPRINT; + +export { + option delimiter: string = "_"; + + # BSD licensed + option JA4_enabled: bool = T; + option JA4_raw: bool = F; + + # FoxIO license required for JA4+ + option JA4S_enabled: bool = F; + option JA4S_raw: bool = F; + + option JA4H_enabled: bool = F; + option JA4H_raw: bool = F; + + option JA4L_enabled: bool = F; + + option JA4SSH_enabled: bool = F; + + option JA4T_enabled: bool = F; + option JA4TS_enabled: bool = F; + + option JA4X_enabled: bool = F; +} From 93024738d31715f9b613520e9adbc0d7b1d4d262 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 21 Jul 2025 10:57:45 -0400 Subject: [PATCH 11/13] Update config.sls --- salt/zeek/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/zeek/config.sls b/salt/zeek/config.sls index 7fdbd8560..761c6f7d3 100644 --- a/salt/zeek/config.sls +++ b/salt/zeek/config.sls @@ -150,6 +150,13 @@ plcronscript: - source: salt://zeek/cron/packetloss.sh - mode: 755 +zeekja4cfg: + file.managed: + - name: /opt/so/conf/zeek/config.zeek + - source: salt://zeek/files/config.zeek.ja4 + - user: 937 + - group: 939 + # BPF compilation and configuration {% if ZEEKBPF %} {% set BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + ZEEKBPF|join(" "),cwd='/root') %} From 5dac3ff2a672fde5d7faaef2774bcf4d489bfeea Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 21 Jul 2025 10:58:25 -0400 Subject: [PATCH 12/13] Update enabled.sls --- salt/zeek/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/zeek/enabled.sls b/salt/zeek/enabled.sls index 7d444ff43..ff090428f 100644 --- a/salt/zeek/enabled.sls +++ b/salt/zeek/enabled.sls @@ -34,6 +34,7 @@ so-zeek: - /opt/so/conf/zeek/policy/cve-2020-0601:/opt/zeek/share/zeek/policy/cve-2020-0601:ro - /opt/so/conf/zeek/policy/intel:/opt/zeek/share/zeek/policy/intel:rw - /opt/so/conf/zeek/bpf:/opt/zeek/etc/bpf:ro + - /opt/so/conf/zeek/config.zeek:/opt/zeek/share/zeek/site/packages/ja4/config.zeek:ro {% if DOCKER.containers['so-zeek'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-zeek'].custom_bind_mounts %} - {{ BIND }} From eabca5df1897c9330e641bf71d4854430e2497b8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 21 Jul 2025 11:01:33 -0400 Subject: [PATCH 13/13] Update defaults.yaml --- salt/zeek/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index 1daf77102..d87a67ac6 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -52,6 +52,7 @@ zeek: - policy/frameworks/notice/community-id - policy/protocols/conn/community-id-logging - ja3 + - ja4 - hassh - intel - cve-2020-0601