diff --git a/pillar/top.sls b/pillar/top.sls index 78d7191a0..ec4748469 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -44,8 +44,6 @@ base: - secrets - manager.soc_manager - manager.adv_manager - - idstools.soc_idstools - - idstools.adv_idstools - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash @@ -118,8 +116,6 @@ base: - elastalert.adv_elastalert - manager.soc_manager - manager.adv_manager - - idstools.soc_idstools - - idstools.adv_idstools - soc.soc_soc - soc.adv_soc - kibana.soc_kibana @@ -159,8 +155,6 @@ base: {% endif %} - secrets - healthcheck.standalone - - idstools.soc_idstools - - idstools.adv_idstools - kratos.soc_kratos - kratos.adv_kratos - hydra.soc_hydra diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index b8eb40f99..4f4dc1667 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -35,8 +35,6 @@ 'hydra', 'elasticfleet', 'elastic-fleet-package-registry', - 'idstools', - 'suricata.manager', 'utility' ] %} diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 588c767f1..4358e6d00 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -25,7 +25,6 @@ container_list() { if [ $MANAGERCHECK == 'so-import' ]; then TRUSTED_CONTAINERS=( "so-elasticsearch" - "so-idstools" "so-influxdb" "so-kibana" "so-kratos" @@ -49,7 +48,6 @@ container_list() { "so-elastic-fleet-package-registry" "so-elasticsearch" "so-idh" - "so-idstools" "so-influxdb" "so-kafka" "so-kibana" @@ -69,7 +67,6 @@ container_list() { ) else TRUSTED_CONTAINERS=( - "so-idstools" "so-elasticsearch" "so-logstash" "so-nginx" diff --git a/salt/common/tools/sbin_jinja/so-import-pcap b/salt/common/tools/sbin_jinja/so-import-pcap index b630df015..9171c4bc6 100755 --- a/salt/common/tools/sbin_jinja/so-import-pcap +++ b/salt/common/tools/sbin_jinja/so-import-pcap @@ -85,7 +85,7 @@ function suricata() { docker run --rm \ -v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \ -v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \ - -v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \ + -v /opt/so/rules/suricata/:/etc/suricata/rules:ro \ -v ${LOG_PATH}:/var/log/suricata/:rw \ -v ${NSM_PATH}/:/nsm/:rw \ -v "$PCAP:/input.pcap:ro" \ diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 2d7ad4e1c..456a187d6 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -24,11 +24,6 @@ docker: custom_bind_mounts: [] extra_hosts: [] extra_env: [] - 'so-idstools': - final_octet: 25 - custom_bind_mounts: [] - extra_hosts: [] - extra_env: [] 'so-influxdb': final_octet: 26 port_bindings: diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index dacbf2302..3c4475236 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -41,7 +41,6 @@ docker: forcedType: "[]string" so-elastic-fleet: *dockerOptions so-elasticsearch: *dockerOptions - so-idstools: *dockerOptions so-influxdb: *dockerOptions so-kibana: *dockerOptions so-kratos: *dockerOptions @@ -102,4 +101,4 @@ docker: multiline: True forcedType: "[]string" so-zeek: *dockerOptions - so-kafka: *dockerOptions \ No newline at end of file + so-kafka: *dockerOptions diff --git a/salt/elasticfleet/config.map.jinja b/salt/elasticfleet/config.map.jinja new file mode 100644 index 000000000..b95a3e895 --- /dev/null +++ b/salt/elasticfleet/config.map.jinja @@ -0,0 +1,34 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} + +{# advanced config_yaml options for elasticfleet logstash output #} +{% set ADV_OUTPUT_LOGSTASH_RAW = ELASTICFLEETMERGED.config.outputs.logstash %} +{% set ADV_OUTPUT_LOGSTASH = {} %} +{% for k, v in ADV_OUTPUT_LOGSTASH_RAW.items() %} +{% if v != "" and v is not none %} +{% if k == 'queue_mem_events' %} +{# rename queue_mem_events queue.mem.events #} +{% do ADV_OUTPUT_LOGSTASH.update({'queue.mem.events':v}) %} +{% elif k == 'loadbalance' %} +{% if v %} +{# only include loadbalance config when its True #} +{% do ADV_OUTPUT_LOGSTASH.update({k:v}) %} +{% endif %} +{% else %} +{% do ADV_OUTPUT_LOGSTASH.update({k:v}) %} +{% endif %} +{% endif %} +{% endfor %} + +{% set LOGSTASH_CONFIG_YAML_RAW = [] %} +{% if ADV_OUTPUT_LOGSTASH %} +{% for k, v in ADV_OUTPUT_LOGSTASH.items() %} +{% do LOGSTASH_CONFIG_YAML_RAW.append(k ~ ': ' ~ v) %} +{% endfor %} +{% endif %} + +{% set LOGSTASH_CONFIG_YAML = LOGSTASH_CONFIG_YAML_RAW | join('\\n') if LOGSTASH_CONFIG_YAML_RAW else '' %} diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 0f013e320..a3132d3f4 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -10,6 +10,14 @@ elasticfleet: grid_enrollment: '' defend_filters: enable_auto_configuration: False + outputs: + logstash: + bulk_max_size: '' + worker: '' + queue_mem_events: '' + timeout: '' + loadbalance: False + compression_level: '' subscription_integrations: False auto_upgrade_integrations: False logging: diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 2a4f4d0db..25212bbce 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -38,12 +38,13 @@ so-elastic-fleet-auto-configure-logstash-outputs: {# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #} so-elastic-fleet-auto-configure-logstash-outputs-force: cmd.run: - - name: /usr/sbin/so-elastic-fleet-outputs-update --force --certs + - name: /usr/sbin/so-elastic-fleet-outputs-update --certs - retry: attempts: 4 interval: 30 - onchanges: - x509: etc_elasticfleet_logstash_crt + - x509: elasticfleet_kafka_crt {% endif %} # If enabled, automatically update Fleet Server URLs & ES Connection diff --git a/salt/elasticfleet/integration-defaults.map.jinja b/salt/elasticfleet/integration-defaults.map.jinja index 500a9e63c..69ce7f3af 100644 --- a/salt/elasticfleet/integration-defaults.map.jinja +++ b/salt/elasticfleet/integration-defaults.map.jinja @@ -121,6 +121,9 @@ "phases": { "cold": { "actions": { + "allocate":{ + "number_of_replicas": "" + }, "set_priority": {"priority": 0} }, "min_age": "60d" @@ -137,12 +140,31 @@ "max_age": "30d", "max_primary_shard_size": "50gb" }, + "forcemerge":{ + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, "set_priority": {"priority": 100} }, "min_age": "0ms" }, "warm": { "actions": { + "allocate": { + "number_of_replicas": "" + }, + "forcemerge": { + "max_num_segments": "" + }, + "shrink":{ + "max_primary_shard_size": "", + "method": "COUNT", + "number_of_shards": "" + }, "set_priority": {"priority": 50} }, "min_age": "30d" diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 450e044e6..d78189f96 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -50,6 +50,46 @@ elasticfleet: global: True forcedType: bool helpLink: elastic-fleet.html + outputs: + logstash: + bulk_max_size: + description: The maximum number of events to bulk in a single Logstash request. + global: True + forcedType: int + advanced: True + helpLink: elastic-fleet.html + worker: + description: The number of workers per configured host publishing events. + global: True + forcedType: int + advanced: true + helpLink: elastic-fleet.html + queue_mem_events: + title: queued events + description: The number of events the queue can store. This value should be evenly divisible by the smaller of 'bulk_max_size' to avoid sending partial batches to the output. + global: True + forcedType: int + advanced: True + helpLink: elastic-fleet.html + timeout: + description: The number of seconds to wait for responses from the Logstash server before timing out. Eg 30s + regex: ^[0-9]+s$ + advanced: True + global: True + helpLink: elastic-fleet.html + loadbalance: + description: If true and multiple Logstash hosts are configured, the output plugin load balances published events onto all Logstash hosts. If false, the output plugin sends all events to one host (determined at random) and switches to another host if the selected one becomes unresponsive. + forcedType: bool + advanced: True + global: True + helpLink: elastic-fleet.html + compression_level: + description: The gzip compression level. The compression level must be in the range of 1 (best speed) to 9 (best compression). + regex: ^[1-9]$ + forcedType: int + advanced: True + global: True + helpLink: elastic-fleet.html server: custom_fqdn: description: Custom FQDN for Agents to connect to. One per line. diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 4fa68298c..58baadca5 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -3,13 +3,16 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} +{%- from 'vars/globals.map.jinja' import GLOBALS %} +{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} +{%- from 'elasticfleet/config.map.jinja' import LOGSTASH_CONFIG_YAML %} . /usr/sbin/so-common FORCE_UPDATE=false UPDATE_CERTS=false +LOGSTASH_PILLAR_CONFIG_YAML="{{ LOGSTASH_CONFIG_YAML }}" +LOGSTASH_PILLAR_STATE_FILE="/opt/so/state/esfleet_logstash_config_pillar" while [[ $# -gt 0 ]]; do case $1 in @@ -19,6 +22,7 @@ while [[ $# -gt 0 ]]; do ;; -c| --certs) UPDATE_CERTS=true + FORCE_UPDATE=true shift ;; *) @@ -41,38 +45,45 @@ function update_logstash_outputs() { LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-logstash.key) LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt) LOGSTASHCA=$(openssl x509 -in /etc/pki/tls/certs/intca.crt) + # Revert escaped \\n to \n for jq + LOGSTASH_PILLAR_CONFIG_YAML=$(printf '%b' "$LOGSTASH_PILLAR_CONFIG_YAML") + if SECRETS=$(echo "$logstash_policy" | jq -er '.item.secrets' 2>/dev/null); then if [[ "$UPDATE_CERTS" != "true" ]]; then # Reuse existing secret JSON_STRING=$(jq -n \ --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \ --argjson SECRETS "$SECRETS" \ --argjson SSL_CONFIG "$SSL_CONFIG" \ - '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}') + '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG,"secrets": $SECRETS}') else # Update certs, creating new secret JSON_STRING=$(jq -n \ --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}') + '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}') fi else if [[ "$UPDATE_CERTS" != "true" ]]; then # Reuse existing ssl config JSON_STRING=$(jq -n \ --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \ --argjson SSL_CONFIG "$SSL_CONFIG" \ - '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}') + '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG}') else # Update ssl config JSON_STRING=$(jq -n \ --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}') + '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}') fi fi fi @@ -84,19 +95,42 @@ function update_kafka_outputs() { # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup if kafka_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" --fail 2>/dev/null); then SSL_CONFIG=$(echo "$kafka_policy" | jq -r '.item.ssl') + KAFKAKEY=$(openssl rsa -in /etc/pki/elasticfleet-kafka.key) + KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt) + KAFKACA=$(openssl x509 -in /etc/pki/tls/certs/intca.crt) if SECRETS=$(echo "$kafka_policy" | jq -er '.item.secrets' 2>/dev/null); then - # Update policy when fleet has secrets enabled - JSON_STRING=$(jq -n \ - --arg UPDATEDLIST "$NEW_LIST_JSON" \ - --argjson SSL_CONFIG "$SSL_CONFIG" \ - --argjson SECRETS "$SECRETS" \ - '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}') + if [[ "$UPDATE_CERTS" != "true" ]]; then + # Update policy when fleet has secrets enabled + JSON_STRING=$(jq -n \ + --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --argjson SSL_CONFIG "$SSL_CONFIG" \ + --argjson SECRETS "$SECRETS" \ + '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}') + else + # Update certs, creating new secret + JSON_STRING=$(jq -n \ + --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --arg KAFKAKEY "$KAFKAKEY" \ + --arg KAFKACRT "$KAFKACRT" \ + --arg KAFKACA "$KAFKACA" \ + '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": {"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"secrets": {"ssl":{"key": $KAFKAKEY }}}') + fi else - # Update policy when fleet has secrets disabled or policy hasn't been force updated - JSON_STRING=$(jq -n \ - --arg UPDATEDLIST "$NEW_LIST_JSON" \ - --argjson SSL_CONFIG "$SSL_CONFIG" \ - '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}') + if [[ "$UPDATE_CERTS" != "true" ]]; then + # Update policy when fleet has secrets disabled or policy hasn't been force updated + JSON_STRING=$(jq -n \ + --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --argjson SSL_CONFIG "$SSL_CONFIG" \ + '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}') + else + # Update ssl config + JSON_STRING=$(jq -n \ + --arg UPDATEDLIST "$NEW_LIST_JSON" \ + --arg KAFKAKEY "$KAFKAKEY" \ + --arg KAFKACRT "$KAFKACRT" \ + --arg KAFKACA "$KAFKACA" \ + '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }}') + fi fi # Update Kafka outputs curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq @@ -119,7 +153,7 @@ function update_kafka_outputs() { # Get the current list of kafka outputs & hash them CURRENT_LIST=$(jq -c -r '.item.hosts' <<< "$RAW_JSON") - CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}') declare -a NEW_LIST=() @@ -142,10 +176,19 @@ function update_kafka_outputs() { printf "Failed to query for current Logstash Outputs..." exit 1 fi + # logstash adv config - compare pillar to last state file value + if [[ -f "$LOGSTASH_PILLAR_STATE_FILE" ]]; then + PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML=$(cat "$LOGSTASH_PILLAR_STATE_FILE") + if [[ "$LOGSTASH_PILLAR_CONFIG_YAML" != "$PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML" ]]; then + echo "Logstash pillar config has changed - forcing update" + FORCE_UPDATE=true + fi + echo "$LOGSTASH_PILLAR_CONFIG_YAML" > "$LOGSTASH_PILLAR_STATE_FILE" + fi # Get the current list of Logstash outputs & hash them CURRENT_LIST=$(jq -c -r '.item.hosts' <<< "$RAW_JSON") - CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}') + CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}') declare -a NEW_LIST=() @@ -194,7 +237,7 @@ function update_kafka_outputs() { # Sort & hash the new list of Logstash Outputs NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") -NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') +NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}') # Compare the current & new list of outputs - if different, update the Logstash outputs if [[ "$NEW_HASH" = "$CURRENT_HASH" ]] && [[ "$FORCE_UPDATE" != "true" ]]; then diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 592f47a2b..c9f77aa7d 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -72,6 +72,8 @@ elasticsearch: actions: set_priority: priority: 0 + allocate: + number_of_replicas: "" min_age: 60d delete: actions: @@ -84,11 +86,25 @@ elasticsearch: max_primary_shard_size: 50gb set_priority: priority: 100 + forcemerge: + max_num_segments: "" + shrink: + max_primary_shard_size: "" + method: COUNT + number_of_shards: "" min_age: 0ms warm: actions: set_priority: priority: 50 + forcemerge: + max_num_segments: "" + shrink: + max_primary_shard_size: "" + method: COUNT + number_of_shards: "" + allocate: + number_of_replicas: "" min_age: 30d so-case: index_sorting: false @@ -245,7 +261,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - warm: 7 so-detection: index_sorting: false index_template: @@ -284,6 +299,19 @@ elasticsearch: hot: actions: {} min_age: 0ms + sos-backup: + index_sorting: false + index_template: + composed_of: [] + ignore_missing_component_templates: [] + index_patterns: + - sos-backup-* + priority: 501 + template: + settings: + index: + number_of_replicas: 0 + number_of_shards: 1 so-assistant-chat: index_sorting: false index_template: @@ -584,7 +612,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - warm: 7 so-import: index_sorting: false index_template: @@ -932,7 +959,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - warm: 7 so-hydra: close: 30 delete: 365 @@ -1043,7 +1069,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - warm: 7 so-lists: index_sorting: false index_template: @@ -1127,6 +1152,8 @@ elasticsearch: actions: set_priority: priority: 0 + allocate: + number_of_replicas: "" min_age: 60d delete: actions: @@ -1139,11 +1166,25 @@ elasticsearch: max_primary_shard_size: 50gb set_priority: priority: 100 + forcemerge: + max_num_segments: "" + shrink: + max_primary_shard_size: "" + method: COUNT + number_of_shards: "" min_age: 0ms warm: actions: set_priority: priority: 50 + allocate: + number_of_replicas: "" + forcemerge: + max_num_segments: "" + shrink: + max_primary_shard_size: "" + method: COUNT + number_of_shards: "" min_age: 30d so-logs-detections_x_alerts: index_sorting: false @@ -3123,7 +3164,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - warm: 7 so-logs-system_x_application: index_sorting: false index_template: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 097a53296..7fd4f8329 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -131,6 +131,47 @@ elasticsearch: description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index. global: True helpLink: elasticsearch.html + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + forcedType: string + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^[0-9]+(?:gb|tb|pb)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True cold: min_age: description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier. @@ -144,6 +185,12 @@ elasticsearch: description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. global: True helpLink: elasticsearch.html + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True warm: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. @@ -158,6 +205,52 @@ elasticsearch: forcedType: int global: True helpLink: elasticsearch.html + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^[0-9]+(?:gb|tb|pb)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True delete: min_age: description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion. @@ -287,6 +380,47 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch.html + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + forcedType: string + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^[0-9]+(?:gb|tb|pb)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True warm: min_age: description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier. @@ -314,6 +448,52 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch.html + shrink: + method: + description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size. + options: + - COUNT + - SIZE + global: True + advanced: True + number_of_shards: + title: shard count + description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'. + global: True + forcedType: int + advanced: True + max_primary_shard_size: + title: max shard size + description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'. + regex: ^[0-9]+(?:gb|tb|pb)$ + global: True + forcedType: string + advanced: True + allow_write_after_shrink: + description: Allow writes after shrink. + global: True + forcedType: bool + default: False + advanced: True + forcemerge: + max_num_segments: + description: Reduce the number of segments in each index shard and clean up deleted documents. + global: True + forcedType: int + advanced: True + index_codec: + title: compression + description: Use higher compression for stored fields at the cost of slower performance. + forcedType: bool + global: True + default: False + advanced: True + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True cold: min_age: description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier. @@ -330,6 +510,12 @@ elasticsearch: global: True advanced: True helpLink: elasticsearch.html + allocate: + number_of_replicas: + description: Set the number of replicas. Remains the same as the previous phase by default. + forcedType: int + global: True + advanced: True delete: min_age: description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion. diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 414d8a6b4..2563f8e23 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -61,5 +61,55 @@ {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endif %} + +{# advanced ilm actions #} +{% if settings.policy is defined and settings.policy.phases is defined %} +{% set PHASE_NAMES = ["hot", "warm", "cold"] %} +{% for P in PHASE_NAMES %} +{% if settings.policy.phases[P] is defined and settings.policy.phases[P].actions is defined %} +{% set PHASE = settings.policy.phases[P].actions %} +{# remove allocate action if number_of_replicas isn't configured #} +{% if PHASE.allocate is defined %} +{% if PHASE.allocate.number_of_replicas is not defined or PHASE.allocate.number_of_replicas == "" %} +{% do PHASE.pop('allocate', none) %} +{% endif %} +{% endif %} +{# start shrink action #} +{% if PHASE.shrink is defined %} +{% if PHASE.shrink.method is defined %} +{% if PHASE.shrink.method == 'COUNT' and PHASE.shrink.number_of_shards is defined and PHASE.shrink.number_of_shards %} +{# remove max_primary_shard_size value when doing shrink operation by count vs size #} +{% do PHASE.shrink.pop('max_primary_shard_size', none) %} +{% elif PHASE.shrink.method == 'SIZE' and PHASE.shrink.max_primary_shard_size is defined and PHASE.shrink.max_primary_shard_size %} +{# remove number_of_shards value when doing shrink operation by size vs count #} +{% do PHASE.shrink.pop('number_of_shards', none) %} +{% else %} +{# method isn't defined or missing a required config number_of_shards/max_primary_shard_size #} +{% do PHASE.pop('shrink', none) %} +{% endif %} +{% endif %} +{% endif %} +{# always remove shrink method since its only used for SOC config, not in the actual ilm policy #} +{% if PHASE.shrink is defined %} +{% do PHASE.shrink.pop('method', none) %} +{% endif %} +{# end shrink action #} +{# start force merge #} +{% if PHASE.forcemerge is defined %} +{% if PHASE.forcemerge.index_codec is defined and PHASE.forcemerge.index_codec %} +{% do PHASE.forcemerge.update({'index_codec': 'best_compression'}) %} +{% else %} +{% do PHASE.forcemerge.pop('index_codec', none) %} +{% endif %} +{% if PHASE.forcemerge.max_num_segments is not defined or not PHASE.forcemerge.max_num_segments %} +{# max_num_segments is empty, drop it #} +{% do PHASE.pop('forcemerge', none) %} +{% endif %} +{% endif %} +{# end force merge #} +{% endif %} +{% endfor %} +{% endif %} + {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} {% endfor %} diff --git a/salt/idstools/config.sls b/salt/idstools/config.sls deleted file mode 100644 index cea75ab9a..000000000 --- a/salt/idstools/config.sls +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - idstools.sync_files - -idstoolslogdir: - file.directory: - - name: /opt/so/log/idstools - - user: 939 - - group: 939 - - makedirs: True - -idstools_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://idstools/tools/sbin - - user: 939 - - group: 939 - - file_mode: 755 - -# If this is used, exclude so-rule-update -#idstools_sbin_jinja: -# file.recurse: -# - name: /usr/sbin -# - source: salt://idstools/tools/sbin_jinja -# - user: 939 -# - group: 939 -# - file_mode: 755 -# - template: jinja - -idstools_so-rule-update: - file.managed: - - name: /usr/sbin/so-rule-update - - source: salt://idstools/tools/sbin_jinja/so-rule-update - - user: 939 - - group: 939 - - mode: 755 - - template: jinja - -suricatacustomdirsfile: - file.directory: - - name: /nsm/rules/detect-suricata/custom_file - - user: 939 - - group: 939 - - makedirs: True - -suricatacustomdirsurl: - file.directory: - - name: /nsm/rules/detect-suricata/custom_temp - - user: 939 - - group: 939 - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/defaults.yaml b/salt/idstools/defaults.yaml deleted file mode 100644 index 1be100cec..000000000 --- a/salt/idstools/defaults.yaml +++ /dev/null @@ -1,10 +0,0 @@ -idstools: - enabled: False - config: - urls: [] - ruleset: ETOPEN - oinkcode: "" - sids: - enabled: [] - disabled: [] - modify: [] diff --git a/salt/idstools/disabled.sls b/salt/idstools/disabled.sls deleted file mode 100644 index ab0e10d7a..000000000 --- a/salt/idstools/disabled.sls +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -include: - - idstools.sostatus - -so-idstools: - docker_container.absent: - - force: True - -so-idstools_so-status.disabled: - file.comment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-idstools$ - -so-rule-update: - cron.absent: - - identifier: so-rule-update - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls deleted file mode 100644 index 365b38772..000000000 --- a/salt/idstools/enabled.sls +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} -{% from 'docker/docker.map.jinja' import DOCKER %} -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% set proxy = salt['pillar.get']('manager:proxy') %} - -include: - - idstools.config - - idstools.sostatus - -so-idstools: - docker_container.running: - - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-idstools:{{ GLOBALS.so_version }} - - hostname: so-idstools - - user: socore - - networks: - - sobridge: - - ipv4_address: {{ DOCKER.containers['so-idstools'].ip }} - {% if proxy %} - - environment: - - http_proxy={{ proxy }} - - https_proxy={{ proxy }} - - no_proxy={{ salt['pillar.get']('manager:no_proxy') }} - {% if DOCKER.containers['so-idstools'].extra_env %} - {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - {% elif DOCKER.containers['so-idstools'].extra_env %} - - environment: - {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %} - - {{ XTRAENV }} - {% endfor %} - {% endif %} - - binds: - - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro - - /opt/so/rules/nids/suri:/opt/so/rules/nids/suri:rw - - /nsm/rules/:/nsm/rules/:rw - {% if DOCKER.containers['so-idstools'].custom_bind_mounts %} - {% for BIND in DOCKER.containers['so-idstools'].custom_bind_mounts %} - - {{ BIND }} - {% endfor %} - {% endif %} - - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} - {% if DOCKER.containers['so-idstools'].extra_hosts %} - {% for XTRAHOST in DOCKER.containers['so-idstools'].extra_hosts %} - - {{ XTRAHOST }} - {% endfor %} - {% endif %} - - watch: - - file: idstoolsetcsync - - file: idstools_so-rule-update - -delete_so-idstools_so-status.disabled: - file.uncomment: - - name: /opt/so/conf/so-status/so-status.conf - - regex: ^so-idstools$ - -so-rule-update: - cron.present: - - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download_cron.log 2>&1 - - identifier: so-rule-update - - user: root - - minute: '1' - - hour: '7' - -# order this last to give so-idstools container time to be ready -run_so-rule-update: - cmd.run: - - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download_idstools_state.log 2>&1' - - require: - - docker_container: so-idstools - - onchanges: - - file: idstools_so-rule-update - - file: idstoolsetcsync - - file: synclocalnidsrules - - order: last - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/etc/disable.conf b/salt/idstools/etc/disable.conf deleted file mode 100644 index 84144a495..000000000 --- a/salt/idstools/etc/disable.conf +++ /dev/null @@ -1,16 +0,0 @@ -{%- set disabled_sids = salt['pillar.get']('idstools:sids:disabled', {}) -%} -# idstools - disable.conf - -# Example of disabling a rule by signature ID (gid is optional). -# 1:2019401 -# 2019401 - -# Example of disabling a rule by regular expression. -# - All regular expression matches are case insensitive. -# re:hearbleed -# re:MS(0[7-9]|10)-\d+ -{%- if disabled_sids != None %} -{%- for sid in disabled_sids %} -{{ sid }} -{%- endfor %} -{%- endif %} \ No newline at end of file diff --git a/salt/idstools/etc/enable.conf b/salt/idstools/etc/enable.conf deleted file mode 100644 index 5da0bfc61..000000000 --- a/salt/idstools/etc/enable.conf +++ /dev/null @@ -1,16 +0,0 @@ -{%- set enabled_sids = salt['pillar.get']('idstools:sids:enabled', {}) -%} -# idstools-rulecat - enable.conf - -# Example of enabling a rule by signature ID (gid is optional). -# 1:2019401 -# 2019401 - -# Example of enabling a rule by regular expression. -# - All regular expression matches are case insensitive. -# re:hearbleed -# re:MS(0[7-9]|10)-\d+ -{%- if enabled_sids != None %} -{%- for sid in enabled_sids %} -{{ sid }} -{%- endfor %} -{%- endif %} \ No newline at end of file diff --git a/salt/idstools/etc/modify.conf b/salt/idstools/etc/modify.conf deleted file mode 100644 index 4ea75ada2..000000000 --- a/salt/idstools/etc/modify.conf +++ /dev/null @@ -1,12 +0,0 @@ -{%- set modify_sids = salt['pillar.get']('idstools:sids:modify', {}) -%} -# idstools-rulecat - modify.conf - -# Format: "" "" - -# Example changing the seconds for rule 2019401 to 3600. -#2019401 "seconds \d+" "seconds 3600" -{%- if modify_sids != None %} -{%- for sid in modify_sids %} -{{ sid }} -{%- endfor %} -{%- endif %} \ No newline at end of file diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf deleted file mode 100644 index e4ec611db..000000000 --- a/salt/idstools/etc/rulecat.conf +++ /dev/null @@ -1,23 +0,0 @@ -{%- from 'vars/globals.map.jinja' import GLOBALS -%} -{%- from 'soc/merged.map.jinja' import SOCMERGED -%} ---suricata-version=7.0.3 ---merged=/opt/so/rules/nids/suri/all.rules ---output=/nsm/rules/detect-suricata/custom_temp ---local=/opt/so/rules/nids/suri/local.rules -{%- if GLOBALS.md_engine == "SURICATA" %} ---local=/opt/so/rules/nids/suri/extraction.rules ---local=/opt/so/rules/nids/suri/filters.rules -{%- endif %} ---url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules ---disable=/opt/so/idstools/etc/disable.conf ---enable=/opt/so/idstools/etc/enable.conf ---modify=/opt/so/idstools/etc/modify.conf -{%- if SOCMERGED.config.server.modules.suricataengine.customRulesets %} - {%- for ruleset in SOCMERGED.config.server.modules.suricataengine.customRulesets %} - {%- if 'url' in ruleset %} ---url={{ ruleset.url }} - {%- elif 'file' in ruleset %} ---local={{ ruleset.file }} - {%- endif %} - {%- endfor %} -{%- endif %} diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls deleted file mode 100644 index ac1d51717..000000000 --- a/salt/idstools/init.sls +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'idstools/map.jinja' import IDSTOOLSMERGED %} - -include: -{% if IDSTOOLSMERGED.enabled %} - - idstools.enabled -{% else %} - - idstools.disabled -{% endif %} diff --git a/salt/idstools/map.jinja b/salt/idstools/map.jinja deleted file mode 100644 index 97d12279b..000000000 --- a/salt/idstools/map.jinja +++ /dev/null @@ -1,7 +0,0 @@ -{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one - or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at - https://securityonion.net/license; you may not use this file except in compliance with the - Elastic License 2.0. #} - -{% import_yaml 'idstools/defaults.yaml' as IDSTOOLSDEFAULTS with context %} -{% set IDSTOOLSMERGED = salt['pillar.get']('idstools', IDSTOOLSDEFAULTS.idstools, merge=True) %} diff --git a/salt/idstools/rules/local.rules b/salt/idstools/rules/local.rules deleted file mode 100644 index ac11dfa58..000000000 --- a/salt/idstools/rules/local.rules +++ /dev/null @@ -1 +0,0 @@ -# Add your custom Suricata rules in this file. \ No newline at end of file diff --git a/salt/idstools/soc_idstools.yaml b/salt/idstools/soc_idstools.yaml deleted file mode 100644 index 4f7a53e91..000000000 --- a/salt/idstools/soc_idstools.yaml +++ /dev/null @@ -1,72 +0,0 @@ -idstools: - enabled: - description: Enables or disables the IDStools process which is used by the Detection system. - config: - oinkcode: - description: Enter your registration code or oinkcode for paid NIDS rulesets. - title: Registration Code - global: True - forcedType: string - helpLink: rules.html - ruleset: - description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. Once you have changed the ruleset here, you will need to wait for the rule update to take place (every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Suricata --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.' - global: True - regex: ETPRO\b|ETOPEN\b - helpLink: rules.html - urls: - description: This is a list of additional rule download locations. This feature is currently disabled. - global: True - multiline: True - forcedType: "[]string" - readonly: True - helpLink: rules.html - sids: - disabled: - description: Contains the list of NIDS rules (or regex patterns) disabled across the grid. This setting is readonly; Use the Detections screen to disable rules. - global: True - multiline: True - forcedType: "[]string" - regex: \d*|re:.* - helpLink: managing-alerts.html - readonlyUi: True - advanced: true - enabled: - description: Contains the list of NIDS rules (or regex patterns) enabled across the grid. This setting is readonly; Use the Detections screen to enable rules. - global: True - multiline: True - forcedType: "[]string" - regex: \d*|re:.* - helpLink: managing-alerts.html - readonlyUi: True - advanced: true - modify: - description: Contains the list of NIDS rules (SID "REGEX_SEARCH_TERM" "REGEX_REPLACE_TERM"). This setting is readonly; Use the Detections screen to modify rules. - global: True - multiline: True - forcedType: "[]string" - helpLink: managing-alerts.html - readonlyUi: True - advanced: true - rules: - local__rules: - description: Contains the list of custom NIDS rules applied to the grid. This setting is readonly; Use the Detections screen to adjust rules. - file: True - global: True - advanced: True - title: Local Rules - helpLink: local-rules.html - readonlyUi: True - filters__rules: - description: If you are using Suricata for metadata, then you can set custom filters for that metadata here. - file: True - global: True - advanced: True - title: Filter Rules - helpLink: suricata.html - extraction__rules: - description: If you are using Suricata for metadata, then you can set a list of MIME types for file extraction here. - file: True - global: True - advanced: True - title: Extraction Rules - helpLink: suricata.html diff --git a/salt/idstools/sostatus.sls b/salt/idstools/sostatus.sls deleted file mode 100644 index 408b10742..000000000 --- a/salt/idstools/sostatus.sls +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} - -append_so-idstools_so-status.conf: - file.append: - - name: /opt/so/conf/so-status/so-status.conf - - text: so-idstools - - unless: grep -q so-idstools /opt/so/conf/so-status/so-status.conf - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls deleted file mode 100644 index cdacfaa74..000000000 --- a/salt/idstools/sync_files.sls +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - -idstoolsdir: - file.directory: - - name: /opt/so/conf/idstools/etc - - user: 939 - - group: 939 - - makedirs: True - -idstoolsetcsync: - file.recurse: - - name: /opt/so/conf/idstools/etc - - source: salt://idstools/etc - - user: 939 - - group: 939 - - template: jinja - -rulesdir: - file.directory: - - name: /opt/so/rules/nids/suri - - user: 939 - - group: 939 - - makedirs: True - -# Don't show changes because all.rules can be large -synclocalnidsrules: - file.recurse: - - name: /opt/so/rules/nids/suri/ - - source: salt://idstools/rules/ - - user: 939 - - group: 939 - - show_changes: False - - include_pat: 'E@.rules' diff --git a/salt/idstools/tools/sbin/so-idstools-restart b/salt/idstools/tools/sbin/so-idstools-restart deleted file mode 100755 index f2abbd0a5..000000000 --- a/salt/idstools/tools/sbin/so-idstools-restart +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-restart idstools $1 diff --git a/salt/idstools/tools/sbin/so-idstools-start b/salt/idstools/tools/sbin/so-idstools-start deleted file mode 100755 index e17b5e521..000000000 --- a/salt/idstools/tools/sbin/so-idstools-start +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-start idstools $1 diff --git a/salt/idstools/tools/sbin/so-idstools-stop b/salt/idstools/tools/sbin/so-idstools-stop deleted file mode 100755 index f2d188d06..000000000 --- a/salt/idstools/tools/sbin/so-idstools-stop +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -. /usr/sbin/so-common - -/usr/sbin/so-stop idstools $1 diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update deleted file mode 100755 index 9ac09ed15..000000000 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# if this script isn't already running -if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - - . /usr/sbin/so-common - -{%- from 'vars/globals.map.jinja' import GLOBALS %} -{%- from 'idstools/map.jinja' import IDSTOOLSMERGED %} - -{%- set proxy = salt['pillar.get']('manager:proxy') %} -{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} - -{%- if proxy %} -# Download the rules from the internet - export http_proxy={{ proxy }} - export https_proxy={{ proxy }} - export no_proxy="{{ noproxy }}" -{%- endif %} - - mkdir -p /nsm/rules/suricata - chown -R socore:socore /nsm/rules/suricata -{%- if not GLOBALS.airgap %} -# Download the rules from the internet -{%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} - docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force -{%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} - docker exec so-idstools idstools-rulecat -v --suricata-version 7.0.3 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} -{%- endif %} -{%- endif %} - - - argstr="" - for arg in "$@"; do - argstr="${argstr} \"${arg}\"" - done - - docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}" - -fi diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 479b598f5..5cb7cf736 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -1,15 +1,5 @@ logrotate: config: - /opt/so/log/idstools/*_x_log: - - daily - - rotate 14 - - missingok - - copytruncate - - compress - - create - - extension .log - - dateext - - dateyesterday /opt/so/log/nginx/*_x_log: - daily - rotate 14 diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index 6f0272ef0..e6bdd596e 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -1,12 +1,5 @@ logrotate: config: - "/opt/so/log/idstools/*_x_log": - description: List of logrotate options for this file. - title: /opt/so/log/idstools/*.log - advanced: True - multiline: True - global: True - forcedType: "[]string" "/opt/so/log/nginx/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/nginx/*.log diff --git a/salt/manager/init.sls b/salt/manager/init.sls index f59c33652..7148ea16e 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -206,10 +206,33 @@ git_config_set_safe_dirs: - multivar: - /nsm/rules/custom-local-repos/local-sigma - /nsm/rules/custom-local-repos/local-yara + - /nsm/rules/custom-local-repos/local-suricata - /nsm/securityonion-resources - /opt/so/conf/soc/ai_summary_repos/securityonion-resources - /nsm/airgap-resources/playbooks - /opt/so/conf/soc/playbooks + +surinsmrulesdir: + file.directory: + - name: /nsm/rules/suricata/etopen + - user: 939 + - group: 939 + - makedirs: True + +suriextractionrules: + file.managed: + - name: /nsm/rules/suricata/so_extraction.rules + - source: salt://suricata/files/so_extraction.rules + - user: 939 + - group: 939 + +surifiltersrules: + file.managed: + - name: /nsm/rules/suricata/so_filters.rules + - source: salt://suricata/files/so_filters.rules + - user: 939 + - group: 939 + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/manager/managed_soc_annotations.sls b/salt/manager/managed_soc_annotations.sls index d8f175df6..4357b53a2 100644 --- a/salt/manager/managed_soc_annotations.sls +++ b/salt/manager/managed_soc_annotations.sls @@ -25,13 +25,11 @@ {% set index_settings = es.get('index_settings', {}) %} {% set input = index_settings.get('so-logs', {}) %} {% for k in matched_integration_names %} - {% if k not in index_settings %} - {% set _ = index_settings.update({k: input}) %} - {% endif %} + {% do index_settings.update({k: input}) %} {% endfor %} {% for k in addon_integration_keys %} {% if k not in matched_integration_names and k in index_settings %} - {% set _ = index_settings.pop(k) %} + {% do index_settings.pop(k) %} {% endif %} {% endfor %} {{ data }} @@ -45,14 +43,12 @@ {% set es = data.get('elasticsearch', {}) %} {% set index_settings = es.get('index_settings', {}) %} {% for k in matched_integration_names %} - {% if k not in index_settings %} - {% set input = ADDON_INTEGRATION_DEFAULTS[k] %} - {% set _ = index_settings.update({k: input})%} - {% endif %} + {% set input = ADDON_INTEGRATION_DEFAULTS[k] %} + {% do index_settings.update({k: input})%} {% endfor %} {% for k in addon_integration_keys %} {% if k not in matched_integration_names and k in index_settings %} - {% set _ = index_settings.pop(k) %} + {% do index_settings.pop(k) %} {% endif %} {% endfor %} {{ data }} diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 2b3281fc8..1d966315a 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -604,16 +604,6 @@ function add_kratos_to_minion() { fi } -function add_idstools_to_minion() { - printf '%s\n'\ - "idstools:"\ - " enabled: True"\ - " " >> $PILLARFILE - if [ $? -ne 0 ]; then - log "ERROR" "Failed to add idstools configuration to $PILLARFILE" - return 1 - fi -} function add_elastic_fleet_package_registry_to_minion() { printf '%s\n'\ @@ -751,7 +741,6 @@ function createEVAL() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -772,7 +761,6 @@ function createSTANDALONE() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -789,7 +777,6 @@ function createMANAGER() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -806,7 +793,6 @@ function createMANAGERSEARCH() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -821,7 +807,6 @@ function createIMPORT() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } @@ -906,7 +891,6 @@ function createMANAGERHYPE() { add_soc_to_minion || return 1 add_registry_to_minion || return 1 add_kratos_to_minion || return 1 - add_idstools_to_minion || return 1 add_elastic_fleet_package_registry_to_minion || return 1 } diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index cada91a44..4209ad207 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -425,6 +425,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.160 ]] && up_to_2.4.170 [[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180 [[ "$INSTALLEDVERSION" == 2.4.180 ]] && up_to_2.4.190 + [[ "$INSTALLEDVERSION" == 2.4.190 ]] && up_to_2.4.200 true } @@ -456,6 +457,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.160 ]] && post_to_2.4.170 [[ "$POSTVERSION" == 2.4.170 ]] && post_to_2.4.180 [[ "$POSTVERSION" == 2.4.180 ]] && post_to_2.4.190 + [[ "$POSTVERSION" == 2.4.190 ]] && post_to_2.4.200 true } @@ -635,6 +637,13 @@ post_to_2.4.190() { POSTVERSION=2.4.190 } +post_to_2.4.200() { + echo "Initiating Suricata idstools migration..." + suricata_idstools_removal_post + + POSTVERSION=2.4.200 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -902,6 +911,15 @@ up_to_2.4.190() { INSTALLEDVERSION=2.4.190 } +up_to_2.4.200() { + echo "Backing up idstools config..." + suricata_idstools_removal_pre + + touch /opt/so/state/esfleet_logstash_config_pillar + + INSTALLEDVERSION=2.4.200 +} + add_hydra_pillars() { mkdir -p /opt/so/saltstack/local/pillar/hydra touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls @@ -985,6 +1003,8 @@ rollover_index() { } suricata_idstools_migration() { + # For 2.4.70 + #Backup the pillars for idstools mkdir -p /nsm/backup/detections-migration/idstools rsync -av /opt/so/saltstack/local/pillar/idstools/* /nsm/backup/detections-migration/idstools @@ -1085,6 +1105,209 @@ playbook_migration() { echo "Playbook Migration is complete...." } +suricata_idstools_removal_pre() { +# For SOUPs beginning with 2.4.200 - pre SOUP checks + +# Create syncBlock file +install -d -o 939 -g 939 -m 755 /opt/so/conf/soc/fingerprints +install -o 939 -g 939 -m 644 /dev/null /opt/so/conf/soc/fingerprints/suricataengine.syncBlock +cat > /opt/so/conf/soc/fingerprints/suricataengine.syncBlock << EOF +Suricata ruleset sync is blocked until this file is removed. **CRITICAL** Make sure that you have manually added any custom Suricata rulesets via SOC config before removing this file - review the documentation for more details: https://docs.securityonion.net/en/2.4/nids.html#sync-block +EOF + +# Remove possible symlink & create salt local rules dir +[ -L /opt/so/saltstack/local/salt/suricata/rules ] && rm -f /opt/so/saltstack/local/salt/suricata/rules +install -d -o 939 -g 939 /opt/so/saltstack/local/salt/suricata/rules/ || echo "Failed to create Suricata local rules directory" + +# Backup custom rules & overrides +mkdir -p /nsm/backup/detections-migration/2-4-200 +cp /usr/sbin/so-rule-update /nsm/backup/detections-migration/2-4-200 +cp /opt/so/conf/idstools/etc/rulecat.conf /nsm/backup/detections-migration/2-4-200 + +# Backup so-detection index via reindex +echo "Creating sos-backup index template..." +template_result=$(/sbin/so-elasticsearch-query '_index_template/sos-backup' -X PUT \ + --retry 5 --retry-delay 15 --retry-all-errors \ + -d '{"index_patterns":["sos-backup-*"],"priority":501,"template":{"settings":{"index":{"number_of_replicas":0,"number_of_shards":1}}}}') + +if [[ -z "$template_result" ]] || ! echo "$template_result" | jq -e '.acknowledged == true' > /dev/null 2>&1; then + echo "Error: Failed to create sos-backup index template" + echo "$template_result" + exit 1 +fi + +BACKUP_INDEX="sos-backup-detection-$(date +%Y%m%d-%H%M%S)" +echo "Backing up so-detection index to $BACKUP_INDEX..." +reindex_result=$(/sbin/so-elasticsearch-query '_reindex?wait_for_completion=true' \ + --retry 5 --retry-delay 15 --retry-all-errors \ + -X POST -d "{\"source\": {\"index\": \"so-detection\"}, \"dest\": {\"index\": \"$BACKUP_INDEX\"}}") + +if [[ -z "$reindex_result" ]]; then + echo "Error: Backup of detections failed - no response from Elasticsearch" + exit 1 +elif echo "$reindex_result" | jq -e '.created >= 0' > /dev/null 2>&1; then + echo "Backup complete: $(echo "$reindex_result" | jq -r '.created') documents copied" +elif echo "$reindex_result" | grep -q "index_not_found_exception"; then + echo "so-detection index does not exist, skipping backup" +else + echo "Error: Backup of detections failed" + echo "$reindex_result" + exit 1 +fi + +} + +suricata_idstools_removal_post() { +# For SOUPs beginning with 2.4.200 - post SOUP checks + +echo "Checking idstools configuration for custom modifications..." + +# Normalize and hash file content for consistent comparison +# Args: $1 - file path +# Outputs: SHA256 hash to stdout +# Returns: 0 on success, 1 on failure +hash_normalized_file() { + local file="$1" + + if [[ ! -r "$file" ]]; then + return 1 + fi + + sed -E \ + -e 's/^[[:space:]]+//; s/[[:space:]]+$//' \ + -e '/^$/d' \ + -e 's|--url=http://[^:]+:7788|--url=http://MANAGER:7788|' \ + "$file" | sha256sum | awk '{print $1}' +} + +# Known-default hashes for so-rule-update (ETOPEN ruleset) +KNOWN_SO_RULE_UPDATE_HASHES=( + # 2.4.100+ (suricata 7.0.3, non-airgap) + "5fbd067ced86c8ec72ffb7e1798aa624123b536fb9d78f4b3ad8d3b45db1eae7" # 2.4.100-2.4.190 non-Airgap + # 2.4.90+ airgap (same for 2.4.90 and 2.4.100+) + "61f632c55791338c438c071040f1490066769bcce808b595b5cc7974a90e653a" # 2.4.90+ Airgap + # 2.4.90 (suricata 6.0, non-airgap, comment inside proxy block) + "0380ec52a05933244ab0f0bc506576e1d838483647b40612d5fe4b378e47aedd" # 2.4.90 non-Airgap + # 2.4.10-2.4.80 (suricata 6.0, non-airgap, comment outside proxy block) + "b6e4d1b5a78d57880ad038a9cd2cc6978aeb2dd27d48ea1a44dd866a2aee7ff4" # 2.4.10-2.4.80 non-Airgap + # 2.4.10-2.4.80 airgap + "b20146526ace2b142fde4664f1386a9a1defa319b3a1d113600ad33a1b037dad" # 2.4.10-2.4.80 Airgap + # 2.4.5 and earlier (no pidof check, non-airgap) + "d04f5e4015c348133d28a7840839e82d60009781eaaa1c66f7f67747703590dc" # 2.4.5 non-Airgap +) + +# Known-default hashes for rulecat.conf +KNOWN_RULECAT_CONF_HASHES=( + # 2.4.100+ (suricata 7.0.3) + "302e75dca9110807f09ade2eec3be1fcfc8b2bf6cf2252b0269bb72efeefe67e" # 2.4.100-2.4.190 without SURICATA md_engine + "8029b7718c324a9afa06a5cf180afde703da1277af4bdd30310a6cfa3d6398cb" # 2.4.100-2.4.190 with SURICATA md_engine + # 2.4.80-2.4.90 (suricata 6.0, with --suricata-version and --output) + "4d8b318e6950a6f60b02f307cf27c929efd39652990c1bd0c8820aa8a307e1e7" # 2.4.80-2.4.90 without SURICATA md_engine + "a1ddf264c86c4e91c81c5a317f745a19466d4311e4533ec3a3c91fed04c11678" # 2.4.80-2.4.90 with SURICATA md_engine + # 2.4.50-2.4.70 (/suri/ path, no --suricata-version) + "86e3afb8d0f00c62337195602636864c98580a13ca9cc85029661a539deae6ae" # 2.4.50-2.4.70 without SURICATA md_engine + "5a97604ca5b820a10273a2d6546bb5e00c5122ca5a7dfe0ba0bfbce5fc026f4b" # 2.4.50-2.4.70 with SURICATA md_engine + # 2.4.20-2.4.40 (/nids/ path without /suri/) + "d098ea9ecd94b5cca35bf33543f8ea8f48066a0785221fabda7fef43d2462c29" # 2.4.20-2.4.40 without SURICATA md_engine + "9dbc60df22ae20d65738ba42e620392577857038ba92278e23ec182081d191cd" # 2.4.20-2.4.40 with SURICATA md_engine + # 2.4.5-2.4.10 (/sorules/ path for extraction/filters) + "490f6843d9fca759ee74db3ada9c702e2440b8393f2cfaf07bbe41aaa6d955c3" # 2.4.5-2.4.10 with SURICATA md_engine + # Note: 2.4.5-2.4.10 without SURICATA md_engine has same hash as 2.4.20-2.4.40 without SURICATA md_engine +) + +# Check a config file against known hashes +# Args: $1 - file path, $2 - array name of known hashes +check_config_file() { + local file="$1" + local known_hashes_array="$2" + local file_display_name=$(basename "$file") + + if [[ ! -f "$file" ]]; then + echo "Warning: $file not found" + echo "$file_display_name not found - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + return 1 + fi + + echo "Hashing $file..." + local file_hash + if ! file_hash=$(hash_normalized_file "$file"); then + echo "Warning: Could not read $file" + echo "$file_display_name not readable - manual verification required" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + return 1 + fi + + echo " Hash: $file_hash" + + # Check if hash matches any known default + local -n known_hashes=$known_hashes_array + for known_hash in "${known_hashes[@]}"; do + if [[ "$file_hash" == "$known_hash" ]]; then + echo " Matches known default configuration" + return 0 + fi + done + + # No match - custom configuration detected + echo "Does not match known default - custom configuration detected" + echo "Custom $file_display_name detected (hash: $file_hash)" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + + # If this is so-rule-update, check for ETPRO license code and write out to the syncBlock file + # If ETPRO is enabled, the license code already exists in the so-rule-update script, this is just making it easier to migrate + if [[ "$file_display_name" == "so-rule-update" ]]; then + local etpro_code + etpro_code=$(grep -oP '\-\-etpro=\K[0-9a-fA-F]+' "$file" 2>/dev/null) || true + if [[ -n "$etpro_code" ]]; then + echo "ETPRO code found: $etpro_code" >> /opt/so/conf/soc/fingerprints/suricataengine.syncBlock + fi + fi + + return 1 +} + +# Check so-rule-update and rulecat.conf +SO_RULE_UPDATE="/usr/sbin/so-rule-update" +RULECAT_CONF="/opt/so/conf/idstools/etc/rulecat.conf" + +custom_found=0 + +check_config_file "$SO_RULE_UPDATE" "KNOWN_SO_RULE_UPDATE_HASHES" || custom_found=1 +check_config_file "$RULECAT_CONF" "KNOWN_RULECAT_CONF_HASHES" || custom_found=1 + +# If no custom configs found, remove syncBlock +if [[ $custom_found -eq 0 ]]; then + echo "idstools migration completed successfully - removing Suricata engine syncBlock" + rm -f /opt/so/conf/soc/fingerprints/suricataengine.syncBlock +else + echo "Custom idstools configuration detected - syncBlock remains in place" + echo "Review /opt/so/conf/soc/fingerprints/suricataengine.syncBlock for details" +fi + +echo "Cleaning up idstools" +echo "Stopping and removing the idstools container..." +if [ -n "$(docker ps -q -f name=^so-idstools$)" ]; then + image_name=$(docker ps -a --filter name=^so-idstools$ --format '{{.Image}}' 2>/dev/null || true) + docker stop so-idstools || echo "Warning: failed to stop so-idstools container" + docker rm so-idstools || echo "Warning: failed to remove so-idstools container" + + if [[ -n "$image_name" ]]; then + echo "Removing idstools image: $image_name" + docker rmi "$image_name" || echo "Warning: failed to remove image $image_name" + fi +fi + +echo "Removing idstools symlink and scripts..." +rm -rf /usr/sbin/so-idstools* +sed -i '/^#\?so-idstools$/d' /opt/so/conf/so-status/so-status.conf +crontab -l | grep -v 'so-rule-update' | crontab - + +# Backup the salt master config & manager pillar before editing it +cp /opt/so/saltstack/local/pillar/minions/$MINIONID.sls /nsm/backup/detections-migration/2-4-200/ +cp /etc/salt/master /nsm/backup/detections-migration/2-4-200/ +so-yaml.py remove /opt/so/saltstack/local/pillar/minions/$MINIONID.sls idstools +so-yaml.py removelistitem /etc/salt/master file_roots.base /opt/so/rules/nids + +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap @@ -1131,7 +1354,7 @@ unmount_update() { update_airgap_rules() { # Copy the rules over to update them for airgap. - rsync -a $UPDATE_DIR/agrules/suricata/* /nsm/rules/suricata/ + rsync -a --delete $UPDATE_DIR/agrules/suricata/ /nsm/rules/suricata/etopen/ rsync -a $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/ rsync -a $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/ # Copy the securityonion-resorces repo over for SOC Detection Summaries and checkout the published summaries branch diff --git a/salt/salt/files/engines.conf b/salt/salt/files/engines.conf index 15d55e18f..8192ee201 100644 --- a/salt/salt/files/engines.conf +++ b/salt/salt/files/engines.conf @@ -6,30 +6,6 @@ engines: interval: 60 - pillarWatch: fpa: - - files: - - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls - - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls - pillar: idstools.config.ruleset - default: ETOPEN - actions: - from: - '*': - to: - '*': - - cmd.run: - cmd: /usr/sbin/so-rule-update - - files: - - /opt/so/saltstack/local/pillar/idstools/soc_idstools.sls - - /opt/so/saltstack/local/pillar/idstools/adv_idstools.sls - pillar: idstools.config.oinkcode - default: '' - actions: - from: - '*': - to: - '*': - - cmd.run: - cmd: /usr/sbin/so-rule-update - files: - /opt/so/saltstack/local/pillar/global/soc_global.sls - /opt/so/saltstack/local/pillar/global/adv_global.sls diff --git a/salt/soc/config.sls b/salt/soc/config.sls index 78a495e0a..7e2beefa0 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -215,7 +215,6 @@ socsensoronirepos: - mode: 775 - makedirs: True - create_custom_local_yara_repo_template: git.present: - name: /nsm/rules/custom-local-repos/local-yara @@ -249,6 +248,39 @@ add_readme_custom_local_sigma_repo_template: - context: repo_type: "sigma" +create_custom_local_suricata_repo_template: + git.present: + - name: /nsm/rules/custom-local-repos/local-suricata + - bare: False + - force: True + +add_readme_custom_local_suricata_repo_template: + file.managed: + - name: /nsm/rules/custom-local-repos/local-suricata/README + - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja + - user: 939 + - group: 939 + - template: jinja + - context: + repo_type: "suricata" + +etpro_airgap_folder: + file.directory: + - name: /nsm/rules/custom-local-repos/local-etpro-suricata + - user: 939 + - group: 939 + - makedirs: True + +add_readme_etpro_airgap_template: + file.managed: + - name: /nsm/rules/custom-local-repos/local-etpro-suricata/README + - source: salt://soc/files/soc/detections_custom_repo_template_readme.jinja + - user: 939 + - group: 939 + - template: jinja + - context: + repo_type: "suricata-etpro" + socore_own_custom_repos: file.directory: - name: /nsm/rules/custom-local-repos/ diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b3bbfa659..28db2ef5f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1563,12 +1563,105 @@ soc: disableRegex: [] enableRegex: [] failAfterConsecutiveErrorCount: 10 - communityRulesFile: /nsm/rules/suricata/emerging-all.rules rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state integrityCheckFrequencySeconds: 1200 ignoredSidRanges: - '1100000-1101000' + rulesetSources: + default: + - name: Emerging-Threats + description: "Emerging Threats ruleset - To enable ET Pro, enter your license key below. Leave empty for ET Open (free) rules." + licenseKey: "" + enabled: true + sourceType: url + sourcePath: 'https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz' + urlHash: "https://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz.md5" + license: "BSD" + excludeFiles: + - "*deleted*" + - "*retired*" + proxyURL: "" + proxyUsername: "" + proxyPassword: "" + proxyCACert: "" + insecureSkipVerify: false + readOnly: true + deleteUnreferenced: true + - name: ABUSECH-SSLBL + deleteUnreferenced: true + description: 'Abuse.ch SSL Blacklist' + enabled: false + license: CC0-1.0 + readOnly: true + sourcePath: https://sslbl.abuse.ch/blacklist/sslblacklist_tls_cert.tar.gz + sourceType: url + - name: local-rules + description: "Local rules from files (*.rules) in a directory on the filesystem" + license: "custom" + sourceType: directory + sourcePath: /nsm/rules/custom-local-repos/local-suricata + readOnly: false + deleteUnreferenced: false + enabled: true + - name: SO_FILTERS + deleteUnreferenced: true + description: Filter rules for when Suricata is set as the metadata engine + enabled: false + license: Elastic-2.0 + readOnly: true + sourcePath: /nsm/rules/suricata/so_filters.rules + sourceType: directory + - name: SO_EXTRACTIONS + description: Extraction rules for when Suricata is set as the metadata engine + deleteUnreferenced: true + enabled: false + license: Elastic-2.0 + readOnly: true + sourcePath: /nsm/rules/suricata/so_extraction.rules + sourceType: directory + airgap: + - name: Emerging-Threats + description: "Emerging Threats ruleset - To enable ET Pro on Airgap, review the documentation at https://docs.securityonion.net/suricata" + licenseKey: "" + enabled: true + sourceType: directory + sourcePath: /nsm/rules/suricata/etopen/ + license: "BSD" + excludeFiles: + - "*deleted*" + - "*retired*" + proxyURL: "" + proxyUsername: "" + proxyPassword: "" + proxyCACert: "" + insecureSkipVerify: false + readOnly: true + deleteUnreferenced: true + - name: local-rules + description: "Local rules from files (*.rules) in a directory on the filesystem" + license: "custom" + sourceType: directory + sourcePath: /nsm/rules/custom-local-repos/local-suricata + readOnly: false + deleteUnreferenced: false + enabled: true + - name: SO_FILTERS + deleteUnreferenced: true + description: Filter rules for when Suricata is set as the metadata engine + enabled: false + license: Elastic-2.0 + readOnly: true + sourcePath: /nsm/rules/suricata/so_filters.rules + sourceType: directory + - name: SO_EXTRACTIONS + description: Extraction rules for when Suricata is set as the metadata engine + deleteUnreferenced: true + enabled: false + license: Elastic-2.0 + readOnly: true + sourcePath: /nsm/rules/suricata/so_extraction.rules + sourceType: directory navigator: intervalMinutes: 30 outputPath: /opt/sensoroni/navigator @@ -2559,26 +2652,16 @@ soc: thresholdColorRatioMed: 0.75 thresholdColorRatioMax: 1 availableModels: - - id: sonnet-4 - displayName: Claude Sonnet 4 - contextLimitSmall: 200000 - contextLimitLarge: 1000000 - lowBalanceColorAlert: 500000 - enabled: true - id: sonnet-4.5 - displayName: Claude Sonnet 4.5 + displayName: Claude Sonnet 4.5 ($$$) + origin: USA contextLimitSmall: 200000 contextLimitLarge: 1000000 lowBalanceColorAlert: 500000 enabled: true - - id: gptoss-120b - displayName: GPT-OSS 120B - contextLimitSmall: 128000 - contextLimitLarge: 128000 - lowBalanceColorAlert: 500000 - enabled: true - id: qwen-235b - displayName: QWEN 235B + displayName: QWEN 235B ($) + origin: China contextLimitSmall: 256000 contextLimitLarge: 256000 lowBalanceColorAlert: 500000 diff --git a/salt/soc/enabled.sls b/salt/soc/enabled.sls index aeb05287e..5efb18fa5 100644 --- a/salt/soc/enabled.sls +++ b/salt/soc/enabled.sls @@ -28,7 +28,8 @@ so-soc: - /opt/so/conf/strelka:/opt/sensoroni/yara:rw - /opt/so/conf/sigma:/opt/sensoroni/sigma:rw - /opt/so/rules/elastalert/rules:/opt/sensoroni/elastalert:rw - - /opt/so/rules/nids/suri:/opt/sensoroni/nids:ro + - /opt/so/saltstack/local/salt/suricata/rules:/opt/sensoroni/suricata/rules:rw + - /opt/so/saltstack/local/salt/suricata/files:/opt/sensoroni/suricata/threshold:rw - /opt/so/conf/soc/fingerprints:/opt/sensoroni/fingerprints:rw - /nsm/soc/jobs:/opt/sensoroni/jobs:rw - /nsm/soc/uploads:/nsm/soc/uploads:rw diff --git a/salt/soc/files/soc/detections_custom_repo_template_readme.jinja b/salt/soc/files/soc/detections_custom_repo_template_readme.jinja index 228a467bf..060b8ec6e 100644 --- a/salt/soc/files/soc/detections_custom_repo_template_readme.jinja +++ b/salt/soc/files/soc/detections_custom_repo_template_readme.jinja @@ -45,6 +45,61 @@ Finally, commit it: The next time the Strelka / YARA engine syncs, the new rule should be imported If there are errors, review the sync log to troubleshoot further. +{% elif repo_type == 'suricata' %} +# Suricata Local Custom Rules Repository + +This folder has already been initialized as a git repo +and your Security Onion grid is configured to import any Suricata rule files found here. + +Just add your rule file and commit it. + +For example: + +** Note: If this is your first time making changes to this repo, you may run into the following error: + +fatal: detected dubious ownership in repository at '/nsm/rules/custom-local-repos/local-suricata' +To add an exception for this directory, call: + git config --global --add safe.directory /nsm/rules/custom-local-repos/local-suricata + +This means that the user you are running commands as does not match the user that is used for this git repo (socore). +You will need to make sure your rule files are accessible to the socore user, so either su to socore +or add the exception and then chown the rule files later. + +Also, you will be asked to set some configuration: +``` +Author identity unknown +*** Please tell me who you are. +Run + git config --global user.email "you@example.com" + git config --global user.name "Your Name" +to set your account's default identity. +Omit --global to set the identity only in this repository. +``` + +Run these commands, ommitting the `--global`. + +With that out of the way: + +First, create the rule file with a .rules extension: +`vi my_custom_rules.rules` + +Next, use git to stage the new rule to be committed: +`git add my_custom_rules.rules` + +Finally, commit it: +`git commit -m "Initial commit of my_custom_rule.rules"` + +The next time the Suricata engine syncs, the new rule/s should be imported +If there are errors, review the sync log to troubleshoot further. + +{% elif repo_type == 'suricata-etpro' %} +# Suricata ETPRO - Airgap + +This folder has been initialized for use with ETPRO during Airgap deployment. + +Just add your ETPRO rule/s file to this folder and the Suricata engine will import them. + +If there are errors, review the sync log to troubleshoot further. {% elif repo_type == 'sigma' %} # Sigma Local Custom Rules Repository diff --git a/salt/soc/files/soc/so-detections-backup.py b/salt/soc/files/soc/so-detections-backup.py index 085b1e4c7..0300c15f2 100644 --- a/salt/soc/files/soc/so-detections-backup.py +++ b/salt/soc/files/soc/so-detections-backup.py @@ -6,6 +6,7 @@ # This script queries Elasticsearch for Custom Detections and all Overrides, # and git commits them to disk at $OUTPUT_DIR +import argparse import os import subprocess import json @@ -18,10 +19,10 @@ from datetime import datetime urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # Constants -ES_URL = "https://localhost:9200/so-detection/_search" +DEFAULT_INDEX = "so-detection" +DEFAULT_OUTPUT_DIR = "/nsm/backup/detections/repo" QUERY_DETECTIONS = '{"query": {"bool": {"must": [{"match_all": {}}, {"term": {"so_detection.ruleset": "__custom__"}}]}},"size": 10000}' QUERY_OVERRIDES = '{"query": {"bool": {"must": [{"exists": {"field": "so_detection.overrides"}}]}},"size": 10000}' -OUTPUT_DIR = "/nsm/backup/detections/repo" AUTH_FILE = "/opt/so/conf/elasticsearch/curl.config" def get_auth_credentials(auth_file): @@ -30,9 +31,10 @@ def get_auth_credentials(auth_file): if line.startswith('user ='): return line.split('=', 1)[1].strip().replace('"', '') -def query_elasticsearch(query, auth): +def query_elasticsearch(query, auth, index): + url = f"https://localhost:9200/{index}/_search" headers = {"Content-Type": "application/json"} - response = requests.get(ES_URL, headers=headers, data=query, auth=auth, verify=False) + response = requests.get(url, headers=headers, data=query, auth=auth, verify=False) response.raise_for_status() return response.json() @@ -47,12 +49,12 @@ def save_content(hit, base_folder, subfolder="", extension="txt"): f.write(content) return file_path -def save_overrides(hit): +def save_overrides(hit, output_dir): so_detection = hit["_source"]["so_detection"] public_id = so_detection["publicId"] overrides = so_detection["overrides"] language = so_detection["language"] - folder = os.path.join(OUTPUT_DIR, language, "overrides") + folder = os.path.join(output_dir, language, "overrides") os.makedirs(folder, exist_ok=True) extension = "yaml" if language == "sigma" else "txt" file_path = os.path.join(folder, f"{public_id}.{extension}") @@ -60,20 +62,20 @@ def save_overrides(hit): f.write('\n'.join(json.dumps(override) for override in overrides) if isinstance(overrides, list) else overrides) return file_path -def ensure_git_repo(): - if not os.path.isdir(os.path.join(OUTPUT_DIR, '.git')): +def ensure_git_repo(output_dir): + if not os.path.isdir(os.path.join(output_dir, '.git')): subprocess.run(["git", "config", "--global", "init.defaultBranch", "main"], check=True) - subprocess.run(["git", "-C", OUTPUT_DIR, "init"], check=True) - subprocess.run(["git", "-C", OUTPUT_DIR, "remote", "add", "origin", "default"], check=True) + subprocess.run(["git", "-C", output_dir, "init"], check=True) + subprocess.run(["git", "-C", output_dir, "remote", "add", "origin", "default"], check=True) -def commit_changes(): - ensure_git_repo() - subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.email", "securityonion@local.invalid"], check=True) - subprocess.run(["git", "-C", OUTPUT_DIR, "config", "user.name", "securityonion"], check=True) - subprocess.run(["git", "-C", OUTPUT_DIR, "add", "."], check=True) - status_result = subprocess.run(["git", "-C", OUTPUT_DIR, "status"], capture_output=True, text=True) +def commit_changes(output_dir): + ensure_git_repo(output_dir) + subprocess.run(["git", "-C", output_dir, "config", "user.email", "securityonion@local.invalid"], check=True) + subprocess.run(["git", "-C", output_dir, "config", "user.name", "securityonion"], check=True) + subprocess.run(["git", "-C", output_dir, "add", "."], check=True) + status_result = subprocess.run(["git", "-C", output_dir, "status"], capture_output=True, text=True) print(status_result.stdout) - commit_result = subprocess.run(["git", "-C", OUTPUT_DIR, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True) + commit_result = subprocess.run(["git", "-C", output_dir, "commit", "-m", "Update detections and overrides"], check=False, capture_output=True) if commit_result.returncode == 1: print("No changes to commit.") elif commit_result.returncode == 0: @@ -81,29 +83,41 @@ def commit_changes(): else: commit_result.check_returncode() +def parse_args(): + parser = argparse.ArgumentParser(description="Backup custom detections and overrides from Elasticsearch") + parser.add_argument("--output", "-o", default=DEFAULT_OUTPUT_DIR, + help=f"Output directory for backups (default: {DEFAULT_OUTPUT_DIR})") + parser.add_argument("--index", "-i", default=DEFAULT_INDEX, + help=f"Elasticsearch index to query (default: {DEFAULT_INDEX})") + return parser.parse_args() + def main(): + args = parse_args() + output_dir = args.output + index = args.index + try: timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - print(f"Backing up Custom Detections and all Overrides to {OUTPUT_DIR} - {timestamp}\n") - - os.makedirs(OUTPUT_DIR, exist_ok=True) + print(f"Backing up Custom Detections and all Overrides to {output_dir} - {timestamp}\n") + + os.makedirs(output_dir, exist_ok=True) auth_credentials = get_auth_credentials(AUTH_FILE) username, password = auth_credentials.split(':', 1) auth = HTTPBasicAuth(username, password) - + # Query and save custom detections - detections = query_elasticsearch(QUERY_DETECTIONS, auth)["hits"]["hits"] + detections = query_elasticsearch(QUERY_DETECTIONS, auth, index)["hits"]["hits"] for hit in detections: - save_content(hit, OUTPUT_DIR, hit["_source"]["so_detection"]["language"], "yaml" if hit["_source"]["so_detection"]["language"] == "sigma" else "txt") - + save_content(hit, output_dir, hit["_source"]["so_detection"]["language"], "yaml" if hit["_source"]["so_detection"]["language"] == "sigma" else "txt") + # Query and save overrides - overrides = query_elasticsearch(QUERY_OVERRIDES, auth)["hits"]["hits"] + overrides = query_elasticsearch(QUERY_OVERRIDES, auth, index)["hits"]["hits"] for hit in overrides: - save_overrides(hit) - - commit_changes() - + save_overrides(hit, output_dir) + + commit_changes(output_dir) + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") print(f"Backup Completed - {timestamp}") except Exception as e: diff --git a/salt/soc/files/soc/so-detections-backup_test.py b/salt/soc/files/soc/so-detections-backup_test.py index 3afa11886..4cdc9fa36 100644 --- a/salt/soc/files/soc/so-detections-backup_test.py +++ b/salt/soc/files/soc/so-detections-backup_test.py @@ -57,12 +57,12 @@ class TestBackupScript(unittest.TestCase): mock_response.json.return_value = {'hits': {'hits': []}} mock_response.raise_for_status = MagicMock() mock_get.return_value = mock_response - - response = ds.query_elasticsearch(ds.QUERY_DETECTIONS, self.auth) - + + response = ds.query_elasticsearch(ds.QUERY_DETECTIONS, self.auth, ds.DEFAULT_INDEX) + self.assertEqual(response, {'hits': {'hits': []}}) mock_get.assert_called_once_with( - ds.ES_URL, + f"https://localhost:9200/{ds.DEFAULT_INDEX}/_search", headers={"Content-Type": "application/json"}, data=ds.QUERY_DETECTIONS, auth=self.auth, @@ -81,7 +81,7 @@ class TestBackupScript(unittest.TestCase): @patch('os.makedirs') @patch('builtins.open', new_callable=mock_open) def test_save_overrides(self, mock_file, mock_makedirs): - file_path = ds.save_overrides(self.mock_override_hit) + file_path = ds.save_overrides(self.mock_override_hit, self.output_dir) expected_path = f'{self.output_dir}/sigma/overrides/test_id.yaml' self.assertEqual(file_path, expected_path) mock_makedirs.assert_called_once_with(f'{self.output_dir}/sigma/overrides', exist_ok=True) @@ -90,9 +90,9 @@ class TestBackupScript(unittest.TestCase): @patch('subprocess.run') def test_ensure_git_repo(self, mock_run): mock_run.return_value = MagicMock(returncode=0) - - ds.ensure_git_repo() - + + ds.ensure_git_repo(self.output_dir) + mock_run.assert_has_calls([ call(["git", "config", "--global", "init.defaultBranch", "main"], check=True), call(["git", "-C", self.output_dir, "init"], check=True), @@ -106,9 +106,9 @@ class TestBackupScript(unittest.TestCase): mock_commit_result = MagicMock(returncode=1) # Ensure sufficient number of MagicMock instances for each subprocess.run call mock_run.side_effect = [mock_status_result, mock_commit_result, MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0), MagicMock(returncode=0)] - + print("Running test_commit_changes...") - ds.commit_changes() + ds.commit_changes(self.output_dir) print("Finished test_commit_changes.") mock_run.assert_has_calls([ @@ -120,39 +120,45 @@ class TestBackupScript(unittest.TestCase): ]) @patch('builtins.print') - @patch('so-detections-backup.commit_changes') - @patch('so-detections-backup.save_overrides') - @patch('so-detections-backup.save_content') - @patch('so-detections-backup.query_elasticsearch') - @patch('so-detections-backup.get_auth_credentials') + @patch.object(ds, 'commit_changes') + @patch.object(ds, 'save_overrides') + @patch.object(ds, 'save_content') + @patch.object(ds, 'query_elasticsearch') + @patch.object(ds, 'get_auth_credentials') @patch('os.makedirs') - def test_main(self, mock_makedirs, mock_get_auth, mock_query, mock_save_content, mock_save_overrides, mock_commit, mock_print): + @patch.object(ds, 'parse_args') + def test_main(self, mock_parse_args, mock_makedirs, mock_get_auth, mock_query, mock_save_content, mock_save_overrides, mock_commit, mock_print): + mock_args = MagicMock() + mock_args.output = self.output_dir + mock_args.index = ds.DEFAULT_INDEX + mock_parse_args.return_value = mock_args mock_get_auth.return_value = self.auth_credentials mock_query.side_effect = [ {'hits': {'hits': [{"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}}]}}, {'hits': {'hits': [{"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}}]}} ] - + with patch('datetime.datetime') as mock_datetime: mock_datetime.now.return_value.strftime.return_value = "2024-05-23 20:49:44" ds.main() - + mock_makedirs.assert_called_once_with(self.output_dir, exist_ok=True) mock_get_auth.assert_called_once_with(ds.AUTH_FILE) mock_query.assert_has_calls([ - call(ds.QUERY_DETECTIONS, self.auth), - call(ds.QUERY_OVERRIDES, self.auth) + call(ds.QUERY_DETECTIONS, self.auth, ds.DEFAULT_INDEX), + call(ds.QUERY_OVERRIDES, self.auth, ds.DEFAULT_INDEX) ]) mock_save_content.assert_called_once_with( - {"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}}, - self.output_dir, - "sigma", + {"_source": {"so_detection": {"publicId": "1", "content": "content1", "language": "sigma"}}}, + self.output_dir, + "sigma", "yaml" ) mock_save_overrides.assert_called_once_with( - {"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}} + {"_source": {"so_detection": {"publicId": "2", "overrides": [{"key": "value"}], "language": "suricata"}}}, + self.output_dir ) - mock_commit.assert_called_once() + mock_commit.assert_called_once_with(self.output_dir) mock_print.assert_called() if __name__ == '__main__': diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index e053ce63f..349937983 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -50,17 +50,104 @@ {% do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules.default}) %} {% endif %} -{# set elastalertengine.rulesRepos and strelkaengine.rulesRepos based on airgap or not #} +{# set elastalertengine.rulesRepos, strelkaengine.rulesRepos, and suricataengine.rulesetSources based on airgap or not #} {% if GLOBALS.airgap %} {% do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.airgap}) %} {% do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.airgap}) %} +{#% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#} +{% do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.airgap}) %} +{#% endif %#} {% do SOCMERGED.config.server.update({'airgapEnabled': true}) %} {% else %} {% do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.default}) %} {% do SOCMERGED.config.server.modules.strelkaengine.update({'rulesRepos': SOCMERGED.config.server.modules.strelkaengine.rulesRepos.default}) %} +{#% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is mapping %#} +{% do SOCMERGED.config.server.modules.suricataengine.update({'rulesetSources': SOCMERGED.config.server.modules.suricataengine.rulesetSources.default}) %} +{#% endif %#} {% do SOCMERGED.config.server.update({'airgapEnabled': false}) %} {% endif %} + +{# Define the Detections custom ruleset that should always be present #} +{% set CUSTOM_RULESET = { + 'name': '__custom__', + 'description': 'User-created custom rules created via the Detections module in the SOC UI', + 'sourceType': 'elasticsearch', + 'sourcePath': 'so_detection.ruleset:__custom__', + 'readOnly': false, + 'deleteUnreferenced': false, + 'license': 'Custom', + 'enabled': true +} %} + +{# Always append the custom ruleset to suricataengine.rulesetSources if not already present #} +{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %} +{% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %} +{% set custom_names = SOCMERGED.config.server.modules.suricataengine.rulesetSources | selectattr('name', 'equalto', '__custom__') | list %} +{% if custom_names | length == 0 %} +{% do SOCMERGED.config.server.modules.suricataengine.rulesetSources.append(CUSTOM_RULESET) %} +{% endif %} +{% endif %} +{% endif %} + +{# Enable SO_FILTERS and SO_EXTRACTIONS when Suricata is the metadata engine #} +{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %} +{% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %} +{% for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %} +{% if ruleset.name in ['SO_FILTERS', 'SO_EXTRACTIONS'] and GLOBALS.md_engine == 'SURICATA' %} +{% do ruleset.update({'enabled': true}) %} +{% endif %} +{% endfor %} +{% endif %} +{% endif %} + +{# Transform Emerging-Threats ruleset based on license key #} +{% if SOCMERGED.config.server.modules.suricataengine is defined and SOCMERGED.config.server.modules.suricataengine.rulesetSources is defined %} +{% if SOCMERGED.config.server.modules.suricataengine.rulesetSources is not mapping %} +{% for ruleset in SOCMERGED.config.server.modules.suricataengine.rulesetSources %} +{% if ruleset.name == 'Emerging-Threats' %} +{% if ruleset.licenseKey and ruleset.licenseKey != '' %} +{# License key is defined - transform to ETPRO #} +{% if ruleset.sourceType == 'directory' %} +{# Airgap mode - update directory path #} +{% do ruleset.update({ + 'name': 'ETPRO', + 'sourcePath': '/nsm/rules/custom-local-repos/local-etpro-suricata/etpro.rules.tar.gz', + 'license': 'Commercial' + }) %} +{% else %} +{# Engine Version is hardcoded in the URL - this does not change often: https://community.emergingthreats.net/t/supported-engines/71 #} +{% do ruleset.update({ + 'name': 'ETPRO', + 'sourcePath': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz', + 'urlHash': 'https://rules.emergingthreatspro.com/' ~ ruleset.licenseKey ~ '/suricata-7.0.3/etpro.rules.tar.gz.md5', + 'license': 'Commercial' + }) %} +{% endif %} +{% else %} +{# No license key - explicitly set to ETOPEN #} +{% if ruleset.sourceType == 'directory' %} +{# Airgap mode - update directory path #} +{% do ruleset.update({ + 'name': 'ETOPEN', + 'sourcePath': '/nsm/rules/suricata/etopen/', + 'license': 'BSD' + }) %} +{% else %} +{% do ruleset.update({ + 'name': 'ETOPEN', + 'sourcePath': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz', + 'urlHash': 'https://rules.emergingthreats.net/open/suricata-7.0.3/emerging.rules.tar.gz.md5', + 'license': 'BSD' + }) %} +{% endif %} +{% endif %} +{% endif %} +{% endfor %} +{% endif %} +{% endif %} + + {# set playbookRepos based on airgap or not #} {% if GLOBALS.airgap %} {% do SOCMERGED.config.server.modules.playbook.update({'playbookRepos': SOCMERGED.config.server.modules.playbook.playbookRepos.airgap}) %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index ed3615bb8..11442afba 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -563,6 +563,64 @@ soc: advanced: True forcedType: "[]string" helpLink: detections.html#rule-engine-status + rulesetSources: + default: &serulesetSources + description: "Ruleset sources for Suricata rules. Supports URL downloads and local directories. Refer to the linked documentation for details on how to configure this setting." + global: True + advanced: False + forcedType: "[]{}" + helpLink: suricata.html + syntax: json + uiElements: + - field: name + label: Ruleset Name (This will be the name of the ruleset in the UI) + required: True + readonly: True + - field: description + label: Description + - field: enabled + label: Enabled (If false, existing rules & overrides will be removed) + forcedType: bool + required: True + - field: licenseKey + label: License Key + required: False + - field: sourceType + label: Source Type + required: True + options: + - url + - directory + - field: sourcePath + label: Source Path (full url or directory path) + required: True + - field: excludeFiles + label: Exclude Files (list of file names to exclude, separated by commas) + required: False + - field: license + label: Ruleset License + required: True + - field: readOnly + label: Read Only (Prevents changes to the rule itself - can still be enabled/disabled/tuned) + forcedType: bool + required: False + - field: deleteUnreferenced + label: Delete Unreferenced (Deletes rules that are no longer referenced by ruleset source) + forcedType: bool + required: False + - field: proxyURL + label: HTTP/HTTPS proxy URL for downloading the ruleset. + required: False + - field: proxyUsername + label: Proxy authentication username. + required: False + - field: proxyPassword + label: Proxy authentication password. + required: False + - field: proxyCACert + label: Path to CA certificate file for MITM proxy verification. + required: False + airgap: *serulesetSources navigator: intervalMinutes: description: How often to generate the Navigator Layers. (minutes) @@ -650,6 +708,9 @@ soc: - field: displayName label: Display Name required: True + - field: origin + label: Country of Origin for the Model Training + required: false - field: contextLimitSmall label: Context Limit (Small) forcedType: int diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index 7de1a0fd4..e0b85b7e7 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -10,12 +10,6 @@ {% from 'suricata/map.jinja' import SURICATAMERGED %} {% from 'bpf/suricata.map.jinja' import SURICATABPF, SURICATA_BPF_STATUS, SURICATA_BPF_CALC %} -suridir: - file.directory: - - name: /opt/so/conf/suricata - - user: 940 - - group: 940 - {% if GLOBALS.pcap_engine in ["SURICATA", "TRANSITION"] %} {% from 'bpf/pcap.map.jinja' import PCAPBPF, PCAP_BPF_STATUS, PCAP_BPF_CALC %} # BPF compilation and configuration @@ -28,6 +22,14 @@ suriPCAPbpfcompilationfailure: {% endif %} {% endif %} +suridir: + file.directory: + - name: /opt/so/conf/suricata + - user: 940 + - group: 939 + - mode: 775 + - makedirs: True + # BPF applied to all of Suricata - alerts/metadata/pcap suribpf: file.managed: @@ -89,9 +91,11 @@ suricata_sbin_jinja: suriruledir: file.directory: - - name: /opt/so/conf/suricata/rules + - name: /opt/so/rules/suricata - user: 940 - - group: 940 + - group: 939 + - mode: 775 + - makedirs: True surilogdir: file.directory: @@ -115,14 +119,12 @@ suridatadir: - mode: 770 - makedirs: True -# salt:// would resolve to /opt/so/rules/nids because of the defined file_roots and -# not existing under /opt/so/saltstack/local/salt or /opt/so/saltstack/default/salt surirulesync: file.recurse: - - name: /opt/so/conf/suricata/rules/ - - source: salt://suri/ + - name: /opt/so/rules/suricata/ + - source: salt://suricata/rules/ - user: 940 - - group: 940 + - group: 939 - show_changes: False surilogscript: @@ -155,10 +157,9 @@ suriconfig: surithresholding: file.managed: - name: /opt/so/conf/suricata/threshold.conf - - source: salt://suricata/files/threshold.conf.jinja + - source: salt://suricata/files/threshold.conf - user: 940 - group: 940 - - template: jinja suriclassifications: file.managed: @@ -176,6 +177,14 @@ so-suricata-eve-clean: - template: jinja - source: salt://suricata/cron/so-suricata-eve-clean +so-suricata-rulestats: + file.managed: + - name: /usr/sbin/so-suricata-rulestats + - user: root + - group: root + - mode: 755 + - source: salt://suricata/cron/so-suricata-rulestats + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/suricata/cron/so-suricata-rulestats b/salt/suricata/cron/so-suricata-rulestats new file mode 100644 index 000000000..459ab894f --- /dev/null +++ b/salt/suricata/cron/so-suricata-rulestats @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Query Suricata for ruleset stats and reload time, write to JSON file for Telegraf to consume + +OUTFILE="/opt/so/log/suricata/rulestats.json" +SURICATASC="docker exec so-suricata /opt/suricata/bin/suricatasc" +SOCKET="/var/run/suricata/suricata-command.socket" + +query() { + timeout 10 $SURICATASC -c "$1" "$SOCKET" 2>/dev/null +} + +STATS=$(query "ruleset-stats") +RELOAD=$(query "ruleset-reload-time") +[ -z "$RELOAD" ] && RELOAD='{}' + +# Outputs valid JSON on success, empty on failure +OUTPUT=$(jq -n \ + --argjson stats "$STATS" \ + --argjson reload "$RELOAD" \ + 'if $stats.return == "OK" and ($stats.message[0].rules_loaded | type) == "number" and ($stats.message[0].rules_failed | type) == "number" then + { + rules_loaded: $stats.message[0].rules_loaded, + rules_failed: $stats.message[0].rules_failed, + last_reload: ($reload.message[0].last_reload // ""), + return: "OK" + } + else empty end' 2>/dev/null) + +if [ -n "$OUTPUT" ]; then + echo "$OUTPUT" > "$OUTFILE" +else + echo '{"return":"FAIL"}' > "$OUTFILE" +fi diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 9c9a7a8ed..cdb243465 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -467,7 +467,7 @@ suricata: append: "yes" default-rule-path: /etc/suricata/rules rule-files: - - all.rules + - all-rulesets.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/disabled.sls b/salt/suricata/disabled.sls index 49f8f93bf..e7a75867f 100644 --- a/salt/suricata/disabled.sls +++ b/salt/suricata/disabled.sls @@ -23,6 +23,11 @@ clean_suricata_eve_files: cron.absent: - identifier: clean_suricata_eve_files +# Remove rulestats cron +rulestats: + cron.absent: + - identifier: suricata_rulestats + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/suricata/enabled.sls b/salt/suricata/enabled.sls index 34e9f2e4c..ec521abb3 100644 --- a/salt/suricata/enabled.sls +++ b/salt/suricata/enabled.sls @@ -36,7 +36,7 @@ so-suricata: - /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro - /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro - /opt/so/conf/suricata/classification.config:/etc/suricata/classification.config:ro - - /opt/so/conf/suricata/rules:/etc/suricata/rules:ro + - /opt/so/rules/suricata:/etc/suricata/rules:ro - /opt/so/log/suricata/:/var/log/suricata/:rw - /nsm/suricata/:/nsm/:rw - /nsm/suricata/extracted:/var/log/suricata//filestore:rw @@ -90,6 +90,18 @@ clean_suricata_eve_files: - month: '*' - dayweek: '*' +# Add rulestats cron - runs every minute to query Suricata for rule load status +suricata_rulestats: + cron.present: + - name: /usr/sbin/so-suricata-rulestats > /dev/null 2>&1 + - identifier: suricata_rulestats + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/idstools/rules/extraction.rules b/salt/suricata/files/so_extraction.rules similarity index 99% rename from salt/idstools/rules/extraction.rules rename to salt/suricata/files/so_extraction.rules index 3ebbd41b1..d43812144 100644 --- a/salt/idstools/rules/extraction.rules +++ b/salt/suricata/files/so_extraction.rules @@ -23,4 +23,4 @@ alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestor alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100016; rev:1;) alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100017; rev:1;) alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100018; rev:1;) -alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;) \ No newline at end of file +alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; noalert; sid:1100019; rev:1;) diff --git a/salt/idstools/rules/filters.rules b/salt/suricata/files/so_filters.rules similarity index 99% rename from salt/idstools/rules/filters.rules rename to salt/suricata/files/so_filters.rules index 051d1913f..c49eaec26 100644 --- a/salt/idstools/rules/filters.rules +++ b/salt/suricata/files/so_filters.rules @@ -9,3 +9,4 @@ #config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;) # Example of filtering out a md5 of a file from being in the files log. #config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;) + diff --git a/salt/suricata/files/threshold.conf b/salt/suricata/files/threshold.conf new file mode 100644 index 000000000..a03ac31a3 --- /dev/null +++ b/salt/suricata/files/threshold.conf @@ -0,0 +1,2 @@ +# Threshold configuration generated by Security Onion +# This file is automatically generated - do not edit manually \ No newline at end of file diff --git a/salt/suricata/files/threshold.conf.jinja b/salt/suricata/files/threshold.conf.jinja deleted file mode 100644 index a439dad96..000000000 --- a/salt/suricata/files/threshold.conf.jinja +++ /dev/null @@ -1,35 +0,0 @@ -{% import_yaml 'suricata/thresholding/sids.yaml' as THRESHOLDING %} -{% if THRESHOLDING -%} - - {% for EACH_SID in THRESHOLDING -%} - {% for ACTIONS_LIST in THRESHOLDING[EACH_SID] -%} - {% for EACH_ACTION in ACTIONS_LIST -%} - - {%- if EACH_ACTION == 'threshold' %} -{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, type {{ ACTIONS_LIST[EACH_ACTION].type }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }} - - {%- elif EACH_ACTION == 'rate_filter' %} - {%- if ACTIONS_LIST[EACH_ACTION].new_action not in ['drop','reject'] %} -{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }} - {%- else %} -##### Security Onion does not support drop or reject actions for rate_filter -##### {{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, count {{ ACTIONS_LIST[EACH_ACTION].count }}, seconds {{ ACTIONS_LIST[EACH_ACTION].seconds }}, new_action {{ ACTIONS_LIST[EACH_ACTION].new_action }}, timeout {{ ACTIONS_LIST[EACH_ACTION].timeout }} - {%- endif %} - - {%- elif EACH_ACTION == 'suppress' %} - {%- if ACTIONS_LIST[EACH_ACTION].track is defined %} -{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }}, track {{ ACTIONS_LIST[EACH_ACTION].track }}, ip {{ ACTIONS_LIST[EACH_ACTION].ip }} - {%- else %} -{{ EACH_ACTION }} gen_id {{ ACTIONS_LIST[EACH_ACTION].gen_id }}, sig_id {{ EACH_SID }} - {%- endif %} - - {%- endif %} - - {%- endfor %} - {%- endfor %} - {%- endfor %} - -{%- else %} -##### Navigate to suricata > thresholding > SIDS in SOC to define thresholding - -{%- endif %} diff --git a/salt/suricata/manager.sls b/salt/suricata/manager.sls deleted file mode 100644 index 3d5183556..000000000 --- a/salt/suricata/manager.sls +++ /dev/null @@ -1,30 +0,0 @@ -{% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} - -surilocaldir: - file.directory: - - name: /opt/so/saltstack/local/salt/suricata - - user: socore - - group: socore - - makedirs: True - -ruleslink: - file.symlink: - - name: /opt/so/saltstack/local/salt/suricata/rules - - user: socore - - group: socore - - target: /opt/so/rules/nids/suri - -refresh_salt_master_fileserver_suricata_ruleslink: - salt.runner: - - name: fileserver.update - - onchanges: - - file: ruleslink - -{% else %} - -{{sls}}_state_not_allowed: - test.fail_without_changes: - - name: {{sls}}_state_not_allowed - -{% endif %} diff --git a/salt/suricata/rules/PLACEHOLDER b/salt/suricata/rules/PLACEHOLDER new file mode 100644 index 000000000..e69de29bb diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 79ad9008d..c0a67b0ca 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -21,6 +21,7 @@ telegraf: - sostatus.sh - stenoloss.sh - suriloss.sh + - surirules.sh - zeekcaptureloss.sh - zeekloss.sh standalone: @@ -36,6 +37,7 @@ telegraf: - sostatus.sh - stenoloss.sh - suriloss.sh + - surirules.sh - zeekcaptureloss.sh - zeekloss.sh - features.sh @@ -81,6 +83,7 @@ telegraf: - sostatus.sh - stenoloss.sh - suriloss.sh + - surirules.sh - zeekcaptureloss.sh - zeekloss.sh - features.sh @@ -95,6 +98,7 @@ telegraf: - sostatus.sh - stenoloss.sh - suriloss.sh + - surirules.sh - zeekcaptureloss.sh - zeekloss.sh idh: diff --git a/salt/telegraf/scripts/surirules.sh b/salt/telegraf/scripts/surirules.sh new file mode 100644 index 000000000..f4c6885e1 --- /dev/null +++ b/salt/telegraf/scripts/surirules.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# Read Suricata ruleset stats from JSON file written by so-suricata-rulestats cron job +# JSON format: {"rules_loaded":45879,"rules_failed":1,"last_reload":"2025-12-04T14:10:57+0000","return":"OK"} +# or on failure: {"return":"FAIL"} + +# if this script isn't already running +if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then + + STATSFILE="/var/log/suricata/rulestats.json" + + # Check file exists, is less than 90 seconds old, and has valid data + if [ -f "$STATSFILE" ] && [ $(($(date +%s) - $(stat -c %Y "$STATSFILE"))) -lt 90 ] && jq -e '.return == "OK" and .rules_loaded != null and .rules_failed != null' "$STATSFILE" > /dev/null 2>&1; then + LOADED=$(jq -r '.rules_loaded' "$STATSFILE") + FAILED=$(jq -r '.rules_failed' "$STATSFILE") + RELOAD_TIME=$(jq -r 'if .last_reload then .last_reload else "" end' "$STATSFILE") + + if [ -n "$RELOAD_TIME" ]; then + echo "surirules loaded=${LOADED}i,failed=${FAILED}i,reload_time=\"${RELOAD_TIME}\",status=\"ok\"" + else + echo "surirules loaded=${LOADED}i,failed=${FAILED}i,status=\"ok\"" + fi + else + echo "surirules loaded=0i,failed=0i,status=\"unknown\"" + fi + +fi + +exit 0 diff --git a/salt/top.sls b/salt/top.sls index 9184bd3e3..007c62b59 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -74,8 +74,6 @@ base: - sensoroni - telegraf - firewall - - idstools - - suricata.manager - healthcheck - elasticsearch - elastic-fleet-package-registry @@ -104,8 +102,6 @@ base: - firewall - sensoroni - telegraf - - idstools - - suricata.manager - healthcheck - elasticsearch - logstash @@ -138,8 +134,6 @@ base: - sensoroni - telegraf - backup.config_backup - - idstools - - suricata.manager - elasticsearch - logstash - redis @@ -171,8 +165,6 @@ base: - sensoroni - telegraf - backup.config_backup - - idstools - - suricata.manager - elasticsearch - logstash - redis @@ -200,8 +192,6 @@ base: - sensoroni - telegraf - firewall - - idstools - - suricata.manager - pcap - elasticsearch - elastic-fleet-package-registry diff --git a/setup/so-functions b/setup/so-functions index 3262f6ece..6954696a5 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -656,11 +656,11 @@ check_requirements() { fi if [[ $total_mem_hr -lt $req_mem ]]; then - whiptail_requirements_error "memory" "${total_mem_hr} GB" "${req_mem} GB" if [[ $is_standalone || $is_heavynode ]]; then echo "This install type will fail with less than $req_mem GB of memory. Exiting setup." exit 0 fi + whiptail_requirements_error "memory" "${total_mem_hr} GB" "${req_mem} GB" fi if [[ $is_standalone || $is_heavynode ]]; then if [[ $total_mem_hr -gt 15 && $total_mem_hr -lt 24 ]]; then @@ -816,7 +816,6 @@ create_manager_pillars() { backup_pillar docker_pillar redis_pillar - idstools_pillar kratos_pillar hydra_pillar soc_pillar @@ -1272,11 +1271,6 @@ ls_heapsize() { } -idstools_pillar() { - title "Ading IDSTOOLS pillar options" - touch $adv_idstools_pillar_file -} - nginx_pillar() { title "Creating the NGINX pillar" [[ -z "$TESTING" ]] && return @@ -1452,7 +1446,7 @@ make_some_dirs() { mkdir -p $local_salt_dir/salt/firewall/portgroups mkdir -p $local_salt_dir/salt/firewall/ports - for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idstools idh elastalert stig global kafka versionlock hypervisor vm; do + for THEDIR in bpf pcap elasticsearch ntp firewall redis backup influxdb strelka sensoroni soc docker zeek suricata nginx telegraf logstash soc manager kratos hydra idh elastalert stig global kafka versionlock hypervisor vm; do mkdir -p $local_salt_dir/pillar/$THEDIR touch $local_salt_dir/pillar/$THEDIR/adv_$THEDIR.sls touch $local_salt_dir/pillar/$THEDIR/soc_$THEDIR.sls diff --git a/setup/so-variables b/setup/so-variables index fc253df0a..a0d7aadc1 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -166,12 +166,6 @@ export hydra_pillar_file adv_hydra_pillar_file="$local_salt_dir/pillar/hydra/adv_hydra.sls" export adv_hydra_pillar_file -idstools_pillar_file="$local_salt_dir/pillar/idstools/soc_idstools.sls" -export idstools_pillar_file - -adv_idstools_pillar_file="$local_salt_dir/pillar/idstools/adv_idstools.sls" -export adv_idstools_pillar_file - nginx_pillar_file="$local_salt_dir/pillar/nginx/soc_nginx.sls" export nginx_pillar_file