mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-06 17:22:49 +01:00
Merge remote-tracking branch 'origin/2.4/dev' into idstools-refactor
This commit is contained in:
DefensiveDepth
@@ -256,6 +256,9 @@ base:
|
||||
- minions.adv_{{ grains.id }}
|
||||
- kafka.nodes
|
||||
- kafka.soc_kafka
|
||||
- stig.soc_stig
|
||||
- elasticfleet.soc_elasticfleet
|
||||
- elasticfleet.adv_elasticfleet
|
||||
|
||||
'*_import':
|
||||
- node_data.ips
|
||||
@@ -313,10 +316,12 @@ base:
|
||||
- elasticfleet.adv_elasticfleet
|
||||
- minions.{{ grains.id }}
|
||||
- minions.adv_{{ grains.id }}
|
||||
- stig.soc_stig
|
||||
|
||||
'*_hypervisor':
|
||||
- minions.{{ grains.id }}
|
||||
- minions.adv_{{ grains.id }}
|
||||
- stig.soc_stig
|
||||
|
||||
'*_desktop':
|
||||
- minions.{{ grains.id }}
|
||||
|
||||
@@ -142,6 +142,7 @@
|
||||
),
|
||||
'so-fleet': (
|
||||
ssl_states +
|
||||
stig_states +
|
||||
['logstash', 'nginx', 'healthcheck', 'elasticfleet']
|
||||
),
|
||||
'so-receiver': (
|
||||
|
||||
@@ -9,3 +9,6 @@ fleetartifactdir:
|
||||
- user: 947
|
||||
- group: 939
|
||||
- makedirs: True
|
||||
- recurse:
|
||||
- user
|
||||
- group
|
||||
|
||||
@@ -9,6 +9,9 @@
|
||||
{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
|
||||
{% set node_data = salt['pillar.get']('node_data') %}
|
||||
|
||||
include:
|
||||
- elasticfleet.artifact_registry
|
||||
|
||||
# Add EA Group
|
||||
elasticfleetgroup:
|
||||
group.present:
|
||||
|
||||
@@ -38,6 +38,7 @@ elasticfleet:
|
||||
- elasticsearch
|
||||
- endpoint
|
||||
- fleet_server
|
||||
- filestream
|
||||
- http_endpoint
|
||||
- httpjson
|
||||
- log
|
||||
|
||||
@@ -67,6 +67,8 @@ so-elastic-fleet-auto-configure-artifact-urls:
|
||||
elasticagent_syncartifacts:
|
||||
file.recurse:
|
||||
- name: /nsm/elastic-fleet/artifacts/beats
|
||||
- user: 947
|
||||
- group: 947
|
||||
- source: salt://beats
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"package": {
|
||||
"name": "filestream",
|
||||
"version": ""
|
||||
},
|
||||
"name": "agent-monitor",
|
||||
"namespace": "",
|
||||
"description": "",
|
||||
"policy_ids": [
|
||||
"so-grid-nodes_general"
|
||||
],
|
||||
"output_id": null,
|
||||
"vars": {},
|
||||
"inputs": {
|
||||
"filestream-filestream": {
|
||||
"enabled": true,
|
||||
"streams": {
|
||||
"filestream.generic": {
|
||||
"enabled": true,
|
||||
"vars": {
|
||||
"paths": [
|
||||
"/opt/so/log/agents/agent-monitor.log"
|
||||
],
|
||||
"data_stream.dataset": "agentmonitor",
|
||||
"pipeline": "elasticagent.monitor",
|
||||
"parsers": "",
|
||||
"exclude_files": [
|
||||
"\\.gz$"
|
||||
],
|
||||
"include_files": [],
|
||||
"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- add_fields:\n target: event\n fields:\n module: gridmetrics",
|
||||
"tags": [],
|
||||
"recursive_glob": true,
|
||||
"ignore_older": "72h",
|
||||
"clean_inactive": -1,
|
||||
"harvester_limit": 0,
|
||||
"fingerprint": true,
|
||||
"fingerprint_offset": 0,
|
||||
"fingerprint_length": 64,
|
||||
"file_identity_native": false,
|
||||
"exclude_lines": [],
|
||||
"include_lines": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@
|
||||
],
|
||||
"data_stream.dataset": "import",
|
||||
"custom": "",
|
||||
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.3.3\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.1.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.3.3\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.3.3\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.1.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
|
||||
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.5.4\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.5.4\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.5.4\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
|
||||
"tags": [
|
||||
"import"
|
||||
]
|
||||
|
||||
@@ -23,14 +23,28 @@ function update_logstash_outputs() {
|
||||
}
|
||||
function update_kafka_outputs() {
|
||||
# Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup
|
||||
SSL_CONFIG=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" | jq -r '.item.ssl')
|
||||
|
||||
JSON_STRING=$(jq -n \
|
||||
--arg UPDATEDLIST "$NEW_LIST_JSON" \
|
||||
--argjson SSL_CONFIG "$SSL_CONFIG" \
|
||||
'{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
|
||||
# Update Kafka outputs
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
|
||||
if kafka_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" --fail 2>/dev/null); then
|
||||
SSL_CONFIG=$(echo "$kafka_policy" | jq -r '.item.ssl')
|
||||
if SECRETS=$(echo "$kafka_policy" | jq -er '.item.secrets' 2>/dev/null); then
|
||||
# Update policy when fleet has secrets enabled
|
||||
JSON_STRING=$(jq -n \
|
||||
--arg UPDATEDLIST "$NEW_LIST_JSON" \
|
||||
--argjson SSL_CONFIG "$SSL_CONFIG" \
|
||||
--argjson SECRETS "$SECRETS" \
|
||||
'{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
|
||||
else
|
||||
# Update policy when fleet has secrets disabled or policy hasn't been force updated
|
||||
JSON_STRING=$(jq -n \
|
||||
--arg UPDATEDLIST "$NEW_LIST_JSON" \
|
||||
--argjson SSL_CONFIG "$SSL_CONFIG" \
|
||||
'{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
|
||||
fi
|
||||
# Update Kafka outputs
|
||||
curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
|
||||
else
|
||||
printf "Failed to get current Kafka output policy..."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
{% if GLOBALS.pipeline == "KAFKA" %}
|
||||
|
||||
@@ -5,46 +5,78 @@
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% if GLOBALS.role in ['so-manager', 'so-standalone', 'so-managersearch'] %}
|
||||
{% if GLOBALS.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-managerhype'] %}
|
||||
|
||||
. /usr/sbin/so-common
|
||||
|
||||
force=false
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-f|--force)
|
||||
force=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option $1"
|
||||
echo "Usage: $0 [-f|--force]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Check to make sure that Kibana API is up & ready
|
||||
RETURN_CODE=0
|
||||
wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
|
||||
RETURN_CODE=$?
|
||||
|
||||
if [[ "$RETURN_CODE" != "0" ]]; then
|
||||
printf "Kibana API not accessible, can't setup Elastic Fleet output policy for Kafka..."
|
||||
exit 1
|
||||
echo -e "\nKibana API not accessible, can't setup Elastic Fleet output policy for Kafka...\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
|
||||
KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
|
||||
KAFKAKEY=$(openssl rsa -in /etc/pki/elasticfleet-kafka.key)
|
||||
KAFKACA=$(openssl x509 -in /etc/pki/tls/certs/intca.crt)
|
||||
KAFKA_OUTPUT_VERSION="2.6.0"
|
||||
|
||||
if ! echo "$output" | grep -q "so-manager_kafka"; then
|
||||
KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
|
||||
KAFKAKEY=$(openssl rsa -in /etc/pki/elasticfleet-kafka.key)
|
||||
KAFKACA=$(openssl x509 -in /etc/pki/tls/certs/intca.crt)
|
||||
KAFKA_OUTPUT_VERSION="2.6.0"
|
||||
if ! kafka_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" --fail 2>/dev/null); then
|
||||
# Create a new output policy for Kafka. Default is disabled 'is_default: false & is_default_monitoring: false'
|
||||
JSON_STRING=$( jq -n \
|
||||
--arg KAFKACRT "$KAFKACRT" \
|
||||
--arg KAFKAKEY "$KAFKAKEY" \
|
||||
--arg KAFKACA "$KAFKACA" \
|
||||
--arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
|
||||
--arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
|
||||
'{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 10 }, "topics":[{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
|
||||
)
|
||||
curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null
|
||||
refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
|
||||
|
||||
if ! echo "$refresh_output" | grep -q "so-manager_kafka"; then
|
||||
echo -e "\nFailed to setup Elastic Fleet output policy for Kafka...\n"
|
||||
--arg KAFKACRT "$KAFKACRT" \
|
||||
--arg KAFKAKEY "$KAFKAKEY" \
|
||||
--arg KAFKACA "$KAFKACA" \
|
||||
--arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
|
||||
--arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
|
||||
'{"name":"grid-kafka", "id":"so-manager_kafka","type":"kafka","hosts":[ $MANAGER_IP ],"is_default":false,"is_default_monitoring":false,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topics":[{"topic":"default-securityonion"}],"headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
|
||||
)
|
||||
if ! response=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --fail 2>/dev/null); then
|
||||
echo -e "\nFailed to setup Elastic Fleet output policy for Kafka...\n"
|
||||
exit 1
|
||||
else
|
||||
echo -e "\nSuccessfully setup Elastic Fleet output policy for Kafka...\n"
|
||||
exit 0
|
||||
fi
|
||||
elif kafka_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" --fail 2>/dev/null) && [[ "$force" == "true" ]]; then
|
||||
# force an update to Kafka policy. Keep the current value of Kafka output policy (enabled/disabled).
|
||||
ENABLED_DISABLED=$(echo "$kafka_output" | jq -e .item.is_default)
|
||||
HOSTS=$(echo "$kafka_output" | jq -r '.item.hosts')
|
||||
JSON_STRING=$( jq -n \
|
||||
--arg KAFKACRT "$KAFKACRT" \
|
||||
--arg KAFKAKEY "$KAFKAKEY" \
|
||||
--arg KAFKACA "$KAFKACA" \
|
||||
--arg ENABLED_DISABLED "$ENABLED_DISABLED"\
|
||||
--arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
|
||||
--argjson HOSTS "$HOSTS" \
|
||||
'{"name":"grid-kafka","type":"kafka","hosts":$HOSTS,"is_default":$ENABLED_DISABLED,"is_default_monitoring":$ENABLED_DISABLED,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topics":[{"topic":"default-securityonion"}],"headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
|
||||
)
|
||||
if ! response=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --fail 2>/dev/null); then
|
||||
echo -e "\nFailed to force update to Elastic Fleet output policy for Kafka...\n"
|
||||
exit 1
|
||||
elif echo "$refresh_output" | grep -q "so-manager_kafka"; then
|
||||
echo -e "\nSuccessfully setup Elastic Fleet output policy for Kafka...\n"
|
||||
else
|
||||
echo -e "\nForced update to Elastic Fleet output policy for Kafka...\n"
|
||||
fi
|
||||
|
||||
elif echo "$output" | grep -q "so-manager_kafka"; then
|
||||
else
|
||||
echo -e "\nElastic Fleet output policy for Kafka already exists...\n"
|
||||
fi
|
||||
{% else %}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
elasticsearch:
|
||||
enabled: false
|
||||
version: 8.18.4
|
||||
version: 8.18.6
|
||||
index_clean: true
|
||||
config:
|
||||
action:
|
||||
@@ -284,6 +284,86 @@ elasticsearch:
|
||||
hot:
|
||||
actions: {}
|
||||
min_age: 0ms
|
||||
so-assistant-chat:
|
||||
index_sorting: false
|
||||
index_template:
|
||||
composed_of:
|
||||
- assistant-chat-mappings
|
||||
- assistant-chat-settings
|
||||
data_stream:
|
||||
allow_custom_routing: false
|
||||
hidden: false
|
||||
ignore_missing_component_templates: []
|
||||
index_patterns:
|
||||
- so-assistant-chat*
|
||||
priority: 501
|
||||
template:
|
||||
mappings:
|
||||
date_detection: false
|
||||
dynamic_templates:
|
||||
- strings_as_keyword:
|
||||
mapping:
|
||||
ignore_above: 1024
|
||||
type: keyword
|
||||
match_mapping_type: string
|
||||
settings:
|
||||
index:
|
||||
lifecycle:
|
||||
name: so-assistant-chat-logs
|
||||
mapping:
|
||||
total_fields:
|
||||
limit: 1500
|
||||
number_of_replicas: 0
|
||||
number_of_shards: 1
|
||||
refresh_interval: 1s
|
||||
sort:
|
||||
field: '@timestamp'
|
||||
order: desc
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
actions: {}
|
||||
min_age: 0ms
|
||||
so-assistant-session:
|
||||
index_sorting: false
|
||||
index_template:
|
||||
composed_of:
|
||||
- assistant-session-mappings
|
||||
- assistant-session-settings
|
||||
data_stream:
|
||||
allow_custom_routing: false
|
||||
hidden: false
|
||||
ignore_missing_component_templates: []
|
||||
index_patterns:
|
||||
- so-assistant-session*
|
||||
priority: 501
|
||||
template:
|
||||
mappings:
|
||||
date_detection: false
|
||||
dynamic_templates:
|
||||
- strings_as_keyword:
|
||||
mapping:
|
||||
ignore_above: 1024
|
||||
type: keyword
|
||||
match_mapping_type: string
|
||||
settings:
|
||||
index:
|
||||
lifecycle:
|
||||
name: so-assistant-session-logs
|
||||
mapping:
|
||||
total_fields:
|
||||
limit: 1500
|
||||
number_of_replicas: 0
|
||||
number_of_shards: 1
|
||||
refresh_interval: 1s
|
||||
sort:
|
||||
field: '@timestamp'
|
||||
order: desc
|
||||
policy:
|
||||
phases:
|
||||
hot:
|
||||
actions: {}
|
||||
min_age: 0ms
|
||||
so-endgame:
|
||||
index_sorting: false
|
||||
index_template:
|
||||
@@ -1243,6 +1323,68 @@ elasticsearch:
|
||||
set_priority:
|
||||
priority: 50
|
||||
min_age: 30d
|
||||
so-elastic-agent-monitor:
|
||||
index_sorting: false
|
||||
index_template:
|
||||
composed_of:
|
||||
- event-mappings
|
||||
- so-elastic-agent-monitor
|
||||
- so-fleet_integrations.ip_mappings-1
|
||||
- so-fleet_globals-1
|
||||
- so-fleet_agent_id_verification-1
|
||||
data_stream:
|
||||
allow_custom_routing: false
|
||||
hidden: false
|
||||
index_patterns:
|
||||
- logs-agentmonitor-*
|
||||
priority: 501
|
||||
template:
|
||||
mappings:
|
||||
_meta:
|
||||
managed: true
|
||||
managed_by: security_onion
|
||||
package:
|
||||
name: elastic_agent
|
||||
settings:
|
||||
index:
|
||||
lifecycle:
|
||||
name: so-elastic-agent-monitor-logs
|
||||
mapping:
|
||||
total_fields:
|
||||
limit: 5000
|
||||
number_of_replicas: 0
|
||||
sort:
|
||||
field: '@timestamp'
|
||||
order: desc
|
||||
policy:
|
||||
_meta:
|
||||
managed: true
|
||||
managed_by: security_onion
|
||||
package:
|
||||
name: elastic_agent
|
||||
phases:
|
||||
cold:
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 0
|
||||
min_age: 60d
|
||||
delete:
|
||||
actions:
|
||||
delete: {}
|
||||
min_age: 365d
|
||||
hot:
|
||||
actions:
|
||||
rollover:
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
set_priority:
|
||||
priority: 100
|
||||
min_age: 0ms
|
||||
warm:
|
||||
actions:
|
||||
set_priority:
|
||||
priority: 50
|
||||
min_age: 30d
|
||||
so-logs-elastic_agent_x_apm_server:
|
||||
index_sorting: false
|
||||
index_template:
|
||||
@@ -4031,7 +4173,7 @@ elasticsearch:
|
||||
hot:
|
||||
actions:
|
||||
rollover:
|
||||
max_age: 1d
|
||||
max_age: 30d
|
||||
max_primary_shard_size: 50gb
|
||||
set_priority:
|
||||
priority: 100
|
||||
|
||||
22
salt/elasticsearch/files/ingest/common.ip_validation
Normal file
22
salt/elasticsearch/files/ingest/common.ip_validation
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"processors": [
|
||||
{
|
||||
"convert": {
|
||||
"field": "_ingest._value",
|
||||
"type": "ip",
|
||||
"target_field": "_ingest._temp_ip",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"append": {
|
||||
"field": "temp._valid_ips",
|
||||
"allow_duplicates": false,
|
||||
"value": [
|
||||
"{{{_ingest._temp_ip}}}"
|
||||
],
|
||||
"ignore_failure": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
36
salt/elasticsearch/files/ingest/elasticagent.monitor
Normal file
36
salt/elasticsearch/files/ingest/elasticagent.monitor
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"processors": [
|
||||
{
|
||||
"set": {
|
||||
"field": "event.dataset",
|
||||
"value": "gridmetrics.agents",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"set": {
|
||||
"field": "event.module",
|
||||
"value": "gridmetrics",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"remove": {
|
||||
"field": [
|
||||
"host",
|
||||
"elastic_agent",
|
||||
"agent"
|
||||
],
|
||||
"ignore_missing": true,
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"json": {
|
||||
"field": "message",
|
||||
"add_to_root": true,
|
||||
"ignore_failure": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -24,7 +24,7 @@
|
||||
{ "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
|
||||
{ "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
|
||||
{"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}},
|
||||
{"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint'","description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
|
||||
{"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null","ignore_missing":true, "description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
|
||||
{ "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
|
||||
]
|
||||
}
|
||||
|
||||
@@ -107,61 +107,61 @@
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-firewall",
|
||||
"name": "logs-pfsense.log-1.23.1-firewall",
|
||||
"if": "ctx.event.provider == 'filterlog'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-openvpn",
|
||||
"name": "logs-pfsense.log-1.23.1-openvpn",
|
||||
"if": "ctx.event.provider == 'openvpn'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-ipsec",
|
||||
"name": "logs-pfsense.log-1.23.1-ipsec",
|
||||
"if": "ctx.event.provider == 'charon'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-dhcp",
|
||||
"name": "logs-pfsense.log-1.23.1-dhcp",
|
||||
"if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-unbound",
|
||||
"name": "logs-pfsense.log-1.23.1-unbound",
|
||||
"if": "ctx.event.provider == 'unbound'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-haproxy",
|
||||
"name": "logs-pfsense.log-1.23.1-haproxy",
|
||||
"if": "ctx.event.provider == 'haproxy'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-php-fpm",
|
||||
"name": "logs-pfsense.log-1.23.1-php-fpm",
|
||||
"if": "ctx.event.provider == 'php-fpm'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-squid",
|
||||
"name": "logs-pfsense.log-1.23.1-squid",
|
||||
"if": "ctx.event.provider == 'squid'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-snort",
|
||||
"name": "logs-pfsense.log-1.23.1-snort",
|
||||
"if": "ctx.event.provider == 'snort'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"pipeline": {
|
||||
"name": "logs-pfsense.log-1.23.0-suricata",
|
||||
"name": "logs-pfsense.log-1.23.1-suricata",
|
||||
"if": "ctx.event.provider == 'suricata'"
|
||||
}
|
||||
},
|
||||
@@ -21,7 +21,10 @@
|
||||
{ "rename": { "field": "message2.RA", "target_field": "dns.recursion.available", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.Z", "target_field": "dns.reserved", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.answers", "target_field": "dns.answers.name", "ignore_missing": true } },
|
||||
{ "script": { "lang": "painless", "if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null", "source": "def ips = []; for (item in ctx.dns.answers.name) { if (item =~ /^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$/ || item =~ /^([a-fA-F0-9:]+:+)+[a-fA-F0-9]+$/) { ips.add(item); } } ctx.dns.resolved_ip = ips;" } },
|
||||
{ "foreach": {"field": "dns.answers.name","processor": {"pipeline": {"name": "common.ip_validation"}},"if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null","ignore_failure": true}},
|
||||
{ "foreach": {"field": "temp._valid_ips","processor": {"append": {"field": "dns.resolved_ip","allow_duplicates": false,"value": "{{{_ingest._value}}}","ignore_failure": true}},"ignore_failure": true}},
|
||||
{ "script": { "source": "if (ctx.dns.resolved_ip != null && ctx.dns.resolved_ip instanceof List) {\n ctx.dns.resolved_ip.removeIf(item -> item == null || item.toString().trim().isEmpty());\n }","ignore_failure": true }},
|
||||
{ "remove": {"field": ["temp"], "ignore_missing": true ,"ignore_failure": true } },
|
||||
{ "rename": { "field": "message2.TTLs", "target_field": "dns.ttls", "ignore_missing": true } },
|
||||
{ "rename": { "field": "message2.rejected", "target_field": "dns.query.rejected", "ignore_missing": true } },
|
||||
{ "script": { "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()", "ignore_failure": true } },
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"template": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"agent": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"hostname": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"id": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"last_checkin_status": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"last_checkin": {
|
||||
"type": "date"
|
||||
},
|
||||
"name": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"offline_duration_hours": {
|
||||
"type": "integer"
|
||||
},
|
||||
"policy_id": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"status": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"template": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"so_kind": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"so_operation": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"so_chat": {
|
||||
"properties": {
|
||||
"role": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"content": {
|
||||
"type": "object",
|
||||
"enabled": false
|
||||
},
|
||||
"sessionId": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"createTime": {
|
||||
"type": "date"
|
||||
},
|
||||
"deletedAt": {
|
||||
"type": "date"
|
||||
},
|
||||
"tags": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"tool_use_id": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"userId": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"message": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"type": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"role": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"model": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"contentStr": {
|
||||
"type": "text"
|
||||
},
|
||||
"contentBlocks": {
|
||||
"type": "nested",
|
||||
"enabled": false
|
||||
},
|
||||
"stopReason": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"stopSequence": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"usage": {
|
||||
"properties": {
|
||||
"input_tokens": {
|
||||
"type": "long"
|
||||
},
|
||||
"output_tokens": {
|
||||
"type": "long"
|
||||
},
|
||||
"credits": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"ecs_version": "1.12.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"template": {},
|
||||
"version": 1,
|
||||
"_meta": {
|
||||
"description": "default settings for common Security Onion Assistant indices"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"template": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"so_kind": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"so_session": {
|
||||
"properties": {
|
||||
"title": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"sessionId": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"createTime": {
|
||||
"type": "date"
|
||||
},
|
||||
"deleteTime": {
|
||||
"type": "date"
|
||||
},
|
||||
"tags": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"userId": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"ecs_version": "1.12.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"template": {},
|
||||
"version": 1,
|
||||
"_meta": {
|
||||
"description": "default settings for common Security Onion Assistant indices"
|
||||
}
|
||||
}
|
||||
@@ -1230,6 +1230,10 @@ firewall:
|
||||
portgroups:
|
||||
- elasticsearch_node
|
||||
- elasticsearch_rest
|
||||
managerhype:
|
||||
portgroups:
|
||||
- elasticsearch_node
|
||||
- elasticsearch_rest
|
||||
standalone:
|
||||
portgroups:
|
||||
- elasticsearch_node
|
||||
@@ -1377,6 +1381,10 @@ firewall:
|
||||
portgroups:
|
||||
- elasticsearch_node
|
||||
- elasticsearch_rest
|
||||
managerhype:
|
||||
portgroups:
|
||||
- elasticsearch_node
|
||||
- elasticsearch_rest
|
||||
standalone:
|
||||
portgroups:
|
||||
- elasticsearch_node
|
||||
@@ -1579,6 +1587,9 @@ firewall:
|
||||
portgroups:
|
||||
- redis
|
||||
- elastic_agent_data
|
||||
managerhype:
|
||||
portgroups:
|
||||
- elastic_agent_data
|
||||
self:
|
||||
portgroups:
|
||||
- redis
|
||||
@@ -1696,6 +1707,9 @@ firewall:
|
||||
managersearch:
|
||||
portgroups:
|
||||
- openssh
|
||||
managerhype:
|
||||
portgroups:
|
||||
- openssh
|
||||
standalone:
|
||||
portgroups:
|
||||
- openssh
|
||||
@@ -1758,6 +1772,8 @@ firewall:
|
||||
portgroups: []
|
||||
managersearch:
|
||||
portgroups: []
|
||||
managerhype:
|
||||
portgroups: []
|
||||
standalone:
|
||||
portgroups: []
|
||||
customhostgroup0:
|
||||
|
||||
@@ -91,7 +91,7 @@ COMMIT
|
||||
-A INPUT -m conntrack --ctstate INVALID -j DROP
|
||||
-A INPUT -p icmp -j ACCEPT
|
||||
-A INPUT -j LOGGING
|
||||
{% if GLOBALS.role in ['so-hypervisor', 'so-managerhyper'] -%}
|
||||
{% if GLOBALS.role in ['so-hypervisor', 'so-managerhype'] -%}
|
||||
-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT
|
||||
-A FORWARD -i br0 -o br0 -j ACCEPT
|
||||
{%- endif %}
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
|
||||
{% set kafka_node_type = salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname + ':role') %}
|
||||
|
||||
{% if role in ['manager', 'managersearch', 'standalone'] %}
|
||||
{% if role.startswith('manager') or role == 'standalone' %}
|
||||
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %}
|
||||
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
|
||||
{% endif %}
|
||||
@@ -38,8 +38,8 @@
|
||||
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
|
||||
{% endif %}
|
||||
|
||||
{% if role in ['manager', 'managersearch', 'standalone', 'receiver'] %}
|
||||
{% for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
|
||||
{% if role.startswith('manager') or role in ['standalone', 'receiver'] %}
|
||||
{% for r in ['manager', 'managersearch', 'managerhype', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
|
||||
{% if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
|
||||
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %}
|
||||
{% endif %}
|
||||
@@ -48,11 +48,11 @@
|
||||
|
||||
{% if KAFKA_EXTERNAL_ACCESS %}
|
||||
{# Kafka external access only applies for Kafka nodes with the broker role. #}
|
||||
{% if role in ['manager', 'managersearch', 'standalone', 'receiver'] and 'broker' in kafka_node_type %}
|
||||
{% if role.startswith('manager') or role in ['standalone', 'receiver'] and 'broker' in kafka_node_type %}
|
||||
{% do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.external_kafka.portgroups.append('kafka_external_access') %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
{% endif %}
|
||||
|
||||
{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}
|
||||
{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}
|
||||
|
||||
@@ -22,7 +22,7 @@ kibana:
|
||||
- default
|
||||
- file
|
||||
migrations:
|
||||
discardCorruptObjects: "8.18.4"
|
||||
discardCorruptObjects: "8.18.6"
|
||||
telemetry:
|
||||
enabled: False
|
||||
security:
|
||||
|
||||
@@ -3,8 +3,7 @@
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'libvirt/map.jinja' import LIBVIRTMERGED %}
|
||||
{% from 'salt/map.jinja' import SYSTEMD_UNIT_FILE %}
|
||||
# We do not import GLOBALS in this state because it is called during setup
|
||||
|
||||
down_original_mgmt_interface:
|
||||
cmd.run:
|
||||
@@ -30,6 +29,8 @@ wait_for_br0_ip:
|
||||
- onchanges:
|
||||
- cmd: down_original_mgmt_interface
|
||||
|
||||
{% if grains.role == 'so-hypervisor' %}
|
||||
|
||||
update_mine_functions:
|
||||
file.managed:
|
||||
- name: /etc/salt/minion.d/mine_functions.conf
|
||||
@@ -38,6 +39,10 @@ update_mine_functions:
|
||||
mine_functions:
|
||||
network.ip_addrs:
|
||||
- interface: br0
|
||||
{%- if role in ['so-eval','so-import','so-manager','so-managerhype','so-managersearch','so-standalone'] %}
|
||||
x509.get_pem_entries:
|
||||
- glob_path: '/etc/pki/ca.crt'
|
||||
{% endif %}
|
||||
- onchanges:
|
||||
- cmd: wait_for_br0_ip
|
||||
|
||||
@@ -47,3 +52,5 @@ restart_salt_minion_service:
|
||||
- enable: True
|
||||
- listen:
|
||||
- file: update_mine_functions
|
||||
|
||||
{% endif %}
|
||||
|
||||
@@ -258,3 +258,12 @@ logrotate:
|
||||
- nocompress
|
||||
- create
|
||||
- sharedscripts
|
||||
/opt/so/log/agents/agent-monitor*_x_log:
|
||||
- daily
|
||||
- rotate 14
|
||||
- missingok
|
||||
- compress
|
||||
- create
|
||||
- extension .log
|
||||
- dateext
|
||||
- dateyesterday
|
||||
@@ -168,3 +168,10 @@ logrotate:
|
||||
multiline: True
|
||||
global: True
|
||||
forcedType: "[]string"
|
||||
"/opt/so/log/agents/agent-monitor*_x_log":
|
||||
description: List of logrotate options for this file.
|
||||
title: /opt/so/log/agents/agent-monitor*.log
|
||||
advanced: True
|
||||
multiline: True
|
||||
global: True
|
||||
forcedType: "[]string"
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
|
||||
{% for node_type, node_details in redis_node_data.items() | sort %}
|
||||
{% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch', 'so-fleet'] %}
|
||||
{% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %}
|
||||
{% if node_type.startswith('manager') or node_type in ['standalone', 'receiver'] %}
|
||||
{% for hostname in redis_node_data[node_type].keys() %}
|
||||
{% do LOGSTASH_REDIS_NODES.append({hostname:node_details[hostname].ip}) %}
|
||||
{% endfor %}
|
||||
@@ -47,7 +47,7 @@
|
||||
{% endif %}
|
||||
{# Disable logstash on manager & receiver nodes unless it has an override configured #}
|
||||
{% if not KAFKA_LOGSTASH %}
|
||||
{% if GLOBALS.role in ['so-manager', 'so-receiver'] and GLOBALS.hostname not in KAFKA_LOGSTASH %}
|
||||
{% if GLOBALS.role in ['so-manager', 'so-managerhype', 'so-receiver'] and GLOBALS.hostname not in KAFKA_LOGSTASH %}
|
||||
{% do LOGSTASH_MERGED.update({'enabled': False}) %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
@@ -5,3 +5,12 @@ manager:
|
||||
minute: 0
|
||||
additionalCA: ''
|
||||
insecureSkipVerify: False
|
||||
agent_monitoring:
|
||||
enabled: False
|
||||
config:
|
||||
critical_agents: []
|
||||
custom_kquery:
|
||||
offline_threshold: 5
|
||||
realert_threshold: 5
|
||||
page_size: 250
|
||||
run_interval: 5
|
||||
|
||||
@@ -34,6 +34,26 @@ agents_log_dir:
|
||||
- user
|
||||
- group
|
||||
|
||||
agents_conf_dir:
|
||||
file.directory:
|
||||
- name: /opt/so/conf/agents
|
||||
- user: root
|
||||
- group: root
|
||||
- recurse:
|
||||
- user
|
||||
- group
|
||||
|
||||
{% if MANAGERMERGED.agent_monitoring.config.critical_agents | length > 0 %}
|
||||
critical_agents_patterns:
|
||||
file.managed:
|
||||
- name: /opt/so/conf/agents/critical-agents.txt
|
||||
- contents: {{ MANAGERMERGED.agent_monitoring.config.critical_agents }}
|
||||
{% else %}
|
||||
remove_critical_agents_config:
|
||||
file.absent:
|
||||
- name: /opt/so/conf/agents/critical-agents.txt
|
||||
{% endif %}
|
||||
|
||||
yara_log_dir:
|
||||
file.directory:
|
||||
- name: /opt/so/log/yarasync
|
||||
@@ -127,6 +147,21 @@ so_fleetagent_status:
|
||||
- month: '*'
|
||||
- dayweek: '*'
|
||||
|
||||
so_fleetagent_monitor:
|
||||
{% if MANAGERMERGED.agent_monitoring.enabled %}
|
||||
cron.present:
|
||||
{% else %}
|
||||
cron.absent:
|
||||
{% endif %}
|
||||
- name: /bin/flock -n /opt/so/log/agents/agent-monitor.lock /usr/sbin/so-elastic-agent-monitor
|
||||
- identifier: so_fleetagent_monitor
|
||||
- user: root
|
||||
- minute: '*/{{ MANAGERMERGED.agent_monitoring.config.run_interval }}'
|
||||
- hour: '*'
|
||||
- daymonth: '*'
|
||||
- month: '*'
|
||||
- dayweek: '*'
|
||||
|
||||
socore_own_saltstack_default:
|
||||
file.directory:
|
||||
- name: /opt/so/saltstack/default
|
||||
|
||||
@@ -37,3 +37,44 @@ manager:
|
||||
forcedType: bool
|
||||
global: True
|
||||
helpLink: proxy.html
|
||||
agent_monitoring:
|
||||
enabled:
|
||||
description: Enable monitoring elastic agents for health issues. Can be used to trigger an alert when a 'critical' agent hasn't checked in with fleet for longer than the configured offline threshold.
|
||||
global: True
|
||||
helpLink: elastic-fleet.html
|
||||
forcedType: bool
|
||||
config:
|
||||
critical_agents:
|
||||
description: List of 'critical' agents to log when they haven't checked in longer than the maximum allowed time. If there are no 'critical' agents specified all offline agents will be logged once they reach the offline threshold.
|
||||
global: True
|
||||
multiline: True
|
||||
helpLink: elastic-fleet.html
|
||||
forcedType: "[]string"
|
||||
custom_kquery:
|
||||
description: For more granular control over what agents to monitor for offline|degraded status add a kquery here. It is recommended to create & test within Elastic Fleet first to ensure your agents are targeted correctly using the query. eg 'status:offline AND tags:INFRA'
|
||||
global: True
|
||||
helpLink: elastic-fleet.html
|
||||
forcedType: string
|
||||
advanced: True
|
||||
offline_threshold:
|
||||
description: The maximum allowed time in hours a 'critical' agent has been offline before being logged.
|
||||
global: True
|
||||
helpLink: elastic-fleet.html
|
||||
forcedType: int
|
||||
realert_threshold:
|
||||
description: The time to pass before another alert for an offline agent exceeding the offline_threshold is generated.
|
||||
global: True
|
||||
helpLink: elastic-fleet.html
|
||||
forcedType: int
|
||||
page_size:
|
||||
description: The amount of agents that can be processed per API request to fleet.
|
||||
global: True
|
||||
helpLink: elastic-fleet.html
|
||||
forcedType: int
|
||||
advanced: True
|
||||
run_interval:
|
||||
description: The time in minutes between checking fleet agent statuses.
|
||||
global: True
|
||||
advanced: True
|
||||
helpLink: elastic-fleet.html
|
||||
forcedType: int
|
||||
|
||||
@@ -454,6 +454,7 @@ function add_sensor_to_minion() {
|
||||
echo "sensor:"
|
||||
echo " interface: '$INTERFACE'"
|
||||
echo " mtu: 9000"
|
||||
echo " channels: 1"
|
||||
echo "zeek:"
|
||||
echo " enabled: True"
|
||||
echo " config:"
|
||||
|
||||
@@ -419,6 +419,7 @@ preupgrade_changes() {
|
||||
[[ "$INSTALLEDVERSION" == 2.4.141 ]] && up_to_2.4.150
|
||||
[[ "$INSTALLEDVERSION" == 2.4.150 ]] && up_to_2.4.160
|
||||
[[ "$INSTALLEDVERSION" == 2.4.160 ]] && up_to_2.4.170
|
||||
[[ "$INSTALLEDVERSION" == 2.4.170 ]] && up_to_2.4.180
|
||||
true
|
||||
}
|
||||
|
||||
@@ -448,6 +449,7 @@ postupgrade_changes() {
|
||||
[[ "$POSTVERSION" == 2.4.141 ]] && post_to_2.4.150
|
||||
[[ "$POSTVERSION" == 2.4.150 ]] && post_to_2.4.160
|
||||
[[ "$POSTVERSION" == 2.4.160 ]] && post_to_2.4.170
|
||||
[[ "$POSTVERSION" == 2.4.170 ]] && post_to_2.4.180
|
||||
true
|
||||
}
|
||||
|
||||
@@ -588,9 +590,6 @@ post_to_2.4.160() {
|
||||
}
|
||||
|
||||
post_to_2.4.170() {
|
||||
echo "Regenerating Elastic Agent Installers"
|
||||
/sbin/so-elastic-agent-gen-installers
|
||||
|
||||
# Update kibana default space
|
||||
salt-call state.apply kibana.config queue=True
|
||||
echo "Updating Kibana default space"
|
||||
@@ -599,6 +598,16 @@ post_to_2.4.170() {
|
||||
POSTVERSION=2.4.170
|
||||
}
|
||||
|
||||
post_to_2.4.180() {
|
||||
echo "Regenerating Elastic Agent Installers"
|
||||
/sbin/so-elastic-agent-gen-installers
|
||||
|
||||
# Force update to Kafka output policy
|
||||
/usr/sbin/so-kafka-fleet-output-policy --force
|
||||
|
||||
POSTVERSION=2.4.180
|
||||
}
|
||||
|
||||
repo_sync() {
|
||||
echo "Sync the local repo."
|
||||
su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
|
||||
@@ -850,10 +859,15 @@ up_to_2.4.170() {
|
||||
touch /opt/so/saltstack/local/pillar/$state/adv_$state.sls /opt/so/saltstack/local/pillar/$state/soc_$state.sls
|
||||
done
|
||||
|
||||
|
||||
INSTALLEDVERSION=2.4.170
|
||||
}
|
||||
|
||||
up_to_2.4.180() {
|
||||
# Elastic Update for this release, so download Elastic Agent files
|
||||
determine_elastic_agent_upgrade
|
||||
|
||||
INSTALLEDVERSION=2.4.170
|
||||
INSTALLEDVERSION=2.4.180
|
||||
}
|
||||
|
||||
add_hydra_pillars() {
|
||||
|
||||
254
salt/manager/tools/sbin_jinja/so-elastic-agent-monitor
Normal file
254
salt/manager/tools/sbin_jinja/so-elastic-agent-monitor
Normal file
@@ -0,0 +1,254 @@
|
||||
{%- from 'manager/map.jinja' import MANAGERMERGED -%}
|
||||
{%- set OFFLINE_THRESHOLD_HOURS = MANAGERMERGED.agent_monitoring.config.offline_threshold -%}
|
||||
{%- set PAGE_SIZE = MANAGERMERGED.agent_monitoring.config.page_size -%}
|
||||
{%- set CUSTOM_KQUERY = MANAGERMERGED.agent_monitoring.config.custom_kquery -%}
|
||||
{%- set REALERT_THRESHOLD = MANAGERMERGED.agent_monitoring.config.realert_threshold -%}
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LOG_DIR="/opt/so/log/agents"
|
||||
LOG_FILE="$LOG_DIR/agent-monitor.log"
|
||||
CURL_CONFIG="/opt/so/conf/elasticsearch/curl.config"
|
||||
FLEET_API="http://localhost:5601/api/fleet/agents"
|
||||
{#- When using custom kquery ignore critical agents patterns. Since we want all the results of custom query logged #}
|
||||
{%- if CUSTOM_KQUERY != None and CUSTOM_KQUERY | length > 0 %}
|
||||
CRITICAL_AGENTS_FILE="/dev/null"
|
||||
{%- else %}
|
||||
CRITICAL_AGENTS_FILE="/opt/so/conf/agents/critical-agents.txt"
|
||||
{%- endif %}
|
||||
OFFLINE_THRESHOLD_HOURS={{ OFFLINE_THRESHOLD_HOURS }}
|
||||
REALERT_THRESHOLD={{ REALERT_THRESHOLD }}
|
||||
PAGE_SIZE="{{ PAGE_SIZE }}"
|
||||
|
||||
log_message() {
|
||||
local level="$1"
|
||||
local message="$2"
|
||||
echo "$(date -u +"%Y-%m-%dT%H:%M:%SZ") [$level] $message" >&2
|
||||
}
|
||||
|
||||
matches_critical_pattern() {
|
||||
local hostname="$1"
|
||||
local pattern_file="$2"
|
||||
|
||||
# If critical agents file doesn't exist or is empty, match all
|
||||
if [ ! -f "$pattern_file" ] || [ ! -s "$pattern_file" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local hostname_lower=$(echo "$hostname" | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
while IFS= read -r pattern || [ -n "$pattern" ]; do
|
||||
# empty lines and comments
|
||||
[[ -z "$pattern" || "$pattern" =~ ^[[:space:]]*# ]] && continue
|
||||
|
||||
# cut whitespace
|
||||
pattern=$(echo "$pattern" | xargs)
|
||||
|
||||
local pattern_lower=$(echo "$pattern" | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
# Replace * with bash wildcard
|
||||
local bash_pattern="${pattern_lower//\*/.*}"
|
||||
|
||||
# Check if hostname matches the pattern
|
||||
if [[ "$hostname_lower" =~ ^${bash_pattern}$ ]]; then
|
||||
return 0
|
||||
fi
|
||||
done < "$pattern_file"
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
calculate_offline_hours() {
|
||||
local last_checkin="$1"
|
||||
local current_time=$(date +%s)
|
||||
local checkin_time=$(date -d "$last_checkin" +%s 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$checkin_time" -eq "0" ]; then
|
||||
echo "0"
|
||||
return
|
||||
fi
|
||||
|
||||
local diff=$((current_time - checkin_time))
|
||||
echo $((diff / 3600))
|
||||
}
|
||||
|
||||
check_recent_log_entries() {
|
||||
local agent_hostname="$1"
|
||||
|
||||
if [ ! -f "$LOG_FILE" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local current_time=$(date +%s)
|
||||
local threshold_seconds=$((REALERT_THRESHOLD * 3600))
|
||||
local agent_hostname_lower=$(echo "$agent_hostname" | tr '[:upper:]' '[:lower:]')
|
||||
local most_recent_timestamp=""
|
||||
|
||||
while IFS= read -r line; do
|
||||
[ -z "$line" ] && continue
|
||||
|
||||
local logged_hostname=$(echo "$line" | jq -r '.["agent.hostname"] // empty' 2>/dev/null)
|
||||
local logged_timestamp=$(echo "$line" | jq -r '.["@timestamp"] // empty' 2>/dev/null)
|
||||
|
||||
[ -z "$logged_hostname" ] || [ -z "$logged_timestamp" ] && continue
|
||||
|
||||
local logged_hostname_lower=$(echo "$logged_hostname" | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
if [ "$logged_hostname_lower" = "$agent_hostname_lower" ]; then
|
||||
most_recent_timestamp="$logged_timestamp"
|
||||
fi
|
||||
done < <(tail -n 1000 "$LOG_FILE" 2>/dev/null)
|
||||
|
||||
# If there is agent entry (within last 1000), check the time difference
|
||||
if [ -n "$most_recent_timestamp" ]; then
|
||||
local logged_time=$(date -d "$most_recent_timestamp" +%s 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$logged_time" -ne "0" ]; then
|
||||
local time_diff=$((current_time - logged_time))
|
||||
local hours_diff=$((time_diff / 3600))
|
||||
|
||||
# Skip if last agent timestamp was more recent than realert threshold
|
||||
if ((hours_diff < REALERT_THRESHOLD)); then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Agent has not been logged within realert threshold
|
||||
return 1
|
||||
}
|
||||
|
||||
main() {
|
||||
log_message "INFO" "Starting Fleet agent status check"
|
||||
|
||||
# Check if critical agents file is configured
|
||||
if [ -f "$CRITICAL_AGENTS_FILE" ] && [ -s "$CRITICAL_AGENTS_FILE" ]; then
|
||||
log_message "INFO" "Using critical agents filter from: $CRITICAL_AGENTS_FILE"
|
||||
log_message "INFO" "Patterns: $(grep -v '^#' "$CRITICAL_AGENTS_FILE" 2>/dev/null | xargs | tr ' ' ',')"
|
||||
else
|
||||
log_message "INFO" "No critical agents filter found, monitoring all agents"
|
||||
fi
|
||||
|
||||
log_message "INFO" "Querying Fleet API"
|
||||
|
||||
local page=1
|
||||
local total_agents=0
|
||||
local processed_agents=0
|
||||
local current_timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
{%- if CUSTOM_KQUERY != None and CUSTOM_KQUERY | length > 0 %}
|
||||
log_message "INFO" "Using custom kquery: {{ CUSTOM_KQUERY }}"
|
||||
FLEET_QUERY="${FLEET_API}?kuery={{ CUSTOM_KQUERY | urlencode }}&perPage=${PAGE_SIZE}&page=${page}"
|
||||
{%- else %}
|
||||
log_message "INFO" "Using default query (all offline or degraded agents)"
|
||||
FLEET_QUERY="${FLEET_API}?kuery=status%3Aoffline%20OR%20status%3Adegraded&perPage=${PAGE_SIZE}&page=${page}"
|
||||
{%- endif %}
|
||||
|
||||
while true; do
|
||||
log_message "INFO" "Fetching page $page (${PAGE_SIZE} agents per page)"
|
||||
|
||||
if ! response_body=$(curl -K "$CURL_CONFIG" \
|
||||
-s --fail \
|
||||
"$FLEET_QUERY" \
|
||||
-H 'kbn-xsrf: true' 2>/dev/null); then
|
||||
log_message "ERROR" "Failed to query Fleet API (page $page)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# pagination info
|
||||
current_total=$(echo "$response_body" | jq -r '.total // 0')
|
||||
current_page=$(echo "$response_body" | jq -r '.page // 1')
|
||||
agents_in_page=$(echo "$response_body" | jq -r '.list | length')
|
||||
|
||||
# Update total
|
||||
if [ "$page" -eq 1 ]; then
|
||||
total_agents="$current_total"
|
||||
log_message "INFO" "Found $total_agents total agents across all pages"
|
||||
fi
|
||||
|
||||
log_message "INFO" "Processing page $current_page with $agents_in_page agents"
|
||||
|
||||
# Process agents from current page
|
||||
mapfile -t agents < <(echo "$response_body" | jq -c '.list[]')
|
||||
|
||||
for agent in "${agents[@]}"; do
|
||||
# Grab agent details
|
||||
agent_id=$(echo "$agent" | jq -r '.id // "unknown"')
|
||||
agent_hostname=$(echo "$agent" | jq -r '.local_metadata.host.hostname // "unknown"')
|
||||
agent_name=$(echo "$agent" | jq -r '.local_metadata.host.name // "unknown"')
|
||||
agent_status=$(echo "$agent" | jq -r '.status // "unknown"')
|
||||
last_checkin=$(echo "$agent" | jq -r '.last_checkin // ""')
|
||||
last_checkin_status=$(echo "$agent" | jq -r '.last_checkin_status // "unknown"')
|
||||
policy_id=$(echo "$agent" | jq -r '.policy_id // "unknown"')
|
||||
|
||||
# Only log agents that are offline or degraded (skip inactive agents)
|
||||
# Fleetserver agents can show multiple versions as 'inactive'
|
||||
if [ "$agent_status" = "offline" ] || [ "$agent_status" = "degraded" ]; then
|
||||
# Check if agent matches critical agent patterns (if configured)
|
||||
if ! matches_critical_pattern "$agent_hostname" "$CRITICAL_AGENTS_FILE"; then
|
||||
log_message "WARN" "${agent_hostname^^} is ${agent_status^^}, but does not match configured critical agents patterns. Not logging ${agent_status^^} agent"
|
||||
continue # Skip this agent if it doesn't match any critical agent pattern
|
||||
fi
|
||||
|
||||
offline_hours=$(calculate_offline_hours "$last_checkin")
|
||||
|
||||
if [ "$offline_hours" -lt "$OFFLINE_THRESHOLD_HOURS" ]; then
|
||||
log_message "INFO" "${agent_hostname^^} has been offline for ${offline_hours}h (threshold: ${OFFLINE_THRESHOLD_HOURS}h). Not logging ${agent_status^^} agent until it reaches threshold"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check if this agent was already logged within the realert_threshold
|
||||
if check_recent_log_entries "$agent_hostname"; then
|
||||
log_message "INFO" "Skipping $agent_hostname (status: $agent_status) - already logged within last ${REALERT_THRESHOLD}h"
|
||||
continue
|
||||
fi
|
||||
|
||||
log_entry=$(echo 'null' | jq -c \
|
||||
--arg ts "$current_timestamp" \
|
||||
--arg id "$agent_id" \
|
||||
--arg hostname "$agent_hostname" \
|
||||
--arg name "$agent_name" \
|
||||
--arg status "$agent_status" \
|
||||
--arg last_checkin "$last_checkin" \
|
||||
--arg last_checkin_status "$last_checkin_status" \
|
||||
--arg policy_id "$policy_id" \
|
||||
--arg offline_hours "$offline_hours" \
|
||||
'{
|
||||
"@timestamp": $ts,
|
||||
"agent.id": $id,
|
||||
"agent.hostname": $hostname,
|
||||
"agent.name": $name,
|
||||
"agent.status": $status,
|
||||
"agent.last_checkin": $last_checkin,
|
||||
"agent.last_checkin_status": $last_checkin_status,
|
||||
"agent.policy_id": $policy_id,
|
||||
"agent.offline_duration_hours": ($offline_hours | tonumber)
|
||||
}')
|
||||
|
||||
echo "$log_entry" >> "$LOG_FILE"
|
||||
|
||||
log_message "INFO" "Logged offline agent: $agent_hostname (status: $agent_status, offline: ${offline_hours}h)"
|
||||
fi
|
||||
done
|
||||
|
||||
processed_agents=$((processed_agents + agents_in_page))
|
||||
|
||||
if [ "$agents_in_page" -eq 0 ] || [ "$processed_agents" -ge "$total_agents" ]; then
|
||||
log_message "INFO" "Completed processing all pages. Total processed: $processed_agents agents"
|
||||
break
|
||||
fi
|
||||
|
||||
page=$((page + 1))
|
||||
|
||||
# Limit pagination loops incase of any issues. If agent count is high enough increase page_size in SOC manager.agent_monitoring.config.page_size
|
||||
if [ "$page" -gt 100 ]; then
|
||||
log_message "ERROR" "Reached maximum page limit (100). Issue with script or extremely large fleet deployment. Consider increasing page_size in SOC -> manager.agent_monitoring.config.page_size"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
log_message "INFO" "Fleet agent status check completed. Processed $processed_agents out of $total_agents agents"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -15,6 +15,7 @@ require_manager
|
||||
echo
|
||||
echo "This script will remove the current Elastic Fleet install and all of its data and then rerun Elastic Fleet setup."
|
||||
echo "Deployed Elastic Agents will no longer be enrolled and will need to be reinstalled."
|
||||
echo "Only the Elastic Fleet instance on the Manager will be reinstalled - dedicated Fleet node config will removed and will need to be reinstalled."
|
||||
echo "This script should only be used as a last resort to reinstall Elastic Fleet."
|
||||
echo
|
||||
echo "If you would like to proceed, then type AGREE and press ENTER."
|
||||
|
||||
@@ -196,19 +196,23 @@ http {
|
||||
}
|
||||
|
||||
location / {
|
||||
auth_request /auth/sessions/whoami;
|
||||
auth_request_set $userid $upstream_http_x_kratos_authenticated_identity_id;
|
||||
proxy_set_header x-user-id $userid;
|
||||
proxy_pass http://{{ GLOBALS.manager }}:9822/;
|
||||
proxy_read_timeout 300;
|
||||
proxy_connect_timeout 300;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Proxy "";
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "Upgrade";
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
auth_request /auth/sessions/whoami;
|
||||
auth_request_set $userid $upstream_http_x_kratos_authenticated_identity_id;
|
||||
proxy_set_header x-user-id $userid;
|
||||
proxy_pass http://{{ GLOBALS.manager }}:9822/;
|
||||
proxy_read_timeout 300;
|
||||
proxy_connect_timeout 300;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Proxy "";
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "Upgrade";
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
proxy_request_buffering off;
|
||||
}
|
||||
|
||||
location ~ ^/auth/.*?(login|oidc/callback) {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
|
||||
# NTP server list
|
||||
{%- for SERVER in NTPCONFIG.servers %}
|
||||
server {{ SERVER }} iburst
|
||||
server {{ SERVER }} iburst maxpoll 10
|
||||
{%- endfor %}
|
||||
|
||||
# Config options
|
||||
@@ -9,3 +9,5 @@ driftfile /var/lib/chrony/drift
|
||||
makestep 1.0 3
|
||||
rtcsync
|
||||
logdir /var/log/chrony
|
||||
port 0
|
||||
cmdport 0
|
||||
@@ -18,11 +18,19 @@ include:
|
||||
|
||||
# This directory needs to exist regardless of whether STENO is enabled or not, in order for
|
||||
# Sensoroni to be able to look at old steno PCAP data
|
||||
|
||||
# if stenographer has never run as the pcap engine no 941 user is created, so we use socore as a placeholder.
|
||||
# /nsm/pcap is empty until stenographer is used as pcap engine
|
||||
{% set pcap_id = 941 %}
|
||||
{% set user_list = salt['user.list_users']() %}
|
||||
{% if GLOBALS.pcap_engine == "SURICATA" and 'stenographer' not in user_list %}
|
||||
{% set pcap_id = 939 %}
|
||||
{% endif %}
|
||||
pcapdir:
|
||||
file.directory:
|
||||
- name: /nsm/pcap
|
||||
- user: 941
|
||||
- group: 941
|
||||
- user: {{ pcap_id }}
|
||||
- group: {{ pcap_id }}
|
||||
- makedirs: True
|
||||
|
||||
pcapoutdir:
|
||||
|
||||
@@ -26,9 +26,9 @@
|
||||
'rocky-devel.repo',
|
||||
'rocky-extras.repo',
|
||||
'rocky.repo',
|
||||
'oracle-linux-ol9',
|
||||
'uek-ol9',
|
||||
'virt-oll9'
|
||||
'oracle-linux-ol9.repo',
|
||||
'uek-ol9.repo',
|
||||
'virt-ol9.repo'
|
||||
]
|
||||
%}
|
||||
{% else %}
|
||||
|
||||
@@ -95,7 +95,7 @@ enable_startup_states:
|
||||
- unless: pgrep so-setup
|
||||
|
||||
# prior to 2.4.30 this managed file would restart the salt-minion service when updated
|
||||
# since this file is currently only adding a sleep timer on service start
|
||||
# since this file is currently only adding a delay service start
|
||||
# it is not required to restart the service
|
||||
salt_minion_service_unit_file:
|
||||
file.managed:
|
||||
|
||||
4
salt/sensor/defaults.yaml
Normal file
4
salt/sensor/defaults.yaml
Normal file
@@ -0,0 +1,4 @@
|
||||
sensor:
|
||||
interface: bond0
|
||||
mtu: 9000
|
||||
channels: 1
|
||||
@@ -9,6 +9,8 @@
|
||||
# in the software, and you may not remove or obscure any functionality in the
|
||||
# software that is protected by the license key."
|
||||
|
||||
{% from 'sensor/map.jinja' import SENSORMERGED %}
|
||||
|
||||
{% if 'vrt' in salt['pillar.get']('features') and salt['grains.get']('salt-cloud', {}) %}
|
||||
|
||||
include:
|
||||
@@ -28,3 +30,18 @@ execute_checksum:
|
||||
- name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable
|
||||
- onchanges:
|
||||
- file: offload_script
|
||||
|
||||
combine_bond_script:
|
||||
file.managed:
|
||||
- name: /usr/sbin/so-combine-bond
|
||||
- source: salt://sensor/tools/sbin_jinja/so-combine-bond
|
||||
- mode: 755
|
||||
- template: jinja
|
||||
- defaults:
|
||||
CHANNELS: {{ SENSORMERGED.channels }}
|
||||
|
||||
execute_combine_bond:
|
||||
cmd.run:
|
||||
- name: /usr/sbin/so-combine-bond
|
||||
- onlyif:
|
||||
- ip link show bond0
|
||||
|
||||
7
salt/sensor/map.jinja
Normal file
7
salt/sensor/map.jinja
Normal file
@@ -0,0 +1,7 @@
|
||||
{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
Elastic License 2.0. #}
|
||||
|
||||
{% import_yaml 'sensor/defaults.yaml' as SENSORDEFAULTS %}
|
||||
{% set SENSORMERGED = salt['pillar.get']('sensor', SENSORDEFAULTS.sensor, merge=True) %}
|
||||
@@ -7,3 +7,9 @@ sensor:
|
||||
description: Maximum Transmission Unit (MTU) of the sensor monitoring interface.
|
||||
helpLink: network.html
|
||||
readonly: True
|
||||
channels:
|
||||
description: Set the size of the nic channels. This is rarely changed from 1
|
||||
helpLink: network.html
|
||||
forcedType: int
|
||||
node: True
|
||||
advanced: True
|
||||
|
||||
70
salt/sensor/tools/sbin_jinja/so-combine-bond
Normal file
70
salt/sensor/tools/sbin_jinja/so-combine-bond
Normal file
@@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script to find all interfaces of bond0 and set channel parameters
|
||||
# Compatible with Oracle Linux 9, Ubuntu, and Debian
|
||||
|
||||
. /usr/sbin/so-common
|
||||
|
||||
# Number of channels to set
|
||||
CHANNELS={{ CHANNELS }}
|
||||
|
||||
# Exit on any error
|
||||
set -e
|
||||
|
||||
# Check if running as root
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if bond0 exists
|
||||
if ! ip link show bond0 &>/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Function to get slave interfaces - works across distributions
|
||||
get_bond_slaves() {
|
||||
local bond_name="$1"
|
||||
local slaves=""
|
||||
|
||||
# Method 1: Try /sys/class/net first (most reliable)
|
||||
if [ -f "/sys/class/net/$bond_name/bonding/slaves" ]; then
|
||||
slaves=$(cat "/sys/class/net/$bond_name/bonding/slaves" 2>/dev/null)
|
||||
fi
|
||||
|
||||
# Method 2: Try /proc/net/bonding (older systems)
|
||||
if [ -z "$slaves" ] && [ -f "/proc/net/bonding/$bond_name" ]; then
|
||||
slaves=$(grep "Slave Interface:" "/proc/net/bonding/$bond_name" 2>/dev/null | awk '{print $3}' | tr '\n' ' ')
|
||||
fi
|
||||
|
||||
# Method 3: Parse ip link output (universal fallback)
|
||||
if [ -z "$slaves" ]; then
|
||||
slaves=$(ip -o link show | grep "master $bond_name" | awk -F': ' '{print $2}' | cut -d'@' -f1 | tr '\n' ' ')
|
||||
fi
|
||||
|
||||
echo "$slaves"
|
||||
}
|
||||
|
||||
# Get slave interfaces
|
||||
SLAVES=$(get_bond_slaves bond0)
|
||||
|
||||
if [ -z "$SLAVES" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Process each slave interface
|
||||
for interface in $SLAVES; do
|
||||
# Skip if interface doesn't exist
|
||||
if ! ip link show "$interface" &>/dev/null; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Try combined mode first
|
||||
if ethtool -L "$interface" combined $CHANNELS &>/dev/null; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Fall back to separate rx/tx
|
||||
ethtool -L "$interface" rx $CHANNELS tx $CHANNELS &>/dev/null || true
|
||||
done
|
||||
|
||||
exit 0
|
||||
@@ -18,6 +18,7 @@ sensoroniagentconf:
|
||||
- group: 939
|
||||
- mode: 600
|
||||
- template: jinja
|
||||
- show_changes: False
|
||||
|
||||
analyzersdir:
|
||||
file.directory:
|
||||
@@ -43,6 +44,22 @@ analyzerscripts:
|
||||
- source: salt://sensoroni/files/analyzers
|
||||
- show_changes: False
|
||||
|
||||
templatesdir:
|
||||
file.directory:
|
||||
- name: /opt/so/conf/sensoroni/templates
|
||||
- user: 939
|
||||
- group: 939
|
||||
- makedirs: True
|
||||
|
||||
sensoronitemplates:
|
||||
file.recurse:
|
||||
- name: /opt/so/conf/sensoroni/templates
|
||||
- source: salt://sensoroni/files/templates
|
||||
- user: 939
|
||||
- group: 939
|
||||
- file_mode: 664
|
||||
- show_changes: False
|
||||
|
||||
sensoroni_sbin:
|
||||
file.recurse:
|
||||
- name: /usr/sbin
|
||||
|
||||
@@ -34,6 +34,8 @@ sensoroni:
|
||||
api_version: community
|
||||
localfile:
|
||||
file_path: []
|
||||
malwarebazaar:
|
||||
api_key:
|
||||
otx:
|
||||
base_url: https://otx.alienvault.com/api/v1/
|
||||
api_key:
|
||||
@@ -49,12 +51,16 @@ sensoroni:
|
||||
live_flow: False
|
||||
mailbox_email_address:
|
||||
message_source_id:
|
||||
threatfox:
|
||||
api_key:
|
||||
urlscan:
|
||||
base_url: https://urlscan.io/api/v1/
|
||||
api_key:
|
||||
enabled: False
|
||||
visibility: public
|
||||
timeout: 180
|
||||
urlhaus:
|
||||
api_key:
|
||||
virustotal:
|
||||
base_url: https://www.virustotal.com/api/v3/search?query=
|
||||
api_key:
|
||||
|
||||
@@ -22,6 +22,7 @@ so-sensoroni:
|
||||
- /nsm/pcapout:/nsm/pcapout:rw
|
||||
- /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro
|
||||
- /opt/so/conf/sensoroni/analyzers:/opt/sensoroni/analyzers:rw
|
||||
- /opt/so/conf/sensoroni/templates:/opt/sensoroni/templates:ro
|
||||
- /opt/so/log/sensoroni:/opt/sensoroni/logs:rw
|
||||
- /nsm/suripcap/:/nsm/suripcap:rw
|
||||
{% if DOCKER.containers['so-sensoroni'].custom_bind_mounts %}
|
||||
|
||||
@@ -35,15 +35,15 @@ Many analyzers require authentication, via an API key or similar. The table belo
|
||||
[EchoTrail](https://www.echotrail.io/docs/quickstart) |✓|
|
||||
[EmailRep](https://emailrep.io/key) |✓|
|
||||
[Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/setting-up-authentication.html) |✓|
|
||||
[GreyNoise](https://www.greynoise.io/plans/community) |✓|
|
||||
[GreyNoise (community)](https://www.greynoise.io/plans/community) |✗|
|
||||
[LocalFile](https://github.com/Security-Onion-Solutions/securityonion/tree/fix/sublime_analyzer_documentation/salt/sensoroni/files/analyzers/localfile) |✗|
|
||||
[Malware Hash Registry](https://hash.cymru.com/docs_whois) |✗|
|
||||
[MalwareBazaar](https://bazaar.abuse.ch/) |✗|
|
||||
[MalwareBazaar](https://bazaar.abuse.ch/) |✓|
|
||||
[Pulsedive](https://pulsedive.com/api/) |✓|
|
||||
[Spamhaus](https://www.spamhaus.org/dbl/) |✗|
|
||||
[Sublime Platform](https://sublime.security) |✓|
|
||||
[ThreatFox](https://threatfox.abuse.ch/) |✗|
|
||||
[Urlhaus](https://urlhaus.abuse.ch/) |✗|
|
||||
[ThreatFox](https://threatfox.abuse.ch/) |✓|
|
||||
[Urlhaus](https://urlhaus.abuse.ch/) |✓|
|
||||
[Urlscan](https://urlscan.io/docs/api/) |✓|
|
||||
[VirusTotal](https://developers.virustotal.com/reference/overview) |✓|
|
||||
[WhoisLookup](https://github.com/meeb/whoisit) |✗|
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
# EchoTrail
|
||||
|
||||
|
||||
## Description
|
||||
Submit a filename, hash, commandline to EchoTrail for analysis
|
||||
|
||||
## Configuration Requirements
|
||||
|
||||
In SOC, navigate to `Administration`, toggle `Show all configurable settings, including advanced settings.`, and navigate to `sensoroni` -> `analyzers` -> `echotrail`.
|
||||
|
||||
|
||||
|
||||
The following configuration options are available for:
|
||||
|
||||
``api_key`` - API key used for communication with the Echotrail API (Required)
|
||||
|
||||
This value should be set in the ``sensoroni`` pillar, like so:
|
||||
|
||||
```
|
||||
sensoroni:
|
||||
analyzers:
|
||||
echotrail:
|
||||
api_key: $yourapikey
|
||||
```
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"name": "Echotrail",
|
||||
"version": "0.1",
|
||||
"author": "Security Onion Solutions",
|
||||
"description": "This analyzer queries Echotrail to see if a related filename, hash, or commandline is considered malicious.",
|
||||
"supportedTypes" : ["filename","hash","commandline"],
|
||||
"baseUrl": "https://api.echotrail.io/insights/"
|
||||
}
|
||||
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import helpers
|
||||
import argparse
|
||||
|
||||
|
||||
# for test usage:
|
||||
# python3 echotrail.py '{"artifactType":"hash", "value":"438b6ccd84f4dd32d9684ed7d58fd7d1e5a75fe3f3d12ab6c788e6bb0ffad5e7"}'
|
||||
# You will need to provide an API key in the .yaml file.
|
||||
def checkConfigRequirements(conf):
|
||||
if not conf['api_key']:
|
||||
sys.exit(126)
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def sendReq(conf, observ_value):
|
||||
# send a get requests using a user-provided API key and the API url
|
||||
url = conf['base_url'] + observ_value
|
||||
headers = {'x-api-key': conf['api_key']}
|
||||
response = requests.request('GET', url=url, headers=headers)
|
||||
return response.json()
|
||||
|
||||
|
||||
def prepareResults(raw):
|
||||
# checking for the 'filenames' key alone does
|
||||
# not work when querying by filename.
|
||||
# So, we can account for a hash query, a filename query,
|
||||
# and anything else with these if statements.
|
||||
if 'filenames' in raw.keys():
|
||||
summary = raw['filenames'][0][0]
|
||||
elif 'tags' in raw.keys():
|
||||
summary = raw['tags'][0][0]
|
||||
else:
|
||||
summary = 'inconclusive'
|
||||
status = 'info'
|
||||
return {'response': raw, 'summary': summary, 'status': status}
|
||||
|
||||
|
||||
def analyze(conf, input):
|
||||
# put all of our methods together and return a properly formatted output.
|
||||
checkConfigRequirements(conf)
|
||||
meta = helpers.loadMetadata(__file__)
|
||||
data = helpers.parseArtifact(input)
|
||||
helpers.checkSupportedType(meta, data['artifactType'])
|
||||
response = sendReq(conf, data['value'])
|
||||
return prepareResults(response)
|
||||
|
||||
|
||||
def main():
|
||||
dir = os.path.dirname(os.path.realpath(__file__))
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Search Echotrail for a given artifact')
|
||||
parser.add_argument(
|
||||
'artifact', help='the artifact represented in JSON format')
|
||||
parser.add_argument('-c', '--config', metavar='CONFIG_FILE', default=dir + '/echotrail.yaml',
|
||||
help='optional config file to use instead of the default config file')
|
||||
args = parser.parse_args()
|
||||
if args.artifact:
|
||||
results = analyze(helpers.loadConfig(args.config), args.artifact)
|
||||
print(json.dumps(results))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,3 +0,0 @@
|
||||
base_url: "{{ salt['pillar.get']('sensoroni:analyzers:echotrail:base_url', 'https://api.echotrail.io/insights/') }}"
|
||||
api_key: "{{ salt['pillar.get']('sensoroni:analyzers:echotrail:api_key', '') }}"
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
from io import StringIO
|
||||
import sys
|
||||
from unittest.mock import patch, MagicMock
|
||||
import unittest
|
||||
import echotrail
|
||||
|
||||
|
||||
class TestEchoTrailMethods(unittest.TestCase):
|
||||
def test_main_success(self):
|
||||
with patch('sys.stdout', new=StringIO()) as mock_cmd:
|
||||
with patch('echotrail.analyze', new=MagicMock(return_value={'test': 'val'})) as mock:
|
||||
sys.argv = ["test", "test"]
|
||||
echotrail.main()
|
||||
expected = '{"test": "val"}\n'
|
||||
self.assertEqual(mock_cmd.getvalue(), expected)
|
||||
mock.assert_called_once()
|
||||
|
||||
def test_main_missing_input(self):
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_stderr:
|
||||
sys.argv = ["cmd"]
|
||||
echotrail.main()
|
||||
self.assertEqual(mock_stderr.getvalue(), "usage: cmd [-h] [-c CONFIG_FILE] artifact\ncmd: error: the following arguments are required: artifact\n")
|
||||
sysmock.assert_called_once()
|
||||
|
||||
def test_checkConfigRequirements(self):
|
||||
conf = {'base_url': 'https://www.randurl.xyz/', 'api_key': ''}
|
||||
with self.assertRaises(SystemExit) as cm:
|
||||
echotrail.checkConfigRequirements(conf)
|
||||
self.assertEqual(cm.exception.code, 126)
|
||||
|
||||
def test_sendReq(self):
|
||||
with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock:
|
||||
response = echotrail.sendReq(conf={'base_url': 'https://www.randurl.xyz/', 'api_key': 'randkey'}, observ_value='example_data')
|
||||
self.assertIsNotNone(response)
|
||||
mock.assert_called_once()
|
||||
|
||||
def test_prepareResults_noinput(self):
|
||||
raw = {}
|
||||
sim_results = {'response': raw,
|
||||
'status': 'info', 'summary': 'inconclusive'}
|
||||
results = echotrail.prepareResults(raw)
|
||||
self.assertEqual(results, sim_results)
|
||||
|
||||
def test_prepareResults_none(self):
|
||||
raw = {'query_status': 'no_result'}
|
||||
sim_results = {'response': raw,
|
||||
'status': 'info', 'summary': 'inconclusive'}
|
||||
results = echotrail.prepareResults(raw)
|
||||
self.assertEqual(results, sim_results)
|
||||
|
||||
def test_prepareResults_filenames(self):
|
||||
raw = {'filenames': [["abc.exe", "def.exe"], ["abc.exe", "def.exe"]]}
|
||||
sim_results = {'response': raw,
|
||||
'status': 'info', 'summary': 'abc.exe'}
|
||||
results = echotrail.prepareResults(raw)
|
||||
self.assertEqual(results, sim_results)
|
||||
|
||||
def test_prepareResults_tags(self):
|
||||
raw = {'tags': [["tag1", "tag2"], ["tag1", "tag2"]]}
|
||||
sim_results = {'response': raw,
|
||||
'status': 'info', 'summary': 'tag1'}
|
||||
results = echotrail.prepareResults(raw)
|
||||
self.assertEqual(results, sim_results)
|
||||
|
||||
def test_analyze(self):
|
||||
sendReqOutput = {'threat': 'no_result'}
|
||||
input = '{"artifactType":"hash", "value":"1234"}'
|
||||
prepareResultOutput = {'response': '',
|
||||
'summary': 'inconclusive', 'status': 'info'}
|
||||
conf = {"api_key": "xyz"}
|
||||
|
||||
with patch('echotrail.sendReq', new=MagicMock(return_value=sendReqOutput)) as mock:
|
||||
with patch('echotrail.prepareResults', new=MagicMock(return_value=prepareResultOutput)) as mock2:
|
||||
results = echotrail.analyze(conf, input)
|
||||
self.assertEqual(results["summary"], "inconclusive")
|
||||
mock2.assert_called_once()
|
||||
mock.assert_called_once()
|
||||
@@ -1,2 +0,0 @@
|
||||
requests>=2.31.0
|
||||
pyyaml>=6.0
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "Greynoise IP Analyzer",
|
||||
"version": "0.1",
|
||||
"version": "0.2",
|
||||
"author": "Security Onion Solutions",
|
||||
"description": "This analyzer queries Greynoise for context around an IP address",
|
||||
"supportedTypes" : ["ip"]
|
||||
|
||||
@@ -7,6 +7,10 @@ import argparse
|
||||
|
||||
|
||||
def checkConfigRequirements(conf):
|
||||
# Community API doesn't require API key
|
||||
if conf.get('api_version') == 'community':
|
||||
return True
|
||||
# Other API versions require API key
|
||||
if "api_key" not in conf or len(conf['api_key']) == 0:
|
||||
sys.exit(126)
|
||||
else:
|
||||
@@ -17,10 +21,12 @@ def sendReq(conf, meta, ip):
|
||||
url = conf['base_url']
|
||||
if conf['api_version'] == 'community':
|
||||
url = url + 'v3/community/' + ip
|
||||
elif conf['api_version'] == 'investigate' or 'automate':
|
||||
# Community API doesn't use API key
|
||||
response = requests.request('GET', url=url)
|
||||
elif conf['api_version'] in ['investigate', 'automate']:
|
||||
url = url + 'v2/noise/context/' + ip
|
||||
headers = {"key": conf['api_key']}
|
||||
response = requests.request('GET', url=url, headers=headers)
|
||||
headers = {"key": conf['api_key']}
|
||||
response = requests.request('GET', url=url, headers=headers)
|
||||
return response.json()
|
||||
|
||||
|
||||
|
||||
@@ -31,13 +31,31 @@ class TestGreynoiseMethods(unittest.TestCase):
|
||||
greynoise.checkConfigRequirements(conf)
|
||||
self.assertEqual(cm.exception.code, 126)
|
||||
|
||||
def test_checkConfigRequirements_community_no_key(self):
|
||||
conf = {"api_version": "community"}
|
||||
# Should not raise exception for community version
|
||||
result = greynoise.checkConfigRequirements(conf)
|
||||
self.assertTrue(result)
|
||||
|
||||
def test_checkConfigRequirements_investigate_no_key(self):
|
||||
conf = {"api_version": "investigate"}
|
||||
with self.assertRaises(SystemExit) as cm:
|
||||
greynoise.checkConfigRequirements(conf)
|
||||
self.assertEqual(cm.exception.code, 126)
|
||||
|
||||
def test_checkConfigRequirements_investigate_with_key(self):
|
||||
conf = {"api_version": "investigate", "api_key": "test_key"}
|
||||
result = greynoise.checkConfigRequirements(conf)
|
||||
self.assertTrue(result)
|
||||
|
||||
def test_sendReq_community(self):
|
||||
with patch('requests.request', new=MagicMock(return_value=MagicMock())) as mock:
|
||||
meta = {}
|
||||
conf = {"base_url": "https://myurl/", "api_key": "abcd1234", "api_version": "community"}
|
||||
conf = {"base_url": "https://myurl/", "api_version": "community"}
|
||||
ip = "192.168.1.1"
|
||||
response = greynoise.sendReq(conf=conf, meta=meta, ip=ip)
|
||||
mock.assert_called_once_with("GET", headers={'key': 'abcd1234'}, url="https://myurl/v3/community/192.168.1.1")
|
||||
# Community API should not include headers
|
||||
mock.assert_called_once_with("GET", url="https://myurl/v3/community/192.168.1.1")
|
||||
self.assertIsNotNone(response)
|
||||
|
||||
def test_sendReq_investigate(self):
|
||||
@@ -115,3 +133,16 @@ class TestGreynoiseMethods(unittest.TestCase):
|
||||
results = greynoise.analyze(conf, artifactInput)
|
||||
self.assertEqual(results["summary"], "suspicious")
|
||||
mock.assert_called_once()
|
||||
|
||||
def test_analyze_community_no_key(self):
|
||||
output = {"ip": "8.8.8.8", "noise": "false", "riot": "true",
|
||||
"classification": "benign", "name": "Google Public DNS",
|
||||
"link": "https://viz.gn.io", "last_seen": "2022-04-26",
|
||||
"message": "Success"}
|
||||
artifactInput = '{"value":"8.8.8.8","artifactType":"ip"}'
|
||||
conf = {"base_url": "myurl/", "api_version": "community"}
|
||||
with patch('greynoise.greynoise.sendReq', new=MagicMock(return_value=output)) as mock:
|
||||
results = greynoise.analyze(conf, artifactInput)
|
||||
self.assertEqual(results["summary"], "harmless")
|
||||
self.assertEqual(results["status"], "ok")
|
||||
mock.assert_called_once()
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user