CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-07 09:42:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'mkrmerge' into escluster

Browse Source
This commit is contained in:
Mike Reeves
2020-11-24 10:29:57 -05:00
committed by GitHub
parent 57d8f25422 1e32a01657
commit ddca9563e5
171 changed files with 2613 additions and 3181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
salt/elasticsearch/files/ingest/osquery.query_result
View File

@@ -6,7 +6,7 @@
{ "gsub": { "field": "message2.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } },
{ "rename": { "if": "ctx.message2.columns?.eventid != null", "field": "message2.columns", "target_field": "winlog", "ignore_missing": true } },
{ "json": { "field": "winlog.data", "target_field": "temp", "ignore_failure": true } },
{ "rename": { "field": "temp.Data", "target_field": "winlog.event_data", "ignore_missing": true } },
{ "rename": { "field": "temp.EventData", "target_field": "winlog.event_data", "ignore_missing": true } },
{ "rename": { "field": "winlog.source", "target_field": "winlog.channel", "ignore_missing": true } },
{ "rename": { "field": "winlog.eventid", "target_field": "winlog.event_id", "ignore_missing": true } },
{ "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } },
@@ -22,4 +22,4 @@
{ "set": { "field": "event.dataset", "value": "{{osquery.result.name}}", "override": false} },
{ "pipeline": { "name": "common" } }
]
}
}

33
salt/elasticsearch/files/ingest/strelka.file
View File

@@ -6,15 +6,27 @@
{ "rename": { "field": "message2.scan", "target_field": "scan", "ignore_missing": true } },
{ "rename": { "field": "message2.request", "target_field": "request", "ignore_missing": true } },
{ "rename": { "field": "scan.hash", "target_field": "hash", "ignore_missing": true } },
{ "rename": { "field": "scan.exiftool", "target_field": "exiftool", "ignore_missing": true } },
{ "grok": { "if": "ctx.request?.attributes?.filename != null", "field": "request.attributes.filename", "patterns": ["-%{WORD:log.id.fuid}-"], "ignore_failure": true } },
{ "foreach":
{
"if": "ctx.scan?.exiftool?.keys !=null",
"field": "scan.exiftool.keys",
"processor":{
"if": "ctx.exiftool?.keys !=null",
"field": "exiftool.keys",
"processor": {
"append": {
"field": "scan.exiftool",
"value": "{{_ingest._value.key}}={{_ingest._value.value}}"
}
}
}
},
{ "foreach":
{
"if": "ctx.exiftool?.keys !=null",
"field": "exiftool.keys",
"processor": {
"set": {
"field": "scan.exiftool.{{_ingest._value.key}}",
"field": "exiftool.{{_ingest._value.key}}",
"value": "{{_ingest._value.value}}"
}
}
@@ -32,6 +44,14 @@
}
}
},
{ "set": { "if": "ctx.exiftool?.SourceFile != null", "field": "file.source", "value": "{{exiftool.SourceFile}}", "ignore_failure": true }},
{ "set": { "if": "ctx.exiftool?.FilePermissions != null", "field": "file.permissions", "value": "{{exiftool.FilePermissions}}", "ignore_failure": true }},
{ "set": { "if": "ctx.exiftool?.FileName != null", "field": "file.name", "value": "{{exiftool.FileName}}", "ignore_failure": true }},
{ "set": { "if": "ctx.exiftool?.FileModifyDate != null", "field": "file.mtime", "value": "{{exiftool.FileModifyDate}}", "ignore_failure": true }},
{ "set": { "if": "ctx.exiftool?.FileAccessDate != null", "field": "file.accessed", "value": "{{exiftool.FileAccessDate}}", "ignore_failure": true }},
{ "set": { "if": "ctx.exiftool?.FileInodeChangeDate != null", "field": "file.ctime", "value": "{{exiftool.FileInodeChangeDate}}", "ignore_failure": true }},
{ "set": { "if": "ctx.exiftool?.FileDirectory != null", "field": "file.directory", "value": "{{exiftool.FileDirectory}}", "ignore_failure": true }},
{ "set": { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem", "value": "{{exiftool.Subsystem}}", "ignore_failure": true }},
{ "set": { "if": "ctx.scan?.yara?.matches != null", "field": "rule.name", "value": "{{scan.yara.matches.0}}" }},
{ "set": { "if": "ctx.scan?.yara?.matches != null", "field": "dataset", "value": "alert", "override": true }},
{ "rename": { "field": "file.flavors.mime", "target_field": "file.mime_type", "ignore_missing": true }},
@@ -42,7 +62,8 @@
{ "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 70 && ctx.rule?.score <=89", "field": "event.severity", "value": 3, "override": true } },
{ "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 90", "field": "event.severity", "value": 4, "override": true } },
{ "set": { "field": "observer.name", "value": "{{agent.name}}" }},
{ "remove": { "field": ["host", "path", "message", "scan.exiftool.keys", "scan.yara.meta"], "ignore_missing": true } },
{ "convert" : { "field" : "scan.exiftool","type": "string", "ignore_missing":true }},
{ "remove": { "field": ["host", "path", "message", "exiftool", "scan.yara.meta"], "ignore_missing": true } },
{ "pipeline": { "name": "common" } }
]
}

22
salt/elasticsearch/files/ingest/syslog
View File

@@ -12,9 +12,25 @@
"ignore_failure": true
}
},
{ "grok": { "field": "message", "patterns": ["<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}: %{GREEDYDATA:real_message}"], "ignore_failure": false } },
{ "set": { "if": "ctx.source.application == 'filterlog'", "field": "dataset", "value": "firewall" } },
{ "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog" } },
{
"grok":
{
"field": "message",
"patterns": [
"^<%{INT:syslog.priority}>%{DATA:syslog.timestamp} %{WORD:source.application}: %{GREEDYDATA:real_message}$",
"^%{SYSLOGTIMESTAMP:syslog.timestamp} %{SYSLOGHOST:syslog.host} %{SYSLOGPROG:syslog.program}: CEF:0\\|%{DATA:vendor}\\|%{DATA:product}\\|%{GREEDYDATA:message2}$"
],
"ignore_failure": true
}
},
{ "set": { "if": "ctx.source?.application == 'filterlog'", "field": "dataset", "value": "firewall", "ignore_failure": true } },
{ "set": { "if": "ctx.vendor != null", "field": "module", "value": "{{ vendor }}", "ignore_failure": true } },
{ "set": { "if": "ctx.product != null", "field": "dataset", "value": "{{ product }}", "ignore_failure": true } },
{ "set": { "field": "ingest.timestamp", "value": "{{ @timestamp }}" } },
{ "date": { "if": "ctx.syslog?.timestamp != null", "field": "syslog.timestamp", "target_field": "@timestamp", "formats": ["MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601", "UNIX"], "ignore_failure": true } },
{ "remove": { "field": ["pid", "program"], "ignore_missing": true, "ignore_failure": true } },
{ "pipeline": { "if": "ctx.vendor != null && ctx.product != null", "name": "{{ vendor }}.{{ product }}", "ignore_failure": true } },
{ "pipeline": { "if": "ctx.dataset == 'firewall'", "name": "filterlog", "ignore_failure": true } },
{ "pipeline": { "name": "common" } }
]
}

34
salt/elasticsearch/files/ingest/sysmon
View File

@@ -30,40 +30,40 @@
{ "rename": { "field": "winlog.event_data.DestinationHostname", "target_field": "destination.hostname", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.DestinationIp", "target_field": "destination.ip", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.DestinationPort", "target_field": "destination.port", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.image", "target_field": "process.executable", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.image", "target_field": "process.executable", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.Image", "target_field": "process.executable", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.processID", "target_field": "process.pid", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.ProcessID", "target_field": "process.pid", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.processGuid", "target_field": "process.entity_id", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.processID", "target_field": "process.pid", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.ProcessId", "target_field": "process.pid", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.processGuid", "target_field": "process.entity_id", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.ProcessGuid", "target_field": "process.entity_id", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.commandLine", "target_field": "process.command_line", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.commandLine", "target_field": "process.command_line", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.CommandLine", "target_field": "process.command_line", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.currentDirectory", "target_field": "process.working_directory", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.currentDirectory", "target_field": "process.working_directory", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.CurrentDirectory", "target_field": "process.working_directory", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.description", "target_field": "process.pe.description", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.description", "target_field": "process.pe.description", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.Description", "target_field": "process.pe.description", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.product", "target_field": "process.pe.product", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.product", "target_field": "process.pe.product", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.Product", "target_field": "process.pe.product", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.company", "target_field": "process.pe.company", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.company", "target_field": "process.pe.company", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.Company", "target_field": "process.pe.company", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.originalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.originalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.OriginalFileName", "target_field": "process.pe.original_file_name", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.fileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.fileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.FileVersion", "target_field": "process.pe.file_version", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.parentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.parentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.ParentCommandLine", "target_field": "process.parent.command_line", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.parentImage", "target_field": "process.parent.executable", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.parentImage", "target_field": "process.parent.executable", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.ParentImage", "target_field": "process.parent.executable", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.parentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.parentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.ParentProcessGuid", "target_field": "process.parent.entity_id", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.parentProcessId", "target_field": "process.ppid", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.parentProcessId", "target_field": "process.ppid", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.ParentProcessId", "target_field": "process.ppid", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.Protocol", "target_field": "network.transport", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.Protocol", "target_field": "network.transport", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.User", "target_field": "user.name", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.SourceHostname", "target_field": "source.hostname", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.SourceIp", "target_field": "source.ip", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.SourcePort", "target_field": "source.port", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.targetFilename", "target_field": "file.target", "ignore_missing": true } },
{ "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } }
{ "rename": { "field": "winlog.event_data.TargetFilename", "target_field": "file.target", "ignore_missing": true } }
]
}

8
salt/elasticsearch/files/so-elasticsearch-pipelines
View File

@@ -28,9 +28,9 @@ COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 240 ]]; do
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
{% else %}
curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
{% endif %}
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
@@ -52,9 +52,9 @@ cd ${ELASTICSEARCH_INGEST_PIPELINES}
echo "Loading pipelines..."
{% if grains['role'] in ['so-node','so-heavynode'] %}
for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
{% else %}
for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
{% endif %}
echo

6
salt/elasticsearch/init.sls
View File

@@ -215,13 +215,17 @@ so-elasticsearch:
- /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro
- /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
- /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro
- watch:
- file: cacertz
- file: esyml
- file: esingestconf
- file: so-elasticsearch-pipelines-file
append_so-elasticsearch_so-status.conf:
file.append:
- name: /opt/so/conf/so-status/so-status.conf
- text: so-elasticsearch
so-elasticsearch-pipelines-file:
file.managed:
- name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines

9
salt/elasticsearch/templates/so/so-common-template.json
View File

@@ -379,9 +379,14 @@
}
}
},
"scan":{
"scan":{
"type":"object",
"dynamic": true
"dynamic": true,
"properties":{
"exiftool":{
"type":"text"
}
}
},
"server":{
"type":"object",