CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into foxtrot

Browse Source
This commit is contained in:
William Wernert
2021-03-09 10:11:04 -05:00
parent ba71b2fbc8 734d25b1ac
commit f0e089b6bf
46 changed files with 137 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pillar/logstash/manager.sls
View File

@@ -3,6 +3,7 @@ logstash:
pipelines:
manager:
config:
- so/0008_input_fleet_livequery.conf.jinja
- so/0009_input_beats.conf
- so/0010_input_hhbeats.conf
- so/9999_output_redis.conf.jinja

1
pillar/logstash/search.sls
View File

@@ -8,6 +8,7 @@ logstash:
- so/9002_output_import.conf.jinja
- so/9034_output_syslog.conf.jinja
- so/9100_output_osquery.conf.jinja
- so/9101_output_osquery_livequery.conf.jinja
- so/9400_output_suricata.conf.jinja
- so/9500_output_beats.conf.jinja
- so/9600_output_ossec.conf.jinja

12
salt/common/tools/sbin/so-elastic-clear
View File

@@ -50,11 +50,7 @@ done
if [ $SKIP -ne 1 ]; then
# List indices
echo
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -k -L https://{{ NODEIP }}:9200/_cat/indices?v
{% else %}
curl -L {{ NODEIP }}:9200/_cat/indices?v
{% endif %}
echo
# Inform user we are about to delete all data
echo
@@ -93,18 +89,10 @@ fi
# Delete data
echo "Deleting data..."
{% if grains['role'] in ['so-node','so-heavynode'] %}
INDXS=$(curl -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
{% else %}
INDXS=$(curl -s -XGET -L {{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
{% endif %}
for INDX in ${INDXS}
do
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
{% else %}
curl -XDELETE -L "{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
{% endif %}
done
#Start Logstash/Filebeat

5
salt/common/tools/sbin/so-elasticsearch-indices-rw
View File

@@ -21,6 +21,5 @@ THEHIVEESPORT=9400
echo "Removing read only attributes for indices..."
echo
for p in $ESPORT $THEHIVEESPORT; do
curl -XPUT -H "Content-Type: application/json" -L http://$IP:$p/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi;
done
curl -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi;
curl -XPUT -H "Content-Type: application/json" -L http://$IP:9400/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute. Please ensure Elasticsearch is running.";fi;

8
salt/common/tools/sbin/so-elasticsearch-pipeline-stats
View File

@@ -19,15 +19,7 @@
. /usr/sbin/so-common
if [ "$1" == "" ]; then
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
{% else %}
curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines"
{% endif %}
else
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
{% else %}
curl -s -L {{ NODEIP }}:9200/_nodes/stats | jq .nodes | jq ".[] | .ingest.pipelines.\"$1\""
{% endif %}
fi

8
salt/common/tools/sbin/so-elasticsearch-pipelines-list
View File

@@ -17,15 +17,7 @@
{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
. /usr/sbin/so-common
if [ "$1" == "" ]; then
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
{% else %}
curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
{% endif %}
else
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
{% else %}
curl -s -L {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
{% endif %}
fi

8
salt/common/tools/sbin/so-elasticsearch-templates-list
View File

@@ -17,15 +17,7 @@
{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
. /usr/sbin/so-common
if [ "$1" == "" ]; then
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_template/* | jq 'keys'
{% else %}
curl -s -L {{ NODEIP }}:9200/_template/* | jq 'keys'
{% endif %}
else
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -s -k -L https://{{ NODEIP }}:9200/_template/$1 | jq
{% else %}
curl -s -L {{ NODEIP }}:9200/_template/$1 | jq
{% endif %}
fi

8
salt/common/tools/sbin/so-elasticsearch-templates-load
View File

@@ -30,11 +30,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 240 ]]; do
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
{% else %}
curl --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
{% endif %}
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -55,11 +51,7 @@ cd ${ELASTICSEARCH_TEMPLATES}
echo "Loading templates..."
{% if grains['role'] in ['so-node','so-heavynode'] %}
for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl -k ${ELASTICSEARCH_AUTH} -s -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
{% else %}
for i in *; do TEMPLATE=$(echo $i | cut -d '-' -f2); echo "so-$TEMPLATE"; curl ${ELASTICSEARCH_AUTH} -s -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/so-$TEMPLATE -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done
{% endif %}
echo
cd - >/dev/null

53
salt/common/tools/sbin/so-features-enable
View File

@@ -1,53 +0,0 @@
#!/bin/bash
# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
. /usr/sbin/so-common
. /usr/sbin/so-image-common
local_salt_dir=/opt/so/saltstack/local
cat << EOF
This program will switch from the open source version of the Elastic Stack to the Features version licensed under the Elastic license.
If you proceed, then we will download new Docker images and restart services.
Please review the Elastic license:
https://raw.githubusercontent.com/elastic/elasticsearch/master/licenses/ELASTIC-LICENSE.txt
Please also note that, if you have a distributed deployment and continue with this change, Elastic traffic between nodes will change from encrypted to cleartext!
(We expect to support Elastic Features Security at some point in the future.)
Do you agree to the terms of the Elastic license and understand the note about encryption?
If so, type AGREE to accept the Elastic license and continue. Otherwise, just press Enter to exit this program without making any changes.
EOF
read INPUT
if [ "$INPUT" != "AGREE" ]; then
exit
fi
echo "Please wait while switching to Elastic Features."
require_manager
TRUSTED_CONTAINERS=( \
"so-elasticsearch" \
"so-filebeat" \
"so-kibana" \
"so-logstash" )
update_docker_containers "features" "-features"
# Modify global.sls to enable Features
sed -i 's/features: False/features: True/' $local_salt_dir/pillar/global.sls

4
salt/common/tools/sbin/so-index-list
View File

@@ -15,8 +15,4 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl -X GET -k -L https://localhost:9200/_cat/indices?v
{% else %}
curl -X GET -L localhost:9200/_cat/indices?v
{% endif %}

10
salt/common/tools/sbin/soup
View File

@@ -612,16 +612,6 @@ if [ $is_airgap -eq 0 ]; then
else
update_registry
update_docker_containers "soup"
FEATURESCHECK=$(lookup_pillar features elastic)
if [[ "$FEATURESCHECK" == "True" ]]; then
TRUSTED_CONTAINERS=(
"so-elasticsearch"
"so-filebeat"
"so-kibana"
"so-logstash"
)
update_docker_containers "features" "-features"
fi
fi
echo ""
echo "Stopping Salt Minion service."

4
salt/curator/files/curator.yml
View File

@@ -12,11 +12,11 @@ client:
- {{elasticsearch}}
port: 9200
url_prefix:
{% if grains['role'] in ['so-node', 'so-heavynode'] %} use_ssl: True{% else %} use_ssl: False{% endif %}
use_ssl: True
certificate:
client_cert:
client_key:
{% if grains['role'] in ['so-node', 'so-heavynode'] %} ssl_no_validate: True{% else %} ssl_no_validate: False{% endif %}
ssl_no_validate: True
http_auth:
timeout: 30
master_only: False

4
salt/elastalert/defaults.yaml
View File

@@ -16,8 +16,8 @@ elastalert:
#aws_region: us-east-1
#profile: test
#es_url_prefix: elasticsearch
#use_ssl: True
#verify_certs: True
use_ssl: true
verify_certs: false
#es_send_get_body_as: GET
#es_username: someusername
#es_password: somepassword

3
salt/elastalert/init.sls
View File

@@ -104,8 +104,9 @@ elastaconf:
wait_for_elasticsearch:
module.run:
- http.wait_for_successful_query:
- url: 'http://{{MANAGER}}:9200/_cat/indices/.kibana*'
- url: 'https://{{MANAGER}}:9200/_cat/indices/.kibana*'
- wait_for: 180
- verify_ssl: False
so-elastalert:
docker_container.running:

12
salt/elasticsearch/files/elasticsearch.yml
View File

@@ -9,12 +9,6 @@
{%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %}
cluster.name: "{{ ESCLUSTERNAME }}"
network.host: 0.0.0.0
# minimum_master_nodes need to be explicitly set when bound on a public IP
# set to 1 to allow single node clusters
# Details: https://github.com/elastic/elasticsearch/pull/17288
#discovery.zen.minimum_master_nodes: 1
# This is a test -- if this is here, then the volume is mounted correctly.
path.logs: /var/log/elasticsearch
action.destructive_requires_name: true
transport.bind_host: 0.0.0.0
@@ -25,17 +19,12 @@ cluster.routing.allocation.disk.watermark.low: 95%
cluster.routing.allocation.disk.watermark.high: 98%
cluster.routing.allocation.disk.watermark.flood_stage: 98%
xpack.ml.enabled: false
{%- if grains['role'] in ['so-node','so-heavynode'] %}
xpack.security.enabled: true
{%- else %}
xpack.security.enabled: false
{%- endif %}
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: none
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
{%- if grains['role'] in ['so-node','so-heavynode'] %}
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.client_authentication: none
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
@@ -46,7 +35,6 @@ xpack.security.authc:
username: anonymous_user
roles: superuser
authz_exception: true
{%- endif %}
node.name: {{ grains.host }}
script.max_compilations_rate: 1000/1m
{%- if TRUECLUSTER is sameas true %}

16
salt/elasticsearch/files/ingest/osquery.live_query Normal file
View File

@@ -0,0 +1,16 @@
{
"description" : "osquery live query",
"processors" : [
{
"script": {
"lang": "painless",
"source": "def dict = ['columns': new HashMap()]; for (entry in ctx['rows'].entrySet()) { dict['columns'][entry.getKey()] = entry.getValue(); } ctx['result'] = dict; "
}
},
{ "remove": { "field": [ "rows" ], "ignore_missing": true, "ignore_failure": true } },
{ "rename": { "field": "distributed_query_execution_id", "target_field": "result.query_id", "ignore_missing": true } },
{ "rename": { "field": "computer_name", "target_field": "host.hostname", "ignore_missing": true } },
{ "pipeline": { "name": "osquery.normalize" } },
{ "pipeline": { "name": "common" } }
]
}

14
salt/elasticsearch/files/ingest/osquery.normalize Normal file
View File

@@ -0,0 +1,14 @@
{
"description" : "osquery normalize",
"processors" : [
{ "rename": { "field": "result.columns.cmdline", "target_field": "process.command_line", "ignore_missing": true } },
{ "rename": { "field": "result.columns.cwd", "target_field": "process.working_directory", "ignore_missing": true } },
{ "rename": { "field": "result.columns.name", "target_field": "process.name", "ignore_missing": true } },
{ "rename": { "field": "result.columns.path", "target_field": "process.executable", "ignore_missing": true } },
{ "rename": { "field": "result.columns.pid", "target_field": "process.pid", "ignore_missing": true } },
{ "rename": { "field": "result.columns.parent", "target_field": "process.ppid", "ignore_missing": true } },
{ "rename": { "field": "result.columns.uid", "target_field": "user.id", "ignore_missing": true } },
{ "rename": { "field": "result.columns.username", "target_field": "user.name", "ignore_missing": true } },
{ "rename": { "field": "result.columns.gid", "target_field": "group.id", "ignore_missing": true } }
]
}

17
salt/elasticsearch/files/ingest/osquery.query_result
View File

@@ -1,24 +1,19 @@
{
"description" : "osquery",
"processors" : [
{ "json": { "field": "message", "target_field": "message2", "ignore_failure": true } },
{ "gsub": { "field": "message2.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } },
{ "rename": { "if": "ctx.message2.columns?.eventid != null", "field": "message2.columns", "target_field": "winlog", "ignore_missing": true } },
{ "json": { "field": "message", "target_field": "result", "ignore_failure": true } },
{ "gsub": { "field": "result.columns.data", "pattern": "\\\\xC2\\\\xAE", "replacement": "", "ignore_missing": true } },
{ "rename": { "if": "ctx.result.columns?.eventid != null", "field": "result.columns", "target_field": "winlog", "ignore_missing": true } },
{ "json": { "field": "winlog.data", "target_field": "unparsed", "ignore_failure": true} },
{ "set": { "if": "!(ctx.unparsed?.EventData instanceof Map)", "field": "error.eventdata_parsing", "value": true, "ignore_failure": true } },
{ "rename": { "if": "!(ctx.error?.eventdata_parsing == true)", "field": "unparsed.EventData", "target_field": "winlog.event_data", "ignore_missing": true, "ignore_failure": true } },
{ "rename": { "field": "winlog.source", "target_field": "winlog.channel", "ignore_missing": true } },
{ "rename": { "field": "winlog.eventid", "target_field": "winlog.event_id", "ignore_missing": true } },
{ "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } },
{ "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } },
{
"script": {
"lang": "painless",
"source": "def dict = ['result': new HashMap()]; for (entry in ctx['message2'].entrySet()) { dict['result'][entry.getKey()] = entry.getValue(); } ctx['osquery'] = dict; "
}
},
{ "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')", "name":"win.eventlogs" } },
{ "set": { "field": "event.module", "value": "osquery", "override": false } },
{ "set": { "field": "event.dataset", "value": "{{osquery.result.name}}", "override": false} },
{ "set": { "field": "event.dataset", "value": "{{result.name}}", "override": false} },
{ "pipeline": { "if": "!(ctx.containsKey('winlog'))", "name": "osquery.normalize" } },
{ "pipeline": { "name": "common" } }
]
}

8
salt/elasticsearch/files/so-elasticsearch-pipelines
View File

@@ -27,11 +27,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 240 ]]; do
{% if grains['role'] in ['so-node','so-heavynode'] %}
curl ${ELASTICSEARCH_AUTH} -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
{% else %}
curl ${ELASTICSEARCH_AUTH} --output /dev/null --silent --head --fail -L http://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
{% endif %}
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -51,11 +47,7 @@ fi
cd ${ELASTICSEARCH_INGEST_PIPELINES}
echo "Loading pipelines..."
{% if grains['role'] in ['so-node','so-heavynode'] %}
for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -k -XPUT -L https://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
{% else %}
for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT -L http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done
{% endif %}
echo
cd - >/dev/null

17
salt/elasticsearch/files/sotls.yml
View File

@@ -1,17 +0,0 @@
keystore.path: /usr/share/elasticsearch/config/sokeys
keystore.password: changeit
keystore.algorithm: SunX509
truststore.path: /etc/pki/java/cacerts
truststore.password: changeit
truststore.algorithm: PKIX
protocols:
- TLSv1.2
ciphers:
- TLS_RSA_WITH_AES_128_CBC_SHA256
- TLS_RSA_WITH_AES_256_GCM_SHA384
transport.encrypted: true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
http.encrypted: true
{%- else %}
http.encrypted: false
{%- endif %}

11
salt/elasticsearch/init.sls
View File

@@ -140,14 +140,6 @@ esyml:
- group: 939
- template: jinja
sotls:
file.managed:
- name: /opt/so/conf/elasticsearch/sotls.yml
- source: salt://elasticsearch/files/sotls.yml
- user: 930
- group: 939
- template: jinja
#sync templates to /opt/so/conf/elasticsearch/templates
{% for TEMPLATE in TEMPLATES %}
es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
@@ -199,7 +191,7 @@ so-elasticsearch:
{% if TRUECLUSTER is sameas false or (TRUECLUSTER is sameas true and not salt['pillar.get']('nodestab', {})) %}
- discovery.type=single-node
{% endif %}
- ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }}
- ES_JAVA_OPTS=-Xms{{ esheap }} -Xmx{{ esheap }} -Des.transport.cname_in_publish_address=true
ulimits:
- memlock=-1:-1
- nofile=65536:65536
@@ -221,7 +213,6 @@ so-elasticsearch:
- /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
- /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
- /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
- /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro
- watch:
- file: cacertz
- file: esyml

4
salt/elasticsearch/templates/so/so-common-template.json
View File

@@ -365,6 +365,10 @@
"request":{
"type":"object",
"dynamic": true
},
"result":{
"type":"object",
"dynamic": true
},
"rfb":{
"type":"object",

2
salt/filebeat/etc/filebeat.yml
View File

@@ -260,7 +260,7 @@ output.{{ type }}:
{%- if grains['role'] in ["so-eval", "so-import"] %}
output.elasticsearch:
enabled: true
hosts: ["{{ MANAGER }}:9200"]
hosts: ["https://{{ MANAGER }}:9200"]
pipelines:
- pipeline: "%{[module]}.%{[dataset]}"
indices:

1
salt/filebeat/init.sls
View File

@@ -13,7 +13,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls in allowed_states %}
{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
{% set LOCALHOSTNAME = salt['grains.get']('host') %}

53
salt/kibana/bin/keepkibanahappy.sh
View File

@@ -1,53 +0,0 @@
{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
# Wait for ElasticSearch to come up, so that we can query for version infromation
echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 30 ]]; do
curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
break
else
((COUNT+=1))
sleep 1
echo -n "."
fi
done
if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
echo
echo -e "Connection attempt timed out. Unable to connect to ElasticSearch. \nPlease try: \n -checking log(s) in /var/log/elasticsearch/\n -running 'sudo docker ps' \n -running 'sudo so-elastic-restart'"
echo
exit
fi
# Make sure Kibana is running
MAX_WAIT=240
# Check to see if Kibana is available
wait_step=0
until curl -s -XGET -L http://{{ ES }}:5601 > /dev/null ; do
wait_step=$(( ${wait_step} + 1 ))
echo "Waiting on Kibana...Attempt #$wait_step"
if [ ${wait_step} -gt ${MAX_WAIT} ]; then
echo "ERROR: Kibana not available for more than ${MAX_WAIT} seconds."
exit 5
fi
sleep 1s;
done
# Apply Kibana template
echo
echo "Applying Kibana template..."
curl -s -XPUT -L http://{{ ES }}:9200/_template/kibana \
-H 'Content-Type: application/json' \
-d'{"index_patterns" : ".kibana", "settings": { "number_of_shards" : 1, "number_of_replicas" : 0 }, "mappings" : { "search": {"properties": {"hits": {"type": "integer"}, "version": {"type": "integer"}}}}}'
echo
curl -s -XPUT -L "{{ ES }}:9200/.kibana/_settings" \
-H 'Content-Type: application/json' \
-d'{"index" : {"number_of_replicas" : 0}}'
echo

6
salt/kibana/bin/so-kibana-config-load
View File

@@ -14,5 +14,9 @@ cp /opt/so/conf/kibana/saved_objects.ndjson.template /opt/so/conf/kibana/saved_o
# SOCtopus and Manager
sed -i "s/PLACEHOLDER/{{ MANAGER }}/g" /opt/so/conf/kibana/saved_objects.ndjson
## This hackery will be removed if using Elastic Auth ##
# Let's snag a cookie from Kibana
THECOOKIE=$(curl -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
# Load saved objects
curl -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1
curl -b "sid=$THECOOKIE" -L -X POST "localhost:5601/api/saved_objects/_import?overwrite=true" -H "kbn-xsrf: true" --form file=@/opt/so/conf/kibana/saved_objects.ndjson > /dev/null 2>&1

8
salt/kibana/etc/kibana.yml
View File

@@ -1,11 +1,11 @@
---
# Default Kibana configuration from kibana-docker.
{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
server.name: kibana
server.host: "0"
server.basePath: /kibana
elasticsearch.hosts: [ "http://{{ ES }}:9200" ]
elasticsearch.hosts: [ "https://{{ ES }}:9200" ]
elasticsearch.ssl.verificationMode: none
#kibana.index: ".kibana"
#elasticsearch.username: elastic
#elasticsearch.password: changeme
@@ -14,3 +14,7 @@ elasticsearch.requestTimeout: 90000
logging.dest: /var/log/kibana/kibana.log
telemetry.enabled: false
security.showInsecureClusterWarning: false
xpack.security.authc.providers:
anonymous.anonymous1:
order: 0
credentials: "elasticsearch_anonymous_user"

19
salt/logstash/pipelines/config/so/0008_input_fleet_livequery.conf.jinja Normal file
View File

@@ -0,0 +1,19 @@
{%- set MANAGER = salt['grains.get']('master') %}
{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
input {
redis {
host => '{{ MANAGER }}'
port => 6379
data_type => 'pattern_channel'
key => 'results_*'
type => 'live_query'
add_field => {
"module" => "osquery"
"dataset" => "live_query"
}
threads => {{ THREADS }}
batch_count => {{ BATCH }}
}
}

3
salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "zeek" and "import" not in [tags] {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-zeek"
template => "/templates/so-zeek-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

3
salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if "import" in [tags] {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-import"
template => "/templates/so-import-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

3
salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [event_type] == "sflow" {
elasticsearch {
@@ -12,10 +11,8 @@ output {
template_name => "so-flow"
template => "/templates/so-flow-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

3
salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [event_type] == "ids" and "import" not in [tags] {
elasticsearch {
@@ -12,10 +11,8 @@ output {
template_name => "so-ids"
template => "/templates/so-ids-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

3
salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "syslog" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-syslog"
template => "/templates/so-syslog-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

5
salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
View File

@@ -3,9 +3,8 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "osquery" {
if [module] =~ "osquery" and "live_query" not in [dataset] {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"
@@ -13,10 +12,8 @@ output {
template_name => "so-osquery"
template => "/templates/so-osquery-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

43
salt/logstash/pipelines/config/so/9101_output_osquery_livequery.conf.jinja Normal file
View File

@@ -0,0 +1,43 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
filter {
if [type] =~ "live_query" {
mutate {
rename => {
"[host][hostname]" => "computer_name"
}
}
prune {
blacklist_names => ["host"]
}
split {
field => "rows"
}
}
}
output {
if [type] =~ "live_query" {
elasticsearch {
pipeline => "osquery.live_query"
hosts => "{{ ES }}"
index => "so-osquery"
template_name => "so-osquery"
template => "/templates/so-osquery-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

3
salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [dataset] =~ "firewall" {
elasticsearch {
@@ -12,10 +11,8 @@ output {
template_name => "so-firewall"
template => "/templates/so-firewall-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

3
salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "suricata" and "import" not in [tags] {
elasticsearch {
@@ -12,10 +11,8 @@ output {
index => "so-ids"
template_name => "so-ids"
template => "/templates/so-ids-template.json"
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

3
salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if "beat-ext" in [tags] and "import" not in [tags] {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-beats"
template => "/templates/so-beats-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

3
salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "ossec" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-ossec"
template => "/templates/so-ossec-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

3
salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
View File

@@ -3,7 +3,6 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
output {
if [module] =~ "strelka" {
elasticsearch {
@@ -13,10 +12,8 @@ output {
template_name => "so-strelka"
template => "/templates/so-strelka-template.json"
template_overwrite => true
{%- if grains['role'] in ['so-node','so-heavynode'] %}
ssl => true
ssl_certificate_verification => false
{%- endif %}
}
}
}

1
salt/soc/files/soc/hunt.queries.json
View File

@@ -42,6 +42,7 @@
{ "name": "MYSQL", "description": "MYSQL grouped by command", "query": "event.dataset:mysql | groupby mysql.command"},
{ "name": "NOTICE", "description": "Zeek notice logs grouped by note and message", "query": "event.dataset:notice | groupby notice.note notice.message"},
{ "name": "NTLM", "description": "NTLM grouped by computer name", "query": "event.dataset:ntlm | groupby ntlm.server.dns.name"},
{ "name": "Osquery Live Queries", "description": "Osquery Live Query results grouped by computer name", "query": "event.dataset:live_query | groupby host.hostname"},
{ "name": "PE", "description": "PE files list", "query": "event.dataset:pe | groupby file.machine file.os file.subsystem"},
{ "name": "RADIUS", "description": "RADIUS grouped by username", "query": "event.dataset:radius | groupby user.name.keyword"},
{ "name": "RDP", "description": "RDP grouped by client name", "query": "event.dataset:rdp | groupby client.name"},

5
salt/soc/files/soc/soc.json
View File

@@ -1,7 +1,6 @@
{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
{%- set SENSORONIKEY = salt['pillar.get']('global:sensoronikey', '') %}
{%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
{%- set FEATURES = salt['pillar.get']('elastic:features', False) %}
{%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
{%- import_json "soc/files/soc/alerts.queries.json" as alerts_queries %}
{%- import_json "soc/files/soc/alerts.actions.json" as alerts_actions %}
@@ -31,7 +30,7 @@
"hostUrl": "http://{{ MANAGERIP }}:4434/"
},
"elastic": {
"hostUrl": "http://{{ MANAGERIP }}:9200",
"hostUrl": "https://{{ MANAGERIP }}:9200",
{%- if salt['pillar.get']('nodestab', {}) %}
"remoteHostUrls": [
{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
@@ -45,7 +44,7 @@
},
"sostatus": {
"refreshIntervalMs": 30000,
"offlineThresholdMs": 60000
"offlineThresholdMs": 900000
},
{% if THEHIVEKEY != '' %}
"thehive": {

2
salt/soctopus/files/SOCtopus.conf
View File

@@ -6,7 +6,7 @@
[es]
es_url = http://{{MANAGER}}:9200
es_url = https://{{MANAGER}}:9200
es_ip = {{MANAGER}}
es_user = YOURESUSER
es_pass = YOURESPASS

2
salt/telegraf/etc/telegraf.conf
View File

@@ -622,7 +622,7 @@
# ## specify a list of one or more Elasticsearch servers
# # you can add username and password to your url to use basic authentication:
# # servers = ["http://user:pass@localhost:9200"]
servers = ["http://{{ MANAGER }}:9200"]
servers = ["https://{{ MANAGER }}:9200"]
{% elif grains['role'] in ['so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode'] %}
[[inputs.elasticsearch]]
servers = ["https://{{ NODEIP }}:9200"]

7
salt/utility/bin/crossthestreams
View File

@@ -1,7 +1,6 @@
#!/bin/bash
{% set ES = salt['pillar.get']('manager:mainip', '') %}
{% set MANAGER = salt['grains.get']('master') %}
{% set FEATURES = salt['pillar.get']('elastic:features', False) %}
{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
# Wait for ElasticSearch to come up, so that we can query for version infromation
@@ -9,7 +8,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 30 ]]; do
curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -29,7 +28,7 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
fi
echo "Applying cross cluster search config..."
curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
-H 'Content-Type: application/json' \
-d "{\"persistent\": {\"search\": {\"remote\": {\"{{ MANAGER }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"
@@ -37,7 +36,7 @@ echo "Applying cross cluster search config..."
{%- if TRUECLUSTER is sameas false %}
{%- if salt['pillar.get']('nodestab', {}) %}
{%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
curl -XPUT -L http://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"{{ SN }}": {"skip_unavailable": "true", "seeds": ["{{ SN.split('_')|first }}:9300"]}}}}}'
{%- endfor %}
{%- endif %}
{%- endif %}

4
salt/utility/bin/eval
View File

@@ -6,7 +6,7 @@ echo -n "Waiting for ElasticSearch..."
COUNT=0
ELASTICSEARCH_CONNECTED="no"
while [[ "$COUNT" -le 30 ]]; do
curl --output /dev/null --silent --head --fail -L http://{{ ES }}:9200
curl -k --output /dev/null --silent --head --fail -L https://{{ ES }}:9200
if [ $? -eq 0 ]; then
ELASTICSEARCH_CONNECTED="yes"
echo "connected!"
@@ -26,6 +26,6 @@ if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
fi
echo "Applying cross cluster search config..."
curl -s -XPUT -L http://{{ ES }}:9200/_cluster/settings \
curl -s -k -XPUT -L https://{{ ES }}:9200/_cluster/settings \
-H 'Content-Type: application/json' \
-d "{\"persistent\": {\"search\": {\"remote\": {\"{{ grains.host }}\": {\"seeds\": [\"127.0.0.1:9300\"]}}}}}"