CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 09:12:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

merge with dev and resolve conflicts

Browse Source
This commit is contained in:
m0duspwnens
2020-07-10 12:20:14 -04:00
parent 0a1b5f29eb bbef7955b2
commit 24b8f81e38
26 changed files with 454 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
pillar/logstash/eval.sls
View File

@@ -16,6 +16,13 @@ logstash:
- so/9600_output_ossec.conf.jinja
- so/9700_output_strelka.conf.jinja
templates:
- so/so-beats-template.json
- so/so-beats-template.json.jinja
- so/so-common-template.json
- so/so-zeek-template.json
- so/so-firewall-template.json.jinja
- so/so-ids-template.json.jinja
- so/so-import-template.json.jinja
- so/so-osquery-template.json.jinja
- so/so-ossec-template.json.jinja
- so/so-strelka-template.json.jinja
- so/so-syslog-template.json.jinja
- so/so-zeek-template.json.jinja

10
pillar/logstash/search.sls
View File

@@ -12,5 +12,13 @@ logstash:
- so/9600_output_ossec.conf.jinja
- so/9700_output_strelka.conf.jinja
templates:
- so/so-beats-template.json.jinja
- so/so-common-template.json
- so/so-zeek-template.json
- so/so-firewall-template.json.jinja
- so/so-ids-template.json.jinja
- so/so-import-template.json.jinja
- so/so-osquery-template.json.jinja
- so/so-ossec-template.json.jinja
- so/so-strelka-template.json.jinja
- so/so-syslog-template.json.jinja
- so/so-zeek-template.json.jinja

6
salt/curator/files/action/delete.yml
View File

@@ -1,8 +1,4 @@
{%- if grains['role'] in ['so-node', 'so-searchnode', 'so-heavynode'] %}
{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit', '') -%}
{%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %}
{%- set log_size_limit = salt['pillar.get']('manager:log_size_limit', '') -%}
{%- endif %}
{%- set log_size_limit = salt['pillar.get']('elasticsearch:log_size_limit', '') -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"

29
salt/curator/files/action/so-beats-close.yml Normal file
View File

@@ -0,0 +1,29 @@
{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-beats:close', 30) -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: close
description: >-
Close Beats indices older than {{cur_close_days}} days.
options:
delete_aliases: False
timeout_override:
continue_if_exception: False
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-beats.*|so-beats.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{cur_close_days}}
exclude:

12
salt/curator/files/action/close.yml → salt/curator/files/action/so-firewall-close.yml
View File

@@ -1,9 +1,4 @@
{%- if grains['role'] in ['so-node', 'so-searchnode', 'so-heavynode'] %}
{%- set cur_close_days = salt['pillar.get']('elasticsearch:cur_close_days', '') -%}
{%- elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone'] %}
{%- set cur_close_days = salt['pillar.get']('manager:cur_close_days', '') -%}
{%- endif -%}
{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-firewall:close', 30) -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
@@ -15,8 +10,7 @@ actions:
1:
action: close
description: >-
Close indices older than {{cur_close_days}} days (based on index name), for logstash-
prefixed indices.
Close Firewall indices older than {{cur_close_days}} days.
options:
delete_aliases: False
timeout_override:
@@ -25,7 +19,7 @@ actions:
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-.*|so-.*)$'
value: '^(logstash-firewall.*|so-firewall.*)$'
- filtertype: age
source: name
direction: older

29
salt/curator/files/action/so-ids-close.yml Normal file
View File

@@ -0,0 +1,29 @@
{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-ids:close', 30) -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: close
description: >-
Close IDS indices older than {{cur_close_days}} days.
options:
delete_aliases: False
timeout_override:
continue_if_exception: False
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-ids.*|so-ids.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{cur_close_days}}
exclude:

29
salt/curator/files/action/so-import-close.yml Normal file
View File

@@ -0,0 +1,29 @@
{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-import:close', 30) -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: close
description: >-
Close Import indices older than {{cur_close_days}} days.
options:
delete_aliases: False
timeout_override:
continue_if_exception: False
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-import.*|so-import.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{cur_close_days}}
exclude:

29
salt/curator/files/action/so-osquery-close.yml Normal file
View File

@@ -0,0 +1,29 @@
{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-osquery:close', 30) -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: close
description: >-
Close osquery indices older than {{cur_close_days}} days.
options:
delete_aliases: False
timeout_override:
continue_if_exception: False
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-osquery.*|so-osquery.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{cur_close_days}}
exclude:

29
salt/curator/files/action/so-ossec-close.yml Normal file
View File

@@ -0,0 +1,29 @@
{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-ossec:close', 30) -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: close
description: >-
Close ossec indices older than {{cur_close_days}} days.
options:
delete_aliases: False
timeout_override:
continue_if_exception: False
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-ossec.*|so-ossec.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{cur_close_days}}
exclude:

29
salt/curator/files/action/so-strelka-close.yml Normal file
View File

@@ -0,0 +1,29 @@
{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-strelka:close', 30) -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: close
description: >-
Close Strelka indices older than {{cur_close_days}} days.
options:
delete_aliases: False
timeout_override:
continue_if_exception: False
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-strelka.*|so-strelka.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{cur_close_days}}
exclude:

29
salt/curator/files/action/so-syslog-close.yml Normal file
View File

@@ -0,0 +1,29 @@
{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-syslog:close', 30) -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: close
description: >-
Close syslog indices older than {{cur_close_days}} days.
options:
delete_aliases: False
timeout_override:
continue_if_exception: False
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-syslog.*|so-syslog.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{cur_close_days}}
exclude:

29
salt/curator/files/action/so-zeek-close.yml Normal file
View File

@@ -0,0 +1,29 @@
{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settins:so-zeek:close', 30) -%}
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: close
description: >-
Close Zeek indices older than {{cur_close_days}} days.
options:
delete_aliases: False
timeout_override:
continue_if_exception: False
disable_action: False
filters:
- filtertype: pattern
kind: regex
value: '^(logstash-zeek.*|so-zeek.*)$'
- filtertype: age
source: name
direction: older
timestring: '%Y.%m.%d'
unit: days
unit_count: {{cur_close_days}}
exclude:

16
salt/curator/init.sls
View File

@@ -30,18 +30,10 @@ curlogdir:
- user: 934
- group: 939
curcloseconf:
file.managed:
- name: /opt/so/conf/curator/action/close.yml
- source: salt://curator/files/action/close.yml
- user: 934
- group: 939
- template: jinja
curdelconf:
file.managed:
- name: /opt/so/conf/curator/action/delete.yml
- source: salt://curator/files/action/delete.yml
actionconfs:
file.recurse:
- name: /opt/so/conf/curator/action
- source: salt://curator/files/action
- user: 934
- group: 939
- template: jinja

26
salt/elasticsearch/files/elasticsearch.yml
View File

@@ -1,6 +1,11 @@
{% if grains['role'] == 'so-manager' or grains['role'] == 'so-eval' %}
{%- set esclustername = salt['pillar.get']('manager:esclustername', '') %}
cluster.name: "{{ esclustername }}"
{%- set NODE_ROUTE_TYPE = salt['pillar.get']('elasticsearch:node_route_type', 'hot') %}
{%- if salt['pillar.get']('elasticsearch:hot_warm_enabled') or salt['pillar.get']('elasticsearch:true_cluster') %}
{%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:true_cluster_name', '') %}
{%- else %}
{%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername', '') %}
{%- endif %}
{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
cluster.name: "{{ ESCLUSTERNAME }}"
network.host: 0.0.0.0
# minimum_master_nodes need to be explicitly set when bound on a public IP
@@ -10,20 +15,13 @@ discovery.zen.minimum_master_nodes: 1
# This is a test -- if this is here, then the volume is mounted correctly.
path.logs: /var/log/elasticsearch
action.destructive_requires_name: true
{%- else %}
{%- set esclustername = salt['grains.get']('host', '') %}
{%- set nodeip = salt['pillar.get']('elasticsearch:mainip', '') -%}
cluster.name: "{{ esclustername }}"
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 1
path.logs: /var/log/elasticsearch
action.destructive_requires_name: true
transport.bind_host: 0.0.0.0
transport.publish_host: {{ nodeip }}
transport.publish_host: {{ NODEIP }}
transport.publish_port: 9300
{%- endif %}
cluster.routing.allocation.disk.threshold_enabled: true
cluster.routing.allocation.disk.watermark.low: 95%
cluster.routing.allocation.disk.watermark.high: 98%
cluster.routing.allocation.disk.watermark.flood_stage: 98%
script.max_compilations_rate: 1000/1m
node.attr.box_type: {{ NODE_ROUTE_TYPE }}
node.name: {{ ESCLUSTERNAME }}
script.max_compilations_rate: 1000/1m

13
salt/logstash/pipelines/templates/so/so-beats-template.json.jinja Normal file
View File

@@ -0,0 +1,13 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-beats:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-beats:refresh', '30s') %}
{
"index_patterns": ["so-beats-*"],
"version": 50001,
"order": 11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

11
salt/logstash/pipelines/templates/so/so-common-template.json
View File

@@ -1,15 +1,16 @@
{
"index_patterns": ["so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*"],
"version":50001,
"order" : 10,
"order":10,
"settings":{
"number_of_replicas":0,
"number_of_shards":1,
"index.refresh_interval":"30s"
"index.refresh_interval":"30s",
"index.routing.allocation.require.box_type":"hot"
},
"mappings":{
"dynamic": false,
"date_detection": false,
"dynamic":false,
"date_detection":false,
"properties":{
"@timestamp":{
"type":"date"
@@ -19,7 +20,7 @@
},
"osquery":{
"type":"object",
"dynamic": true
"dynamic":true
},
"geoip":{
"dynamic":true,

13
salt/logstash/pipelines/templates/so/so-firewall-template.json.jinja Normal file
View File

@@ -0,0 +1,13 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-firewall:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-firewall:refresh', '30s') %}
{
"index_patterns": ["so-firewall-*"],
"version":50001,
"order":11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

13
salt/logstash/pipelines/templates/so/so-ids-template.json.jinja Normal file
View File

@@ -0,0 +1,13 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-ids:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-ids:refresh', '30s') %}
{
"index_patterns": ["so-ids-*"],
"version":50001,
"order":11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

13
salt/logstash/pipelines/templates/so/so-import-template.json.jinja Normal file
View File

@@ -0,0 +1,13 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-import:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-import:refresh', '30s') %}
{
"index_patterns": ["so-import-*"],
"version":50001,
"order": 11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

13
salt/logstash/pipelines/templates/so/so-osquery-template.json.jinja Normal file
View File

@@ -0,0 +1,13 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-osquery:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-osquery:refresh', '30s') %}
{
"index_patterns": ["so-osquery-*"],
"version":50001,
"order":11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

13
salt/logstash/pipelines/templates/so/so-ossec-template.json.jinja Normal file
View File

@@ -0,0 +1,13 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-ossec:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-ossec:refresh', '30s') %}
{
"index_patterns": ["so-ossec-*"],
"version":50001,
"order":11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

13
salt/logstash/pipelines/templates/so/so-strelka-template.json.jinja Normal file
View File

@@ -0,0 +1,13 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-strelka:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-strelka:refresh', '30s') %}
{
"index_patterns": ["so-strelka-*"],
"version":50001,
"order":11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

14
salt/logstash/pipelines/templates/so/so-syslog-template.json.jinja Normal file
View File

@@ -0,0 +1,14 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-syslog:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-syslog:refresh', '30s') %}
{
"index_patterns": ["so-syslog-*"],
"version":50001,
"order":11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

10
salt/logstash/pipelines/templates/so/so-zeek-template.json
View File

@@ -1,10 +0,0 @@
{
"index_patterns": ["so-zeek-*"],
"version":50001,
"order" : 11,
"settings":{
"number_of_replicas":0,
"number_of_shards":1,
"index.refresh_interval":"30s"
}
}

13
salt/logstash/pipelines/templates/so/so-zeek-template.json.jinja Normal file
View File

@@ -0,0 +1,13 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-zeek:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-zeek:refresh', '30s') %}
{
"index_patterns": ["so-zeek-*"],
"version":50001,
"order":11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

67
setup/so-functions
View File

@@ -1013,8 +1013,66 @@ manager_static() {
"strelka:"\
" enabled: $STRELKA"\
" rules: $STRELKARULES"\
"curator:"\
" hot_warm: False"\
"elastic:"\
" features: False" > "$static_pillar"
" features: False"\
"elasticsearch:"\
" replicas: 0"\
" true_cluster: False"\
" true_cluster_name: so"\
" discovery_nodes: 1"\
" hot_warm_enabled: False"\
" cluster_routing_allocation_disk.threshold_enabled: true"\
" cluster_routing_allocation_disk_watermark_low: 95%"\
" cluster_routing_allocation_disk_watermark_high: 98%"\
" cluster_routing_allocation_disk_watermark_flood_stage: 98%"\
" index_settings:"\
" so-beats:"\
" shards: 1"\
" warm: 7"\
" close: 30"\
" delete: 365"\
" so-firewall:"\
" shards: 1"\
" warm: 7"\
" close: 30"\
" delete: 365"\
" so-ids:"\
" shards: 1"\
" warm: 7"\
" close: 30"\
" delete: 365"\
" so-import:"\
" shards: 1"\
" warm: 7"\
" close: 73000"\
" delete: 73001"\
" so-osquery:"\
" shards: 1"\
" warm: 7"\
" close: 30"\
" delete: 365"\
" so-ossec:"\
" shards: 1"\
" warm: 7"\
" close: 30"\
" delete: 365"\
" so-strelka:"\
" shards: 1"\
" warm: 7"\
" close: 30"\
" delete: 365"\
" so-syslog:"\
" shards: 1"\
" warm: 7"\
" close: 30"\
" delete: 365"\
" so-zeek:"\
" shards: 5"\
" warm: 7"\
" close: 365"\
" delete: 45" > "$static_pillar"
printf '%s\n' '----' >> "$setup_log" 2>&1
cat "$static_pillar" >> "$setup_log" 2>&1
@@ -1066,12 +1124,7 @@ elasticsearch_pillar() {
" node_type: $NODETYPE"\
" es_port: $node_es_port"\
" log_size_limit: $log_size_limit"\
" cur_close_days: $CURCLOSEDAYS"\
" route_type: hot"\
" index_settings:"\
" so-zeek:"\
" shards: 5"\
" replicas: 0"\
" node_route_type: hot"\
"" >> "$pillar_file"
if [ "$install_type" != 'EVAL' ] && [ "$install_type" != 'HELIXSENSOR' ] && [ "$install_type" != 'MANAGERSEARCH' ] && [ "$install_type" != 'STANDALONE' ]; then