diff --git a/salt/common/files/log-rotate.conf b/salt/common/files/log-rotate.conf
index f8a16a038..fd53d9983 100644
--- a/salt/common/files/log-rotate.conf
+++ b/salt/common/files/log-rotate.conf
@@ -18,6 +18,7 @@
/opt/so/log/filebeat/*.log
/opt/so/log/telegraf/*.log
/opt/so/log/redis/*.log
+/opt/so/log/stenographer/*.log
/opt/so/log/salt/so-salt-minion-check
/opt/so/log/salt/minion
/opt/so/log/salt/master
diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common
index ef53ce60f..1a4ca233f 100755
--- a/salt/common/tools/sbin/so-image-common
+++ b/salt/common/tools/sbin/so-image-common
@@ -16,7 +16,7 @@
# along with this program. If not, see .
# NOTE: This script depends on so-common
-IMAGEREPO=securityonion
+IMAGEREPO=security-onion-solutions
container_list() {
MANAGERCHECK=$1
@@ -103,7 +103,7 @@ update_docker_containers() {
local PROGRESS_CALLBACK=$3
local LOG_FILE=$4
- local CONTAINER_REGISTRY=quay.io
+ local CONTAINER_REGISTRY=ghcr.io
local SIGNPATH=/root/sosigs
if [ -z "$CURLTYPE" ]; then
diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml
index 1ad65c43f..4a7260bc6 100644
--- a/salt/elasticsearch/files/elasticsearch.yml
+++ b/salt/elasticsearch/files/elasticsearch.yml
@@ -7,6 +7,7 @@
{%- else %}
{%- set ESCLUSTERNAME = salt['pillar.get']('elasticsearch:esclustername') %}
{%- endif %}
+{%- set NODE_ROLES = salt['pillar.get']('elasticsearch:node_roles', ['data', 'ingest']) %}
cluster.name: "{{ ESCLUSTERNAME }}"
network.host: 0.0.0.0
@@ -24,24 +25,24 @@ cluster.routing.allocation.disk.threshold_enabled: true
cluster.routing.allocation.disk.watermark.low: 95%
cluster.routing.allocation.disk.watermark.high: 98%
cluster.routing.allocation.disk.watermark.flood_stage: 98%
-{%- if FEATURES is sameas true %}
#xpack.security.enabled: false
-#xpack.security.http.ssl.enabled: false
-#xpack.security.transport.ssl.enabled: false
-#xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
-#xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
-#xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt
-#xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
-#xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
-#xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt
-#xpack.security.transport.ssl.verification_mode: none
-#xpack.security.http.ssl.client_authentication: none
+xpack.security.transport.ssl.enabled: true
+xpack.security.transport.ssl.verification_mode: none
+xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
+xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
+xpack.security.transport.ssl.certificate_authorities: [ "/usr/share/elasticsearch/config/ca.crt" ]
+{%- if grains['role'] in ['so-node','so-heavynode'] %}
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.client_authentication: none
+xpack.security.http.ssl.key: /usr/share/elasticsearch/config/elasticsearch.key
+xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/elasticsearch.crt
+xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/ca.crt
+{%- endif %}
#xpack.security.authc:
# anonymous:
# username: anonymous_user
# roles: superuser
# authz_exception: true
-{%- endif %}
node.name: {{ grains.host }}
script.max_compilations_rate: 1000/1m
{%- if TRUECLUSTER is sameas true %}
@@ -55,7 +56,7 @@ discovery.seed_hosts:
{%- endfor %}
{%- endif %}
{%- else %}
-node.roles: [ data, ingest ]
+node.roles: {{ NODE_ROLES }}
node.attr.box_type: {{ NODE_ROUTE_TYPE }}
discovery.seed_hosts:
- {{ grains.master }}
diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls
index 089a21e1c..ae8976e1d 100644
--- a/salt/elasticsearch/init.sls
+++ b/salt/elasticsearch/init.sls
@@ -23,12 +23,6 @@
{% set TRUECLUSTER = salt['pillar.get']('elasticsearch:true_cluster', False) %}
{% set MANAGERIP = salt['pillar.get']('global:managerip') %}
-{% if FEATURES is sameas true %}
- {% set FEATUREZ = "-features" %}
-{% else %}
- {% set FEATUREZ = '' %}
-{% endif %}
-
{% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %}
{% set esclustername = salt['pillar.get']('manager:esclustername') %}
{% set esheap = salt['pillar.get']('manager:esheap') %}
@@ -186,7 +180,7 @@ eslogdir:
so-elasticsearch:
docker_container.running:
- - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}{{ FEATUREZ }}
+ - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elasticsearch:{{ VERSION }}
- hostname: elasticsearch
- name: so-elasticsearch
- user: elasticsearch
@@ -220,7 +214,13 @@ so-elasticsearch:
- /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
- /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
- /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
+ {% if ismanager %}
- /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro
+ {% else %}
+ - /etc/ssl/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
+ {% endif %}
+ - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
+ - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
- /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
- /opt/so/conf/elasticsearch/sotls.yml:/usr/share/elasticsearch/config/sotls.yml:ro
- watch:
diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index 799a37337..1b902d542 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -11,6 +11,10 @@
{%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %}
{%- set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) -%}
{%- set FLEETNODE = salt['pillar.get']('global:fleet_node', False) -%}
+{%- set FBMEMEVENTS = salt['pillar.get']('filebeat:mem_events', 2048) -%}
+{%- set FBMEMFLUSHMINEVENTS = salt['pillar.get']('filebeat:mem_flush_min_events', 2048) -%}
+{%- set FBLSWORKERS = salt['pillar.get']('filebeat:ls_workers', 1) -%}
+{%- set FBLSBULKMAXSIZE = salt['pillar.get']('filebeat:ls_bulk_max_size', 2048) -%}
name: {{ HOSTNAME }}
@@ -290,7 +294,10 @@ output.logstash:
hosts: ["{{ MANAGER }}:5644"]
# Number of workers per Logstash host.
- #worker: 1
+ worker: {{ FBLSWORKERS }}
+
+ # Number of records to send to Logstash input at a time
+ bulk_max_size: {{ FBLSBULKMAXSIZE }}
# Set gzip compression level.
#compression_level: 3
@@ -491,3 +498,6 @@ setup.template.enabled: false
#http.host: localhost
# Port on which the HTTP endpoint will bind. Default is 5066.
+
+queue.mem.events: {{ FBMEMEVENTS }}
+queue.mem.flush.min_events: {{ FBMEMFLUSHMINEVENTS }}
diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls
index 47dc2f3a9..eb5d8f351 100644
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -64,7 +64,7 @@ filebeatconfsync:
OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }}
so-filebeat:
docker_container.running:
- - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}{{ FEATURES }}
+ - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
- hostname: so-filebeat
- user: root
- extra_hosts: {{ MANAGER }}:{{ MANAGERIP }},{{ LOCALHOSTNAME }}:{{ LOCALHOSTIP }}
diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls
index 7d6b541cd..1b5d05e57 100644
--- a/salt/kibana/init.sls
+++ b/salt/kibana/init.sls
@@ -73,7 +73,7 @@ kibanabin:
# Start the kibana docker
so-kibana:
docker_container.running:
- - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}{{ FEATURES }}
+ - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-kibana:{{ VERSION }}
- hostname: kibana
- user: kibana
- environment:
@@ -104,7 +104,7 @@ wait_for_kibana:
module.run:
- http.wait_for_successful_query:
- url: "http://{{MANAGER}}:5601/api/saved_objects/_find?type=config"
- - wait_for: 180
+ - wait_for: 900
- onchanges:
- file: kibanadashtemplate
diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls
index 61f533865..e37713a4e 100644
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -146,7 +146,7 @@ lslogdir:
so-logstash:
docker_container.running:
- - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}{{ FEATURES }}
+ - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logstash:{{ VERSION }}
- hostname: so-logstash
- name: so-logstash
- user: logstash
diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf
index 2233b5c25..3d89628e4 100644
--- a/salt/telegraf/etc/telegraf.conf
+++ b/salt/telegraf/etc/telegraf.conf
@@ -670,47 +670,52 @@
{% if grains['role'] in ['so-manager', 'so-managersearch'] %}
[[inputs.exec]]
commands = [
- "/scripts/redis.sh",
- "/scripts/influxdbsize.sh"
+ "/scripts/redis.sh",
+ "/scripts/influxdbsize.sh"
]
- data_format = "influx"
+ data_format = "influx"
+ ## Timeout for each command to complete.
+ timeout = "15s"
{% elif grains['role'] in ['so-sensor', 'so-heavynode'] %}
[[inputs.exec]]
commands = [
- "/scripts/stenoloss.sh",
- "/scripts/suriloss.sh",
- "/scripts/checkfiles.sh",
- "/scripts/zeekloss.sh",
- "/scripts/zeekcaptureloss.sh",
- "/scripts/oldpcap.sh"
+ "/scripts/stenoloss.sh",
+ "/scripts/suriloss.sh",
+ "/scripts/checkfiles.sh",
+ "/scripts/zeekloss.sh",
+ "/scripts/zeekcaptureloss.sh",
+ "/scripts/oldpcap.sh"
]
- data_format = "influx"
+ data_format = "influx"
+ timeout = "15s"
{% elif grains['role'] == 'so-standalone' %}
[[inputs.exec]]
commands = [
- "/scripts/redis.sh",
- "/scripts/influxdbsize.sh",
- "/scripts/stenoloss.sh",
- "/scripts/suriloss.sh",
- "/scripts/checkfiles.sh",
- "/scripts/zeekloss.sh",
- "/scripts/zeekcaptureloss.sh",
- "/scripts/oldpcap.sh"
+ "/scripts/redis.sh",
+ "/scripts/influxdbsize.sh",
+ "/scripts/stenoloss.sh",
+ "/scripts/suriloss.sh",
+ "/scripts/checkfiles.sh",
+ "/scripts/zeekloss.sh",
+ "/scripts/zeekcaptureloss.sh",
+ "/scripts/oldpcap.sh"
]
- data_format = "influx"
+ data_format = "influx"
+ timeout = "15s"
{% elif grains['role'] == 'so-eval' %}
[[inputs.exec]]
commands = [
- "/scripts/redis.sh",
- "/scripts/stenoloss.sh",
- "/scripts/suriloss.sh",
- "/scripts/checkfiles.sh",
- "/scripts/zeekloss.sh",
- "/scripts/zeekcaptureloss.sh",
- "/scripts/oldpcap.sh",
- "/scripts/influxdbsize.sh"
+ "/scripts/redis.sh",
+ "/scripts/stenoloss.sh",
+ "/scripts/suriloss.sh",
+ "/scripts/checkfiles.sh",
+ "/scripts/zeekloss.sh",
+ "/scripts/zeekcaptureloss.sh",
+ "/scripts/oldpcap.sh",
+ "/scripts/influxdbsize.sh"
]
- data_format = "influx"
+ data_format = "influx"
+ timeout = "15s"
{% elif grains['role'] == 'so-helix' %}
[[inputs.exec]]
commands = [
@@ -723,19 +728,16 @@
"/scripts/helixeps.sh"
]
data_format = "influx"
+ timeout = "15s"
{% endif %}
-#
-# ## Timeout for each command to complete.
-# timeout = "5s"
-#
-# ## measurement name suffix (for separating different commands)
-# name_suffix = "_mycollector"
-#
-# ## Data format to consume.
-# ## Each data format has its own unique set of configuration options, read
-# ## more about them here:
-# ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+ ## measurement name suffix (for separating different commands)
+ # name_suffix = "_mycollector"
+
+ ## Data format to consume.
+ ## Each data format has its own unique set of configuration options, read
+ ## more about them here:
+ ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
diff --git a/salt/telegraf/init.sls b/salt/telegraf/init.sls
index fb4cacbc6..81513eee2 100644
--- a/salt/telegraf/init.sls
+++ b/salt/telegraf/init.sls
@@ -48,6 +48,7 @@ so-telegraf:
- HOST_MOUNT_PREFIX=/host
- GODEBUG=x509ignoreCN=0
- network_mode: host
+ - init: True
- binds:
- /opt/so/log/telegraf:/var/log/telegraf:rw
- /opt/so/conf/telegraf/etc/telegraf.conf:/etc/telegraf/telegraf.conf:ro
diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls
index d6c7ac15e..cf40f739a 100644
--- a/salt/wazuh/init.sls
+++ b/salt/wazuh/init.sls
@@ -96,8 +96,8 @@ wazuhmgrwhitelist:
# Check to see if Wazuh API port is available
wazuhportavailable:
- cmd.run:
- -name: netstat -utanp | grep ":55000" | grep -qv docker && PROCESS=$(netstat -utanp | grep ":55000" | uniq) && echo "Another process ($PROCESS) appears to be using port 55000. Please terminate this process, or reboot to ensure a clean state so that the Wazuh API can start properly." && exit 1 || exit 0
+ cmd.run:
+ - name: netstat -utanp | grep ":55000" | grep -qv docker && PROCESS=$(netstat -utanp | grep ":55000" | uniq) && echo "Another process ($PROCESS) appears to be using port 55000. Please terminate this process, or reboot to ensure a clean state so that the Wazuh API can start properly." && exit 1 || exit 0
so-wazuh:
docker_container.running:
diff --git a/setup/automation/distributed-net-centos-manager b/setup/automation/distributed-net-centos-manager
new file mode 100644
index 000000000..cbf803dd2
--- /dev/null
+++ b/setup/automation/distributed-net-centos-manager
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=7
+BASICSURI=7
+# BLOGS=
+#BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-manager
+install_type=MANAGER
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=IP
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
diff --git a/setup/automation/distributed-net-centos-search b/setup/automation/distributed-net-centos-search
new file mode 100644
index 000000000..aec7afd31
--- /dev/null
+++ b/setup/automation/distributed-net-centos-search
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+# ALLOW_CIDR=0.0.0.0/0
+# ALLOW_ROLE=a
+# BASICZEEK=7
+# BASICSURI=7
+# BLOGS=
+# BNICS=eth1
+# ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+# GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-search
+install_type=SEARCHNODE
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+# MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+MSRV=distributed-manager
+MSRVIP=10.66.166.42
+# MTU=
+# NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+# OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+# PLAYBOOK=1
+# REDIRECTHOST=
+# REDIRECTINFO=IP
+# RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+# STRELKA=1
+# THEHIVE=1
+# WAZUH=1
+# WEBUSER=onionuser@somewhere.invalid
+# WEBPASSWD1=0n10nus3r
+# WEBPASSWD2=0n10nus3r
diff --git a/setup/automation/distributed-net-centos-sensor b/setup/automation/distributed-net-centos-sensor
new file mode 100644
index 000000000..4cc3f6a75
--- /dev/null
+++ b/setup/automation/distributed-net-centos-sensor
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+# ALLOW_CIDR=0.0.0.0/0
+# ALLOW_ROLE=a
+BASICZEEK=2
+BASICSURI=2
+# BLOGS=
+BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+# GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-sensor
+install_type=SENSOR
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+# MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+MSRV=distributed-manager
+MSRVIP=10.66.166.42
+# MTU=
+# NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+# NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+# OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+# PLAYBOOK=1
+# REDIRECTHOST=
+# REDIRECTINFO=IP
+# RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+# STRELKA=1
+# THEHIVE=1
+# WAZUH=1
+# WEBUSER=onionuser@somewhere.invalid
+# WEBPASSWD1=0n10nus3r
+# WEBPASSWD2=0n10nus3r
diff --git a/setup/automation/distributed-net-ubuntu-manager b/setup/automation/distributed-net-ubuntu-manager
new file mode 100644
index 000000000..706a3d556
--- /dev/null
+++ b/setup/automation/distributed-net-ubuntu-manager
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=7
+BASICSURI=7
+# BLOGS=
+#BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-manager
+install_type=MANAGER
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=ens18
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=IP
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
diff --git a/setup/automation/distributed-net-ubuntu-search b/setup/automation/distributed-net-ubuntu-search
new file mode 100644
index 000000000..68d21e80e
--- /dev/null
+++ b/setup/automation/distributed-net-ubuntu-search
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+# ALLOW_CIDR=0.0.0.0/0
+# ALLOW_ROLE=a
+# BASICZEEK=7
+# BASICSURI=7
+# BLOGS=
+# BNICS=eth1
+# ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+# GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-search
+install_type=SEARCHNODE
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+# MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=ens18
+# MSEARCH=
+MSRV=distributed-manager
+MSRVIP=10.66.166.42
+# MTU=
+# NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+# OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+# PLAYBOOK=1
+# REDIRECTHOST=
+# REDIRECTINFO=IP
+# RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+# STRELKA=1
+# THEHIVE=1
+# WAZUH=1
+# WEBUSER=onionuser@somewhere.invalid
+# WEBPASSWD1=0n10nus3r
+# WEBPASSWD2=0n10nus3r
diff --git a/setup/automation/distributed-net-ubuntu-sensor b/setup/automation/distributed-net-ubuntu-sensor
new file mode 100644
index 000000000..572a1fa52
--- /dev/null
+++ b/setup/automation/distributed-net-ubuntu-sensor
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+# ALLOW_CIDR=0.0.0.0/0
+# ALLOW_ROLE=a
+BASICZEEK=2
+BASICSURI=2
+# BLOGS=
+BNICS=ens19
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+# GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=distributed-sensor
+install_type=SENSOR
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+# MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=ens18
+# MSEARCH=
+MSRV=distributed-manager
+MSRVIP=10.66.166.42
+# MTU=
+# NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+# NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+# OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+# PLAYBOOK=1
+# REDIRECTHOST=
+# REDIRECTINFO=IP
+# RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+SOREMOTEPASS1=onionuser
+SOREMOTEPASS2=onionuser
+# STRELKA=1
+# THEHIVE=1
+# WAZUH=1
+# WEBUSER=onionuser@somewhere.invalid
+# WEBPASSWD1=0n10nus3r
+# WEBPASSWD2=0n10nus3r
diff --git a/setup/automation/import-iso b/setup/automation/import-iso
new file mode 100644
index 000000000..7eb455b89
--- /dev/null
+++ b/setup/automation/import-iso
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+TESTING=true
+
+address_type=DHCP
+ADMINUSER=onionuser
+ADMINPASS1=onionuser
+ADMINPASS2=onionuser
+ALLOW_CIDR=0.0.0.0/0
+ALLOW_ROLE=a
+BASICZEEK=2
+BASICSURI=2
+# BLOGS=
+#BNICS=eth1
+ZEEKVERSION=ZEEK
+# CURCLOSEDAYS=
+# EVALADVANCED=BASIC
+GRAFANA=1
+# HELIXAPIKEY=
+HNMANAGER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12
+HNSENSOR=inherit
+HOSTNAME=import
+install_type=IMPORT
+# LSINPUTBATCHCOUNT=
+# LSINPUTTHREADS=
+# LSPIPELINEBATCH=
+# LSPIPELINEWORKERS=
+MANAGERADV=BASIC
+MANAGERUPDATES=1
+# MDNS=
+# MGATEWAY=
+# MIP=
+# MMASK=
+MNIC=eth0
+# MSEARCH=
+# MSRV=
+# MTU=
+NIDS=Suricata
+# NODE_ES_HEAP_SIZE=
+# NODE_LS_HEAP_SIZE=
+NODESETUP=NODEBASIC
+NSMSETUP=BASIC
+NODEUPDATES=MANAGER
+# OINKCODE=
+OSQUERY=1
+# PATCHSCHEDULEDAYS=
+# PATCHSCHEDULEHOURS=
+PATCHSCHEDULENAME=auto
+PLAYBOOK=1
+# REDIRECTHOST=
+REDIRECTINFO=IP
+RULESETUP=ETOPEN
+# SHARDCOUNT=
+# SKIP_REBOOT=
+# SOREMOTEPASS1=onionuser
+# SOREMOTEPASS2=onionuser
+STRELKA=1
+THEHIVE=1
+WAZUH=1
+WEBUSER=onionuser@somewhere.invalid
+WEBPASSWD1=0n10nus3r
+WEBPASSWD2=0n10nus3r
diff --git a/setup/so-functions b/setup/so-functions
index 7fba83ea5..1df065e71 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -17,6 +17,7 @@
# README - DO NOT DEFINE GLOBAL VARIABLES IN THIS FILE. Instead use so-variables.
+### Begin Logging Section ###
log() {
msg=$1
level=${2:-I}
@@ -41,51 +42,7 @@ logCmd() {
info "Executing command: $cmd"
$cmd >> "$setup_log" 2>&1
}
-
-filter_unused_nics() {
-
- if [[ $MNIC ]]; then local grep_string="$MNIC\|bond0"; else local grep_string="bond0"; fi
-
- # If we call this function and NICs have already been assigned to the bond interface then add them to the grep search string
- if [[ $BNICS ]]; then
- grep_string="$grep_string"
- for BONDNIC in "${BNICS[@]}"; do
- grep_string="$grep_string\|$BONDNIC"
- done
- fi
-
- # Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use)
- filtered_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string" | sed 's/ //g')
- readarray -t filtered_nics <<< "$filtered_nics"
-
- nic_list=()
- for nic in "${filtered_nics[@]}"; do
- case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in
- 1)
- nic_list+=("$nic" "Link UP " "OFF")
- ;;
- 0)
- nic_list+=("$nic" "Link DOWN " "OFF")
- ;;
- *)
- nic_list+=("$nic" "Link UNKNOWN " "OFF")
- ;;
- esac
- done
-
- export nic_list
-}
-
-calculate_useable_cores() {
-
- # Calculate reasonable core usage
- local cores_for_zeek=$(( (num_cpu_cores/2) - 1 ))
- local lb_procs_round
- lb_procs_round=$(printf "%.0f\n" $cores_for_zeek)
-
- if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi
- export lb_procs
-}
+### End Logging Section ###
airgap_rules() {
# Copy the rules for suricata if using Airgap
@@ -99,16 +56,6 @@ airgap_rules() {
cp -Rv /root/SecurityOnion/agrules/strelka /nsm/repo/rules/
}
-analyze_system() {
- title "System Characteristics"
- logCmd "uptime"
- logCmd "uname -a"
- logCmd "free -h"
- logCmd "lscpu"
- logCmd "df -h"
- logCmd "ip a"
-}
-
accept_salt_key_remote() {
systemctl restart salt-minion
@@ -146,24 +93,6 @@ addtotab_generate_templates() {
}
-# $5 => (optional) password variable
-so_add_user() {
- local username=$1
- local uid=$2
- local gid=$3
- local home_dir=$4
- if [ "$5" ]; then local pass=$5; fi
-
- echo "Add $username user" >> "$setup_log" 2>&1
- groupadd --gid "$gid" "$username"
- useradd -m --uid "$uid" --gid "$gid" --home-dir "$home_dir" "$username"
-
- # If a password has been passed in, set the password
- if [ "$pass" ]; then
- echo "$username":"$pass" | chpasswd --crypt-method=SHA512
- fi
-}
-
add_socore_user_manager() {
so_add_user "socore" "939" "939" "/opt/so" >> "$setup_log" 2>&1
}
@@ -172,29 +101,6 @@ add_soremote_user_manager() {
so_add_user "soremote" "947" "947" "/home/soremote" "$SOREMOTEPASS1" >> "$setup_log" 2>&1
}
-wait_for_file() {
- local filename=$1
- local max_attempts=$2 # this is multiplied by the wait interval, so make sure it isn't too large
- local cur_attempts=0
- local wait_interval=$3
- local total_time=$(( max_attempts * wait_interval ))
- local date
- date=$(date)
-
- while [[ $cur_attempts -lt $max_attempts ]]; do
- if [ -f "$filename" ]; then
- echo "File $filename found at $date" >> "$setup_log" 2>&1
- return 0
- else
- ((cur_attempts++))
- echo "File $filename does not exist; waiting ${wait_interval}s then checking again ($cur_attempts/$max_attempts)..." >> "$setup_log" 2>&1
- sleep "$wait_interval"
- fi
- done
- echo "Could not find $filename after waiting ${total_time}s" >> "$setup_log" 2>&1
- return 1
-}
-
add_web_user() {
wait_for_file /opt/so/conf/kratos/db/db.sqlite 30 5
{
@@ -204,22 +110,25 @@ add_web_user() {
} >> "/root/so-user-add.log" 2>&1
}
-# Create an secrets pillar so that passwords survive re-install
-secrets_pillar(){
- if [ ! -f $local_salt_dir/pillar/secrets.sls ]; then
- echo "Creating Secrets Pillar" >> "$setup_log" 2>&1
- mkdir -p $local_salt_dir/pillar
- printf '%s\n'\
- "secrets:"\
- " mysql: $MYSQLPASS"\
- " playbook_db: $PLAYBOOKDBPASS"\
- " playbook_admin: $PLAYBOOKADMINPASS"\
- " playbook_automation: $PLAYBOOKAUTOMATIONPASS"\
- " grafana_admin: $GRAFANAPASS"\
- " fleet: $FLEETPASS"\
- " fleet_jwt: $FLEETJWT"\
- " fleet_enroll-secret: False" > $local_salt_dir/pillar/secrets.sls
- fi
+analyze_system() {
+ title "System Characteristics"
+ logCmd "uptime"
+ logCmd "uname -a"
+ logCmd "free -h"
+ logCmd "lscpu"
+ logCmd "df -h"
+ logCmd "ip a"
+}
+
+calculate_useable_cores() {
+
+ # Calculate reasonable core usage
+ local cores_for_zeek=$(( (num_cpu_cores/2) - 1 ))
+ local lb_procs_round
+ lb_procs_round=$(printf "%.0f\n" $cores_for_zeek)
+
+ if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi
+ export lb_procs
}
check_admin_pass() {
@@ -825,6 +734,27 @@ check_requirements() {
fi
}
+check_sos_appliance() {
+ # Lets see if this is a SOS Appliance
+ if [ -f "/etc/SOSMODEL"]; then
+ local MODEL=$(cat /etc/SOSMODEL)
+ echo "Found SOS Model $MODEL"
+ echo "sosmodel: $MODEL" >> /etc/salt/grains
+ fi
+}
+
+compare_main_nic_ip() {
+ if [[ "$MAINIP" != "$MNIC_IP" ]]; then
+ read -r -d '' message <<- EOM
+ The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC).
+
+ This is not a supported configuration, please remediate and rerun setup.
+ EOM
+ whiptail --title "Security Onion Setup" --msgbox "$message" 10 75
+ kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1
+ fi
+}
+
compare_versions() {
manager_ver=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" cat /etc/soversion)
@@ -1219,6 +1149,95 @@ download_repo_tarball() {
} >> "$setup_log" 2>&1
}
+elasticsearch_pillar() {
+
+ local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls
+
+ # Create the node pillar
+ printf '%s\n'\
+ "elasticsearch:"\
+ " mainip: '$MAINIP'"\
+ " mainint: '$MNIC'"\
+ " esheap: '$NODE_ES_HEAP_SIZE'" >> "$pillar_file"
+ if [ -n "$ESCLUSTERNAME" ]; then
+ printf '%s\n'\
+ " esclustername: $ESCLUSTERNAME" >> "$pillar_file"
+ else
+ printf '%s\n'\
+ " esclustername: {{ grains.host }}" >> "$pillar_file"
+ fi
+ printf '%s\n'\
+ " node_type: '$NODETYPE'"\
+ " es_port: $node_es_port"\
+ " log_size_limit: $log_size_limit"\
+ " node_route_type: 'hot'"\
+ "" >> "$pillar_file"
+
+ printf '%s\n'\
+ "logstash_settings:"\
+ " ls_pipeline_batch_size: $LSPIPELINEBATCH"\
+ " ls_input_threads: $LSINPUTTHREADS"\
+ " lsheap: $NODE_LS_HEAP_SIZE"\
+ " ls_pipeline_workers: $num_cpu_cores"\
+ "" >> "$pillar_file"
+
+}
+
+es_heapsize() {
+
+ # Determine ES Heap Size
+ if [ "$total_mem" -lt 8000 ] ; then
+ ES_HEAP_SIZE="600m"
+ elif [ "$total_mem" -ge 100000 ]; then
+ # Set a max of 25GB for heap size
+ # https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html
+ ES_HEAP_SIZE="25000m"
+ else
+ # Set heap size to 25% of available memory
+ ES_HEAP_SIZE=$(( total_mem / 4 ))"m"
+ fi
+ export ES_HEAP_SIZE
+
+ if [[ "$install_type" =~ ^(EVAL|MANAGERSEARCH|STANDALONE|IMPORT)$ ]]; then
+ NODE_ES_HEAP_SIZE=ES_HEAP_SIZE
+ export NODE_ES_HEAP_SIZE
+ fi
+}
+
+filter_unused_nics() {
+
+ if [[ $MNIC ]]; then local grep_string="$MNIC\|bond0"; else local grep_string="bond0"; fi
+
+ # If we call this function and NICs have already been assigned to the bond interface then add them to the grep search string
+ if [[ $BNICS ]]; then
+ grep_string="$grep_string"
+ for BONDNIC in "${BNICS[@]}"; do
+ grep_string="$grep_string\|$BONDNIC"
+ done
+ fi
+
+ # Finally, set filtered_nics to any NICs we aren't using (and ignore interfaces that aren't of use)
+ filtered_nics=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "$grep_string" | sed 's/ //g')
+ readarray -t filtered_nics <<< "$filtered_nics"
+
+ nic_list=()
+ for nic in "${filtered_nics[@]}"; do
+ case $(cat "/sys/class/net/${nic}/carrier" 2>/dev/null) in
+ 1)
+ nic_list+=("$nic" "Link UP " "OFF")
+ ;;
+ 0)
+ nic_list+=("$nic" "Link DOWN " "OFF")
+ ;;
+ *)
+ nic_list+=("$nic" "Link UNKNOWN " "OFF")
+ ;;
+ esac
+ done
+
+ export nic_list
+}
+
fireeye_pillar() {
local fireeye_pillar_path=$local_salt_dir/pillar/fireeye
@@ -1369,6 +1388,33 @@ import_registry_docker() {
fi
}
+# Set Logstash heap size based on total memory
+ls_heapsize() {
+
+ if [ "$total_mem" -ge 32000 ]; then
+ LS_HEAP_SIZE='1000m'
+ return
+ fi
+
+ case "$install_type" in
+ 'MANAGERSEARCH' | 'HEAVYNODE' | 'HELIXSENSOR' | 'STANDALONE')
+ LS_HEAP_SIZE='1000m'
+ ;;
+ 'EVAL')
+ LS_HEAP_SIZE='700m'
+ ;;
+ *)
+ LS_HEAP_SIZE='500m'
+ ;;
+ esac
+ export LS_HEAP_SIZE
+
+ if [[ "$install_type" =~ ^(EVAL|MANAGERSEARCH|STANDALONE)$ ]]; then
+ NODE_LS_HEAP_SIZE=LS_HEAP_SIZE
+ export NODE_LS_HEAP_SIZE
+ fi
+}
+
manager_pillar() {
local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls
@@ -1608,6 +1654,11 @@ manager_global() {
printf '%s\n' '----' >> "$setup_log" 2>&1
}
+mark_version() {
+ # Drop a file with the current version
+ echo "$SOVERSION" > /etc/soversion
+}
+
minio_generate_keys() {
local charSet="[:graph:]"
@@ -1669,40 +1720,6 @@ network_setup() {
} >> "$setup_log" 2>&1
}
-elasticsearch_pillar() {
-
- local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls
-
- # Create the node pillar
- printf '%s\n'\
- "elasticsearch:"\
- " mainip: '$MAINIP'"\
- " mainint: '$MNIC'"\
- " esheap: '$NODE_ES_HEAP_SIZE'" >> "$pillar_file"
- if [ -n "$ESCLUSTERNAME" ]; then
- printf '%s\n'\
- " esclustername: $ESCLUSTERNAME" >> "$pillar_file"
- else
- printf '%s\n'\
- " esclustername: {{ grains.host }}" >> "$pillar_file"
- fi
- printf '%s\n'\
- " node_type: '$NODETYPE'"\
- " es_port: $node_es_port"\
- " log_size_limit: $log_size_limit"\
- " node_route_type: 'hot'"\
- "" >> "$pillar_file"
-
- printf '%s\n'\
- "logstash_settings:"\
- " ls_pipeline_batch_size: $LSPIPELINEBATCH"\
- " ls_input_threads: $LSINPUTTHREADS"\
- " lsheap: $NODE_LS_HEAP_SIZE"\
- " ls_pipeline_workers: $num_cpu_cores"\
- "" >> "$pillar_file"
-
-}
-
parse_install_username() {
# parse out the install username so things copy correctly
INSTALLUSERNAME=${SUDO_USER:-${USER}}
@@ -2140,6 +2157,24 @@ salt_firstcheckin() {
salt-call state.show_top >> /dev/null 2>&1 # send output to /dev/null because we don't actually care about the ouput
}
+# Create an secrets pillar so that passwords survive re-install
+secrets_pillar(){
+ if [ ! -f $local_salt_dir/pillar/secrets.sls ]; then
+ echo "Creating Secrets Pillar" >> "$setup_log" 2>&1
+ mkdir -p $local_salt_dir/pillar
+ printf '%s\n'\
+ "secrets:"\
+ " mysql: $MYSQLPASS"\
+ " playbook_db: $PLAYBOOKDBPASS"\
+ " playbook_admin: $PLAYBOOKADMINPASS"\
+ " playbook_automation: $PLAYBOOKAUTOMATIONPASS"\
+ " grafana_admin: $GRAFANAPASS"\
+ " fleet: $FLEETPASS"\
+ " fleet_jwt: $FLEETJWT"\
+ " fleet_enroll-secret: False" > $local_salt_dir/pillar/secrets.sls
+ fi
+}
+
set_base_heapsizes() {
es_heapsize
ls_heapsize
@@ -2155,18 +2190,6 @@ set_main_ip() {
MNIC_IP=$(ip a s "$MNIC" | grep -oE 'inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' | cut -d' ' -f2)
}
-compare_main_nic_ip() {
- if [[ "$MAINIP" != "$MNIC_IP" ]]; then
- read -r -d '' message <<- EOM
- The IP being routed by Linux is not the IP address assigned to the management interface ($MNIC).
-
- This is not a supported configuration, please remediate and rerun setup.
- EOM
- whiptail --title "Security Onion Setup" --msgbox "$message" 10 75
- kill -SIGINT "$(ps --pid $$ -oppid=)"; exit 1
- fi
-}
-
# Add /usr/sbin to everyone's path
set_path() {
echo "complete -cf sudo" > /etc/profile.d/securityonion.sh
@@ -2440,6 +2463,24 @@ set_updates() {
fi
}
+# $5 => (optional) password variable
+so_add_user() {
+ local username=$1
+ local uid=$2
+ local gid=$3
+ local home_dir=$4
+ if [ "$5" ]; then local pass=$5; fi
+
+ echo "Add $username user" >> "$setup_log" 2>&1
+ groupadd --gid "$gid" "$username"
+ useradd -m --uid "$uid" --gid "$gid" --home-dir "$home_dir" "$username"
+
+ # If a password has been passed in, set the password
+ if [ "$pass" ]; then
+ echo "$username":"$pass" | chpasswd --crypt-method=SHA512
+ fi
+}
+
steno_pillar() {
local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls
@@ -2451,11 +2492,6 @@ steno_pillar() {
}
-mark_version() {
- # Drop a file with the current version
- echo "$SOVERSION" > /etc/soversion
-}
-
update_sudoers_for_testing() {
if [ -n "$TESTING" ]; then
info "Ensuring $INSTALLUSERNAME has password-less sudo access for automated testing purposes."
@@ -2507,56 +2543,29 @@ use_turbo_proxy() {
fi
}
-# Set Logstash heap size based on total memory
-ls_heapsize() {
+wait_for_file() {
+ local filename=$1
+ local max_attempts=$2 # this is multiplied by the wait interval, so make sure it isn't too large
+ local cur_attempts=0
+ local wait_interval=$3
+ local total_time=$(( max_attempts * wait_interval ))
+ local date
+ date=$(date)
- if [ "$total_mem" -ge 32000 ]; then
- LS_HEAP_SIZE='1000m'
- return
- fi
-
- case "$install_type" in
- 'MANAGERSEARCH' | 'HEAVYNODE' | 'HELIXSENSOR' | 'STANDALONE')
- LS_HEAP_SIZE='1000m'
- ;;
- 'EVAL')
- LS_HEAP_SIZE='700m'
- ;;
- *)
- LS_HEAP_SIZE='500m'
- ;;
- esac
- export LS_HEAP_SIZE
-
- if [[ "$install_type" =~ ^(EVAL|MANAGERSEARCH|STANDALONE)$ ]]; then
- NODE_LS_HEAP_SIZE=LS_HEAP_SIZE
- export NODE_LS_HEAP_SIZE
- fi
+ while [[ $cur_attempts -lt $max_attempts ]]; do
+ if [ -f "$filename" ]; then
+ echo "File $filename found at $date" >> "$setup_log" 2>&1
+ return 0
+ else
+ ((cur_attempts++))
+ echo "File $filename does not exist; waiting ${wait_interval}s then checking again ($cur_attempts/$max_attempts)..." >> "$setup_log" 2>&1
+ sleep "$wait_interval"
+ fi
+ done
+ echo "Could not find $filename after waiting ${total_time}s" >> "$setup_log" 2>&1
+ return 1
}
-
-es_heapsize() {
-
- # Determine ES Heap Size
- if [ "$total_mem" -lt 8000 ] ; then
- ES_HEAP_SIZE="600m"
- elif [ "$total_mem" -ge 100000 ]; then
- # Set a max of 25GB for heap size
- # https://www.elastic.co/guide/en/elasticsearch/guide/current/heap-sizing.html
- ES_HEAP_SIZE="25000m"
- else
- # Set heap size to 25% of available memory
- ES_HEAP_SIZE=$(( total_mem / 4 ))"m"
- fi
- export ES_HEAP_SIZE
-
- if [[ "$install_type" =~ ^(EVAL|MANAGERSEARCH|STANDALONE|IMPORT)$ ]]; then
- NODE_ES_HEAP_SIZE=ES_HEAP_SIZE
- export NODE_ES_HEAP_SIZE
- fi
-}
-
-
whiptail_prog_new_message() {
local message=$1
set_progress_str "$percentage" "$message"
diff --git a/setup/so-setup b/setup/so-setup
index 0da0b9f8d..68800c85b 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -604,6 +604,7 @@ set_redirect >> $setup_log 2>&1
set_progress_str 8 'Initializing Salt minion'
configure_minion "$minion_type" >> $setup_log 2>&1
+ check_sos_appliance >> $setup_log 2>&1
update_sudoers_for_testing >> $setup_log 2>&1