diff --git a/salt/common/init.sls b/salt/common/init.sls index 82ac4a062..09d71114b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -28,20 +28,83 @@ salttmp: - group: 939 - makedirs: True -# Install packages needed for the sensor -sensorpkgs: +# Install epel +{% if grains['os'] == 'CentOS' %} +epel: pkg.installed: - - skip_suggestions: False + - skip_suggestions: True + - pkgs: + - epel-release +{% endif %} + +# Install common packages +{% if grains['os'] != 'CentOS' %} +commonpkgs: + pkg.installed: + - skip_suggestions: True + - pkgs: + - apache2-utils + - wget + - ntpdate + - jq + - python3-docker + - docker-ce + - curl + - ca-certificates + - software-properties-common + - apt-transport-https + - openssl + - netcat + - python3-mysqldb + - sqlite3 + - argon2 + - libssl-dev + - python3-dateutil + - python3-m2crypto + - python3-mysqldb +heldpackages: + pkg.installed: + - pkgs: + - containerd.io: 1.2.13-2 + - docker-ce: 5:19.03.9~3-0~ubuntu-bionic + - hold: True + - update_holds: True + +{% else %} +commonpkgs: + pkg.installed: + - skip_suggestions: True - pkgs: - wget + - ntpdate + - bind-utils - jq - {% if grains['os'] != 'CentOS' %} - - apache2-utils - {% else %} - - net-tools - tcpdump - httpd-tools - {% endif %} + - net-tools + - curl + - sqlite + - argon2 + - mariadb-devel + - nmap-ncat + - python3 + - python36-docker + - python36-dateutil + - python36-m2crypto + - python36-mysql + - yum-utils + - device-mapper-persistent-data + - lvm2 + - openssl + +heldpackages: + pkg.installed: + - pkgs: + - containerd.io: 1.2.13-3.2.el7 + - docker-ce: 3:19.03.9-3.el7 + - hold: True + - update_holds: True +{% endif %} # Always keep these packages up to date diff --git a/salt/common/tools/sbin/so-elastalert-create b/salt/common/tools/sbin/so-elastalert-create index fbe9527a7..0270503bf 100755 --- a/salt/common/tools/sbin/so-elastalert-create +++ b/salt/common/tools/sbin/so-elastalert-create @@ -166,8 +166,7 @@ cat << EOF What elasticsearch index do you want to use? Below are the default Index Patterns used in Security Onion: -*:logstash-* -*:logstash-beats-* +*:so-ids-* *:elastalert_status* EOF diff --git a/salt/curator/files/action/close.yml b/salt/curator/files/action/close.yml index dfe5519e8..a65e9af3d 100644 --- a/salt/curator/files/action/close.yml +++ b/salt/curator/files/action/close.yml @@ -24,9 +24,8 @@ actions: disable_action: False filters: - filtertype: pattern - kind: prefix - value: logstash- - exclude: + kind: regex + value: '^(logstash-.*|so-.*)$' - filtertype: age source: name direction: older diff --git a/salt/curator/files/action/delete.yml b/salt/curator/files/action/delete.yml index e6f2f3833..030bbbfac 100644 --- a/salt/curator/files/action/delete.yml +++ b/salt/curator/files/action/delete.yml @@ -20,8 +20,8 @@ actions: disable_action: False filters: - filtertype: pattern - kind: prefix - value: logstash- + kind: regex + value: '^(logstash-.*|so-.*)$' - filtertype: space source: creation_date use_age: True diff --git a/salt/curator/files/bin/so-curator-closed-delete-delete b/salt/curator/files/bin/so-curator-closed-delete-delete index b0ec62424..3d397defc 100755 --- a/salt/curator/files/bin/so-curator-closed-delete-delete +++ b/salt/curator/files/bin/so-curator-closed-delete-delete @@ -33,17 +33,17 @@ LOG="/opt/so/log/curator/so-curator-closed-delete.log" # Check for 2 conditions: # 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT? -# 2. Are there any closed logstash- indices that we can delete? +# 2. Are there any closed logstash-, or so- indices that we can delete? # If both conditions are true, keep on looping until one of the conditions is false. while [[ $(du -hs --block-size=1GB /nsm/elasticsearch/nodes | awk '{print $1}' ) -gt "{{LOG_SIZE_LIMIT}}" ]] && -curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^ close logstash-" > /dev/null; do +curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E "^ close (logstash-|so-)" > /dev/null; do # We need to determine OLDEST_INDEX. - # First, get the list of closed indices that are prefixed with "logstash-". + # First, get the list of closed indices that are prefixed with "logstash-" or "so-". # For example: logstash-ids-YYYY.MM.DD # Then, sort by date by telling sort to use hyphen as delimiter and then sort on the third field. # Finally, select the first entry in that sorted list. - OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep "^ close logstash-" | awk '{print $2}' | sort -t- -k3 | head -1) + OLDEST_INDEX=$(curl -s {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/_cat/indices | grep -E "^ close (logstash-|so-)" | awk '{print $2}' | sort -t- -k3 | head -1) # Now that we've determined OLDEST_INDEX, ask Elasticsearch to delete it. curl -XDELETE {{ELASTICSEARCH_HOST}}:{{ELASTICSEARCH_PORT}}/${OLDEST_INDEX} diff --git a/salt/elastalert/files/rules/so/nids2hive.yaml b/salt/elastalert/files/rules/so/nids2hive.yaml index 68a173fcd..097511d56 100644 --- a/salt/elastalert/files/rules/so/nids2hive.yaml +++ b/salt/elastalert/files/rules/so/nids2hive.yaml @@ -40,7 +40,7 @@ hive_alert_config: title: '{match[rule][name]}' type: 'NIDS' source: 'SecurityOnion' - description: "`Hunting Pivot:` \n\n \n\n `Kibana Dashboard:` \n\n \n\n `IPs: `{match[source][ip]}:{match[source][port]} --> {match[destination][ip]}:{match[destination][port]} \n\n `Signature:`{match[rule][rule]}" + description: "`Hunting Pivot:` \n\n \n\n `Kibana Dashboard - Signature Drilldown:` \n\n \n\n `Kibana Dashboard - Community_ID:` \n\n \n\n `IPs: `{match[source][ip]}:{match[source][port]} --> {match[destination][ip]}:{match[destination][port]} \n\n `Signature:`{match[rule][rule]}" severity: 2 tags: ['{match[rule][uuid]}','{match[source][ip]}','{match[destination][ip]}'] tlp: 3 diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 73f3c9239..271ef40cf 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -22,3 +22,7 @@ transport.bind_host: 0.0.0.0 transport.publish_host: {{ nodeip }} transport.publish_port: 9300 {%- endif %} +cluster.routing.allocation.disk.threshold_enabled: true +cluster.routing.allocation.disk.watermark.low: 95% +cluster.routing.allocation.disk.watermark.high: 98% +cluster.routing.allocation.disk.watermark.flood_stage: 98% diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index e70d5e2d8..a65742f99 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -38,7 +38,7 @@ { "rename": { "field": "module", "target_field": "event.module", "ignore_missing": true } }, { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_missing": true } }, { "rename": { "field": "category", "target_field": "event.category", "ignore_missing": true } }, - { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_missing": true } }, + { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } }, { "remove": { "field": [ "index_name_prefix", "message2", "type" ], diff --git a/salt/elasticsearch/files/ingest/osquery.query_result b/salt/elasticsearch/files/ingest/osquery.query_result index e9cdbe2d3..5b37655f9 100644 --- a/salt/elasticsearch/files/ingest/osquery.query_result +++ b/salt/elasticsearch/files/ingest/osquery.query_result @@ -31,7 +31,7 @@ { "rename": { "field": "message3.columns.remote_port", "target_field": "remote.port", "ignore_missing": true } }, { "rename": { "field": "message3.columns.process_name", "target_field": "process.name", "ignore_missing": true } }, { "rename": { "field": "message3.columns.eventid", "target_field": "event.code", "ignore_missing": true } }, - { "set": { "if": "ctx.message3.columns.?data != null", "field": "dataset", "value": "wel-{{message3.columns.source}}", "override": true } }, + { "set": { "if": "ctx.message3.columns?.data != null", "field": "dataset", "value": "wel-{{message3.columns.source}}", "override": true } }, { "rename": { "field": "message3.columns.winlog.EventData.SubjectUserName", "target_field": "user.name", "ignore_missing": true } }, { "rename": { "field": "message3.columns.winlog.EventData.destinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, { "rename": { "field": "message3.columns.winlog.EventData.destinationIp", "target_field": "destination.ip", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common index c31625db6..b0ac0d12a 100644 --- a/salt/elasticsearch/files/ingest/zeek.common +++ b/salt/elasticsearch/files/ingest/zeek.common @@ -6,7 +6,8 @@ { "dot_expander": { "field": "id.orig_h", "path": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.orig_p", "path": "message2", "ignore_failure": true } }, { "dot_expander": { "field": "id.resp_h", "path": "message2", "ignore_failure": true } }, - { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + { "dot_expander": { "field": "id.resp_p", "path": "message2", "ignore_failure": true } }, + {"community_id": {"if": "ctx.network?.transport != null", "field":["message2.id.orig_h","message2.id.orig_p","message2.id.resp_h","message2.id.resp_p","network.transport"],"target_field":"network.community_id"}}, { "rename": { "field": "message2.id.orig_h", "target_field": "source.ip", "ignore_missing": true } }, { "rename": { "field": "message2.id.orig_p", "target_field": "source.port", "ignore_missing": true } }, { "rename": { "field": "message2.id.resp_h", "target_field": "destination.ip", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.http b/salt/elasticsearch/files/ingest/zeek.http index a1354044c..3368e45e1 100644 --- a/salt/elasticsearch/files/ingest/zeek.http +++ b/salt/elasticsearch/files/ingest/zeek.http @@ -29,6 +29,7 @@ { "script": { "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", "ignore_failure": true } }, { "script": { "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()", "ignore_failure": true } }, { "script": { "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", "ignore_failure": true } }, + { "set": { "field": "network.transport", "value": "tcp" } }, { "pipeline": { "name": "zeek.common" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.notice b/salt/elasticsearch/files/ingest/zeek.notice index 4e54f325d..b662393f6 100644 --- a/salt/elasticsearch/files/ingest/zeek.notice +++ b/salt/elasticsearch/files/ingest/zeek.notice @@ -6,7 +6,7 @@ { "rename": { "field": "message2.fuid", "target_field": "log.id.fuid", "ignore_missing": true } }, { "rename": { "field": "message2.mime", "target_field": "file.mimetype", "ignore_missing": true } }, { "rename": { "field": "message2.desc", "target_field": "file.description", "ignore_missing": true } }, - { "rename": { "field": "message2.proto", "target_field": "network.protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } }, { "rename": { "field": "message2.note", "target_field": "notice.note", "ignore_missing": true } }, { "rename": { "field": "message2.msg", "target_field": "notice.message", "ignore_missing": true } }, { "rename": { "field": "message2.sub", "target_field": "notice.sub_message", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/zeek.smtp b/salt/elasticsearch/files/ingest/zeek.smtp index 473b4cce5..9bfb1e3e1 100644 --- a/salt/elasticsearch/files/ingest/zeek.smtp +++ b/salt/elasticsearch/files/ingest/zeek.smtp @@ -25,6 +25,7 @@ { "rename": { "field": "message2.tls", "target_field": "smtp.tls", "ignore_missing": true } }, { "rename": { "field": "message2.fuids", "target_field": "log.id.fuids", "ignore_missing": true } }, { "rename": { "field": "message2.is_webmail", "target_field": "smtp.is_webmail", "ignore_missing": true } }, + { "set": { "field": "network.transport", "value": "tcp" } }, { "pipeline": { "name": "zeek.common" } } ] } diff --git a/salt/elasticsearch/files/so-elasticsearch-pipelines b/salt/elasticsearch/files/so-elasticsearch-pipelines index b1b6db158..514054359 100755 --- a/salt/elasticsearch/files/so-elasticsearch-pipelines +++ b/salt/elasticsearch/files/so-elasticsearch-pipelines @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +RETURN_CODE=0 ELASTICSEARCH_HOST=$1 ELASTICSEARCH_PORT=9200 @@ -46,7 +47,9 @@ fi cd ${ELASTICSEARCH_INGEST_PIPELINES} echo "Loading pipelines..." -for i in *; do echo $i; curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null; echo; done +for i in *; do echo $i; RESPONSE=$(curl ${ELASTICSEARCH_AUTH} -XPUT http://${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ingest/pipeline/$i -H 'Content-Type: application/json' -d@$i 2>/dev/null); echo $RESPONSE; if [[ "$RESPONSE" == *"error"* ]]; then RETURN_CODE=1; fi; done echo cd - >/dev/null + +exit $RETURN_CODE \ No newline at end of file diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 6b76e622c..76770e2bd 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -32,7 +32,7 @@ "dateRangeMinutes": 1440, "mostRecentlyUsedLimit": 5, "eventFields": { - "default": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid" ], + "default": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "log.id.uid", "network.community_id" ], "bro_conn": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "protocol", "service", "log.id.uid" ], "bro_dce_rpc": ["soc_timestamp", "source.ip", "source.port", "destination.ip", "destination.port", "endpoint", "named_pipe", "operation", "log.id.uid" ], "bro_dhcp": ["soc_timestamp", "source.ip", "destination.ip", "domain_name", "hostname", "message_types", "log.id.uid" ], diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index f2415d010..e5878cb70 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -1,9 +1,9 @@ -{%- set ip = salt['pillar.get']('static:masterip', '') %} +{%- set MASTER = salt['pillar.get']('master:url_base', '') %} {%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') %} {%- set CORTEXKEY = salt['pillar.get']('static:cortexorguserkey', '') %} [es] -es_url = http://{{ip}}:9200 +es_url = http://{{MASTER}}:9200 es_user = YOURESUSER es_pass = YOURESPASS es_index_pattern = so-* @@ -11,7 +11,7 @@ es_verifycert = no [cortex] auto_analyze_alerts = no -cortex_url = https://{{ip}}/cortex/ +cortex_url = https://{{MASTER}}/cortex/ cortex_key = {{ CORTEXKEY }} supported_analyzers = Urlscan_io_Search,CERTatPassiveDNS @@ -32,7 +32,7 @@ grr_user = YOURGRRUSER grr_pass = YOURGRRPASS [hive] -hive_url = https://{{ip}}/thehive/ +hive_url = https://{{MASTER}}/thehive/ hive_key = {{ HIVEKEY }} hive_tlp = 3 hive_verifycert = no @@ -59,7 +59,7 @@ slack_url = YOURSLACKWORKSPACE slack_webhook = YOURSLACKWEBHOOK [playbook] -playbook_url = https://{{ip}}/playbook +playbook_url = https://{{MASTER}}/playbook playbook_key = de6639318502476f2fa5aa06f43f51fb389a3d7f playbook_verifycert = no diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 8bdbd8274..5767531f4 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -112,5 +112,5 @@ strelka_filestream: strelka_zeek_extracted_sync: cron.present: - user: root - - name: mv /nsm/zeek/extracted/complete/* /nsm/strelka + - name: '[ -d /nsm/zeek/extracted/complete/ ] && mv /nsm/zeek/extracted/complete/* /nsm/strelka/ > /dev/null 2>&1' - minute: '*' diff --git a/setup/automation/pm_standalone_defaults b/setup/automation/pm_standalone_defaults new file mode 100644 index 000000000..b5a6258ff --- /dev/null +++ b/setup/automation/pm_standalone_defaults @@ -0,0 +1,75 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +TESTING=true + +address_type=DHCP +ADMINUSER=onionuser +ADMINPASS1=onionuser +ADMINPASS2=onionuser +BASICBRO=7 +BASICSURI=7 +# BLOGS= +BNICS=eth1 +BROVERSION=ZEEK +# CURCLOSEDAYS= +# EVALADVANCED=BASIC +GRAFANA=1 +# HELIXAPIKEY= +HNMASTER=10.0.0.0/8,192.168.0.0/16,172.16.0.0/12 +HNSENSOR=inherit +HOSTNAME=standalone +install_type=STANDALONE +# LSINPUTBATCHCOUNT= +# LSINPUTTHREADS= +# LSPIPELINEBATCH= +# LSPIPELINEWORKERS= +MASTERADV=BASIC +MASTERUPDATES=1 +# MDNS= +# MGATEWAY= +# MIP= +# MMASK= +MNIC=eth0 +# MSEARCH= +# MSRV= +# MTU= +NAVIGATOR=1 +NIDS=Suricata +# NODE_ES_HEAP_SIZE= +# NODE_LS_HEAP_SIZE= +NODESETUP=NODEBASIC +NSMSETUP=BASIC +NODEUPDATES=MASTER +# OINKCODE= +OSQUERY=1 +# PATCHSCHEDULEDAYS= +# PATCHSCHEDULEHOURS= +PATCHSCHEDULENAME=auto +PLAYBOOK=1 +# REDIRECTHOST= +REDIRECTINFO=IP +RULESETUP=ETOPEN +# SHARDCOUNT= +SOREMOTEPASS1=onionuser +SOREMOTEPASS2=onionuser +STRELKA=1 +THEHIVE=1 +WAZUH=1 +WEBUSER=onionuser@somewhere.invalid +WEBPASSWD1=onionuser +WEBPASSWD2=onionuser \ No newline at end of file diff --git a/setup/proxies/docker.conf b/setup/proxies/docker.conf new file mode 100644 index 000000000..9ab2c4b4c --- /dev/null +++ b/setup/proxies/docker.conf @@ -0,0 +1,2 @@ +[Service] +ExecStart=/usr/bin/dockerd /usr/bin/dockerd -H fd:// --registry-mirror "$proxy_addr" diff --git a/setup/so-functions b/setup/so-functions index 922046d5e..7a1e7ec80 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -19,7 +19,7 @@ source ./so-whiptail source ./so-variables source ./so-common-functions -SOVERSION=1.3.0 +SOVERSION=1.4.0 accept_salt_key_remote() { systemctl restart salt-minion @@ -514,7 +514,7 @@ detect_os() { # Install bind-utils so the host command exists if ! command -v host > /dev/null 2>&1; then echo "Installing required packages to run installer" - yum -y install bind-utils >> "$setup_log" 2>&1 + yum -y install bind-utils yum-plugin-versionlock >> "$setup_log" 2>&1 fi @@ -550,6 +550,9 @@ detect_os() { disable_onion_user() { # Disable the default account cause security. usermod -L onion + + # Remove the automated setup script from crontab, if it exists + crontab -u onion -r } disable_misc_network_features() { @@ -580,7 +583,9 @@ docker_install() { { yum clean expire-cache; yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo; - yum -y install docker-ce; + yum -y install docker-ce-19.03.9-3.el7 containerd.io-1.2.6-3.el7; + yum versionlock docker-ce-19.03.9-3.el7; + yum versionlock containerd.io-1.2.6-3.el7 } >> "$setup_log" 2>&1 else @@ -616,9 +621,10 @@ docker_registry() { echo "Setting up Docker Registry" >> "$setup_log" 2>&1 mkdir -p /etc/docker >> "$setup_log" 2>&1 # Make the host use the master docker registry + if [ -n "$TURBO" ]; then local proxy="$TURBO"; else local proxy="https://$MSRV"; fi printf '%s\n'\ "{"\ - " \"registry-mirrors\": [\"https://$MSRV:5000\"]"\ + " \"registry-mirrors\": [ \"$proxy:5000\" ]"\ "}" > /etc/docker/daemon.json echo "Docker Registry Setup - Complete" >> "$setup_log" 2>&1 @@ -683,7 +689,7 @@ docker_seed_registry() { # Tag it with the new registry destination docker tag soshybridhunter/"$i" "$HOSTNAME":5000/soshybridhunter/"$i" docker push "$HOSTNAME":5000/soshybridhunter/"$i" - docker rmi soshybridhunter/"$i" + #docker rmi soshybridhunter/"$i" } >> "$setup_log" 2>&1 done else @@ -948,6 +954,54 @@ node_pillar() { cat "$pillar_file" >> "$setup_log" 2>&1 } +parse_options() { + case "$1" in + --turbo=*) + local proxy + proxy=$(echo "$1" | tr -d '"' | awk -F'--turbo=' '{print $2}') + proxy_url="http://$proxy" + TURBO="$proxy_url" + ;; + --proxy=*) + local proxy + proxy=$(echo "$1" | tr -d '"' | awk -F'--proxy=' '{print $2}') + + local proxy_protocol + proxy_protocol=$(echo "$proxy" | awk 'match($0, /http|https/) { print substr($0, RSTART, RLENGTH) }') + + if [[ ! $proxy_protocol =~ ^(http|https)$ ]]; then + echo "Invalid proxy protocol" + echo "Ignoring proxy" + return + fi + + if [[ $2 == --proxy-user=* && $3 == --proxy-pass=* ]]; then + local proxy_user + local proxy_password + proxy_user=$(echo "$2" | tr -d '"' | awk -F'--proxy-user=' '{print $2}') + proxy_password=$(echo "$3" | tr -d '"' | awk -F'--proxy-pass=' '{print $2}') + + local proxy_addr + proxy_addr=$(echo "$proxy" | awk -F'http\:\/\/|https\:\/\/' '{print $2}') + + export http_proxy="${proxy_protocol}://${proxy_user}:${proxy_password}@${proxy_addr}" + + elif [[ (-z $2 || -z $3) && (-n $2 || -n $3) || ( -n $2 && -n $3 && ($2 != --proxy-user=* || $3 != --proxy-pass=*) ) ]]; then + echo "Invalid options passed for proxy. Order is --proxy-user= --proxy-pass=" + echo "Ignoring proxy" + return + + else + export http_proxy="$proxy" + fi + + export {https,ftp,rsync,all}_proxy="$http_proxy" + ;; + *) + echo "Invalid option" + esac +} + patch_pillar() { local pillar_file=$temp_install_dir/pillar/minions/$MINION_ID.sls @@ -1068,7 +1122,7 @@ saltify() { yum -y update exclude=salt*; systemctl enable salt-minion; } >> "$setup_log" 2>&1 - echo "exclude=salt*" >> /etc/yum.conf + yum versionlock salt* else DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1 @@ -1229,8 +1283,6 @@ set_progress_str() { '----'\ "$percentage% - ${progress_bar_text^^}"\ "----" >> "$setup_log" 2>&1 - - sleep 5 } sensor_pillar() { @@ -1445,6 +1497,23 @@ update_packages() { fi } +use_turbo_proxy() { + if [[ ! $install_type =~ ^(MASTER|EVAL|HELIXSENSOR|MASTERSEARCH|STANDALONE)$ ]]; then + echo "turbo is not supported on this install type" >> $setup_log 2>&1 + return + fi + + if [[ $OS == 'centos' ]]; then + printf '%s\n' "proxy=${TURBO}:3142" >> /etc/yum.conf + else + printf '%s\n'\ + "Acquire {"\ + " HTTP::proxy \"${TURBO}:3142\";"\ + " HTTPS::proxy \"${TURBO}:3142\";"\ + "}" > /etc/apt/apt.conf.d/proxy.conf + fi +} + ls_heapsize() { if [ "$total_mem" -ge 32000 ]; then diff --git a/setup/so-setup b/setup/so-setup index 406d69763..d18f60089 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -24,12 +24,49 @@ source ./so-variables setup_type=$1 export setup_type +automation=$2 + +automated=no + +echo "---- Starting setup at $(date -u) ----" >> $setup_log 2>&1 + +function progress() { + if [ $automated == no ]; then + whiptail --title "Security Onion Install" --gauge 'Please wait while installing' 6 60 0 + fi +} + +if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then + echo "Preselecting variable values based on automated setup: $automation" >> $setup_log 2>&1 + source automation/$automation + automated=yes + + echo "Checking network configuration" >> $setup_log 2>&1g + ip a >> $setup_log 2>&1 + + attempt=1 + attempts=60 + ip a | grep "$MNIC:" | grep "state UP" >> $setup_log 2>&1 + while [ $? -ne 0 ]; do + ip a >> $setup_log 2>&1 + if [ $attempt -gt $attempts ]; then + echo "Network unavailable - setup cannot continue" >> $setup_log 2>&1 + exit 1 + fi + echo "Waiting for network to come up (attempt $attempt of $attempts)" >> $setup_log 2>&1 + attempt=$((attempt + 1)) + sleep 10; + ip a | grep "$MNIC:" | grep "state UP" >> $setup_log 2>&1 + done + echo "Network is up on $MNIC" >> $setup_log 2>&1 +fi + case "$setup_type" in iso | network) # Accepted values - echo "Beginning Security Onion $setup_type install" + echo "Beginning Security Onion $setup_type install" >> $setup_log 2>&1 ;; *) - echo "Invalid install type, must be 'iso' or 'network'" + echo "Invalid install type, must be 'iso' or 'network'" | tee $setup_log exit 1 ;; esac @@ -37,9 +74,13 @@ esac # Allow execution of SO tools during setup export PATH=$PATH:../salt/common/tools/sbin -date -u > $setup_log 2>&1 - got_root + +if [[ $# -gt 1 ]]; then + set -- "${@:2}" + parse_options "$@" >> $setup_log 2>&1 +fi + detect_os if [ "$OS" == ubuntu ]; then @@ -48,10 +89,10 @@ fi setterm -blank 0 -if (whiptail_you_sure); then +if [ "$setup_type" == 'iso' ] || (whiptail_you_sure); then true else - echo "User cancelled setup." >> $setup_log 2>&1 + echo "User cancelled setup." | tee $setup_log whiptail_cancel fi @@ -241,6 +282,10 @@ fi whiptail_make_changes +if [[ -n "$TURBO" ]]; then + use_turbo_proxy +fi + if [[ "$setup_type" == 'iso' ]]; then # Init networking so rest of install works set_hostname_iso @@ -485,7 +530,7 @@ fi set_progress_str 95 'Verifying setup' salt-call -l info state.highstate >> $setup_log 2>&1 -} | whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0 +} | progress success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}') if [[ "$success" = 0 ]]; then diff --git a/setup/so-whiptail b/setup/so-whiptail index cfe00b67b..72455fc9e 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -958,7 +958,7 @@ whiptail_setup_complete() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Finished $install_type install. Press ENTER to reboot." 8 75 + whiptail --title "Security Onion Setup" --msgbox "Finished $install_type install. Press Ok to reboot." 8 75 install_cleanup >> $setup_log 2>&1 } @@ -967,7 +967,7 @@ whiptail_setup_failed() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $setup_log for details. Press ENTER to reboot." 8 75 + whiptail --title "Security Onion Setup" --msgbox "Install had a problem. Please see $setup_log for details. Press Ok to reboot." 8 75 install_cleanup >> $setup_log 2>&1 } @@ -1048,7 +1048,7 @@ whiptail_you_sure() { [ -n "$TESTING" ] && return - whiptail --title "Security Onion Setup" --yesno "Are you sure you want to install Security Onion over the internet?" 8 75 + whiptail --title "Security Onion Setup" --yesno "Are you sure you want to continue a network install of Security Onion?" 8 75 local exitstatus=$? return $exitstatus diff --git a/so-setup-network b/so-setup-network index ae9af4ffa..2528ff14b 100755 --- a/so-setup-network +++ b/so-setup-network @@ -15,6 +15,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -cd setup +cd setup || exit -./so-setup network +./so-setup network "$@"