diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 3548a7f0d..a837950e4 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -34,7 +34,7 @@ 'influxdb', 'soc', 'kratos', - 'elastic-fleet', + 'elasticfleet', 'firewall', 'idstools', 'suricata.manager', @@ -105,7 +105,7 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'elastic-fleet' + 'elasticfleet' ], 'so-manager': [ 'salt.master', @@ -118,7 +118,7 @@ 'influxdb', 'soc', 'kratos', - 'elastic-fleet', + 'elasticfleet', 'firewall', 'idstools', 'suricata.manager', @@ -137,7 +137,7 @@ 'influxdb', 'soc', 'kratos', - 'elastic-fleet', + 'elasticfleet', 'firewall', 'manager', 'idstools', @@ -166,7 +166,7 @@ 'influxdb', 'soc', 'kratos', - 'elastic-fleet', + 'elasticfleet', 'firewall', 'idstools', 'suricata.manager', diff --git a/salt/common/tools/sbin/so-elastic-agent-gen-installers b/salt/common/tools/sbin/so-elastic-agent-gen-installers index 131292dab..128f894e4 100755 --- a/salt/common/tools/sbin/so-elastic-agent-gen-installers +++ b/salt/common/tools/sbin/so-elastic-agent-gen-installers @@ -24,11 +24,11 @@ mkdir -p /tmp/elastic-agent-workspace for OS in "${CONTAINERGOOS[@]}" do printf "\n\nGenerating $OS Installer..." - cp /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz + cp /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz docker run -e CGO_ENABLED=0 -e GOOS=$OS \ --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \ --mount type=bind,source=/tmp/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ - --mount type=bind,source=/opt/so/saltstack/local/salt/elastic-fleet/files/so_agent-installers/,target=/output/ \ + --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_$OS printf "\n $OS Installer Generated..." done diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index bc65161fa..7930e05d6 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -17,7 +17,9 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http: # Disable certain Features from showing up in the Kibana UI echo -echo "Setting up default Security Onion package policies for Elastic Agent..." +echo "Disable certain Features from showing up in the Kibana UI" +so-kibana-space-defaults +echo # Suricata logs echo @@ -71,7 +73,7 @@ echo # Kratos logs echo echo "Setting up Kratos package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- rename:\n fields:\n - from: \"audience\"\n to: \"event.dataset\"\n ignore_missing: true\n- add_fields:\n when:\n not: \n has_fields: ['event.dataset']\n target: ''\n fields:\n event.dataset: access", "custom": "pipeline: kratos" }}}}}}' +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos" }}}}}}' echo # RITA Logs diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 9c2d60eca..8334490d5 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -91,19 +91,19 @@ printf '%s\n'\ "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State -salt-call state.apply elastic-fleet queue=True +salt-call state.apply elasticfleet queue=True # Load Elastic Fleet integrations /usr/sbin/so-elastic-fleet-integration-policy-load # Temp -wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-darwin-x86_64.tar.gz -wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-linux-x86_64.tar.gz -wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-windows-x86_64.tar.gz +wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-darwin-x86_64.tar.gz +wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-linux-x86_64.tar.gz +wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-windows-x86_64.tar.gz #git clone -b 2.4-so-elastic-agent https://github.com/Security-Onion-Solutions/securityonion-image.git #cd securityonion-image/so-elastic-agent-builder #docker build -t so-elastic-agent-builder . so-elastic-agent-gen-installers -salt-call state.apply elastic-fleet.install_agent_grid queue=True +salt-call state.apply elasticfleet.install_agent_grid queue=True diff --git a/salt/common/tools/sbin/so-kibana-space-defaults b/salt/common/tools/sbin/so-kibana-space-defaults index 9175a36bc..430054e06 100755 --- a/salt/common/tools/sbin/so-kibana-space-defaults +++ b/salt/common/tools/sbin/so-kibana-space-defaults @@ -13,6 +13,6 @@ echo "Setting up default Space:" {% if HIGHLANDER %} curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch"]} ' >> /opt/so/log/kibana/misc.log {% else %} -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet","fleetv2","securitySolutionCases"]} ' >> /opt/so/log/kibana/misc.log +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCases"]} ' >> /opt/so/log/kibana/misc.log {% endif %} echo diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion index b565f5a31..d14955e61 100755 --- a/salt/common/tools/sbin/so-minion +++ b/salt/common/tools/sbin/so-minion @@ -23,7 +23,7 @@ if [[ $# -lt 1 ]]; then echo " accept: Accepts a new key and adds the minion files" echo " delete: Removes the key and deletes the minion files" echo " reject: Rejects a key" - echo " test: Ingest test data" + echo " test: Perform minion test" echo "" exit 1 fi diff --git a/salt/elastic-fleet/files/so_agent-installers/readme b/salt/elasticfleet/files/so_agent-installers/readme similarity index 100% rename from salt/elastic-fleet/files/so_agent-installers/readme rename to salt/elasticfleet/files/so_agent-installers/readme diff --git a/salt/elastic-fleet/init.sls b/salt/elasticfleet/init.sls similarity index 100% rename from salt/elastic-fleet/init.sls rename to salt/elasticfleet/init.sls diff --git a/salt/elastic-fleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls similarity index 86% rename from salt/elastic-fleet/install_agent_grid.sls rename to salt/elasticfleet/install_agent_grid.sls index 2f848ac2e..c4c389cea 100644 --- a/salt/elastic-fleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -9,7 +9,7 @@ run_installer: cmd.script: - - name: salt://elastic-fleet/files/so_agent-installers/so-elastic-agent_linux + - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux - args: -token={{ GRIDNODETOKEN }} {% endif %} diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml new file mode 100644 index 000000000..0e111feca --- /dev/null +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -0,0 +1,18 @@ +elasticfleet: + server: + endpoints_enrollment: + description: Endpoint enrollment key. + global: True + helpLink: elastic-fleet.html + es_token: + description: Elastic auth token. + global: True + helpLink: elastic-fleet.html + grid_enrollment: + description: Grid enrollment key. + global: True + helpLink: elastic-fleet.html + url: + description: Agent connection URL. + global: True + helpLink: elastic-fleet.html \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/kratos b/salt/elasticsearch/files/ingest/kratos index f8dcf53e0..9551dad24 100644 --- a/salt/elasticsearch/files/ingest/kratos +++ b/salt/elasticsearch/files/ingest/kratos @@ -1,7 +1,9 @@ { "description" : "kratos", "processors" : [ - { "set": { "field": "event.dataset", "value": "access" } }, + {"set":{"field":"audience","value":"access","override":false,"ignore_failure":true}}, + {"set":{"field":"event.dataset","ignore_empty_value":true,"ignore_failure":true,"value":"kratos.{{{audience}}}","media_type":"text/plain"}}, + {"set":{"field":"event.action","ignore_failure":true,"copy_from":"msg" }}, { "pipeline": { "name": "common" } } ] -} +} \ No newline at end of file diff --git a/salt/idh/init.sls b/salt/idh/init.sls index d1ba5ce33..2cf22c358 100644 --- a/salt/idh/init.sls +++ b/salt/idh/init.sls @@ -74,8 +74,6 @@ so-idh: - file: opencanary_config - require: - file: opencanary_config - - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} append_so-idh_so-status.conf: file.append: diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index e3a13c2f2..52d018354 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -95,7 +95,7 @@ so-nginx: - /opt/so/log/nginx/:/var/log/nginx:rw - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw - - /opt/so/saltstack/local/salt/elastic-fleet/files/so_agent-installers/:/opt/socore/html/packages + - /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/:/opt/socore/html/packages {% if grains.role in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone', 'so-import'] %} - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index bf31fff27..4a521f12c 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -210,19 +210,19 @@ chownilogstashelasticfleetp8: # Create Symlinks to the keys so I can distribute it to all the things elasticfleetdircerts: file.directory: - - name: /opt/so/saltstack/local/salt/elastic-fleet/files/certs + - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs - makedirs: True efkeylink: file.symlink: - - name: /opt/so/saltstack/local/salt/elastic-fleet/files/certs/elasticfleet.p8 + - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet.p8 - target: /etc/pki/elasticfleet.p8 - user: socore - group: socore efcrtlink: file.symlink: - - name: /opt/so/saltstack/local/salt/elastic-fleet/files/certs/elasticfleet.crt + - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet.crt - target: /etc/pki/elasticfleet.crt - user: socore - group: socore diff --git a/salt/top.sls b/salt/top.sls index a07e16013..2c6ad266f 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -59,7 +59,7 @@ base: {%- endif %} - schedule - docker_clean - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid '*_eval and G@saltversion:{{saltversion}}': - match: compound @@ -147,7 +147,7 @@ base: - schedule - soctopus - playbook - - elastic-fleet + - elasticfleet - docker_clean '*_standalone and G@saltversion:{{saltversion}}': @@ -198,7 +198,7 @@ base: - schedule - soctopus - playbook - - elastic-fleet + - elasticfleet - docker_clean '*_searchnode and G@saltversion:{{saltversion}}': @@ -215,7 +215,7 @@ base: - logstash {%- endif %} - schedule - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid - docker_clean '*_managersearch and G@saltversion:{{saltversion}}': @@ -257,7 +257,7 @@ base: - schedule - soctopus - playbook - - elastic-fleet + - elasticfleet - docker_clean '*_heavynode and G@saltversion:{{saltversion}}': @@ -286,7 +286,7 @@ base: - zeek {%- endif %} - schedule - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid - docker_clean '*_import and G@saltversion:{{saltversion}}': @@ -317,7 +317,7 @@ base: - suricata - zeek - schedule - - elastic-fleet + - elasticfleet - docker_clean '*_receiver and G@saltversion:{{saltversion}}': @@ -333,7 +333,7 @@ base: - redis {%- endif %} - schedule - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid - docker_clean '*_idh and G@saltversion:{{saltversion}}': @@ -343,7 +343,7 @@ base: - telegraf - firewall - schedule - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid - docker_clean - idh diff --git a/setup/so-setup b/setup/so-setup index 814fc6e79..88a2fd045 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -58,6 +58,10 @@ while [[ $# -gt 0 ]]; do esac done +# Preserve old setup/error logs +[ -f "$error_log" ] && mv "$error_log" "$error_log.$(date +%Y-%m-%dT%H:%M:%S)" +[ -f "$setup_log" ] && mv "$setup_log" "$setup_log.$(date +%Y-%m-%dT%H:%M:%S)" + # Let's see what OS we are dealing with here detect_os @@ -134,9 +138,7 @@ title "Checking to see if install has run before" if [[ -f /root/accept_changes ]]; then is_reinstall=true whiptail_reinstall - info "Old setup detected. Moving the last setup.log to setup.log.bak" - mv "$setup_log" "$setup_log.bak" - [ -f "$error_log" ] && mv "$error_log" "$error_log.bak" + info "Old setup detected. Preparing for reinstallation." reinstall_init reset_proxy fi @@ -267,7 +269,7 @@ if ! [[ -f $install_opt_file ]]; then if (whiptail_you_sure); then true else - error "User cancelled setup." + info "User cancelled setup." whiptail_cancel fi # If this is an analyst install lets streamline the process. diff --git a/setup/so-whiptail b/setup/so-whiptail index 4ed473381..f58f65fa2 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -976,7 +976,7 @@ whiptail_manager_unreachable() { Run the following on the manager: - so-firewall-minion --role=$install_type --ip=$MAINIP + sudo so-firewall-minion --role=$install_type --ip=$MAINIP Would you like to retry? EOM @@ -1271,38 +1271,54 @@ whiptail_setup_complete() { [ -n "$TESTING" ] && return - if [[ -n "$REDIRECTIT" && $is_manager = true ]]; then + + if [[ $waitforstate ]]; then + # Manager-type Nodes - Install Summary if [[ -n $ALLOW_CIDR ]]; then local sentence_prefix="Access" else local sentence_prefix="Run so-allow to access" fi - local accessMessage="\n${sentence_prefix} the web interface at: https://${REDIRECTIT}\n" - elif [[ $is_idh ]]; then - local accessMessage="\nSSH for this node has been moved to TCP/2222, accessible only from the Manager node.\n" + + read -r -d '' message <<- EOM + ${install_type} setup is now complete! + + ${sentence_prefix} the Security Onion Console (SOC) web interface by navigating to: + https://${REDIRECTIT} + + Then login with the following username and password. + + SOC Username: ${WEBUSER} + SOC Password: Use the password that was entered during setup + + Press TAB and then the ENTER key to exit this screen. + EOM + whiptail --title "$whiptail_title" --msgbox "$message" 24 75 --scrolltext else - local accessMessage="" + if [[ $is_idh ]]; then + local accessMessage="\nSSH for this node has been moved to TCP/2222, accessible only from the Manager node.\n" + else + local accessMessage="" + fi + MINIONFINGERPRINT=$(salt-call --local key.finger --out=newline_values_only) + read -r -d '' message <<- EOM + ${install_type} initialization is now complete! + + To finish configuration, open the Security Onion Console web interface + and navigate to Administration -> Grid Members. + + Then find this node in the Pending Members list, + click the Review button, and then click the Accept button. + + Node Hostname: $HOSTNAME + Node Fingerprint: + $MINIONFINGERPRINT + $accessMessage + Press TAB and then the ENTER key to exit this screen. + EOM + + whiptail --title "$whiptail_title" --msgbox "$message" 24 75 --scrolltext fi - - MINIONFINGERPRINT=$(salt-call --local key.finger --out=newline_values_only) - read -r -d '' message <<- EOM - ${install_type} initialization is now complete! - - To finish configuration, open the Security Onion Console web interface - and navigate to Administration -> Grid Members. - - Then find this node in the Pending Members list, - click the Review button, and then click the Accept button. - - Node Hostname: $HOSTNAME - Node Fingerprint: - $MINIONFINGERPRINT - - $accessMessage - Press TAB and then the ENTER key to exit this screen. - EOM - - whiptail --title "$whiptail_title" --msgbox "$message" 24 75 --scrolltext } whiptail_setup_failed() {