From cbd710bcf25ffa515d2d60341a801e58bf77f0e1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 30 Mar 2020 19:27:56 -0400 Subject: [PATCH] Replaced auth system with new identity management system. --- pillar/docker/config.sls | 15 +- pillar/top.sls | 3 - salt/common/init.sls | 8 - salt/common/nginx/nginx.conf.so-eval | 75 +++++----- salt/common/nginx/nginx.conf.so-master | 111 +++++++------- salt/common/nginx/nginx.conf.so-mastersearch | 140 +++++++++--------- salt/common/tools/sbin/so-playbook-sync | 2 +- salt/common/tools/sbin/so-user-add | 17 +-- .../files/bin/so-curator-closed-delete | 4 +- .../files/registry/scripts/so-docker-download | 5 +- salt/top.sls | 11 +- setup/so-functions | 64 +++++++- setup/so-setup | 46 ++++-- setup/so-whiptail | 50 +++++++ upgrade/so-update-functions | 4 +- 15 files changed, 329 insertions(+), 226 deletions(-) diff --git a/pillar/docker/config.sls b/pillar/docker/config.sls index f3259dfc0..423910b4c 100644 --- a/pillar/docker/config.sls +++ b/pillar/docker/config.sls @@ -17,10 +17,9 @@ eval: - so-grafana {% endif %} - so-dockerregistry - - so-sensoroni + - so-soc + - so-kratos - so-idstools - - so-auth-api - - so-auth-ui {% if OSQUERY != '0' %} - so-mysql - so-fleet @@ -89,12 +88,11 @@ master_search: containers: - so-core - so-telegraf - - so-sensoroni + - so-soc + - so-kratos - so-acng - so-idstools - so-redis - - so-auth-api - - so-auth-ui - so-logstash - so-elasticsearch - so-curator @@ -135,12 +133,11 @@ master: - so-influxdb - so-grafana {% endif %} - - so-sensoroni + - so-soc + - so-kratos - so-acng - so-idstools - so-redis - - so-auth-api - - so-auth-ui - so-elasticsearch - so-logstash - so-kibana diff --git a/pillar/top.sls b/pillar/top.sls index 7ebd8ada2..35621b6c2 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -21,7 +21,6 @@ base: - static - firewall.* - data.* - - auth - minions.{{ grains.id }} '*_master': @@ -33,7 +32,6 @@ base: - firewall.* - data.* - brologs - - auth - logstash - logstash.eval - healthcheck.eval @@ -63,5 +61,4 @@ base: - static - firewall.* - data.* - - auth - minions.{{ grains.id }} diff --git a/salt/common/init.sls b/salt/common/init.sls index 6e8a3ea65..7ce884392 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -104,13 +104,6 @@ nginxconf: - template: jinja - source: salt://common/nginx/nginx.conf.{{ grains.role }} -copyindex: - file.managed: - - name: /opt/so/conf/nginx/index.html - - user: 939 - - group: 939 - - source: salt://common/nginx/index.html - nginxlogdir: file.directory: - name: /opt/so/log/nginx/ @@ -133,7 +126,6 @@ so-core: - binds: - /opt/so:/opt/so:rw - /opt/so/conf/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - /opt/so/conf/nginx/index.html:/opt/socore/html/index.html:ro - /opt/so/log/nginx/:/var/log/nginx:rw - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval index e322ab59d..89e6fe46b 100644 --- a/salt/common/nginx/nginx.conf.so-eval +++ b/salt/common/nginx/nginx.conf.so-eval @@ -4,7 +4,6 @@ # * Official English Documentation: http://nginx.org/en/docs/ # * Official Russian Documentation: http://nginx.org/ru/docs/ -user nginx; worker_processes auto; error_log /var/log/nginx/error.log; pid /run/nginx.pid; @@ -107,10 +106,42 @@ http { # Load configuration files for the default server block. #include /etc/nginx/default.d/*.conf; - #location / { - # try_files $uri $uri.html /index.html; - # } + location ~* (^/login/|^/js/.*|^/css/.*|^/images/.*) { + proxy_pass http://{{ masterip }}:9822; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + location / { + auth_request /auth/sessions/whoami; + proxy_pass http://{{ masterip }}:9822/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + + location /auth/ { + rewrite /auth/(.*) /$1 break; + proxy_pass http://{{ masterip }}:4433/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } location /grafana/ { rewrite /grafana/(.*) /$1 break; proxy_pass http://{{ masterip }}:3000/; @@ -124,7 +155,7 @@ http { } location /kibana/ { - auth_request /so-auth/api/auth/; + auth_request /auth/sessions/whoami; rewrite /kibana/(.*) /$1 break; proxy_pass http://{{ masterip }}:5601/; proxy_read_timeout 90; @@ -162,7 +193,7 @@ http { location /navigator/ { - auth_request /so-auth/api/auth/; + auth_request /auth/sessions/whoami; proxy_pass http://{{ masterip }}:4200/navigator/; proxy_read_timeout 90; proxy_connect_timeout 90; @@ -219,22 +250,8 @@ http { } - location /sensoroni/ { - auth_request /so-auth/api/auth/; - proxy_pass http://{{ masterip }}:9822/; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - - } - - location /kibana/app/sensoroni/ { - rewrite ^/kibana/app/sensoroni/(.*) /sensoroni/$1 permanent; + location /kibana/app/soc/ { + rewrite ^/kibana/app/soc/(.*) /soc/$1 permanent; } location /kibana/app/fleet/ { @@ -255,23 +272,11 @@ http { proxy_set_header Proxy ""; } - location /so-auth/loginpage/ { - proxy_pass http://{{ masterip }}:4242/; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /so-auth/api/ { - proxy_pass http://{{ masterip }}:5656/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $host; - } - error_page 401 = @error401; location @error401 { add_header Set-Cookie "NSREDIRECT=http://{{ masterip }}$request_uri;Domain={{ masterip }};Path=/;Max-Age=60000"; - return 302 http://{{ masterip }}/so-auth/loginpage/; + return 302 /auth/self-service/browser/flows/login; } error_page 404 /404.html; diff --git a/salt/common/nginx/nginx.conf.so-master b/salt/common/nginx/nginx.conf.so-master index ed9dfb253..89e6fe46b 100644 --- a/salt/common/nginx/nginx.conf.so-master +++ b/salt/common/nginx/nginx.conf.so-master @@ -4,7 +4,6 @@ # * Official English Documentation: http://nginx.org/en/docs/ # * Official Russian Documentation: http://nginx.org/ru/docs/ -user nginx; worker_processes auto; error_log /var/log/nginx/error.log; pid /run/nginx.pid; @@ -59,9 +58,9 @@ http { # } #} server { - listen 80 default_server; - server_name _; - return 301 https://$host$request_uri; + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; } {% if FLEET_MASTER %} @@ -107,13 +106,45 @@ http { # Load configuration files for the default server block. #include /etc/nginx/default.d/*.conf; - #location / { - # try_files $uri $uri.html /index.html; - # } + location ~* (^/login/|^/js/.*|^/css/.*|^/images/.*) { + proxy_pass http://{{ masterip }}:9822; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + location / { + auth_request /auth/sessions/whoami; + proxy_pass http://{{ masterip }}:9822/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + + location /auth/ { + rewrite /auth/(.*) /$1 break; + proxy_pass http://{{ masterip }}:4433/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } location /grafana/ { - rewrite /grafana/(.*) /$1 break; - proxy_pass http://{{ masterip }}:3000/; + rewrite /grafana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:3000/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -124,9 +155,9 @@ http { } location /kibana/ { - auth_request /so-auth/api/auth/; - rewrite /kibana/(.*) /$1 break; - proxy_pass http://{{ masterip }}:5601/; + auth_request /auth/sessions/whoami; + rewrite /kibana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:5601/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -150,7 +181,7 @@ http { } location /playbook/ { - proxy_pass http://{{ masterip }}:3200/playbook/; + proxy_pass http://{{ masterip }}:3200/playbook/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -160,9 +191,10 @@ http { } + location /navigator/ { - auth_request /so-auth/api/auth/; - proxy_pass http://{{ masterip }}:4200/navigator/; + auth_request /auth/sessions/whoami; + proxy_pass http://{{ masterip }}:4200/navigator/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -184,10 +216,10 @@ http { } location /thehive/ { - proxy_pass http://{{ masterip }}:9000/thehive/; + proxy_pass http://{{ masterip }}:9000/thehive/; proxy_read_timeout 90; proxy_connect_timeout 90; - proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_http_version 1.1; # this is essential for chunked responses to work proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -196,19 +228,19 @@ http { } location /cortex/ { - proxy_pass http://{{ masterip }}:9001/cortex/; + proxy_pass http://{{ masterip }}:9001/cortex/; proxy_read_timeout 90; proxy_connect_timeout 90; - proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_http_version 1.1; # this is essential for chunked responses to work proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; } - + location /soctopus/ { - proxy_pass http://{{ masterip }}:7000/; + proxy_pass http://{{ masterip }}:7000/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -218,22 +250,8 @@ http { } - location /sensoroni/ { - auth_request /so-auth/api/auth/; - proxy_pass http://{{ masterip }}:9822/; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - - } - - location /kibana/app/sensoroni/ { - rewrite ^/kibana/app/sensoroni/(.*) /sensoroni/$1 permanent; + location /kibana/app/soc/ { + rewrite ^/kibana/app/soc/(.*) /soc/$1 permanent; } location /kibana/app/fleet/ { @@ -244,36 +262,21 @@ http { rewrite ^/kibana/app/soctopus/(.*) /soctopus/$1 permanent; } - location /sensoroniagents/ { - proxy_pass http://{{ masterip }}:9822/; + proxy_pass http://{{ masterip }}:9822/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - - } - - - location /so-auth/loginpage/ { - proxy_pass http://{{ masterip }}:4242/; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /so-auth/api/ { - proxy_pass http://{{ masterip }}:5656/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $host; } error_page 401 = @error401; location @error401 { add_header Set-Cookie "NSREDIRECT=http://{{ masterip }}$request_uri;Domain={{ masterip }};Path=/;Max-Age=60000"; - return 302 http://{{ masterip }}/so-auth/loginpage/; + return 302 /auth/self-service/browser/flows/login; } error_page 404 /404.html; diff --git a/salt/common/nginx/nginx.conf.so-mastersearch b/salt/common/nginx/nginx.conf.so-mastersearch index 1eb4e8e5c..89e6fe46b 100644 --- a/salt/common/nginx/nginx.conf.so-mastersearch +++ b/salt/common/nginx/nginx.conf.so-mastersearch @@ -4,7 +4,6 @@ # * Official English Documentation: http://nginx.org/en/docs/ # * Official Russian Documentation: http://nginx.org/ru/docs/ -user nginx; worker_processes auto; error_log /var/log/nginx/error.log; pid /run/nginx.pid; @@ -59,9 +58,9 @@ http { # } #} server { - listen 80 default_server; - server_name _; - return 301 https://$host$request_uri; + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; } {% if FLEET_MASTER %} @@ -107,13 +106,45 @@ http { # Load configuration files for the default server block. #include /etc/nginx/default.d/*.conf; - #location / { - # try_files $uri $uri.html /index.html; - # } + location ~* (^/login/|^/js/.*|^/css/.*|^/images/.*) { + proxy_pass http://{{ masterip }}:9822; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + location / { + auth_request /auth/sessions/whoami; + proxy_pass http://{{ masterip }}:9822/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + + location /auth/ { + rewrite /auth/(.*) /$1 break; + proxy_pass http://{{ masterip }}:4433/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } location /grafana/ { - rewrite /grafana/(.*) /$1 break; - proxy_pass http://{{ masterip }}:3000/; + rewrite /grafana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:3000/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -124,9 +155,9 @@ http { } location /kibana/ { - auth_request /so-auth/api/auth/; - rewrite /kibana/(.*) /$1 break; - proxy_pass http://{{ masterip }}:5601/; + auth_request /auth/sessions/whoami; + rewrite /kibana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:5601/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -136,8 +167,21 @@ http { } - location /playbook/ { - proxy_pass http://{{ masterip }}:3200/playbook/; + location /nodered/ { + proxy_pass http://{{ masterip }}:1880/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Proxy ""; + + } + + location /playbook/ { + proxy_pass http://{{ masterip }}:3200/playbook/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -147,9 +191,10 @@ http { } + location /navigator/ { - auth_request /so-auth/api/auth/; - proxy_pass http://{{ masterip }}:4200/navigator/; + auth_request /auth/sessions/whoami; + proxy_pass http://{{ masterip }}:4200/navigator/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -171,10 +216,10 @@ http { } location /thehive/ { - proxy_pass http://{{ masterip }}:9000/thehive/; + proxy_pass http://{{ masterip }}:9000/thehive/; proxy_read_timeout 90; proxy_connect_timeout 90; - proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_http_version 1.1; # this is essential for chunked responses to work proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -183,31 +228,19 @@ http { } location /cortex/ { - proxy_pass http://{{ masterip }}:9001/cortex/; + proxy_pass http://{{ masterip }}:9001/cortex/; proxy_read_timeout 90; proxy_connect_timeout 90; - proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_http_version 1.1; # this is essential for chunked responses to work proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; } - - location /cyberchef/ { - proxy_pass http://{{ masterip }}:9080/; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_http_version 1.1; # this is essential for chunked responses to work - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - - } - + location /soctopus/ { - proxy_pass http://{{ masterip }}:7000/; + proxy_pass http://{{ masterip }}:7000/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -217,22 +250,8 @@ http { } - location /sensoroni/ { - auth_request /so-auth/api/auth/; - proxy_pass http://{{ masterip }}:9822/; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; - - } - - location /kibana/app/sensoroni/ { - rewrite ^/kibana/app/sensoroni/(.*) /sensoroni/$1 permanent; + location /kibana/app/soc/ { + rewrite ^/kibana/app/soc/(.*) /soc/$1 permanent; } location /kibana/app/fleet/ { @@ -243,36 +262,21 @@ http { rewrite ^/kibana/app/soctopus/(.*) /soctopus/$1 permanent; } - location /sensoroniagents/ { - proxy_pass http://{{ masterip }}:9822/; + proxy_pass http://{{ masterip }}:9822/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - - } - - - location /so-auth/loginpage/ { - proxy_pass http://{{ masterip }}:4242/; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /so-auth/api/ { - proxy_pass http://{{ masterip }}:5656/; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $host; } error_page 401 = @error401; location @error401 { add_header Set-Cookie "NSREDIRECT=http://{{ masterip }}$request_uri;Domain={{ masterip }};Path=/;Max-Age=60000"; - return 302 http://{{ masterip }}/so-auth/loginpage/; + return 302 /auth/self-service/browser/flows/login; } error_page 404 /404.html; @@ -284,4 +288,4 @@ http { } } -} \ No newline at end of file +} diff --git a/salt/common/tools/sbin/so-playbook-sync b/salt/common/tools/sbin/so-playbook-sync index f4c2c456e..8b2817eaa 100755 --- a/salt/common/tools/sbin/so-playbook-sync +++ b/salt/common/tools/sbin/so-playbook-sync @@ -17,4 +17,4 @@ . /usr/sbin/so-common -docker exec so-soctopus python3 playbook_play-sync.py +docker exec so-soctopus python3 playbook_play-sync.py >> /opt/so/log/soctopus/so-playbook-sync.log 2>&1 diff --git a/salt/common/tools/sbin/so-user-add b/salt/common/tools/sbin/so-user-add index 930e02d7d..dd02d993f 100755 --- a/salt/common/tools/sbin/so-user-add +++ b/salt/common/tools/sbin/so-user-add @@ -1,17 +1,2 @@ #!/bin/bash -USERNAME=$1 - -# Make sure a username is provided -[ $# -eq 0 ] && { echo "Usage: $0 username"; exit 1; } - -# If the file is there already lets create it otherwise add the user -if [ ! -f /opt/so/conf/nginx/.htpasswd ]; then - - # Create the password file - htpasswd -c /opt/so/conf/nginx/.htpasswd $USERNAME - -else - - htpasswd /opt/so/conf/nginx/.htpasswd $USERNAME - -fi +so-user add $* \ No newline at end of file diff --git a/salt/curator/files/bin/so-curator-closed-delete b/salt/curator/files/bin/so-curator-closed-delete index 4382a721d..8f6d0a8ea 100755 --- a/salt/curator/files/bin/so-curator-closed-delete +++ b/salt/curator/files/bin/so-curator-closed-delete @@ -34,8 +34,6 @@ #fi # Avoid starting multiple instances -if pgrep -f "so-curator-closed-delete-delete" >/dev/null; then - echo "Script is already running." -else +if ! pgrep -f "so-curator-closed-delete-delete" >/dev/null; then /usr/sbin/so-curator-closed-delete-delete fi diff --git a/salt/master/files/registry/scripts/so-docker-download b/salt/master/files/registry/scripts/so-docker-download index a6c2aa7c5..488b45886 100644 --- a/salt/master/files/registry/scripts/so-docker-download +++ b/salt/master/files/registry/scripts/so-docker-download @@ -1,12 +1,13 @@ #!/bin/bash MASTER={{ MASTER }} -VERSION="HH1.1.4" +VERSION="HH1.2.1" TRUSTED_CONTAINERS=( \ "so-core:$VERSION" \ "so-cyberchef:$VERSION" \ "so-acng:$VERSION" \ -"so-sensoroni:$VERSION" \ +"so-soc:$VERSION" \ +"so-kratos:$VERSION" \ "so-fleet:$VERSION" \ "so-soctopus:$VERSION" \ "so-steno:$VERSION" \ diff --git a/salt/top.sls b/salt/top.sls index 10ef82f9a..456e09420 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -52,10 +52,9 @@ base: - registry - master - common - - sensoroni + - soc - firewall - idstools - - auth - healthcheck {%- if FLEETMASTER or FLEETNODE %} - mysql @@ -98,12 +97,11 @@ base: - ssl - registry - common - - sensoroni + - soc - firewall - master - idstools - redis - - auth {%- if FLEETMASTER or FLEETNODE %} - mysql {%- endif %} @@ -192,7 +190,6 @@ base: - firewall - sensor - master - - auth {%- if FLEETMASTER or FLEETNODE %} - fleet.install_package {%- endif %} @@ -203,13 +200,11 @@ base: - ssl - registry - common - - sensoroni - - auth + - soc - firewall - master - idstools - redis - - auth {%- if FLEETMASTER or FLEETNODE %} - mysql {%- endif %} diff --git a/setup/so-functions b/setup/so-functions index 38b38572c..4a129d884 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -78,6 +78,29 @@ add_socore_user_notmaster() { } +wait_for_identity_db_to_exist() { + MAXATTEMPTS=30 + attempts=0 + while [[ $attempts -lt $MAXATTEMPTS ]]; do + # Check and see if the DB file is in there + if [ -f /opt/so/conf/kratos/db/db.sqlite ]; then + echo "Database file exists at $(date)" + attempts=$MAXATTEMPTS + else + echo "Identity database does not yet exist; waiting 5 seconds and will check again ($attempts/$MAXATTEMPTS)..." + sleep 5 + attempts=$((attempts+1)) + fi + done +} + +add_web_user() { + wait_for_identity_db_to_exist + echo "Attempting to add administrator user for web interface..." + echo "$WEBPASSWD1" | /usr/sbin/so-user add $WEBUSER + echo "Add user result: $?" +} + # Create an auth pillar so that passwords survive re-install auth_pillar(){ @@ -219,6 +242,16 @@ check_socore_pass() { } +check_web_pass() { + + if [ $WEBPASSWD1 == $WEBPASSWD2 ]; then + WPMATCH=yes + else + whiptail_passwords_dont_match + fi + +} + checkin_at_boot() { echo "Enabling checkin at boot" >> $SETUPLOG 2>&1 echo "startup_states: highstate" >> /etc/salt/minion @@ -539,7 +572,8 @@ docker_seed_registry() { "so-navigator:$VERSION" \ "so-playbook:$VERSION" \ "so-redis:$VERSION" \ - "so-sensoroni:$VERSION" \ + "so-soc:$VERSION" \ + "so-kratos:$VERSION" \ "so-soctopus:$VERSION" \ "so-steno:$VERSION" \ #"so-strelka:$VERSION" \ @@ -556,7 +590,7 @@ docker_seed_registry() { "so-idstools:$VERSION" \ "so-logstash:$VERSION" \ "so-redis:$VERSION" \ - "so-sensoroni:$VERSION" \ + #"so-sensoroni:$VERSION" \ "so-steno:$VERSION" \ "so-suricata:$VERSION" \ "so-telegraf:$VERSION" \ @@ -651,6 +685,7 @@ generate_passwords(){ CORTEXKEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) CORTEXORGUSERKEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) SENSORONIKEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) + KRATOSKEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) } get_filesystem_nsm(){ @@ -690,6 +725,14 @@ get_main_ip() { } +get_redirect() { + whiptail_set_redirect_info + whiptail_set_redirect + if [ $REDIRECTINFO == 'OTHER' ]; then + whiptail_set_redirect_host + fi +} + got_root() { # Make sure you are root @@ -801,6 +844,18 @@ master_pillar() { echo " thehive: $THEHIVE" >> $PILLARFILE echo " playbook: $PLAYBOOK" >> $PILLARFILE echo "" >> $PILLARFILE + echo "kratos:" >> $PILLARFILE + if [[ $REDIRECTINFO == 'OTHER' ]]; then + REDIRECTIT=$REDIRECT + elif [[ $REDIRECTINFO == 'IP' ]]; then + REDIRECTIT=$MAINIP + elif [[ $REDIRECTINFO == 'HOSTNAME' ]]; then + REDIRECTIT=$HOSTNAME + fi + echo " kratoskey: $KRATOSKEY" >> $PILLARFILE + echo " redirect: $REDIRECTIT" >> $PILLARFILE + echo "" >> $PILLARFILE + } @@ -974,6 +1029,7 @@ saltify() { yum -y install wget https://repo.saltstack.com/py3/redhat/salt-py3-repo-latest-2.el7.noarch.rpm cp /etc/yum.repos.d/salt-py3-latest.repo /etc/yum.repos.d/salt-py3-2019-2.repo sed -i 's/latest/2019.2/g' /etc/yum.repos.d/salt-py3-2019-2.repo + yum -y install sqlite3 argon2 curl jq openssl # Download Ubuntu Keys in case master updates = 1 mkdir -p /opt/so/gpg wget --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/apt/ubuntu/16.04/amd64/latest/SALTSTACK-GPG-KEY.pub @@ -1191,11 +1247,11 @@ EOF # Initialize the new repos apt-get update >> $SETUPLOG 2>&1 if [ $OSVER != "xenial" ]; then - apt-get -y install salt-minion=2019.2.3+ds-1 salt-common=2019.2.3+ds-1 python3-dateutil python3-m2crypto >> $SETUPLOG 2>&1 + apt-get -y install salt-minion=2019.2.3+ds-1 salt-common=2019.2.3+ds-1 python3-dateutil python3-m2crypto sqlite3 argon2 curl jq openssl >> $SETUPLOG 2>&1 apt-mark hold salt-minion salt-common else # Need to add python packages here - apt-get -y install salt-minion=2019.2.3+ds-1 salt-common=2019.2.3+ds-1 python-dateutil python-m2crypto >> $SETUPLOG 2>&1 + apt-get -y install salt-minion=2019.2.3+ds-1 salt-common=2019.2.3+ds-1 python-dateutil python-m2crypto sqlite3 argon2 curl jq openssl >> $SETUPLOG 2>&1 apt-mark hold salt-minion salt-common fi else diff --git a/setup/so-setup b/setup/so-setup index e6a3f4708..88ae45c9b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -300,6 +300,15 @@ if (whiptail_you_sure) ; then check_socore_pass done + # Get a password for the web admin user + whiptail_create_web_user + WPMATCH=no + while [ $WPMATCH != yes ]; do + whiptail_create_web_user_password1 + whiptail_create_web_user_password2 + check_web_pass + done + get_redirect # Last Chance to back out whiptail_make_changes set_hostname @@ -376,21 +385,23 @@ if (whiptail_you_sure) ; then docker_seed_registry >> $SETUPLOG 2>&1 echo -e "XXX\n43\nInstalling Common Components... \nXXX" salt-call state.apply common >> $SETUPLOG 2>&1 + echo -e "XXX\n44\nInstalling SOC... \nXXX" + salt-call state.apply soc >> $SETUPLOG 2>&1 echo -e "XXX\n45\nApplying firewall rules... \nXXX" salt-call state.apply firewall >> $SETUPLOG 2>&1 salt-call state.apply master >> $SETUPLOG 2>&1 salt-call state.apply idstools >> $SETUPLOG 2>&1 - echo -e "XXX\n40\nInstalling Redis... \nXXX" + echo -e "XXX\n46\nInstalling Redis... \nXXX" salt-call state.apply redis >> $SETUPLOG 2>&1 if [[ $OSQUERY == '1' ]]; then - echo -e "XXX\n41\nInstalling MySQL... \nXXX" + echo -e "XXX\n48\nInstalling MySQL... \nXXX" salt-call state.apply mysql >> $SETUPLOG 2>&1 fi if [[ $WAZUH == '1' ]]; then - echo -e "XXX\n68\nInstalling Wazuh... \nXXX" + echo -e "XXX\n48\nInstalling Wazuh... \nXXX" salt-call state.apply wazuh >> $SETUPLOG 2>&1 fi - echo -e "XXX\n45\nInstalling Elastic Components... \nXXX" + echo -e "XXX\n49\nInstalling Elastic Components... \nXXX" salt-call state.apply elasticsearch >> $SETUPLOG 2>&1 salt-call state.apply logstash >> $SETUPLOG 2>&1 salt-call state.apply kibana >> $SETUPLOG 2>&1 @@ -419,7 +430,9 @@ if (whiptail_you_sure) ; then echo -e "XX\n97\nFinishing touches... \nXXX" filter_unused_nics >> $SETUPLOG 2>&1 network_setup >> $SETUPLOG 2>&1 - echo -e "XXX\n98\nVerifying Setup... \nXXX" + echo -e "XXX\n98\nAdding user to SOC... \nXXX" + add_web_user >> $SETUPLOG 2>&1 + echo -e "XXX\n99\nVerifying Setup... \nXXX" salt-call state.highstate >> $SETUPLOG 2>&1 } |whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0 GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}') @@ -570,6 +583,15 @@ if (whiptail_you_sure) ; then check_socore_pass done fi + # Get a password for the web admin user + whiptail_create_web_user + WPMATCH=no + while [ $WPMATCH != yes ]; do + whiptail_create_web_user_password1 + whiptail_create_web_user_password2 + check_web_pass + done + get_redirect whiptail_make_changes set_hostname set_version @@ -616,14 +638,10 @@ if (whiptail_you_sure) ; then master_pillar >> $SETUPLOG 2>&1 echo "** Generating the patch pillar **" >> $SETUPLOG patch_pillar >> $SETUPLOG 2>&1 - - echo -e "XXX\n7\nConfiguring minion... \nXXX" configure_minion $TYPE >> $SETUPLOG 2>&1 echo -e "XXX\n7\nSetting the node type to $TYPE... \nXXX" set_node_type >> $SETUPLOG 2>&1 - - echo -e "XXX\n7\nSearch node pillar... \nXXX" node_pillar >> $SETUPLOG 2>&1 echo -e "XXX\n8\nCreating firewall policies... \nXXX" @@ -650,12 +668,13 @@ if (whiptail_you_sure) ; then echo -e "XXX\n25\nInstalling master components... \nXXX" salt-call state.apply master >> $SETUPLOG 2>&1 salt-call state.apply idstools >> $SETUPLOG 2>&1 - + echo -e "XXX\n26\nInstalling SOC... \nXXX" + salt-call state.apply soc >> $SETUPLOG 2>&1 if [[ $OSQUERY == '1' ]]; then salt-call state.apply mysql >> $SETUPLOG 2>&1 fi if [[ $WAZUH == '1' ]]; then - echo -e "XXX\n65\nInstalling Wazuh components... \nXXX" + echo -e "XXX\n27\nInstalling Wazuh components... \nXXX" salt-call state.apply wazuh >> $SETUPLOG 2>&1 fi echo -e "XXX\n35\nInstalling ElasticSearch... \nXXX" @@ -700,10 +719,11 @@ if (whiptail_you_sure) ; then echo -e "XXX\n95\nSetting checkin to run on boot... \nXXX" checkin_at_boot >> $SETUPLOG 2>&1 echo -e "XX\n97\nFinishing touches... \nXXX" - salt-call state.apply auth >> $SETUPLOG 2>&1 filter_unused_nics >> $SETUPLOG 2>&1 network_setup >> $SETUPLOG 2>&1 - echo -e "XXX\n98\nVerifying Setup... \nXXX" + echo -e "XXX\n98\nAdding user to SOC... \nXXX" + add_web_user >> $SETUPLOG 2>&1 + echo -e "XXX\n99\nVerifying Setup... \nXXX" salt-call state.highstate >> $SETUPLOG 2>&1 } |whiptail --title "Hybrid Hunter Install" --gauge "Please wait while installing" 6 60 0 GOODSETUP=$(tail -10 $SETUPLOG | grep Failed | awk '{ print $2}') diff --git a/setup/so-whiptail b/setup/so-whiptail index 2769150cd..fa5ee1699 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -158,6 +158,34 @@ whiptail_create_socore_user_password2() { } +whiptail_create_web_user() { + + WEBUSER=$(whiptail --title "Security Onion Install" --inputbox \ + "Please enter an email address to create an administrator account for the web interface." 10 60 3>&1 1>&2 2>&3) + +} + + +whiptail_create_web_user_password1() { + + WEBPASSWD1=$(whiptail --title "Security Onion Install" --passwordbox \ + "Enter a password for $WEBUSER" 10 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + +whiptail_create_web_user_password2() { + + WEBPASSWD2=$(whiptail --title "Security Onion Install" --passwordbox \ + "Re-enter a password for $WEBUSER" 10 60 3>&1 1>&2 2>&3) + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus + +} + + whiptail_cur_close_days() { CURCLOSEDAYS=$(whiptail --title "Security Onion Setup" --inputbox \ @@ -683,6 +711,28 @@ whiptail_set_hostname() { } +whiptail_set_redirect() { + REDIRECTINFO=$(whiptail --title "Security Onion Setup" --radiolist \ + "Choose the access method for the web interface:" 20 75 4 \ + "IP" "Use IP to access the web interface" ON \ + "HOSTNAME" "Use Hostname ($HOSTNAME) to access the web interface" OFF \ + "OTHER" "Use a different name like a FQDN or Load Balancer" OFF 3>&1 1>&2 2>&3 ) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + +whiptail_set_redirect_host() { + REDIRECTHOST=$(whiptail --title "Security Onion Setup" --inputbox \ + "Enter the Hostname or IP you would like to use for the web interface." 10 75 $HOSTNAME 3>&1 1>&2 2>&3) + local exitstatus=$? + whiptail_check_exitstatus $exitstatus +} + +whiptail_set_redirect_info() { + whiptail --title "Security Onion Setup" --msgbox "The following selection refers to accessing the web interface. \n +For security reasons, we use strict cookie enforcement." 10 75 +} + whiptail_setup_complete() { whiptail --title "Security Onion Setup" --msgbox "Finished installing this as an $INSTALLTYPE. Press Enter to reboot." 8 75 diff --git a/upgrade/so-update-functions b/upgrade/so-update-functions index 71e8eac69..ef7bb4597 100644 --- a/upgrade/so-update-functions +++ b/upgrade/so-update-functions @@ -51,12 +51,13 @@ if [ $MASTERCHECK != 'so-helix' ]; then "so-idstools:$BUILD$UPDATEVERSION" \ "so-influxdb:$BUILD$UPDATEVERSION" \ "so-kibana:$BUILD$UPDATEVERSION" \ + "so-kratos:$BUILD$UPDATEVERSION" \ "so-logstash:$BUILD$UPDATEVERSION" \ "so-mysql:$BUILD$UPDATEVERSION" \ "so-navigator:$BUILD$UPDATEVERSION" \ "so-playbook:$BUILD$UPDATEVERSION" \ "so-redis:$BUILD$UPDATEVERSION" \ - "so-sensoroni:$BUILD$UPDATEVERSION" \ + "so-soc:$BUILD$UPDATEVERSION" \ "so-soctopus:$BUILD$UPDATEVERSION" \ "so-steno:$BUILD$UPDATEVERSION" \ "so-strelka:$BUILD$UPDATEVERSION" \ @@ -73,7 +74,6 @@ if [ $MASTERCHECK != 'so-helix' ]; then "so-idstools:$BUILD$UPDATEVERSION" \ "so-logstash:$BUILD$UPDATEVERSION" \ "so-redis:$BUILD$UPDATEVERSION" \ - "so-sensoroni:$BUILD$UPDATEVERSION" \ "so-steno:$BUILD$UPDATEVERSION" \ "so-suricata:$BUILD$UPDATEVERSION" \ "so-telegraf:$BUILD$UPDATEVERSION" \