diff --git a/pillar/docker/config.sls b/pillar/docker/config.sls new file mode 100644 index 000000000..f3259dfc0 --- /dev/null +++ b/pillar/docker/config.sls @@ -0,0 +1,205 @@ +{% set OSQUERY = salt['pillar.get']('master:osquery', '0') %} +{% set WAZUH = salt['pillar.get']('master:wazuh', '0') %} +{% set THEHIVE = salt['pillar.get']('master:thehive', '0') %} +{% set PLAYBOOK = salt['pillar.get']('master:playbook', '0') %} +{% set FREQSERVER = salt['pillar.get']('master:freq', '0') %} +{% set DOMAINSTATS = salt['pillar.get']('master:domainstats', '0') %} +{% set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %} +{% set GRAFANA = salt['pillar.get']('master:grafana', '0') %} + + +eval: + containers: + - so-core + - so-telegraf + {% if GRAFANA == '1' %} + - so-influxdb + - so-grafana + {% endif %} + - so-dockerregistry + - so-sensoroni + - so-idstools + - so-auth-api + - so-auth-ui + {% if OSQUERY != '0' %} + - so-mysql + - so-fleet + - so-redis + {% endif %} + - so-elasticsearch + - so-logstash + - so-kibana + - so-steno + - so-suricata + - so-zeek + - so-curator + - so-elastalert + {% if WAZUH != '0' %} + - so-wazuh + {% endif %} + - so-soctopus + {% if THEHIVE != '0' %} + - so-thehive + - so-thehive-es + - so-cortex + {% endif %} + {% if PLAYBOOK != '0' %} + - so-playbook + - so-navigator + {% endif %} + {% if FREQSERVER != '0' %} + - so-freqserver + {% endif %} + {% if DOMAINSTATS != '0' %} + - so-domainstats + {% endif %} +heavy_node: + containers: + - so-core + - so-telegraf + - so-redis + - so-logstash + - so-elasticsearch + - so-curator + - so-steno + - so-suricata + - so-wazuh + - so-filebeat + {% if BROVER != 'SURICATA' %} + - so-zeek + {% endif %} +helix: + containers: + - so-core + - so-telegraf + - so-idstools + - so-steno + - so-zeek + - so-redis + - so-logstash + - so-filebeat +hot_node: + containers: + - so-core + - so-telegraf + - so-logstash + - so-elasticsearch + - so-curator +master_search: + containers: + - so-core + - so-telegraf + - so-sensoroni + - so-acng + - so-idstools + - so-redis + - so-auth-api + - so-auth-ui + - so-logstash + - so-elasticsearch + - so-curator + - so-kibana + - so-elastalert + - so-filebeat + - so-soctopus + {% if OSQUERY != '0' %} + - so-mysql + - so-fleet + - so-redis + {% endif %} + {% if WAZUH != '0' %} + - so-wazuh + {% endif %} + - so-soctopus + {% if THEHIVE != '0' %} + - so-thehive + - so-thehive-es + - so-cortex + {% endif %} + {% if PLAYBOOK != '0' %} + - so-playbook + - so-navigator + {% endif %} + {% if FREQSERVER != '0' %} + - so-freqserver + {% endif %} + {% if DOMAINSTATS != '0' %} + - so-domainstats + {% endif %} +master: + containers: + - so-dockerregistry + - so-core + - so-telegraf + {% if GRAFANA == '1' %} + - so-influxdb + - so-grafana + {% endif %} + - so-sensoroni + - so-acng + - so-idstools + - so-redis + - so-auth-api + - so-auth-ui + - so-elasticsearch + - so-logstash + - so-kibana + - so-elastalert + - so-filebeat + {% if OSQUERY != '0' %} + - so-mysql + - so-fleet + - so-redis + {% endif %} + {% if WAZUH != '0' %} + - so-wazuh + {% endif %} + - so-soctopus + {% if THEHIVE != '0' %} + - so-thehive + - so-thehive-es + - so-cortex + {% endif %} + {% if PLAYBOOK != '0' %} + - so-playbook + - so-navigator + {% endif %} + {% if FREQSERVER != '0' %} + - so-freqserver + {% endif %} + {% if DOMAINSTATS != '0' %} + - so-domainstats + {% endif %} +parser_node: + containers: + - so-core + - so-telegraf + - so-logstash +search_node: + containers: + - so-core + - so-telegraf + - so-logstash + - so-elasticsearch + - so-curator + - so-filebeat + {% if WAZUH != '0' %} + - so-wazuh + {% endif %} +sensor: + containers: + - so-core + - so-telegraf + - so-steno + - so-suricata + {% if BROVER != 'SURICATA' %} + - so-zeek + {% endif %} + - so-wazuh + - so-filebeat +warm_node: + containers: + - so-core + - so-telegraf + - so-elasticsearch + diff --git a/pillar/firewall/search_nodes.sls b/pillar/firewall/search_nodes.sls new file mode 100644 index 000000000..d6563b873 --- /dev/null +++ b/pillar/firewall/search_nodes.sls @@ -0,0 +1,2 @@ +search_nodes: + - 127.0.0.1 diff --git a/pillar/firewall/storage_nodes.sls b/pillar/firewall/storage_nodes.sls deleted file mode 100644 index ffa970320..000000000 --- a/pillar/firewall/storage_nodes.sls +++ /dev/null @@ -1,2 +0,0 @@ -storage_nodes: - - 127.0.0.1 diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls new file mode 100644 index 000000000..654afd2b3 --- /dev/null +++ b/pillar/logstash/eval.sls @@ -0,0 +1,4 @@ +logstash: + pipelines: + eval: + config: "/usr/share/logstash/pipelines/eval/*.conf" diff --git a/pillar/logstash/helix.sls b/pillar/logstash/helix.sls new file mode 100644 index 000000000..e396a7aad --- /dev/null +++ b/pillar/logstash/helix.sls @@ -0,0 +1,4 @@ +logstash: + pipelines: + helix: + config: "/usr/share/logstash/pipelines/helix/*.conf" diff --git a/pillar/logstash/master.sls b/pillar/logstash/master.sls new file mode 100644 index 000000000..3be98f6b9 --- /dev/null +++ b/pillar/logstash/master.sls @@ -0,0 +1,4 @@ +logstash: + pipelines: + master: + config: "/usr/share/logstash/pipelines/master/*.conf" diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls new file mode 100644 index 000000000..0eca8571f --- /dev/null +++ b/pillar/logstash/search.sls @@ -0,0 +1,4 @@ +logstash: + pipelines: + search: + config: "/usr/share/logstash/pipelines/search/*.conf" diff --git a/pillar/thresholding/pillar.example b/pillar/thresholding/pillar.example new file mode 100644 index 000000000..705cb606c --- /dev/null +++ b/pillar/thresholding/pillar.example @@ -0,0 +1,44 @@ +thresholding: + sids: + 8675309: + - threshold: + gen_id: 1 + type: threshold + track: by_src + count: 10 + seconds: 10 + - threshold: + gen_id: 1 + type: limit + track: by_dst + count: 100 + seconds: 30 + - rate_filter: + gen_id: 1 + track: by_rule + count: 50 + seconds: 30 + new_action: alert + timeout: 30 + - suppress: + gen_id: 1 + track: by_either + ip: 10.10.3.7 + 11223344: + - threshold: + gen_id: 1 + type: limit + track: by_dst + count: 10 + seconds: 10 + - rate_filter: + gen_id: 1 + track: by_src + count: 50 + seconds: 20 + new_action: pass + timeout: 60 + - suppress: + gen_id: 1 + track: by_src + ip: 10.10.3.0/24 diff --git a/pillar/thresholding/pillar.usage b/pillar/thresholding/pillar.usage new file mode 100644 index 000000000..1626433b1 --- /dev/null +++ b/pillar/thresholding/pillar.usage @@ -0,0 +1,20 @@ +thresholding: + sids: + : + - threshold: + gen_id: + type: + track: + count: + seconds: + - rate_filter: + gen_id: + track: + count: + seconds: + new_action: + timeout: + - suppress: + gen_id: + track: + ip: diff --git a/pillar/top.sls b/pillar/top.sls index 17bf33e02..42d40ec10 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -1,38 +1,55 @@ base: '*': - patch.needs_restarting + - docker.config + + 'G@role:so-mastersearch or G@role:so-heavynode': + - match: compound + - logstash.master + - logstash.search 'G@role:so-sensor': - - sensors.{{ grains.id }} - static - firewall.* - brologs + - minions.{{ grains.id }} + + 'G@role:so-master or G@role:so-mastersearch': + - match: compound + - static + - firewall.* + - data.* + - auth + - minions.{{ grains.id }} 'G@role:so-master': - - masters.{{ grains.id }} - - static - - firewall.* - - data.* - - auth + - logstash.master 'G@role:so-eval': - - masters.{{ grains.id }} - static - firewall.* - data.* - brologs - auth + - logstash.eval + - minions.{{ grains.id }} 'G@role:so-node': - - nodes.{{ grains.id }} - static - firewall.* + - minions.{{ grains.id }} + + 'G@role:so-heavynode': + - static + - firewall.* + - brologs + - minions.{{ grains.id }} 'G@role:so-helix': - - masters.{{ grains.id }} - - sensors.{{ grains.id }} - static - firewall.* - fireeye - - static - brologs + - logstash.helix + - static + - minions.{{ grains.id }} diff --git a/salt/auth/init.sls b/salt/auth/init.sls new file mode 100644 index 000000000..abbe514d3 --- /dev/null +++ b/salt/auth/init.sls @@ -0,0 +1,30 @@ +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} + +so-auth-api-dir: + file.directory: + - name: /opt/so/conf/auth/api + - user: 939 + - group: 939 + - makedirs: True + +so-auth-api: + docker_container.running: + - image: {{ MASTER }}:5000/soshybridhunter/so-auth-api:{{ VERSION }} + - hostname: so-auth-api + - name: so-auth-api + - environment: + - BASE_PATH: "/so-auth/api" + - AUTH_TOKEN_TIMEOUT: 32400 + - binds: + - /opt/so/conf/auth/api:/data + - port_bindings: + - 0.0.0.0:5656:5656 + +so-auth-ui: + docker_container.running: + - image: {{ MASTER }}:5000/soshybridhunter/so-auth-ui:{{ VERSION }} + - hostname: so-auth-ui + - name: so-auth-ui + - port_bindings: + - 0.0.0.0:4242:80 diff --git a/salt/common/grafana/etc/dashboards/dashboard.yml b/salt/common/grafana/etc/dashboards/dashboard.yml index 8bb7615ea..80d356c8e 100644 --- a/salt/common/grafana/etc/dashboards/dashboard.yml +++ b/salt/common/grafana/etc/dashboards/dashboard.yml @@ -17,13 +17,13 @@ providers: editable: true options: path: /etc/grafana/grafana_dashboards/forward_nodes -- name: 'Storage Nodes' - folder: 'Storage Nodes' +- name: 'Search Nodes' + folder: 'Search Nodes' type: file disableDeletion: false editable: true options: - path: /etc/grafana/grafana_dashboards/storage_nodes + path: /etc/grafana/grafana_dashboards/search_nodes {%- else %} - name: 'Security Onion' folder: 'Eval Mode' diff --git a/salt/common/grafana/grafana_dashboards/eval/eval.json b/salt/common/grafana/grafana_dashboards/eval/eval.json index 8dd5532d1..069226d3c 100644 --- a/salt/common/grafana/grafana_dashboards/eval/eval.json +++ b/salt/common/grafana/grafana_dashboards/eval/eval.json @@ -1395,7 +1395,7 @@ "condition": "AND", "key": "container_name", "operator": "=", - "value": "so-bro" + "value": "so-zeek" } ] } @@ -1913,7 +1913,7 @@ "condition": "AND", "key": "container_name", "operator": "=", - "value": "so-bro" + "value": "so-zeek" } ] } diff --git a/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json b/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json index 83a1fc9e6..8e35246eb 100644 --- a/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json +++ b/salt/common/grafana/grafana_dashboards/forward_nodes/sensor.json @@ -1396,7 +1396,7 @@ "condition": "AND", "key": "container_name", "operator": "=", - "value": "so-bro" + "value": "so-zeek" } ] } @@ -1901,7 +1901,7 @@ "condition": "AND", "key": "container_name", "operator": "=", - "value": "so-bro" + "value": "so-zeek" } ] } diff --git a/salt/common/grafana/grafana_dashboards/storage_nodes/storage.json b/salt/common/grafana/grafana_dashboards/search_nodes/searchnode.json similarity index 99% rename from salt/common/grafana/grafana_dashboards/storage_nodes/storage.json rename to salt/common/grafana/grafana_dashboards/search_nodes/searchnode.json index 2f9a1111a..12688e15c 100644 --- a/salt/common/grafana/grafana_dashboards/storage_nodes/storage.json +++ b/salt/common/grafana/grafana_dashboards/search_nodes/searchnode.json @@ -12,7 +12,7 @@ } ] }, - "description": "This Dashboard provides a general overview of a Storage Node", + "description": "This Dashboard provides a general overview of a Search Node", "editable": true, "gnetId": 2381, "graphTooltip": 0, @@ -3433,7 +3433,7 @@ ] }, "timezone": "browser", - "title": "Storage Node - {{ SERVERNAME }} Overview", + "title": "Search Node - {{ SERVERNAME }} Overview", "uid": "{{ UID }}", "version": 3 } diff --git a/salt/common/init.sls b/salt/common/init.sls index e34431a46..4ae78f57b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -1,4 +1,6 @@ -{%- set GRAFANA = salt['pillar.get']('master:grafana', '0') %} +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} +{% set GRAFANA = salt['pillar.get']('master:grafana', '0') %} # Add socore Group socoregroup: group.present: @@ -114,16 +116,9 @@ nginxtmp: - group: 939 - makedirs: True -# Start the core docker -so-coreimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-core:HH1.1.3 - so-core: docker_container.running: - - require: - - so-coreimage - - image: docker.io/soshybridhunter/so-core:HH1.1.3 + - image: {{ MASTER }}:5000/soshybridhunter/so-core:{{ VERSION }} - hostname: so-core - user: socore - binds: @@ -175,15 +170,9 @@ tgrafconf: - template: jinja - source: salt://common/telegraf/etc/telegraf.conf -so-telegrafimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-telegraf:HH1.1.0 - so-telegraf: docker_container.running: - - require: - - so-telegrafimage - - image: docker.io/soshybridhunter/so-telegraf:HH1.1.0 + - image: {{ MASTER }}:5000/soshybridhunter/so-telegraf:{{ VERSION }} - environment: - HOST_PROC=/host/proc - HOST_ETC=/host/etc @@ -214,7 +203,7 @@ so-telegraf: - /opt/so/conf/telegraf/etc/telegraf.conf - /opt/so/conf/telegraf/scripts -# If its a master or eval lets install the back end for now +# If its a master or eval lets install the back end for now {% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' and GRAFANA == 1 %} # Influx DB @@ -236,15 +225,9 @@ influxdbconf: - template: jinja - source: salt://common/influxdb/etc/influxdb.conf -so-influximage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-influxdb:HH1.1.0 - so-influxdb: docker_container.running: - - require: - - so-influximage - - image: docker.io/soshybridhunter/so-influxdb:HH1.1.0 + - image: {{ MASTER }}:5000/soshybridhunter/so-influxdb:{{ VERSION }} - hostname: influxdb - environment: - INFLUXDB_HTTP_LOG_ENABLED=false @@ -303,7 +286,7 @@ grafanadashfndir: grafanadashsndir: file.directory: - - name: /opt/so/conf/grafana/grafana_dashboards/storage_nodes + - name: /opt/so/conf/grafana/grafana_dashboards/search_nodes - user: 939 - group: 939 - makedirs: True @@ -360,13 +343,13 @@ dashboard-{{ SN }}: {% if salt['pillar.get']('nodestab', False) %} {%- for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %} -dashboard-{{ SN }}: +dashboardsearch-{{ SN }}: file.managed: - - name: /opt/so/conf/grafana/grafana_dashboards/storage_nodes/{{ SN }}-Node.json + - name: /opt/so/conf/grafana/grafana_dashboards/search_nodes/{{ SN }}-Node.json - user: 939 - group: 939 - template: jinja - - source: salt://common/grafana/grafana_dashboards/storage_nodes/storage.json + - source: salt://common/grafana/grafana_dashboards/search_nodes/searchnode.json - defaults: SERVERNAME: {{ SN }} MANINT: {{ SNDATA.manint }} @@ -400,14 +383,9 @@ dashboard-{{ SN }}: {% endfor %} {% endif %} -# Install the docker. This needs to be behind nginx at some point -so-grafanaimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-grafana:HH1.1.0 - so-grafana: docker_container.running: - - image: docker.io/soshybridhunter/so-grafana:HH1.1.0 + - image: {{ MASTER }}:5000/soshybridhunter/so-grafana:{{ VERSION }} - hostname: grafana - user: socore - binds: diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval index b5cf6ef5a..701f558e7 100644 --- a/salt/common/nginx/nginx.conf.so-eval +++ b/salt/common/nginx/nginx.conf.so-eval @@ -58,9 +58,9 @@ http { # } #} server { - listen 80 default_server; - server_name _; - return 301 https://$host$request_uri; + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; } @@ -88,8 +88,8 @@ http { # } location /grafana/ { - rewrite /grafana/(.*) /$1 break; - proxy_pass http://{{ masterip }}:3000/; + rewrite /grafana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:3000/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -100,10 +100,9 @@ http { } location /kibana/ { - auth_basic "Security Onion"; - auth_basic_user_file /opt/so/conf/nginx/.htpasswd; - rewrite /kibana/(.*) /$1 break; - proxy_pass http://{{ masterip }}:5601/; + auth_request /so-auth/api/auth/; + rewrite /kibana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:5601/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -114,7 +113,7 @@ http { } location /playbook/ { - proxy_pass http://{{ masterip }}:3200/playbook/; + proxy_pass http://{{ masterip }}:3200/playbook/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -126,9 +125,8 @@ http { location /navigator/ { - auth_basic "Security Onion"; - auth_basic_user_file /opt/so/conf/nginx/.htpasswd; - proxy_pass http://{{ masterip }}:4200/navigator/; + auth_request /so-auth/api/auth/; + proxy_pass http://{{ masterip }}:4200/navigator/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -139,7 +137,7 @@ http { } location /api/ { - proxy_pass https://{{ masterip }}:8080/api/; + proxy_pass https://{{ masterip }}:8080/api/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Upgrade $http_upgrade; @@ -152,7 +150,7 @@ http { } location /fleet/ { - proxy_pass https://{{ masterip }}:8080/fleet/; + proxy_pass https://{{ masterip }}:8080/fleet/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -163,10 +161,10 @@ http { } location /thehive/ { - proxy_pass http://{{ masterip }}:9000/thehive/; + proxy_pass http://{{ masterip }}:9000/thehive/; proxy_read_timeout 90; proxy_connect_timeout 90; - proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_http_version 1.1; # this is essential for chunked responses to work proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -175,10 +173,10 @@ http { } location /cortex/ { - proxy_pass http://{{ masterip }}:9001/cortex/; + proxy_pass http://{{ masterip }}:9001/cortex/; proxy_read_timeout 90; proxy_connect_timeout 90; - proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_http_version 1.1; # this is essential for chunked responses to work proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -186,20 +184,8 @@ http { } - location /cyberchef/ { - proxy_pass http://{{ masterip }}:9080/; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_http_version 1.1; # this is essential for chunked responses to work - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - - } - location /soctopus/ { - proxy_pass http://{{ masterip }}:7000/; + proxy_pass http://{{ masterip }}:7000/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -210,17 +196,16 @@ http { } location /sensoroni/ { - auth_basic "Security Onion"; - auth_basic_user_file /opt/so/conf/nginx/.htpasswd; - proxy_pass http://{{ masterip }}:9822/; + auth_request /so-auth/api/auth/; + proxy_pass http://{{ masterip }}:9822/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "Upgrade"; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; } @@ -237,15 +222,34 @@ http { } location /sensoroniagents/ { - proxy_pass http://{{ masterip }}:9822/; + proxy_pass http://{{ masterip }}:9822/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy ""; - } + + location /so-auth/loginpage/ { + proxy_pass http://{{ masterip }}:4242/; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /so-auth/api/ { + proxy_pass http://{{ masterip }}:5656/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + } + + error_page 401 = @error401; + + location @error401 { + add_header Set-Cookie "NSREDIRECT=http://{{ masterip }}$request_uri;Domain={{ masterip }};Path=/;Max-Age=60000"; + return 302 http://{{ masterip }}/so-auth/loginpage/; + } + error_page 404 /404.html; location = /40x.html { } diff --git a/salt/common/nginx/nginx.conf.so-heavynode b/salt/common/nginx/nginx.conf.so-heavynode new file mode 100644 index 000000000..39688f3df --- /dev/null +++ b/salt/common/nginx/nginx.conf.so-heavynode @@ -0,0 +1,89 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + root /usr/share/nginx/html; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location / { + } + + error_page 404 /404.html; + location = /40x.html { + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + } + } + +# Settings for a TLS enabled server. +# +# server { +# listen 443 ssl http2 default_server; +# listen [::]:443 ssl http2 default_server; +# server_name _; +# root /usr/share/nginx/html; +# +# ssl_certificate "/etc/pki/nginx/server.crt"; +# ssl_certificate_key "/etc/pki/nginx/private/server.key"; +# ssl_session_cache shared:SSL:1m; +# ssl_session_timeout 10m; +# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_prefer_server_ciphers on; +# +# # Load configuration files for the default server block. +# include /etc/nginx/default.d/*.conf; +# +# location / { +# } +# +# error_page 404 /404.html; +# location = /40x.html { +# } +# +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# } +# } + +} diff --git a/salt/common/nginx/nginx.conf.so-master b/salt/common/nginx/nginx.conf.so-master index 265413fa2..2c836962c 100644 --- a/salt/common/nginx/nginx.conf.so-master +++ b/salt/common/nginx/nginx.conf.so-master @@ -88,7 +88,7 @@ http { # } location /grafana/ { - rewrite /grafana/(.*) /$1 break; + rewrite /grafana/(.*) /$1 break; proxy_pass http://{{ masterip }}:3000/; proxy_read_timeout 90; proxy_connect_timeout 90; @@ -100,9 +100,8 @@ http { } location /kibana/ { - auth_basic "Security Onion"; - auth_basic_user_file /opt/so/conf/nginx/.htpasswd; - rewrite /kibana/(.*) /$1 break; + auth_request /so-auth/api/auth/; + rewrite /kibana/(.*) /$1 break; proxy_pass http://{{ masterip }}:5601/; proxy_read_timeout 90; proxy_connect_timeout 90; @@ -125,8 +124,7 @@ http { } location /navigator/ { - auth_basic "Security Onion"; - auth_basic_user_file /opt/so/conf/nginx/.htpasswd; + auth_request /so-auth/api/auth/; proxy_pass http://{{ masterip }}:4200/navigator/; proxy_read_timeout 90; proxy_connect_timeout 90; @@ -151,9 +149,8 @@ http { } location /fleet/ { - rewrite /fleet/(.*) /$1 break; - auth_basic "Security Onion"; - auth_basic_user_file /opt/so/conf/nginx/.htpasswd; + rewrite /fleet/(.*) /$1 break; + auth_request /so-auth/api/auth/; proxy_pass https://{{ masterip }}:8080/; proxy_read_timeout 90; proxy_connect_timeout 90; @@ -188,18 +185,6 @@ http { } - location /cyberchef/ { - proxy_pass http://{{ masterip }}:9080/; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_http_version 1.1; # this is essential for chunked responses to work - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - - } - location /soctopus/ { proxy_pass http://{{ masterip }}:7000/; proxy_read_timeout 90; @@ -212,8 +197,7 @@ http { } location /sensoroni/ { - auth_basic "Security Onion"; - auth_basic_user_file /opt/so/conf/nginx/.htpasswd; + auth_request /so-auth/api/auth/; proxy_pass http://{{ masterip }}:9822/; proxy_read_timeout 90; proxy_connect_timeout 90; @@ -250,6 +234,26 @@ http { } + + location /so-auth/loginpage/ { + proxy_pass http://{{ masterip }}:4242/; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /so-auth/api/ { + proxy_pass http://{{ masterip }}:5656/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + } + + error_page 401 = @error401; + + location @error401 { + add_header Set-Cookie "NSREDIRECT=http://{{ masterip }}$request_uri;Domain={{ masterip }};Path=/;Max-Age=60000"; + return 302 http://{{ masterip }}/so-auth/loginpage/; + } + error_page 404 /404.html; location = /40x.html { } diff --git a/salt/common/nginx/nginx.conf.so-mastersearch b/salt/common/nginx/nginx.conf.so-mastersearch new file mode 100644 index 000000000..1bd0ebd2f --- /dev/null +++ b/salt/common/nginx/nginx.conf.so-mastersearch @@ -0,0 +1,278 @@ +{%- set masterip = salt['pillar.get']('master:mainip', '') %} +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + #server { + # listen 80 default_server; + # listen [::]:80 default_server; + # server_name _; + # root /opt/socore/html; + # index index.html; + + # Load configuration files for the default server block. + #include /etc/nginx/default.d/*.conf; + + # location / { + # } + + # error_page 404 /404.html; + # location = /40x.html { + # } + + # error_page 500 502 503 504 /50x.html; + # location = /50x.html { + # } + #} + server { + listen 80 default_server; + server_name _; + return 301 https://$host$request_uri; + } + + +# Settings for a TLS enabled server. + + server { + listen 443 ssl http2 default_server; + #listen [::]:443 ssl http2 default_server; + server_name _; + root /opt/socore/html; + index index.html; + + ssl_certificate "/etc/pki/nginx/server.crt"; + ssl_certificate_key "/etc/pki/nginx/server.key"; + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 10m; + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; + + # Load configuration files for the default server block. + #include /etc/nginx/default.d/*.conf; + + #location / { + # try_files $uri $uri.html /index.html; + # } + + location /grafana/ { + rewrite /grafana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:3000/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /kibana/ { + auth_request /so-auth/api/auth/; + rewrite /kibana/(.*) /$1 break; + proxy_pass http://{{ masterip }}:5601/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /playbook/ { + proxy_pass http://{{ masterip }}:3200/playbook/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /navigator/ { + auth_request /so-auth/api/auth/; + proxy_pass http://{{ masterip }}:4200/navigator/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /api/ { + proxy_pass https://{{ masterip }}:8080/api/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /fleet/ { + rewrite /fleet/(.*) /$1 break; + auth_request /so-auth/api/auth/; + proxy_pass https://{{ masterip }}:8080/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /thehive/ { + proxy_pass http://{{ masterip }}:9000/thehive/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /cortex/ { + proxy_pass http://{{ masterip }}:9001/cortex/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /cyberchef/ { + proxy_pass http://{{ masterip }}:9080/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_http_version 1.1; # this is essential for chunked responses to work + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /soctopus/ { + proxy_pass http://{{ masterip }}:7000/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /sensoroni/ { + auth_request /so-auth/api/auth/; + proxy_pass http://{{ masterip }}:9822/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + } + + location /kibana/app/sensoroni/ { + rewrite ^/kibana/app/sensoroni/(.*) /sensoroni/$1 permanent; + } + + location /kibana/app/fleet/ { + rewrite ^/kibana/app/fleet/(.*) /fleet/$1 permanent; + } + + location /kibana/app/soctopus/ { + rewrite ^/kibana/app/soctopus/(.*) /soctopus/$1 permanent; + } + + + location /sensoroniagents/ { + proxy_pass http://{{ masterip }}:9822/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + + location /so-auth/loginpage/ { + proxy_pass http://{{ masterip }}:4242/; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /so-auth/api/ { + proxy_pass http://{{ masterip }}:5656/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + } + + error_page 401 = @error401; + + location @error401 { + add_header Set-Cookie "NSREDIRECT=http://{{ masterip }}$request_uri;Domain={{ masterip }};Path=/;Max-Age=60000"; + return 302 http://{{ masterip }}/so-auth/loginpage/; + } + + error_page 404 /404.html; + location = /40x.html { + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + } + } + +} diff --git a/salt/common/scripts/dockernet.sh b/salt/common/scripts/dockernet.sh old mode 100644 new mode 100755 diff --git a/salt/common/telegraf/etc/telegraf.conf b/salt/common/telegraf/etc/telegraf.conf index af9941bfa..7d65073a1 100644 --- a/salt/common/telegraf/etc/telegraf.conf +++ b/salt/common/telegraf/etc/telegraf.conf @@ -15,6 +15,7 @@ {%- set MASTER = grains['master'] %} {% set NODEIP = salt['pillar.get']('node:mainip', '') %} +{% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} # Global tags can be specified here in key="value" format. @@ -86,6 +87,7 @@ ############################################################################### # Configuration for sending metrics to InfluxDB +{% if grains['role'] != 'so-helix' %} [[outputs.influxdb]] ## The full HTTP or UDP URL for your InfluxDB instance. ## @@ -148,7 +150,52 @@ ## integer values. Enabling this option will result in field type errors if ## existing data has been written. # influx_uint_support = false +{% else %} +# A plugin that can transmit metrics over HTTP +[[outputs.http]] + ## URL is the address to send metrics to + url = "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload" + ## Timeout for HTTP message + # timeout = "5s" + + ## HTTP method, one of: "POST" or "PUT" + method = "POST" + + ## HTTP Basic Auth credentials + # username = "username" + # password = "pa$$word" + + ## OAuth2 Client Credentials Grant + # client_id = "clientid" + # client_secret = "secret" + # token_url = "https://indentityprovider/oauth2/v1/token" + # scopes = ["urn:opc:idm:__myscopes__"] + + ## Optional TLS Config + # tls_ca = "/etc/telegraf/ca.pem" + # tls_cert = "/etc/telegraf/cert.pem" + # tls_key = "/etc/telegraf/key.pem" + ## Use TLS but skip chain & host verification + # insecure_skip_verify = false + + ## Data format to output. + ## Each data format has it's own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + data_format = "json" + + ## HTTP Content-Encoding for write request body, can be set to "gzip" to + ## compress body or "identity" to apply no encoding. + content_encoding = "gzip" + + ## Additional HTTP headers + [outputs.http.headers] + # # Should be set manually to "application/json" for json data_format + Content-Type = "application/json; charset=utf-8" + Authorization = "{{ HELIX_API_KEY }}" + +{% endif %} ############################################################################### # PROCESSOR PLUGINS # ############################################################################### @@ -647,6 +694,17 @@ "/scripts/influxdbsize.sh" ] data_format = "influx" +{% elif grains['role'] == 'so-helix' %} +[[inputs.exec]] + commands = [ + "/scripts/stenoloss.sh", + "/scripts/suriloss.sh", + "/scripts/checkfiles.sh", + "/scripts/broloss.sh", + "/scripts/oldpcap.sh", + "/scripts/helixeps.sh" + ] + data_format = "influx" {% endif %} # diff --git a/salt/common/telegraf/scripts/broloss.sh b/salt/common/telegraf/scripts/broloss.sh index a7bec4dc1..9fcf2d527 100644 --- a/salt/common/telegraf/scripts/broloss.sh +++ b/salt/common/telegraf/scripts/broloss.sh @@ -1,7 +1,7 @@ #!/bin/bash -BROLOG=$(tac /host/nsm/bro/logs/packetloss.log | head -2) -declare RESULT=($BROLOG) +ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2) +declare RESULT=($ZEEKLOG) CURRENTDROP=${RESULT[3]} PASTDROP=${RESULT[9]} DROPPED=$(($CURRENTDROP - $PASTDROP)) diff --git a/salt/common/telegraf/scripts/helixeps.sh b/salt/common/telegraf/scripts/helixeps.sh new file mode 100644 index 000000000..eee4f65c3 --- /dev/null +++ b/salt/common/telegraf/scripts/helixeps.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +PREVCOUNTFILE='/tmp/helixevents.txt' +EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.pipelines.helix.events.out')" + +if [ ! -z "$EVENTCOUNTCURRENT" ]; then + + if [ -f "$PREVCOUNTFILE" ]; then + EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` + else + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + exit 0 + fi + + echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE + EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) + if [ "$EVENTS" -lt 0 ]; then + EVENTS=0 + fi + + echo "helixeps eps=${EVENTS%%.*}" + +fi + +exit 0 diff --git a/salt/common/tools/sbin/so-allow b/salt/common/tools/sbin/so-allow old mode 100644 new mode 100755 index 1685e386a..61df47fd0 --- a/salt/common/tools/sbin/so-allow +++ b/salt/common/tools/sbin/so-allow @@ -1,42 +1,101 @@ #!/bin/bash -got_root() { - # Make sure you are root - if [ "$(id -u)" -ne 0 ]; then - echo "This script must be run using sudo!" - exit 1 - fi +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . -} +. /usr/sbin/so-common -got_root +SKIP=0 -echo "This program allows you to add a firewall rule to allow connections from a new IP address." -echo "" -echo "Choose the role for the IP or Range you would like to add" -echo "" -echo "[a] - Analyst - ports 80/tcp and 443/tcp" -echo "[b] - Logstash Beat - port 5044/tcp" -echo "[o] - Osquery endpoint - port 8080/tcp" -echo "[w] - Wazuh endpoint - port 1514" -echo "" -echo "Please enter your selection (a - analyst, b - beats, o - osquery, w - wazuh):" -read ROLE -echo "Enter a single ip address or range to allow (example: 10.10.10.10 or 10.10.0.0/16):" -read IP +while getopts "abowi:" OPTION +do + case $OPTION in + + h) + usage + exit 0 + ;; + a) + FULLROLE="analyst" + SKIP=1 + ;; + b) + FULLROLE="beats_endpoint" + SKIP=1 + ;; + i) IP=$OPTARG + ;; + o) + FULLROLE="osquery_endpoint" + SKIP=1 + ;; + w) + FULLROLE="wazuh_endpoint" + SKIP=1 + ;; + esac +done + +if [ "$SKIP" -eq 0 ]; then + + echo "This program allows you to add a firewall rule to allow connections from a new IP address." + echo "" + echo "Choose the role for the IP or Range you would like to add" + echo "" + echo "[a] - Analyst - ports 80/tcp and 443/tcp" + echo "[b] - Logstash Beat - port 5044/tcp" + echo "[o] - Osquery endpoint - port 8080/tcp" + echo "[w] - Wazuh endpoint - port 1514" + echo "" + echo "Please enter your selection (a - analyst, b - beats, o - osquery, w - wazuh):" + read ROLE + echo "Enter a single ip address or range to allow (example: 10.10.10.10 or 10.10.0.0/16):" + read IP + + if [ "$ROLE" == "a" ]; then + FULLROLE=analyst + elif [ "$ROLE" == "b" ]; then + FULLROLE=beats_endpoint + elif [ "$ROLE" == "o" ]; then + FULLROLE=osquery_endpoint + elif [ "$ROLE" == "w" ]; then + FULLROLE=wazuh_endpoint + else + echo "I don't recognize that role" + exit 1 + fi -if [ "$ROLE" == "a" ]; then - FULLROLE=analyst -elif [ "$ROLE" == "b" ]; then - FULLROLE=beats_endpoint -elif [ "$ROLE" == "o" ]; then - FULLROLE=osquery_endpoint -elif [ "$ROLE" == "w" ]; then - FULLROLE=wazuh_endpoint -else - echo "I don't recognize that role" - exit 1 fi echo "Adding $IP to the $FULLROLE role. This can take a few seconds" /opt/so/saltstack/pillar/firewall/addfirewall.sh $FULLROLE $IP + +# Check if Wazuh enabled +if grep -q -R "wazuh: 1" /opt/so/saltstack/pillar/*; then + # If analyst, add to Wazuh AR whitelist + if [ "$FULLROLE" == "analyst" ]; then + WAZUH_MGR_CFG="/opt/so/wazuh/etc/ossec.conf" + if ! grep -q "$IP" $WAZUH_MGR_CFG ; then + DATE=`date` + sed -i 's/<\/ossec_config>//' $WAZUH_MGR_CFG + sed -i '/^$/N;/^\n$/D' $WAZUH_MGR_CFG + echo -e "\n \n $IP\n \n" >> $WAZUH_MGR_CFG + echo "Added whitelist entry for $IP in $WAZUH_MGR_CFG." + echo + echo "Restarting OSSEC Server..." + /usr/sbin/so-wazuh-restart + fi + fi +fi diff --git a/salt/common/tools/sbin/so-auth-restart b/salt/common/tools/sbin/so-auth-restart new file mode 100644 index 000000000..8659b1e3a --- /dev/null +++ b/salt/common/tools/sbin/so-auth-restart @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart auth $1 + diff --git a/salt/common/tools/sbin/so-auth-start b/salt/common/tools/sbin/so-auth-start new file mode 100644 index 000000000..5330f662d --- /dev/null +++ b/salt/common/tools/sbin/so-auth-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start auth $1 diff --git a/salt/common/tools/sbin/so-auth-stop b/salt/common/tools/sbin/so-auth-stop new file mode 100644 index 000000000..5ca6db7e2 --- /dev/null +++ b/salt/common/tools/sbin/so-auth-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop auth $1 diff --git a/salt/common/tools/sbin/so-bpf-compile b/salt/common/tools/sbin/so-bpf-compile new file mode 100755 index 000000000..44c5b8249 --- /dev/null +++ b/salt/common/tools/sbin/so-bpf-compile @@ -0,0 +1,37 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "$#" -lt 2 ]; then + cat 1>&2 <. + +. /usr/sbin/so-common + +salt-call state.highstate diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common new file mode 100755 index 000000000..759f78f18 --- /dev/null +++ b/salt/common/tools/sbin/so-common @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Check for prerequisites +if [ "$(id -u)" -ne 0 ]; then + echo "This script must be run using sudo!" + exit 1 +fi + +# Define a banner to separate sections +banner="=========================================================================" + +header() { + echo + printf '%s\n' "$banner" "$*" "$banner" +} diff --git a/salt/common/tools/sbin/so-cortex-restart b/salt/common/tools/sbin/so-cortex-restart new file mode 100755 index 000000000..ef0e3e4fe --- /dev/null +++ b/salt/common/tools/sbin/so-cortex-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart cortex $1 diff --git a/salt/common/tools/sbin/so-cortex-start b/salt/common/tools/sbin/so-cortex-start new file mode 100755 index 000000000..a08969cab --- /dev/null +++ b/salt/common/tools/sbin/so-cortex-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start cortex $1 diff --git a/salt/common/tools/sbin/so-cortex-stop b/salt/common/tools/sbin/so-cortex-stop new file mode 100755 index 000000000..a13d1e2e3 --- /dev/null +++ b/salt/common/tools/sbin/so-cortex-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop cortex $1 diff --git a/salt/common/tools/sbin/so-curator-restart b/salt/common/tools/sbin/so-curator-restart new file mode 100755 index 000000000..6babd0bba --- /dev/null +++ b/salt/common/tools/sbin/so-curator-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart curator $1 diff --git a/salt/common/tools/sbin/so-curator-start b/salt/common/tools/sbin/so-curator-start new file mode 100755 index 000000000..308171f66 --- /dev/null +++ b/salt/common/tools/sbin/so-curator-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start curator $1 diff --git a/salt/common/tools/sbin/so-curator-stop b/salt/common/tools/sbin/so-curator-stop new file mode 100755 index 000000000..f815868ee --- /dev/null +++ b/salt/common/tools/sbin/so-curator-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop curator $1 diff --git a/salt/common/tools/sbin/so-elastalert-create b/salt/common/tools/sbin/so-elastalert-create new file mode 100755 index 000000000..fbe9527a7 --- /dev/null +++ b/salt/common/tools/sbin/so-elastalert-create @@ -0,0 +1,1000 @@ +#!/bin/bash + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Original written by Bryant Treacle +# https://raw.githubusercontent.com/bryant-treacle/Elastalert-Rule-Generator/master/so-elastalert-create +# Modified by Doug Burks +# +# Purpose: This script will allow you to test your elastalert rule without entering the Docker container. + +###################################################### +# Universal Rule Options # +###################################################### + +################################# +# Function for Main Menu # +################################# + +main_menu() +{ +while true; do + +rule_type_select_prompt + read rule_type + + if [ $rule_type = "1" ] ; then + rule_name_prompt + index_name_prompt + cardinality_rule_prompt + realert_prompt + filter_options_prompt + alert_options_prompt + final_prompt + elif [ $rule_type = "2" ] ; then + rule_name_prompt + index_name_prompt + blacklist_rule_prompt + realert_prompt + filter_options_prompt + alert_options_prompt + final_prompt + elif [ $rule_type = "3" ] ; then + rule_name_prompt + index_name_prompt + whitelist_rule_prompt + realert_prompt + filter_options_prompt + alert_options_prompt + final_prompt + elif [ $rule_type = "4" ] ; then + rule_name_prompt + index_name_prompt + frequency_rule_prompt + realert_prompt + filter_options_prompt + alert_options_prompt + final_prompt + elif [ $rule_type = "5" ] ; then + rule_name_prompt + index_name_prompt + change_rule_prompt + realert_prompt + filter_options_prompt + alert_options_prompt + final_prompt + elif [ $rule_type = "6" ] ; then + rule_name_prompt + index_name_prompt + spike_rule_prompt + realert_prompt + filter_options_prompt + alert_options_prompt + final_prompt + elif [ $rule_type = "7" ] ; then + rule_name_prompt + index_name_prompt + new_term_rule_prompt + realert_prompt + filter_options_prompt + alert_options_prompt + final_prompt + elif [ $rule_type = "8" ] ; then + rule_name_prompt + index_name_prompt + flatline_rule_prompt + realert_prompt + filter_options_prompt + alert_options_prompt + final_prompt + elif [ $rule_type = "9" ] ; then + exit + fi +done +} + +############################# +# Rule Type # +############################# +rule_type_select_prompt() +{ +cat << EOF + +This script will help automate the creation of Elastalert Rules. +Please choose the rule you would like to build. + +For Cardinality rules: Press 1 +For Blacklist rules: Press 2 +For Whitelist rules: Press 3 +For Frequency rules: Press 4 +For Change rules: Press 5 +For Spike rules: Press 6 +For New Term rules: Press 7 +For Flatline rules: Press 8 +To Exit: Press 9 + +EOF + +} + +############################# +# Rule Name # +############################# +rule_name_prompt() +{ + +cat << EOF +The rule name will appear in the subject of the alerts and be the name of the yaml rule file. + +What do you want to name the rule? + +EOF + + read raw_rulename + rulename=$(echo ${raw_rulename,,} | sed 's/ /_/g') + +cat << EOF >> "$rulename.yaml" +# Elasticsearch Host +es_host: elasticsearch +es_port: 9200 + +# (Required) +# Rule name, must be unique +name: $raw_rulename + +EOF +} + +############################# +# Index Name # +############################# +index_name_prompt() +{ +cat << EOF + +What elasticsearch index do you want to use? +Below are the default Index Patterns used in Security Onion: + +*:logstash-* +*:logstash-beats-* +*:elastalert_status* + +EOF + read indexname +cat << EOF >> "$rulename.yaml" + +# (Required) +# Index to search, wildcard supported +index: "$indexname" + +EOF +} + +############################# +# Alert Options # +############################# +alert_options_prompt() +{ +cat << EOF + +By default, all matches will be written back to the elastalert index. +Please choose from the below options. + + - For Email: Press 1 + - For Slack: Press 2 + - For the default (debug): Press 3 +EOF + +read alertoption + + if [ $alertoption = "1" ] ; then + echo "Please enter the email address you want to send the alerts to. Note: Ensure the Master Server is configured for SMTP." + read emailaddress + cat << EOF >> "$rulename.yaml" +# (Required) +# The alert is use when a match is found +alert: +- email + +# (required, email specific) +# a list of email addresses to send alerts to +email: + - $emailaddress +EOF + + elif [ $alertoption = "2" ] ; then + + echo "The webhook URL that includes your auth data and the ID of the channel (room) you want to post to." + echo "Go to the Incoming Webhooks section in your Slack account https://XXXXX.slack.com/services/new/incoming-webhook," + echo "choose the channel, click ‘Add Incoming Webhooks Integration’ and copy the resulting URL. You can use a list of URLs to send to multipe channels." + echo "" + echo "Please enter the webhook URL below:" + echo "" + read webhookurl + + cat << EOF >> "$rulename.yaml" +# (Required) +# The alert is use when a match is found +alert: +- slack + +# (required,Slack specific) +# Enter the webhook URL below +slack: + - $webhookurl + +EOF + + else + echo "Using default alert type of debug. Alerts will only be written to the *:elastalert_status* index." + cat << EOF >> "$rulename.yaml" +# (Required) +# The alert is use when a match is found +alert: +- debug + +EOF + + fi +} + +############################# +# Filter Options # +############################# +filter_options_prompt() +{ +cat << EOF + +By default this script will use a wildcard search that will include all logs for the index chosen above. +Would you like to use a specific filter? (Y/N) + +EOF + + read filteroption + if [ ${filteroption,,} = "y" ] ; then + echo "This script will allow you to generate basic filters. For complex filters visit https://elastalert.readthedocs.io/en/latest/recipes/writing_filters.html" + echo "" + echo "Term: Allows you to match a value in a field. For example you can select the field source_ip and the value 192.168.1.1" + echo "or choose a specific log type you want the rule to apply to ie. field_type: event_type and the field_value bro_http" + echo "" + echo "Wildcard: Allows you to use the wildcard * in the field_value. For example field_type: useragent and field_value: *Mozilla* " + echo "" + echo "Please choose from the following filter types." + echo "" + echo "term or wildcard" + read filter_type + if [ ${filter_type,,} = "term" ] ; then + echo "What field do you want to filter on?" + read field_name + echo "What is the value for the $field_name field." + read field_value + + cat << EOF >> "$rulename.yaml" +#(Required) +# A list of Elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: + +- term: + $field_name: "$field_value" + +EOF + elif [ ${filter_type,,} = "wildcard" ] ; then + echo "What field do you want to use?" + read field_name + echo "What is the value for the $field_name field." + read field_value + cat << EOF >> "$rulename.yaml" +#(Required) +# A list of Elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: + +- wildcard: + $field_name: "$field_value" + +EOF + fi + else + + cat << EOF >> "$rulename.yaml" +#(Required) +# A list of Elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: + +- wildcard: + event_type: "*" + +EOF + fi + +} + +############################ +# Re-alert Options # +############################ +realert_prompt() +{ +echo "The realert option allows you to ignore repeating alerts for a given period of time." +echo "Would you like to set a realert timeframe? (Y/N)" + read realert_option + + if [ ${realert_option,,} = "y" ] ; then + echo "Please choose from the following units of measure:" + echo "" + echo " - weeks, days, hours, minutes, or seconds" + read realert_unit_of_measure + echo "Please enter the number of $realert_unit_of_measure you want to use." + read realert_timeframe + + cat << EOF >> "$rulename.yaml" +# This option allows you to ignore repeating alerts for a period of time. +realert: + $realert_unit_of_measure: $realert_timeframe + +EOF + fi +} + +####################### +# Final prompt # +####################### +final_prompt() +{ +current_directory=$(pwd) +sleep 1 +echo "Writing rule to the following location:" +echo "" +echo " $current_directory/$rulename.yaml" +echo "" +sleep 1 +echo "Complete!" +sleep 1 +} + + +################################### +# Functions for Cardinality Rules # +################################### +cardinality_rule_prompt() +{ + echo "The Cardinality rule will be alert when the maximum or minimum number of unique values for a given field reach a threshold." + echo "What field do you want to be the Cardinality Field?" + echo "" + read cardinality_field + cat << EOF >> "$rulename.yaml" + +# (Required) +# Type of alert. +# The Cardinality rule matches when the total number of unique values for a certain field , within a given timeframe is higher or lower than a threshold. +type: cardinality + +# (Required, cardinality specific) +# Count the number of unique value for this field +cardinality_field: $cardinality_field + +EOF + echo "" + echo "To alert on values LESS than X unique values in the cardinality field: Press 1" + echo "To alert on values GREATER than X unique values in the cardinality field: Press 2" + echo "" + read cardinality_max_min + if [ $cardinality_max_min = "1" ] ; then + echo "The Minimum Cardinality value will alert you when there is less than X unique values in that field." + echo "What is the minimum Cardinality value?" + echo "" + read cardinality_min + + cat << EOF >> "$rulename.yaml" +# (Required, frequency specific) +# Alert when there is less than X unique values +min_cardinality: $cardinality_min + +EOF + + elif [ $cardinality_max_min = "2" ] ; then + echo "The Maximum Cardinality value will alert you when there is more than X unique values." + echo "What is the maximum Cardinality value?" + echo "" + read cardinality_max + + cat << EOF >> "$rulename.yaml" +# (Required, frequency specific) +# Alert when there is more than X unique values +max_cardinality: $cardinality_max + +EOF + + fi + echo "" + echo "The Cardinality Timeframe is defined as the number of unique values in the most recent X hours." + echo "" + echo "Below are the available units of measure for the timeframe field:" + echo " - weeks, days, hours, minutes, or seconds" + echo "What unit of measure do you want to use?" + read timeframe_units + echo "Please enter the number of $timeframe_units you want to use." + read timeframe + + cat << EOF >> "$rulename.yaml" +# (Required, frequency specific) +# The cardinality is defined as the number of unique values for the most recent 4 hours + +timeframe: + $timeframe_units: $timeframe + +EOF + +echo "The query_key counts by this field. For each unique value of the query_key field, cardinality will be counted separately." +echo "Would you like to set the query_key parameter? (Y/N)" + read cardinality_rule_options + if [ ${cardinality_rule_options,,} = "y" ] ; then + echo "What field do you want the query_key to be?" + read query_key + cat << EOF >> "$rulename.yaml" +# (Optional, frequency specific) +# query_key: Group cardinality counts by this field. For each unique value of the query_key field, cardinality will be counted separately. +query_key: $query_key + +EOF + fi + +} + +################################# +# Functions for Blacklist Rules # +################################# +blacklist_rule_prompt() +{ + echo "The blacklist rule will compare the values contained in a text file against the compare_key and alert if there is a match." + echo "What field do you want to compare to the blacklist?" + echo "" + read compare_key + echo "The blacklist file should be a text file with a single value per line." + echo "" + echo "The file needs to be accessible by the so-elastalert container." + echo "" + echo "Please enter the full path and filename of the blacklist." + echo "" + read -e blacklist_file_location + + cat << EOF >> "$rulename.yaml" +# (Required) +# Type of alert. +# The Blacklist rule will check a certain field against a blacklist and match if it is in the blacklist +type: blacklist + +# (Required, blacklist) +# The name of the field to use to compare to the blacklist. If the field is null, those events will be ignored. +compare_key: $compare_key + +# (Required, blacklist) +# A list of blacklisted values, and/or a list of paths to flat files which contain the blacklisted values +blacklist: + - "!file $blacklist_file_location" + +EOF + +} + +################################### +# Functions for Whitelist Rules # +################################### +whitelist_rule_prompt() +{ + echo "The whitelist rule will compare the values contained in a text file against the compare_key and alert if there is a match." + echo "What field do you want to compare to the whitelist?" + echo "" + read compare_key + + echo "The whitelist file should be a text file with a single value per line." + echo "" + echo "The file needs to be accessible by the so-elastalert container." + echo "" + echo "Please enter the full path and filename of the whitelist." + echo "" + read -e whitelist_file_location + echo "The ignore_null parameter If true, events without a compare_key field will not match." + echo "" + echo "Please enter true or false for the ignore_null parameter." + read ignore_null + + cat << EOF >> "$rulename.yaml" +# (Required) +# Type of alert. +# the whitelist rule will check a certain field against a whitelist and match if it is in the whitelist +type: whitelist + +# (Required, whitelist) +# The name of the field to use to compare to the whitelist. If the field is null, those events will be ignored. +compare_key: $compare_key + +# (Required, whitelist) +# ignore_null: If true, events without a compare_key field will not match. +ignore_null: $ignore_null + +# (Required, whitelist) +# A list of whitelisted values, and/or a list of paths to flat files which contain the whitelisted values +whitelist: + - "!file $whitelist_file_location" + +EOF +} + +################################### +# Functions for Frequency Rules # +################################### +frequency_rule_prompt() +{ +echo "The Frequency rule matches when there are at least a certain number of events in a given timeframe." +echo "" +echo "Enter the number of events you want to alert on:" + read num_events +echo "" +echo "Below are the available units of measure for the timeframe field:" +echo " - weeks, days, hours, minutes, or seconds" +echo "What unit of measure do you want to use?" + read timeframe_units +echo "Please enter the number of $timeframe_units you want to use." + read timeframe + +cat << EOF >> "$rulename.yaml" + +# (Required) +# Type of alert. +# the frequency rule type alerts when num_events events occur with timeframe time +type: frequency + +# (Required, frequency specific) +# Alert when this many documents matching the query occur within a timeframe +num_events: $num_events + +# (Required, frequency specific) +# num_events must occur within this amount of time to trigger an alert +timeframe: + $timeframe_units: $timeframe + +EOF + +cat << EOF +The frequency rule has the below optional fields: + - use_count_query: if true, ElastALert will poll Elasticsearch using the count api and not download all the matching + documents. This is useful if you only care about the numbers and not the actual data. + - use_terms_query: If true, ElastAlert will make an aggregation query against Elasticsearch to get counts of documents matching + each unique value of the query_key. This will only return the Maximum of terms_size, default 50 unique terms. + + +Would you like to set the optional settings? (Y/N) + +EOF + read frequency_rule_options + + if [ ${frequency_rule_options,,} = "y" ] ; then + echo "To set the use_count_query to true: press 1" + echo "To set the use_terms_query to true: press 2" + read frequency_query_type + if [ $frequency_query_type = "1" ] ; then + + cat << EOF >> "$rulename.yaml" + +# Only count number of records, instead of bringing all data back +use_count_query: true +doc_type: 'doc' + +EOF + elif [ $frequency_query_type = "2" ] ; then + echo "Please enter the query_key:" + read query_key + echo "Please enter the terms size:" + read term_size + + cat << EOF >> "$rulename.yaml" +# Only count number of records, instead of bringing all data back +use_terms_query: true +doc_type: 'doc' + +# Query_key count of documents will be stored independently for each value of query_key +query_key: $query_key + +# Term_size is the maximum number of terms returned per query. Default is 50. +terms_size: $term_size + +EOF + fi + fi + +} + +################################ +# Functions for Change Rules # +################################ +change_rule_prompt() +{ +echo "The change rule will monitor a certain field and match if that field changes." +echo "" +echo "The field must change with respect to the last event with the same query_key." +echo "Below is an example with a query_key of bob and a compare_key of source_ip:" +echo "" +echo " -username bob AND source_ip: 192.168.1.2" +echo " -username bob AND source_ip: 192.168.1.3" +echo "" +echo "The compare_key parameter names of the field to monitor for changes." +echo "Since this is a list of strings, we can have multiple keys. An alert will trigger if any of the fields change." +echo "" +echo "What field do you want to monitor for changes?" + read compare_key +echo "" +echo "The query_key parameter names the field that must be present in all of the events that are checked." +echo "What field do you want be the query_key?" + read query_key +echo "" +echo "The value of compare_key must change in two events that are less than the timeframe apart to trigger an alert." +echo "" +echo "Below are the available units of measure for the timeframe field:" +echo " - weeks, days, hours, minutes, or seconds" +echo "What unit of measure do you want to use?" + read timeframe_units +echo "Please enter the number of $timeframe_units you want to use." + read timeframe + cat << EOF >> "$rulename.yaml" +# (Required) +# Type of alert. +# This rule will monitor a certain field and match if that field changes. +type: change + +# (Required, change specific) +# The field to look for changes in +compare_key: $compare_key + +# (Required, change specific) +# Ignore documents without the compare_key (source_ip) field +ignore_null: true + +# (Required, change specific) +# The change must occur in two documents with the same query_key +query_key: $query_key + +# (Required, change specific) +# The value of compare_key must change in two events that are less than timeframe apart to t$ +timeframe: + $timeframe_units: $timeframe + +EOF + +} + +################################ +# Functions for Spike Rules # +################################ +spike_rule_prompt() +{ +echo "The spike rule matches when the volume of events during a given time period is spike_height times larger or smaller than during the previous time period." +echo "" +echo "Example to detect syn flood attack to public facing webserver:" +echo "Alert when the number of connection states to my web server per hour is twice as many as the previous hour." +echo "" +echo "The spike_height parameter is the ratio of number of events in the last timeframe to the previous timeframe that when hit will trigger an alert." +echo "Note: This value is a multiple!! 2 = 2x as many; 5 = 5x as many" +echo "What do you want the spike_height to be?" + read spike_height +echo "" +echo "What do you want the spike_type parameter to be?" +echo " - up: more than previous timeframe" +echo " - down: less than previous timeframe" +echo " - both: up or down" + read spike_type +echo "Below are the available units of measure for the timeframe field:" +echo " - weeks, days, hours, minutes, or seconds" +echo "What unit of measure do you want to use?" + read timeframe_units +echo "Please enter the number of $timeframe_units you want to use." + read timeframe + cat << EOF >> "$rulename.yaml" +# (Required) +# Type of alert. +# This rule matches when the volume of events during a given time period is spike_height times larger or smaller than during the previous time period. +type: spike + +# (Required, spike specific) +# The ratio of number of events in the last timeframe to the previous timeframe. +spike_height: $spike_height + +# (Required, spike specific) +# The spike being up, down or both +spike_type: $spike_type + +# (Required, spike specific) +# The value of average out the rate of events over this time period. +timeframe: + $timeframe_units: $timeframe + +EOF + +echo "The spike rule has the following optional parameters:" +echo " - field_value: When set, uses the value of the field in the document and not the number of matching documents. Note the value must be a number" +echo " - threshold_ref: The minimum number of events that must exist in the reference window for an alert to trigger." +echo " For example, if spike_height: 3 and threshold_ref: 10, then the ‘reference’ window must contain at least 10 events and the ‘current’ window at least three times that for an alert to be triggered." +echo " - threshold_cur: The minimum number of events that must exist in the current window for an alert to trigger." +echo " For example, if spike_height: 3 and threshold_cur: 60, then an alert will occur if the current window has more than 60 events and the reference window has less than a third as many." +echo "" +echo "Would you like to set one of these parameters? (Y/N)" + read spike_additional_options + if [ ${spike_additional_options,,} = "y" ] ; then + counter=0 + while [ $counter -eq 0 ]; do + counter=$(( $counter + 1 )) + echo "Please choose from the following options:" + echo "For field_value: Press 1" + echo "For threshold_ref: Press 2" + echo "For threshold_cur: Press 3" + echo "To continue: Press 4" + read spike_options_select + if [ $spike_options_select = "1" ] ; then + echo "What field would you like to use?" + read field_value_field + cat << EOF >> "$rulename.yaml" +#(Optional, spike specific) +# field_value: When set, uses the value of the field in the document and not the number of matching documents. +field_value: $spike_options_select + +EOF +# reset the counter for the while loop + counter=0 + elif [ $spike_options_select = "2" ] ; then + echo "What would you like the threshold_ref to be?" + read threshold_ref_field + cat << EOF >> "$rulename.yaml" +#(Optional, spike specific) +# The minimum number of events that must exist in the reference window for an alert to trigger. +threshold_ref: $threshold_ref_field + +EOF +#Reset the counter for the while loop + counter=0 + elif [ $spike_options_select = "3" ] ; then + echo "What would you like the threshold_cur to be?" + read threshold_cur_field + cat << EOF >> "$rulename.yaml" +#(Optional, spike specific +# The minimum number of events that must exist in the current window for an alert to trigger. +threshold_cur: $threshold_cur_field + +EOF +#Reset the counter for the while loop + counter=0 + elif [ $spike_options_select = "4" ] ; then + counter=1 + fi + done + fi +} + +################################### +# Functions for new term Rules # +################################### +new_term_rule_prompt() +{ +echo "This rule matches when a new value appears in a field that has never been seen before." +echo "When ElastAlert starts, it will use an aggregation query to gather all known terms for a list of fields." +echo "" +echo "What field(s) do you want to monitor for new terms?" + read new_term_field +cat << EOF >> "$rulename.yaml" +# (Required) +# Type of alert. +# This rule matches when a new value appears in a field that has never been seen before. +type: new_term + +# (Required, new_term specific) +# Monitor the field ip_address +fields: + - "$new_term_field" + +EOF + +echo "The New Term rule has the following additional options:" +echo " - terms_window_size: The amount of time used for the initial query to find existing terms. No term that has occurred within this time frame will trigger an alert. The default is 30 days." +echo " - window_step_size: When querying for existing terms, split up the time range into steps of this size. This is usefull when covering large timeframes" +echo " - alert_on_missing_field: Whether or not to alert when a field is missing from a document. The default is false." +echo "Would you like to set any of these options? (Y/N)" + read new_term_options + if [ ${new_term_options,,} = "y" ] ; then + counter=0 + while [ $counter -eq 0 ]; do + counter=$(( $counter + 1 )) + echo "Please choose from the following options:" + echo "" + echo "For terms_window_size: Press 1" + echo "For window_step_size: Press 2" + echo "For alert_on_missing_field: Press 3" + echo "To continue: Press 4" + read new_term_loop_option + if [ $new_term_loop_option = "1" ] ; then + echo "Below are the available units of measure for the terms_window_size field:" + echo " - weeks, days, hours, minutes, or seconds" + echo "What unit of measure do you want to use?" + read timeframe_units + echo "Please enter the number of $timeframe_units you want to use." + read timeframe + cat << EOF >> "$rulename.yaml" + +# (Optional, new_term specific) +# This means that we will query 90 days worth of data when ElastAlert starts to find which values of ip_address already exist +# If they existed in the last 90 days, no alerts will be triggered for them when they appear +terms_window_size: + $timeframe_units: $timeframe + +EOF +#Reset the while loop counter + counter=0 + elif [ $new_term_loop_option = "2" ] ; then + echo "Below are the available units of measure for the window_step_size field:" + echo " - weeks, days, hours, minutes, or seconds" + echo "What unit of measure do you want to use?" + read timeframe_units + echo "Please enter the number of $timeframe_units you want to use." + read timeframe + cat << EOF >> "$rulename.yaml" + +# (Optional, new_term specific) +# This means that we will query 90 days worth of data when ElastAlert starts to find which values of ip_address alr$ +# If they existed in the last 90 days, no alerts will be triggered for them when they appear +window_step_size: + $timeframe_units: $timeframe + +EOF +#Reset the while loop counter + counter=0 + elif [ $new_term_loop_option = "3" ] ; then + echo "Please enter either true or false for the alert_on_missing_field." + read alert_on_missing_field_option + cat << EOF >> "$rulename.yaml" +# (Optional, new_term specific) +# Whether or not to alert when a field is missing from a document. The default is false. +alert_on_missing_field: $alert_on_missing_field_option + +EOF +#reset the while loop counter + counter=0 + elif [ $new_term_loop_options = "4" ] ; then + counter=1 + fi + done + fi +} + +################################### +# Functions for Flat line Rules # +################################### +flatline_rule_prompt() +{ +echo "flatline: This rule matches when the total number of events is under a given threshold for a time period." +echo "" +echo "Please enter the minimum threshold of events." + read threshold +echo "Below are the available units of measure for the timeframe field:" +echo " - weeks, days, hours, minutes, or seconds" +echo "What unit of measure do you want to use?" + read timeframe_units +echo "Please enter the number of $timeframe_units you want to use." + read timeframe +echo "" + cat << EOF >> "$rulename.yaml" +# (Required) +# Type of alert. +# flatline: This rule matches when the total number of events is under a given threshold for a time period. +type: flatline + +# (Required, spike specific) +# threshold: The minimum number of events for an alert not to be triggered. +threshold: $threshold + +# (Required, spike specific) +# The value of average out the rate of events over this time period. +timeframe: + $timeframe_units: $timeframe + +EOF + +echo "The flatline rule has the following additional options:" +echo "" +echo " - use_count_query: If true, ElastAlert will poll Elasticsearch using the count api, and not download all of the matching documents." +echo " - use_terms_query: If true, ElastAlert will make an aggregation query against Elasticsearch to get counts of documents matching each unique value of query_key. " +echo " - terms_size: When used with use_terms_query, this is the maximum number of terms returned per query. Default is 50." +echo " - query_key: With flatline rule, query_key means that an alert will be triggered if any value of query_key has been seen at least once and then falls below the threshold." +echo " - forget_keys: Only valid when used with query_key. If this is set to true, ElastAlert will “forget” about the query_key value that triggers an alert, therefore preventing any more alerts for it until it’s seen again." +echo "" +echo "Would you like to set any of theses options? (Y/N)" + read flatline_option + if [ ${flatline_option,,} = "y" ] ; then + counter=0 + while [ $counter -eq 0 ]; do + counter=$(( $counter + 1 )) + echo "Please choose from the following options:" + echo "" + echo "For use_count_query: Press 1" + echo "For use_term_query: Press 2" + echo "For terms_size: Press 3" + echo "To continue: Press 4" + echo "" + read flatline_option_select + + if [ $flatline_option_select = "1" ] ; then + echo "Please enter true or false for the use_count_query field." + read use_count_query + cat << EOF >> "$rulename.yaml" + +# (Optional, flatline specific) +# use_count_query: If true, ElastAlert will poll Elasticsearch using the count api, and not download all of the matching documents. +use_count_query: $use_count_query +doc_type: 'doc' + +EOF +#Reset counter for while loop + counter=0 + + elif [ $flatline_option_select = "2" ] ; then + echo "Please enter true or false for the use_terms_query." + read use_terms_query + echo "Please enter the query_key field." + read query_key + echo "The forget_keys when set to true will, elastalert will forget about the query_key value, preventing any more alerts for it until it is seen again" + echo "" + echo "Please enter true of false for the forget_keys field." + read forget_keys + cat << EOF >> "$rulename.yaml" + +# (Optional, flatline specific) +# Use_terms_query: If true, ElastAlert will make an aggregation query against Elasticsearch to get counts of documents matching each unique value of query_key." +use_terms_query: $use_terms_query + +# (Optional, flatline specific) +# Query_key means that an alert will be triggered if any value of query_key has been seen at least once and then falls below the threshold." +query_key: $query_key + +# (Optional, flatline specific) +# If this is set to true, ElastAlert will “forget” about the query_key value that triggers an alert, therefore preventing any more alerts for it until it’s seen again. +forget_keys: $forget_keys + +EOF +#Reset counters for while loop + counter=0 + elif [ $flatline_option_select = "3" ] ; then + echo "Please enter the maximum number of terms returned per query, Default is 50" + read terms_size + cat << EOF >> "$rulename.yaml" + +# (Optional, flatline specific) +# When used with use_terms_query, this is the maximum number of terms returned per query. Default is 50. +terms_size: $terms_size + +EOF +#Reset counters for while loop + counter=0 + + elif [ $flatline_option_select = "4" ] ; then + counter=1 + fi + done +fi +} + +########################## +# Start Function # +########################## +main_menu diff --git a/salt/common/tools/sbin/so-elastalert-restart b/salt/common/tools/sbin/so-elastalert-restart new file mode 100755 index 000000000..861820037 --- /dev/null +++ b/salt/common/tools/sbin/so-elastalert-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart elastalert $1 diff --git a/salt/common/tools/sbin/so-elastalert-start b/salt/common/tools/sbin/so-elastalert-start new file mode 100755 index 000000000..b731fcf6f --- /dev/null +++ b/salt/common/tools/sbin/so-elastalert-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start elastalert $1 diff --git a/salt/common/tools/sbin/so-elastalert-stop b/salt/common/tools/sbin/so-elastalert-stop new file mode 100755 index 000000000..900c8ec26 --- /dev/null +++ b/salt/common/tools/sbin/so-elastalert-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop elastalert $1 diff --git a/salt/common/tools/sbin/so-elastalert-test b/salt/common/tools/sbin/so-elastalert-test new file mode 100755 index 000000000..e72d928ed --- /dev/null +++ b/salt/common/tools/sbin/so-elastalert-test @@ -0,0 +1,142 @@ +#!/bin/bash + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Originally written by Bryant Treacle +# https://raw.githubusercontent.com/bryant-treacle/so-elastalert-test-rule/master/so-elastalert-test +# Modified by Doug Burks and Wes Lambert +# +# Purpose: This script will allow you to test your elastalert rule without entering the Docker container. + +. /usr/sbin/so-elastic-common + +OPTIONS="" +SKIP=0 +RESULTS_TO_LOG="n" +RULE_NAME="" +FILE_SAVE_LOCATION="" + +usage() +{ +cat < Write results to specified log file + -o '' Specify Elastalert options ( Ex. --schema-only , --count-only, --days N ) + -r Specify path/name of rule to test + +EOF +} + +while getopts "hal:o:r:" OPTION +do + case $OPTION in + h) + usage + exit 0 + ;; + a) + OPTIONS="--alert" + ;; + l) + RESULTS_TO_LOG="y" + FILE_SAVE_LOCATION=$OPTARG + ;; + + o) + OPTIONS=$OPTARG + ;; + + r) + RULE_NAME=$OPTARG + SKIP=1 + ;; + *) + usage + exit 0 + ;; + esac +done + +docker_exec(){ + if [ ${RESULTS_TO_LOG,,} = "y" ] ; then + docker exec -it so-elastalert bash -c "elastalert-test-rule $RULE_NAME $OPTIONS" > $FILE_SAVE_LOCATION + else + docker exec -it so-elastalert bash -c "elastalert-test-rule $RULE_NAME $OPTIONS" + fi +} + +rule_prompt(){ + CURRENT_RULES=$(find /opt/so/rules/elastalert -name "*.yaml") + echo + echo "This script will allow you to test an Elastalert rule." + echo + echo "Below is a list of active Elastalert rules:" + echo + echo "-----------------------------------" + echo + echo "$CURRENT_RULES" + echo + echo "-----------------------------------" + echo + echo "Note: To test a rule it must be accessible by the Elastalert Docker container." + echo + echo "Make sure to swap the local path (/opt/so/rules/elastalert/) for the docker path (/etc/elastalert/rules/)" + echo "Example: /opt/so/rules/elastalert/nids2hive.yaml would be /etc/elastalert/rules/nids2hive.yaml" + echo + while [ -z $RULE_NAME ]; do + echo "Please enter the file path and rule name you want to test." + read -e RULE_NAME + done +} + +log_save_prompt(){ + RESULTS_TO_LOG="" + while [ -z $RESULTS_TO_LOG ]; do + echo "The results can be rather long. Would you like to write the results to a file? (Y/N)" + read RESULTS_TO_LOG + done +} + +log_path_prompt(){ + while [ -z $FILE_SAVE_LOCATION ]; do + echo "Please enter the file path and file name." + read -e FILE_SAVE_LOCATION + done + echo "Depending on the rule this may take a while." +} + +if [ $SKIP -eq 0 ]; then + rule_prompt + log_save_prompt + if [ ${RESULTS_TO_LOG,,} = "y" ] ; then + log_path_prompt + fi +fi + +docker_exec + +if [ $? -eq 0 ]; then + echo "Test completed successfully!" +else + echo "Something went wrong..." +fi + +echo + + + diff --git a/salt/common/tools/sbin/so-elastic-clear b/salt/common/tools/sbin/so-elastic-clear new file mode 100755 index 000000000..2db400839 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-clear @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +{%- set MASTERIP = salt['pillar.get']('static:masterip', '') -%} +. /usr/sbin/so-common + +SKIP=0 +######################################### +# Options +######################################### +usage() +{ +cat < /dev/null 2>&1 +done + +/usr/sbin/so-logstash-start +/usr/sbin/so-filebeat-start + diff --git a/salt/common/tools/sbin/so-elastic-diagnose b/salt/common/tools/sbin/so-elastic-diagnose new file mode 100755 index 000000000..367a145db --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-diagnose @@ -0,0 +1,33 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Source common settings +. /usr/sbin/so-common + +# Check for log files +for FILE in /opt/so/log/elasticsearch/*.log /opt/so/log/logstash/*.log /opt/so/log/kibana/*.log /opt/so/log/elastalert/*.log /opt/so/log/curator/*.log /opt/so/log/freqserver/*.log /opt/so/log/nginx/*.log; do + +# If file exists, then look for errors or warnings +if [ -f $FILE ]; then + MESSAGE=`grep -i 'ERROR\|FAIL\|WARN' $FILE` + if [ ! -z "$MESSAGE" ]; then + header $FILE + echo $MESSAGE | sed 's/WARN/\nWARN/g' | sed 's/WARNING/\nWARNING/g' | sed 's/ERROR/\nERROR/g' | sort | uniq -c | sort -nr + echo + fi +fi +done diff --git a/salt/common/tools/sbin/so-elastic-download b/salt/common/tools/sbin/so-elastic-download new file mode 100755 index 000000000..020a42f79 --- /dev/null +++ b/salt/common/tools/sbin/so-elastic-download @@ -0,0 +1,46 @@ +#!/bin/bash +MASTER=MASTER +VERSION="HH1.1.4" +TRUSTED_CONTAINERS=( \ +"so-auth-api:$VERSION" \ +"so-auth-ui:$VERSION" \ +"so-core:$VERSION" \ +"so-thehive-cortex:$VERSION" \ +"so-curator:$VERSION" \ +"so-domainstats:$VERSION" \ +"so-elastalert:$VERSION" \ +"so-elasticsearch:$VERSION" \ +"so-filebeat:$VERSION" \ +"so-fleet:$VERSION" \ +"so-fleet-launcher:$VERSION" \ +"so-freqserver:$VERSION" \ +"so-grafana:$VERSION" \ +"so-idstools:$VERSION" \ +"so-influxdb:$VERSION" \ +"so-kibana:$VERSION" \ +"so-logstash:$VERSION" \ +"so-mysql:$VERSION" \ +"so-navigator:$VERSION" \ +"so-playbook:$VERSION" \ +"so-redis:$VERSION" \ +"so-sensoroni:$VERSION" \ +"so-soctopus:$VERSION" \ +"so-steno:$VERSION" \ +#"so-strelka:$VERSION" \ +"so-suricata:$VERSION" \ +"so-telegraf:$VERSION" \ +"so-thehive:$VERSION" \ +"so-thehive-es:$VERSION" \ +"so-wazuh:$VERSION" \ +"so-zeek:$VERSION" ) + +for i in "${TRUSTED_CONTAINERS[@]}" +do + # Pull down the trusted docker image + echo "Downloading $i" + docker pull --disable-content-trust=false docker.io/soshybridhunter/$i + # Tag it with the new registry destination + docker tag soshybridhunter/$i $MASTER:5000/soshybridhunter/$i + docker push $MASTER:5000/soshybridhunter/$i + docker rmi soshybridhunter/$i +done diff --git a/salt/common/tools/sbin/so-elasticsearch-restart b/salt/common/tools/sbin/so-elasticsearch-restart new file mode 100755 index 000000000..42cf8ec40 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart elasticsearch $1 diff --git a/salt/common/tools/sbin/so-elasticsearch-start b/salt/common/tools/sbin/so-elasticsearch-start new file mode 100755 index 000000000..7373c354c --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start elasticsearch $1 diff --git a/salt/common/tools/sbin/so-elasticsearch-stop b/salt/common/tools/sbin/so-elasticsearch-stop new file mode 100755 index 000000000..6ec2acb13 --- /dev/null +++ b/salt/common/tools/sbin/so-elasticsearch-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop elasticsearch $1 diff --git a/salt/common/tools/sbin/so-features-enable b/salt/common/tools/sbin/so-features-enable new file mode 100755 index 000000000..a37743960 --- /dev/null +++ b/salt/common/tools/sbin/so-features-enable @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +VERSION=$(grep soversion /opt/so/saltstack/pillar/static.sls | cut -d':' -f2|sed 's/ //g') +# Modify static.sls to enable Features +sed -i 's/features: False/features: True/' /opt/so/saltstack/pillar/static.sls +SUFFIX="-features" +TRUSTED_CONTAINERS=( \ + "so-elasticsearch:$VERSION$SUFFIX" \ + "so-filebeat:$VERSION$SUFFIX" \ + "so-kibana:$VERSION$SUFFIX" \ + "so-logstash:$VERSION$SUFFIX" ) + +for i in "${TRUSTED_CONTAINERS[@]}" +do + # Pull down the trusted docker image + echo "Downloading $i" + docker pull --disable-content-trust=false docker.io/soshybridhunter/$i + # Tag it with the new registry destination + docker tag soshybridhunter/$i $HOSTNAME:5000/soshybridhunter/$i + docker push $HOSTNAME:5000/soshybridhunter/$i +done +for i in "${TRUSTED_CONTAINERS[@]}" +do + echo "Removing $i locally" + docker rmi soshybridhunter/$i +done diff --git a/salt/common/tools/sbin/so-filebeat-restart b/salt/common/tools/sbin/so-filebeat-restart old mode 100644 new mode 100755 index 85faf7499..04f0a3bea --- a/salt/common/tools/sbin/so-filebeat-restart +++ b/salt/common/tools/sbin/so-filebeat-restart @@ -1,17 +1,20 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -docker stop so-filebeat && sudo docker rm so-filebeat && salt-call state.apply filebeat +. /usr/sbin/so-common + +/usr/sbin/so-restart filebeat $1 diff --git a/salt/common/tools/sbin/so-filebeat-start b/salt/common/tools/sbin/so-filebeat-start old mode 100644 new mode 100755 index e5ce6ed88..e9f36fcf0 --- a/salt/common/tools/sbin/so-filebeat-start +++ b/salt/common/tools/sbin/so-filebeat-start @@ -1,17 +1,20 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -docker rm so-filebeat && salt-call state.apply filebeat +. /usr/sbin/so-common + +/usr/sbin/so-start filebeat $1 diff --git a/salt/common/tools/sbin/so-filebeat-stop b/salt/common/tools/sbin/so-filebeat-stop old mode 100644 new mode 100755 index 3b7419db7..4b7df4e41 --- a/salt/common/tools/sbin/so-filebeat-stop +++ b/salt/common/tools/sbin/so-filebeat-stop @@ -1,17 +1,20 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -docker stop so-filebeat +. /usr/sbin/so-common + +/usr/sbin/so-stop filebeat $1 diff --git a/salt/common/tools/sbin/so-fleet-restart b/salt/common/tools/sbin/so-fleet-restart new file mode 100755 index 000000000..2dfbdc3dd --- /dev/null +++ b/salt/common/tools/sbin/so-fleet-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart fleet $1 diff --git a/salt/common/tools/sbin/so-fleet-start b/salt/common/tools/sbin/so-fleet-start new file mode 100755 index 000000000..d26d01cc9 --- /dev/null +++ b/salt/common/tools/sbin/so-fleet-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start fleet $1 diff --git a/salt/common/tools/sbin/so-fleet-stop b/salt/common/tools/sbin/so-fleet-stop new file mode 100755 index 000000000..94634633b --- /dev/null +++ b/salt/common/tools/sbin/so-fleet-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop fleet $1 diff --git a/salt/common/tools/sbin/so-get-parsed b/salt/common/tools/sbin/so-get-parsed deleted file mode 100644 index 5b299e494..000000000 --- a/salt/common/tools/sbin/so-get-parsed +++ /dev/null @@ -1 +0,0 @@ -sudo docker exec -it so-redis redis-cli llen logstash:unparsed diff --git a/salt/common/tools/sbin/so-get-unparsed b/salt/common/tools/sbin/so-get-unparsed deleted file mode 100644 index 5b299e494..000000000 --- a/salt/common/tools/sbin/so-get-unparsed +++ /dev/null @@ -1 +0,0 @@ -sudo docker exec -it so-redis redis-cli llen logstash:unparsed diff --git a/salt/common/tools/sbin/so-grafana-restart b/salt/common/tools/sbin/so-grafana-restart new file mode 100755 index 000000000..b0af550a4 --- /dev/null +++ b/salt/common/tools/sbin/so-grafana-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart grafana $1 diff --git a/salt/common/tools/sbin/so-grafana-start b/salt/common/tools/sbin/so-grafana-start new file mode 100755 index 000000000..64b9cb3bf --- /dev/null +++ b/salt/common/tools/sbin/so-grafana-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start grafana $1 diff --git a/salt/common/tools/sbin/so-grafana-stop b/salt/common/tools/sbin/so-grafana-stop new file mode 100755 index 000000000..0f1a70f08 --- /dev/null +++ b/salt/common/tools/sbin/so-grafana-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop grafana $1 diff --git a/salt/common/tools/sbin/so-helix-apikey b/salt/common/tools/sbin/so-helix-apikey old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-index-list b/salt/common/tools/sbin/so-index-list new file mode 100755 index 000000000..d241d444d --- /dev/null +++ b/salt/common/tools/sbin/so-index-list @@ -0,0 +1,18 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +curl -X GET "localhost:9200/_cat/indices?v" diff --git a/salt/common/tools/sbin/so-kibana-restart b/salt/common/tools/sbin/so-kibana-restart new file mode 100755 index 000000000..d2c5dbaf9 --- /dev/null +++ b/salt/common/tools/sbin/so-kibana-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart kibana $1 diff --git a/salt/common/tools/sbin/so-kibana-start b/salt/common/tools/sbin/so-kibana-start new file mode 100755 index 000000000..032b18901 --- /dev/null +++ b/salt/common/tools/sbin/so-kibana-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start kibana $1 diff --git a/salt/common/tools/sbin/so-kibana-stop b/salt/common/tools/sbin/so-kibana-stop new file mode 100755 index 000000000..31a64f3b2 --- /dev/null +++ b/salt/common/tools/sbin/so-kibana-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop kibana $1 diff --git a/salt/common/tools/sbin/so-list-index b/salt/common/tools/sbin/so-list-index deleted file mode 100644 index fda5eeb2e..000000000 --- a/salt/common/tools/sbin/so-list-index +++ /dev/null @@ -1 +0,0 @@ -curl -X GET "localhost:9200/_cat/indices?v" diff --git a/salt/common/tools/sbin/so-logstash-get-parsed b/salt/common/tools/sbin/so-logstash-get-parsed new file mode 100755 index 000000000..5560539c8 --- /dev/null +++ b/salt/common/tools/sbin/so-logstash-get-parsed @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +docker exec -it so-redis redis-cli llen logstash:unparsed diff --git a/salt/common/tools/sbin/so-logstash-get-unparsed b/salt/common/tools/sbin/so-logstash-get-unparsed new file mode 100755 index 000000000..5560539c8 --- /dev/null +++ b/salt/common/tools/sbin/so-logstash-get-unparsed @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +docker exec -it so-redis redis-cli llen logstash:unparsed diff --git a/salt/common/tools/sbin/so-logstash-restart b/salt/common/tools/sbin/so-logstash-restart new file mode 100755 index 000000000..dec06f95c --- /dev/null +++ b/salt/common/tools/sbin/so-logstash-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart logstash $1 diff --git a/salt/common/tools/sbin/so-logstash-start b/salt/common/tools/sbin/so-logstash-start new file mode 100755 index 000000000..82331196f --- /dev/null +++ b/salt/common/tools/sbin/so-logstash-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start logstash $1 diff --git a/salt/common/tools/sbin/so-logstash-stop b/salt/common/tools/sbin/so-logstash-stop new file mode 100755 index 000000000..e95083b01 --- /dev/null +++ b/salt/common/tools/sbin/so-logstash-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop logstash $1 diff --git a/salt/common/tools/sbin/so-mysql-restart b/salt/common/tools/sbin/so-mysql-restart new file mode 100755 index 000000000..b29a2e0ae --- /dev/null +++ b/salt/common/tools/sbin/so-mysql-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart mysql $1 diff --git a/salt/common/tools/sbin/so-mysql-start b/salt/common/tools/sbin/so-mysql-start new file mode 100755 index 000000000..e056f7a37 --- /dev/null +++ b/salt/common/tools/sbin/so-mysql-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start mysql $1 diff --git a/salt/common/tools/sbin/so-mysql-stop b/salt/common/tools/sbin/so-mysql-stop new file mode 100755 index 000000000..7393a00a1 --- /dev/null +++ b/salt/common/tools/sbin/so-mysql-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop mysql $1 diff --git a/salt/common/tools/sbin/so-nsm-clear b/salt/common/tools/sbin/so-nsm-clear new file mode 100755 index 000000000..95ded5a4b --- /dev/null +++ b/salt/common/tools/sbin/so-nsm-clear @@ -0,0 +1,76 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +. /usr/sbin/so-common + +SKIP=0 +######################################### +# Options +######################################### +usage() +{ +cat <. + +. /usr/sbin/so-common + +/usr/sbin/so-restart steno $1 diff --git a/salt/common/tools/sbin/so-pcap-start b/salt/common/tools/sbin/so-pcap-start new file mode 100755 index 000000000..595fd54bb --- /dev/null +++ b/salt/common/tools/sbin/so-pcap-start @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start steno $1 diff --git a/salt/common/tools/sbin/so-pcap-stop b/salt/common/tools/sbin/so-pcap-stop new file mode 100755 index 000000000..d539d2f98 --- /dev/null +++ b/salt/common/tools/sbin/so-pcap-stop @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop steno $1 diff --git a/salt/common/tools/sbin/so-playbook-restart b/salt/common/tools/sbin/so-playbook-restart new file mode 100755 index 000000000..8a246c0db --- /dev/null +++ b/salt/common/tools/sbin/so-playbook-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart playbook $1 diff --git a/salt/common/tools/sbin/so-playbook-ruleupdate b/salt/common/tools/sbin/so-playbook-ruleupdate old mode 100644 new mode 100755 index 6e2d16f5d..b29b63cf5 --- a/salt/common/tools/sbin/so-playbook-ruleupdate +++ b/salt/common/tools/sbin/so-playbook-ruleupdate @@ -1 +1,20 @@ -sudo docker exec so-soctopus python3 playbook_bulk-update.py +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +docker exec so-soctopus python3 playbook_bulk-update.py diff --git a/salt/common/tools/sbin/so-playbook-start b/salt/common/tools/sbin/so-playbook-start new file mode 100755 index 000000000..fb5df55f1 --- /dev/null +++ b/salt/common/tools/sbin/so-playbook-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start playbook $1 diff --git a/salt/common/tools/sbin/so-playbook-stop b/salt/common/tools/sbin/so-playbook-stop new file mode 100755 index 000000000..d0a84bab1 --- /dev/null +++ b/salt/common/tools/sbin/so-playbook-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop playbook $1 diff --git a/salt/common/tools/sbin/so-playbook-sync b/salt/common/tools/sbin/so-playbook-sync old mode 100644 new mode 100755 index 3fc13c199..f4c2c456e --- a/salt/common/tools/sbin/so-playbook-sync +++ b/salt/common/tools/sbin/so-playbook-sync @@ -1 +1,20 @@ -sudo docker exec so-soctopus python3 playbook_play-sync.py +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +docker exec so-soctopus python3 playbook_play-sync.py diff --git a/salt/common/tools/sbin/so-redis-count b/salt/common/tools/sbin/so-redis-count old mode 100644 new mode 100755 index 5b299e494..5560539c8 --- a/salt/common/tools/sbin/so-redis-count +++ b/salt/common/tools/sbin/so-redis-count @@ -1 +1,20 @@ -sudo docker exec -it so-redis redis-cli llen logstash:unparsed +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +docker exec -it so-redis redis-cli llen logstash:unparsed diff --git a/salt/common/tools/sbin/so-redis-restart b/salt/common/tools/sbin/so-redis-restart new file mode 100755 index 000000000..e2ec4c2d2 --- /dev/null +++ b/salt/common/tools/sbin/so-redis-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart redis $1 diff --git a/salt/common/tools/sbin/so-redis-start b/salt/common/tools/sbin/so-redis-start new file mode 100755 index 000000000..f64600b05 --- /dev/null +++ b/salt/common/tools/sbin/so-redis-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start redis $1 diff --git a/salt/common/tools/sbin/so-redis-stop b/salt/common/tools/sbin/so-redis-stop new file mode 100755 index 000000000..ac3d2d106 --- /dev/null +++ b/salt/common/tools/sbin/so-redis-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop redis $1 diff --git a/salt/common/tools/sbin/so-restart b/salt/common/tools/sbin/so-restart new file mode 100755 index 000000000..2e3c0a00c --- /dev/null +++ b/salt/common/tools/sbin/so-restart @@ -0,0 +1,37 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Usage: so-restart filebeat | kibana | playbook | thehive + +. /usr/sbin/so-common + +echo $banner +printf "Restarting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" +echo $banner + +if [ "$2" = "--force" ] +then + printf "\nForce-stopping all Salt jobs before proceeding\n\n" + salt-call saltutil.kill_all_jobs +fi + +case $1 in + "cortex") docker stop so-thehive-cortex so-thehive && docker rm so-thehive-cortex so-thehive && salt-call state.apply hive queue=True;; + "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;; + "auth") docker stop so-auth-api; docker stop so-auth-ui; salt-call state.apply auth queue=True;; + *) docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;; +esac diff --git a/salt/common/tools/sbin/so-rule-update b/salt/common/tools/sbin/so-rule-update old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-salt-start b/salt/common/tools/sbin/so-salt-start new file mode 100755 index 000000000..c53a71535 --- /dev/null +++ b/salt/common/tools/sbin/so-salt-start @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +echo $banner +printf "Starting local Salt Minion...\n" +echo $banner + +service salt-minion start +service salt-minion status diff --git a/salt/common/tools/sbin/so-salt-stop b/salt/common/tools/sbin/so-salt-stop new file mode 100755 index 000000000..fa3394cd6 --- /dev/null +++ b/salt/common/tools/sbin/so-salt-stop @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +echo $banner +printf "Stopping local Salt Minion...\n" +echo $banner + +service salt-minion stop +service salt-minion status diff --git a/salt/common/tools/sbin/so-soctopus-restart b/salt/common/tools/sbin/so-soctopus-restart new file mode 100755 index 000000000..3d8f67893 --- /dev/null +++ b/salt/common/tools/sbin/so-soctopus-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart soctopus $1 diff --git a/salt/common/tools/sbin/so-soctopus-start b/salt/common/tools/sbin/so-soctopus-start new file mode 100755 index 000000000..db0a33302 --- /dev/null +++ b/salt/common/tools/sbin/so-soctopus-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start soctopus $1 diff --git a/salt/common/tools/sbin/so-soctopus-stop b/salt/common/tools/sbin/so-soctopus-stop new file mode 100755 index 000000000..3747d6e34 --- /dev/null +++ b/salt/common/tools/sbin/so-soctopus-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop soctopus $1 diff --git a/salt/common/tools/sbin/so-start b/salt/common/tools/sbin/so-start old mode 100644 new mode 100755 index 8ad0326db..889160122 --- a/salt/common/tools/sbin/so-start +++ b/salt/common/tools/sbin/so-start @@ -1 +1,47 @@ -sudo salt-call state.highstate +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Usage: so-start all | filebeat | kibana | playbook | thehive + +. /usr/sbin/so-common + +echo $banner +printf "Starting $1...\n\nThis could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n" +echo $banner + +if [ "$2" = "--force" ] +then + printf "\nForce-stopping all Salt jobs before proceeding\n\n" + salt-call saltutil.kill_all_jobs +fi + +case $1 in + "all") salt-call state.highstate queue=True;; + "steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;; + "auth") + if docker ps | grep -q so-auth-api; then + if docker ps | grep -q so-auth-ui; then + printf "\n$1 is already running!\n\n" + else + docker rm so-auth-api >/dev/null 2>&1; docker rm so-auth-ui >/dev/null 2>&1; salt-call state.apply $1 queue=True + fi + else + docker rm so-auth-api >/dev/null 2>&1; docker rm so-auth-ui >/dev/null 2>&1; salt-call state.apply $1 queue=True + fi + ;; + *) if docker ps | grep -q so-$1; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; +esac diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status new file mode 100755 index 000000000..7f6e01eec --- /dev/null +++ b/salt/common/tools/sbin/so-status @@ -0,0 +1,206 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +{%- set pillar_suffix = ':containers' -%} +{%- if (salt['grains.get']('role') == 'so-mastersearch') -%} + {%- set pillar_val = 'master_search' -%} +{%- elif (salt['grains.get']('role') == 'so-master') -%} + {%- set pillar_val = 'master' -%} +{%- elif (salt['grains.get']('role') == 'so-heavynode') -%} + {%- set pillar_val = 'heavy_node' -%} +{%- elif (salt['grains.get']('role') == 'so-sensor') -%} + {%- set pillar_val = 'sensor' -%} +{%- elif (salt['grains.get']('role') == 'so-eval') -%} + {%- set pillar_val = 'eval' -%} +{%- elif (salt['grains.get']('role') == 'so-helix') -%} + {%- set pillar_val = 'helix' -%} +{%- elif (salt['grains.get']('role') == 'so-node') -%} + {%- if (salt['pillar.get']('node:node_type') == 'parser') -%} + {%- set pillar_val = 'parser_node' -%} + {%- elif (salt['pillar.get']('node:node_type') == 'hot') -%} + {%- set pillar_val = 'hot_node' -%} + {%- elif (salt['pillar.get']('node:node_type') == 'warm') -%} + {%- set pillar_val = 'warm_node' -%} + {%- elif (salt['pillar.get']('node:node_type') == 'search') -%} + {%- set pillar_val = 'search_node' -%} + {%- endif -%} +{%- endif -%} +{%- set pillar_name = pillar_val ~ pillar_suffix -%} +{%- set container_list = salt['pillar.get'](pillar_name) %} + +if ! [ "$(id -u)" = 0 ]; then + echo "This command must be run as root" + exit 1 +fi + +# Constants +ERROR_STRING="ERROR" +SUCCESS_STRING="OK" +PENDING_STRING="PENDING" +MISSING_STRING='MISSING' +declare -a BAD_STATUSES=("removing" "paused" "exited" "dead") +declare -a PENDING_STATUSES=("paused" "created" "restarting") +declare -a GOOD_STATUSES=("running") + +declare -a temp_container_name_list=() +declare -a temp_container_state_list=() + +declare -a container_name_list=() +declare -a container_state_list=() + +declare -a expected_container_list=() + +# {% raw %} + +compare_lists() { + local found=0 + + create_expected_container_list + + if [[ ${#expected_container_list[@]} = 0 ]]; then + container_name_list="${temp_container_name_list[*]}" + container_state_list="${temp_container_state_list[*]}" + return 1 + fi + + for intended_item in "${expected_container_list[@]}"; do + found=0 + for i in "${!temp_container_name_list[@]}"; do + [[ ${temp_container_name_list[$i]} = "$intended_item" ]] \ + && found=1 \ + && container_name_list+=("${temp_container_name_list[$i]}") \ + && container_state_list+=("${temp_container_state_list[$i]}") \ + && break + done + if [[ $found = 0 ]]; then + container_name_list+=("$intended_item") + container_state_list+=("missing") + fi + done +} + +# {% endraw %} + +create_expected_container_list() { + {% for item in container_list%} + expected_container_list+=("{{ item }}") + {% endfor %} +} + +populate_container_lists() { + systemctl is-active --quiet docker + + if [[ $? = 0 ]]; then + mapfile -t docker_raw_list < <(curl -s --unix-socket /var/run/docker.sock http:/containers/json?all=1 \ + | jq -c '.[] | { Name: .Names[0], State: .State }' \ + | tr -d '/{"}') + else + exit 1 + fi + + local container_name="" + local container_state="" + + for line in "${docker_raw_list[@]}"; do + container_name="$( echo $line | sed -e 's/Name:\(.*\),State:\(.*\)/\1/' )" # Get value in the first search group (container names) + container_state="$( echo $line | sed -e 's/Name:\(.*\),State:\(.*\)/\2/' )" # Get value in the second search group (container states) + + temp_container_name_list+=( "${container_name}" ) + temp_container_state_list+=( "${container_state}" ) + done + + compare_lists +} + +parse_status() { + local container_state=${1} + + [[ $container_state = "missing" ]] && printf $MISSING_STRING && return 1 + + for state in "${GOOD_STATUSES[@]}"; do + [[ $container_state = "$state" ]] && printf $SUCCESS_STRING && return 0 + done + + for state in "${PENDING_STATUSES[@]}"; do + [[ $container_state = "$state" ]] && printf $PENDING_STRING && return 0 + done + + # This is technically not needed since the default is error state + for state in "${BAD_STATUSES[@]}"; do + [[ $container_state = "$state" ]] && printf $ERROR_STRING && return 1 + done + + printf $ERROR_STRING && return 1 +} + +# {% raw %} + +print_line() { + local service_name=${1} + local service_state="$( parse_status ${2} )" + local columns=$(tput cols) + local state_color="\e[0m" + + local PADDING_CONSTANT=14 + + if [[ $service_state = "$ERROR_STRING" ]] || [[ $service_state = "$MISSING_STRING" ]]; then + state_color="\e[1;31m" + elif [[ $service_state = "$SUCCESS_STRING" ]]; then + state_color="\e[1;32m" + elif [[ $service_state = "$PENDING_STRING" ]]; then + state_color="\e[1;33m" + fi + + printf " $service_name " + for i in $(seq 0 $(( $columns - $PADDING_CONSTANT - ${#service_name} - ${#service_state} ))); do + printf "-" + done + printf " [ " + printf "${state_color}%b\e[0m" "$service_state" + printf "%s \n" " ]" +} + +main() { + local focus_color="\e[1;34m" + printf "\n" + printf "${focus_color}%b\e[0m" "Checking Docker status\n\n" + + systemctl is-active --quiet docker + if [[ $? = 0 ]]; then + print_line "Docker" "running" + else + print_line "Docker" "exited" + fi + + populate_container_lists + + printf "\n" + printf "${focus_color}%b\e[0m" "Checking container statuses\n\n" + + local num_containers=${#container_name_list[@]} + + for i in $(seq 0 $(($num_containers - 1 ))); do + print_line ${container_name_list[$i]} ${container_state_list[$i]} + done + + printf "\n" +} + +# {% endraw %} + + +main diff --git a/salt/common/tools/sbin/so-stop b/salt/common/tools/sbin/so-stop new file mode 100755 index 000000000..8d5770b64 --- /dev/null +++ b/salt/common/tools/sbin/so-stop @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Usage: so-stop filebeat | kibana | playbook | thehive + +. /usr/sbin/so-common + +echo $banner +printf "Stopping $1...\n" +echo $banner + +case $1 in + "auth") docker stop so-auth-api; docker rm so-auth-api; docker stop so-auth-ui; docker rm so-auth-ui ;; + *) docker stop so-$1 ; docker rm so-$1 ;; +esac + diff --git a/salt/common/tools/sbin/so-suricata-restart b/salt/common/tools/sbin/so-suricata-restart old mode 100644 new mode 100755 index 0fabe198c..151e1a44c --- a/salt/common/tools/sbin/so-suricata-restart +++ b/salt/common/tools/sbin/so-suricata-restart @@ -1,17 +1,20 @@ #!/bin/bash - -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -docker stop so-suricata && sudo docker rm so-suricata && salt-call state.apply suricata +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart suricata $1 diff --git a/salt/common/tools/sbin/so-suricata-start b/salt/common/tools/sbin/so-suricata-start old mode 100644 new mode 100755 index dd9bd8df9..9e04eedfb --- a/salt/common/tools/sbin/so-suricata-start +++ b/salt/common/tools/sbin/so-suricata-start @@ -1,17 +1,20 @@ #!/bin/bash - -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -docker rm so-suricata && salt-call state.apply suricata +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start suricata $1 diff --git a/salt/common/tools/sbin/so-suricata-stop b/salt/common/tools/sbin/so-suricata-stop old mode 100644 new mode 100755 index 8f0383164..7581f9c00 --- a/salt/common/tools/sbin/so-suricata-stop +++ b/salt/common/tools/sbin/so-suricata-stop @@ -1,17 +1,20 @@ #!/bin/bash - -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC # -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -docker stop so-suricata +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop suricata $1 diff --git a/salt/common/tools/sbin/so-bro-restart b/salt/common/tools/sbin/so-tcpreplay old mode 100644 new mode 100755 similarity index 53% rename from salt/common/tools/sbin/so-bro-restart rename to salt/common/tools/sbin/so-tcpreplay index 8161b7cb3..349bb6e84 --- a/salt/common/tools/sbin/so-bro-restart +++ b/salt/common/tools/sbin/so-tcpreplay @@ -1,7 +1,7 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC - +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -14,4 +14,17 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -docker stop so-bro && sudo docker rm so-bro && salt-call state.apply bro + +# Usage: so-tcpreplay "/opt/so/samples/*" + +REPLAY_ENABLED=$(docker images | grep so-tcpreplay) +REPLAY_RUNNING=$(docker ps | grep so-tcpreplay) + +if [ "$REPLAY_ENABLED" != "" ] && [ "$REPLAY_RUNNING" != "" ]; then + docker cp so-tcpreplay:/opt/samples /opt/samples + docker exec -it so-tcpreplay /usr/local/bin/tcpreplay -i bond0 -M10 $1 +else + echo "Replay functionality not enabled! To enable, run `so-tcpreplay-start`" + echo + echo "Note that you will need internet access to download the appropriate components" +fi diff --git a/salt/common/tools/sbin/so-tcpreplay-restart b/salt/common/tools/sbin/so-tcpreplay-restart new file mode 100755 index 000000000..28230c600 --- /dev/null +++ b/salt/common/tools/sbin/so-tcpreplay-restart @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart tcpreplay $1 + diff --git a/salt/common/tools/sbin/so-bro-stop b/salt/common/tools/sbin/so-tcpreplay-start old mode 100644 new mode 100755 similarity index 83% rename from salt/common/tools/sbin/so-bro-stop rename to salt/common/tools/sbin/so-tcpreplay-start index 62bc2e1b1..287404b96 --- a/salt/common/tools/sbin/so-bro-stop +++ b/salt/common/tools/sbin/so-tcpreplay-start @@ -1,7 +1,7 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC - +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -14,4 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -docker stop so-bro + +. /usr/sbin/so-common + +/usr/sbin/so-start tcpreplay $1 diff --git a/salt/common/tools/sbin/so-bro-start b/salt/common/tools/sbin/so-tcpreplay-stop old mode 100644 new mode 100755 similarity index 83% rename from salt/common/tools/sbin/so-bro-start rename to salt/common/tools/sbin/so-tcpreplay-stop index 87a47febe..d12014260 --- a/salt/common/tools/sbin/so-bro-start +++ b/salt/common/tools/sbin/so-tcpreplay-stop @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,4 +14,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -docker rm so-bro && salt-call state.apply bro + +. /usr/sbin/so-common + +/usr/sbin/so-stop tcpreplay $1 + diff --git a/salt/common/tools/sbin/so-thehive-restart b/salt/common/tools/sbin/so-thehive-restart new file mode 100755 index 000000000..08cd8318e --- /dev/null +++ b/salt/common/tools/sbin/so-thehive-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart thehive $1 diff --git a/salt/common/tools/sbin/so-thehive-start b/salt/common/tools/sbin/so-thehive-start new file mode 100755 index 000000000..92fe88bb5 --- /dev/null +++ b/salt/common/tools/sbin/so-thehive-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start thehive $1 diff --git a/salt/common/tools/sbin/so-thehive-stop b/salt/common/tools/sbin/so-thehive-stop new file mode 100755 index 000000000..b326f699c --- /dev/null +++ b/salt/common/tools/sbin/so-thehive-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop thehive $1 diff --git a/salt/common/tools/sbin/so-user-add b/salt/common/tools/sbin/so-user-add old mode 100644 new mode 100755 diff --git a/salt/common/tools/sbin/so-wazuh-restart b/salt/common/tools/sbin/so-wazuh-restart old mode 100644 new mode 100755 index 3183479c4..34e2eee9a --- a/salt/common/tools/sbin/so-wazuh-restart +++ b/salt/common/tools/sbin/so-wazuh-restart @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,4 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -docker stop so-wazuh && sudo docker rm so-wazuh && salt-call state.apply wazuh +. /usr/sbin/so-common + +/usr/sbin/so-restart wazuh $1 diff --git a/salt/common/tools/sbin/so-wazuh-start b/salt/common/tools/sbin/so-wazuh-start old mode 100644 new mode 100755 index 195287314..607f59ae7 --- a/salt/common/tools/sbin/so-wazuh-start +++ b/salt/common/tools/sbin/so-wazuh-start @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,4 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -docker rm so-wazuh && salt-call state.apply wazuh +. /usr/sbin/so-common + +/usr/sbin/so-start wazuh $1 + diff --git a/salt/common/tools/sbin/so-wazuh-stop b/salt/common/tools/sbin/so-wazuh-stop old mode 100644 new mode 100755 index dd64354c7..399d14567 --- a/salt/common/tools/sbin/so-wazuh-stop +++ b/salt/common/tools/sbin/so-wazuh-stop @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,4 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -docker stop so-wazuh +. /usr/sbin/so-common + +/usr/sbin/so-stop wazuh $1 + diff --git a/salt/common/tools/sbin/so-zeek-restart b/salt/common/tools/sbin/so-zeek-restart new file mode 100755 index 000000000..51d90e924 --- /dev/null +++ b/salt/common/tools/sbin/so-zeek-restart @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-restart zeek $1 diff --git a/salt/common/tools/sbin/so-zeek-start b/salt/common/tools/sbin/so-zeek-start new file mode 100755 index 000000000..b3190c319 --- /dev/null +++ b/salt/common/tools/sbin/so-zeek-start @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-start zeek $1 diff --git a/salt/common/tools/sbin/so-zeek-stop b/salt/common/tools/sbin/so-zeek-stop new file mode 100755 index 000000000..d57c91c2f --- /dev/null +++ b/salt/common/tools/sbin/so-zeek-stop @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +/usr/sbin/so-stop zeek $1 diff --git a/salt/curator/init.sls b/salt/curator/init.sls index 74dd47a99..e1cd4829a 100644 --- a/salt/curator/init.sls +++ b/salt/curator/init.sls @@ -1,3 +1,5 @@ +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} {% if grains['role'] == 'so-node' or grains['role'] == 'so-eval' %} # Curator # Create the group @@ -112,15 +114,9 @@ curdel: - month: '*' - dayweek: '*' -so-curatorimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-curator:HH1.1.0 - so-curator: docker_container.running: - - require: - - so-curatorimage - - image: docker.io/soshybridhunter/so-curator:HH1.1.0 + - image: {{ MASTER }}:5000/soshybridhunter/so-curator:{{ VERSION }} - hostname: curator - name: so-curator - user: curator diff --git a/salt/bro/cron/packetloss.sh b/salt/deprecated-bro/cron/packetloss.sh similarity index 100% rename from salt/bro/cron/packetloss.sh rename to salt/deprecated-bro/cron/packetloss.sh diff --git a/salt/deprecated-bro/cron/zeek_clean b/salt/deprecated-bro/cron/zeek_clean new file mode 100644 index 000000000..24bbc218c --- /dev/null +++ b/salt/deprecated-bro/cron/zeek_clean @@ -0,0 +1,64 @@ +#!/bin/bash + +# Delete Zeek Logs based on defined CRIT_DISK_USAGE value + +# Copyright 2014,2015,2016,2017,2018, 2019 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +clean () { + +SENSOR_DIR='/nsm' +CRIT_DISK_USAGE=90 +CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %) +LOG="/nsm/bro/logs/zeek_clean.log" + +if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then + while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; + do + TODAY=$(date -u "+%Y-%m-%d") + + # find the oldest Zeek logs directory and exclude today + OLDEST_DIR=$(ls /nsm/bro/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | grep -v $TODAY | head -n 1) + if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ] + then + echo "$(date) - No old Zeek logs available to clean up in /nsm/bro/logs/" >> $LOG + exit 0 + else + echo "$(date) - Removing directory: /nsm/bro/logs/$OLDEST_DIR" >> $LOG + rm -rf /nsm/bro/logs/"$OLDEST_DIR" + fi + + # find oldest files in extracted directory and exclude today + OLDEST_EXTRACT=$(find /nsm/bro/extracted -type f -printf '%T+ %p\n' 2>/dev/null | sort | grep -v $TODAY | head -n 1) + if [ -z "$OLDEST_EXTRACT" -o "$OLDEST_EXTRACT" == ".." -o "$OLDEST_EXTRACT" == "." ] + then + echo "$(date) - No old extracted files available to clean up in /nsm/bro/extracted/" >> $LOG + else + OLDEST_EXTRACT_DATE=`echo $OLDEST_EXTRACT | awk '{print $1}' | cut -d+ -f1` + OLDEST_EXTRACT_FILE=`echo $OLDEST_EXTRACT | awk '{print $2}'` + echo "$(date) - Removing extracted files for $OLDEST_EXTRACT_DATE" >> $LOG + find /nsm/bro/extracted -type f -printf '%T+ %p\n' | grep $OLDEST_EXTRACT_DATE | awk '{print $2}' |while read FILE + do + echo "$(date) - Removing extracted file: $FILE" >> $LOG + rm -f "$FILE" + done + fi + done +else + echo "$(date) - CRIT_DISK_USAGE value of $CRIT_DISK_USAGE not greater than current usage of $CUR_USAGE..." >> $LOG +fi +} + +clean diff --git a/salt/bro/files/local.bro b/salt/deprecated-bro/files/local.bro similarity index 98% rename from salt/bro/files/local.bro rename to salt/deprecated-bro/files/local.bro index 42112f7ee..afe4b94ca 100644 --- a/salt/bro/files/local.bro +++ b/salt/deprecated-bro/files/local.bro @@ -102,6 +102,9 @@ # is currently considered a preview and therefore not loaded by default. @load base/protocols/smb +# BPF Configuration +@load securityonion/bpfconf + # Add the interface to the log event #@load securityonion/add-interface-to-logs.bro diff --git a/salt/bro/files/local.bro.community b/salt/deprecated-bro/files/local.bro.community similarity index 100% rename from salt/bro/files/local.bro.community rename to salt/deprecated-bro/files/local.bro.community diff --git a/salt/bro/files/node.cfg b/salt/deprecated-bro/files/node.cfg similarity index 100% rename from salt/bro/files/node.cfg rename to salt/deprecated-bro/files/node.cfg diff --git a/salt/bro/init.sls b/salt/deprecated-bro/init.sls similarity index 72% rename from salt/bro/init.sls rename to salt/deprecated-bro/init.sls index 2e6f10f3c..8f36be420 100644 --- a/salt/bro/init.sls +++ b/salt/deprecated-bro/init.sls @@ -1,3 +1,7 @@ +{% set interface = salt['pillar.get']('sensor:interface', 'bond0') %} +{% set BPF_ZEEK = salt['pillar.get']('zeek:bpf') %} +{% set BPF_STATUS = 0 %} + # Bro Salt State # Add Bro group brogroup: @@ -35,6 +39,14 @@ brospooldir: - user: 937 - makedirs: true +# Bro extracted directory +broextractdir: + file.directory: + - name: /nsm/bro/extracted + - user: 937 + - group: 939 + - makedirs: True + brosfafincompletedir: file.directory: - name: /nsm/faf/files/incomplete @@ -71,6 +83,21 @@ plcronscript: - source: salt://bro/cron/packetloss.sh - mode: 755 +zeekcleanscript: + file.managed: + - name: /usr/local/bin/zeek_clean + - source: salt://bro/cron/zeek_clean + - mode: 755 + +/usr/local/bin/zeek_clean: + cron.present: + - user: root + - minute: '*' + - hour: '*' + - daymonth: '*' + - month: '*' + - dayweek: '*' + /usr/local/bin/packetloss.sh: cron.present: - user: root @@ -80,6 +107,32 @@ plcronscript: - month: '*' - dayweek: '*' +# BPF compilation and configuration +{% if BPF_ZEEK %} + {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', interface + ' ' + BPF_ZEEK|join(" ") ) %} + {% if BPF_CALC['stderr'] == "" %} + {% set BPF_STATUS = 1 %} + {% else %} +zeekbpfcompilationfailure: + test.configurable_test_state: + - changes: False + - result: False + - comment: "BPF Syntax Error - Discarding Specified BPF" + {% endif %} +{% endif %} + +zeekbpf: + file.managed: + - name: /opt/so/conf/bro/bpf + - user: 940 + - group: 940 + {% if BPF_STATUS %} + - contents_pillar: zeek:bpf + {% else %} + - contents: + - "ip or not ip" + {% endif %} + # Sync local.bro {% if salt['pillar.get']('static:broversion', '') == 'COMMUNITY' %} localbrosync: @@ -103,6 +156,7 @@ so-bro: - binds: - /nsm/bro/logs:/nsm/bro/logs:rw - /nsm/bro/spool:/nsm/bro/spool:rw + - /nsm/bro/extracted:/nsm/bro/extracted:rw - /opt/so/conf/bro/local.bro:/opt/bro/share/bro/site/local.bro:ro - /opt/so/conf/bro/node.cfg:/opt/bro/etc/node.cfg:ro - /opt/so/conf/bro/policy/securityonion:/opt/bro/share/bro/policy/securityonion:ro @@ -136,8 +190,10 @@ so-bro: - binds: - /nsm/bro/logs:/nsm/bro/logs:rw - /nsm/bro/spool:/nsm/bro/spool:rw + - /nsm/bro/extracted:/nsm/bro/extracted:rw - /opt/so/conf/bro/local.bro:/opt/bro/share/bro/site/local.bro:ro - /opt/so/conf/bro/node.cfg:/opt/bro/etc/node.cfg:ro + - /opt/so/conf/bro/bpf:/opt/bro/share/bro/site/bpf:ro - /opt/so/conf/bro/policy/securityonion:/opt/bro/share/bro/policy/securityonion:ro - /opt/so/conf/bro/policy/custom:/opt/bro/share/bro/policy/custom:ro - /opt/so/conf/bro/policy/intel:/opt/bro/share/bro/policy/intel:rw @@ -146,6 +202,5 @@ so-bro: - file: /opt/so/conf/bro/local.bro - file: /opt/so/conf/bro/node.cfg - file: /opt/so/conf/bro/policy - - + - file: /opt/so/conf/bro/bpf {% endif %} diff --git a/salt/bro/policy/intel/__load__.bro b/salt/deprecated-bro/policy/intel/__load__.bro similarity index 100% rename from salt/bro/policy/intel/__load__.bro rename to salt/deprecated-bro/policy/intel/__load__.bro diff --git a/salt/bro/policy/securityonion/add-interface-to-logs.bro b/salt/deprecated-bro/policy/securityonion/add-interface-to-logs.bro similarity index 100% rename from salt/bro/policy/securityonion/add-interface-to-logs.bro rename to salt/deprecated-bro/policy/securityonion/add-interface-to-logs.bro diff --git a/salt/bro/policy/securityonion/apt1/__load__.bro b/salt/deprecated-bro/policy/securityonion/apt1/__load__.bro similarity index 100% rename from salt/bro/policy/securityonion/apt1/__load__.bro rename to salt/deprecated-bro/policy/securityonion/apt1/__load__.bro diff --git a/salt/bro/policy/securityonion/apt1/apt1-certs.dat b/salt/deprecated-bro/policy/securityonion/apt1/apt1-certs.dat similarity index 100% rename from salt/bro/policy/securityonion/apt1/apt1-certs.dat rename to salt/deprecated-bro/policy/securityonion/apt1/apt1-certs.dat diff --git a/salt/bro/policy/securityonion/apt1/apt1-fqdn.dat b/salt/deprecated-bro/policy/securityonion/apt1/apt1-fqdn.dat similarity index 100% rename from salt/bro/policy/securityonion/apt1/apt1-fqdn.dat rename to salt/deprecated-bro/policy/securityonion/apt1/apt1-fqdn.dat diff --git a/salt/bro/policy/securityonion/apt1/apt1-md5.dat b/salt/deprecated-bro/policy/securityonion/apt1/apt1-md5.dat similarity index 100% rename from salt/bro/policy/securityonion/apt1/apt1-md5.dat rename to salt/deprecated-bro/policy/securityonion/apt1/apt1-md5.dat diff --git a/salt/deprecated-bro/policy/securityonion/bpfconf.bro b/salt/deprecated-bro/policy/securityonion/bpfconf.bro new file mode 100644 index 000000000..595aef8f2 --- /dev/null +++ b/salt/deprecated-bro/policy/securityonion/bpfconf.bro @@ -0,0 +1,106 @@ +##! This script is to support the bpf.conf file like other network monitoring tools use. +##! Please don't try to learn from this script right now, there are a large number of +##! hacks in it to work around bugs discovered in Bro. + +@load base/frameworks/notice + +module BPFConf; + +export { + ## The file that is watched on disk for BPF filter changes. + ## Two templated variables are available; "sensorname" and "interface". + ## They can be used by surrounding the term by doubled curly braces. + const filename = "/opt/bro/share/bro/site/bpf" &redef; + + redef enum Notice::Type += { + ## Invalid filter notice. + InvalidFilter + }; +} + +global filter_parts: vector of string = vector(); +global current_filter_filename = ""; + +type FilterLine: record { + s: string; +}; + +redef enum PcapFilterID += { + BPFConfPcapFilter, +}; + +event BPFConf::line(description: Input::EventDescription, tpe: Input::Event, s: string) + { + local part = sub(s, /[[:blank:]]*#.*$/, ""); + + # We don't want any blank parts. + if ( part != "" ) + filter_parts[|filter_parts|] = part; + } + +event Input::end_of_data(name: string, source:string) + { + if ( name == "bpfconf" ) + { + local filter = join_string_vec(filter_parts, " "); + capture_filters["bpf.conf"] = filter; + if ( Pcap::precompile_pcap_filter(BPFConfPcapFilter, filter) ) + { + PacketFilter::install(); + } + else + { + NOTICE([$note=InvalidFilter, + $msg=fmt("Compiling packet filter from %s failed", filename), + $sub=filter]); + } + + filter_parts=vector(); + } + } + + +function add_filter_file() + { + local real_filter_filename = BPFConf::filename; + + # Support the interface template value. + #if ( SecurityOnion::sensorname != "" ) + # real_filter_filename = gsub(real_filter_filename, /\{\{sensorname\}\}/, SecurityOnion::sensorname); + + # Support the interface template value. + #if ( SecurityOnion::interface != "" ) + # real_filter_filename = gsub(real_filter_filename, /\{\{interface\}\}/, SecurityOnion::interface); + + #if ( /\{\{/ in real_filter_filename ) + # { + # return; + # } + #else + # Reporter::info(fmt("BPFConf filename set: %s (%s)", real_filter_filename, Cluster::node)); + + if ( real_filter_filename != current_filter_filename ) + { + current_filter_filename = real_filter_filename; + Input::add_event([$source=real_filter_filename, + $name="bpfconf", + $reader=Input::READER_RAW, + $mode=Input::REREAD, + $want_record=F, + $fields=FilterLine, + $ev=BPFConf::line]); + } + } + +#event SecurityOnion::found_sensorname(name: string) +# { +# add_filter_file(); +# } + +event bro_init() &priority=5 + { + if ( BPFConf::filename != "" ) + add_filter_file(); + } + + diff --git a/salt/bro/policy/securityonion/conn-add-sensorname.bro b/salt/deprecated-bro/policy/securityonion/conn-add-sensorname.bro similarity index 100% rename from salt/bro/policy/securityonion/conn-add-sensorname.bro rename to salt/deprecated-bro/policy/securityonion/conn-add-sensorname.bro diff --git a/salt/bro/policy/securityonion/file-extraction/__load__.bro b/salt/deprecated-bro/policy/securityonion/file-extraction/__load__.bro similarity index 100% rename from salt/bro/policy/securityonion/file-extraction/__load__.bro rename to salt/deprecated-bro/policy/securityonion/file-extraction/__load__.bro diff --git a/salt/bro/policy/securityonion/file-extraction/extract.bro b/salt/deprecated-bro/policy/securityonion/file-extraction/extract.bro similarity index 100% rename from salt/bro/policy/securityonion/file-extraction/extract.bro rename to salt/deprecated-bro/policy/securityonion/file-extraction/extract.bro diff --git a/salt/bro/policy/securityonion/json-logs/__load__.bro b/salt/deprecated-bro/policy/securityonion/json-logs/__load__.bro similarity index 100% rename from salt/bro/policy/securityonion/json-logs/__load__.bro rename to salt/deprecated-bro/policy/securityonion/json-logs/__load__.bro diff --git a/salt/cyberchef/init.sls b/salt/domainstats/init.sls similarity index 56% rename from salt/cyberchef/init.sls rename to salt/domainstats/init.sls index 202b15037..01e673764 100644 --- a/salt/cyberchef/init.sls +++ b/salt/domainstats/init.sls @@ -1,4 +1,4 @@ -# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,41 +13,39 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Create the cyberchef group -cyberchefgroup: +# Create the group +dstatsgroup: group.present: - - name: cyberchef - - gid: 946 + - name: domainstats + - gid: 936 -# Add the cyberchef user -cyberchef: +# Add user +domainstats: user.present: - - uid: 946 - - gid: 946 - - home: /opt/so/conf/cyberchef + - uid: 936 + - gid: 936 + - home: /opt/so/conf/domainstats + - createhome: False -cyberchefconfdir: +# Create the log directory +dstatslogdir: file.directory: - - name: /opt/so/conf/cyberchef - - user: 946 + - name: /opt/so/log/domainstats + - user: 936 - group: 939 - makedirs: True -cybercheflog: - file.directory: - - name: /opt/so/log/cyberchef - - user: 946 - - group: 946 - - makedirs: True - -so-cyberchefimage: +so-domainstatsimage: cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-cyberchef:HH1.1.3 + - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-domainstats:HH1.0.3 -so-cyberchef: +so-domainstats: docker_container.running: - require: - - so-cyberchefimage - - image: docker.io/soshybridhunter/so-cyberchef:HH1.1.3 - - port_bindings: - - 0.0.0.0:9080:8080 + - so-domainstatsimage + - image: docker.io/soshybridhunter/so-domainstats:HH1.0.3 + - hostname: domainstats + - name: so-domainstats + - user: domainstats + - binds: + - /opt/so/log/domainstats:/var/log/domain_stats diff --git a/salt/elastalert/files/elastalert_config.yaml b/salt/elastalert/files/elastalert_config.yaml index 735ccb190..e71f41bf8 100644 --- a/salt/elastalert/files/elastalert_config.yaml +++ b/salt/elastalert/files/elastalert_config.yaml @@ -82,3 +82,7 @@ writeback_index: elastalert_status # sending the alert until this time period has elapsed alert_time_limit: days: 2 + +index_settings: + shards: 1 + replicas: 0 diff --git a/salt/elastalert/files/modules/so/thehive.py b/salt/elastalert/files/modules/so/thehive.py index 42b6f9e1d..af18b412e 100644 --- a/salt/elastalert/files/modules/so/thehive.py +++ b/salt/elastalert/files/modules/so/thehive.py @@ -1,84 +1,107 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals +# HiveAlerter modified from original at: https://raw.githubusercontent.com/Nclose-ZA/elastalert_hive_alerter/master/elastalert_hive_alerter/hive_alerter.py + import uuid -import re from elastalert.alerts import Alerter from thehive4py.api import TheHiveApi from thehive4py.models import Alert, AlertArtifact, CustomFieldHelper + class TheHiveAlerter(Alerter): """ Use matched data to create alerts containing observables in an instance of TheHive - This is a modified version for use with Security Onion """ required_options = set(['hive_connection', 'hive_alert_config']) - def alert(self, matches): + def get_aggregation_summary_text(self, matches): + text = super(TheHiveAlerter, self).get_aggregation_summary_text(matches) + if text: + text = '```\n{0}```\n'.format(text) + return text + def create_artifacts(self, match): + artifacts = [] + context = {'rule': self.rule, 'match': match} + for mapping in self.rule.get('hive_observable_data_mapping', []): + for observable_type, match_data_key in mapping.items(): + try: + artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) + except KeyError as e: + print(('format string {} fail cause no key {} in {}'.format(e, match_data_key, context))) + return artifacts + + def create_alert_config(self, match): + context = {'rule': self.rule, 'match': match} + alert_config = { + 'artifacts': self.create_artifacts(match), + 'sourceRef': str(uuid.uuid4())[0:6], + 'title': '{rule[name]}'.format(**context) + } + + alert_config.update(self.rule.get('hive_alert_config', {})) + + for alert_config_field, alert_config_value in alert_config.items(): + if alert_config_field == 'customFields': + custom_fields = CustomFieldHelper() + for cf_key, cf_value in alert_config_value.items(): + try: + func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) + except AttributeError: + raise Exception('unsupported custom field type {}'.format(cf_value['type'])) + value = cf_value['value'].format(**context) + func(cf_key, value) + alert_config[alert_config_field] = custom_fields.build() + elif isinstance(alert_config_value, str): + alert_config[alert_config_field] = alert_config_value.format(**context) + elif isinstance(alert_config_value, (list, tuple)): + formatted_list = [] + for element in alert_config_value: + try: + formatted_list.append(element.format(**context)) + except (AttributeError, KeyError, IndexError): + formatted_list.append(element) + alert_config[alert_config_field] = formatted_list + + return alert_config + + def send_to_thehive(self, alert_config): connection_details = self.rule['hive_connection'] - api = TheHiveApi( - connection_details.get('hive_host'), + connection_details.get('hive_host', ''), connection_details.get('hive_apikey', ''), proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), cert=connection_details.get('hive_verify', False)) - for match in matches: - context = {'rule': self.rule, 'match': match} + alert = Alert(**alert_config) + response = api.create_alert(alert) + if response.status_code != 201: + raise Exception('alert not successfully created in TheHive\n{}'.format(response.text)) + + def alert(self, matches): + if self.rule.get('hive_alert_config_type', 'custom') != 'classic': + for match in matches: + alert_config = self.create_alert_config(match) + self.send_to_thehive(alert_config) + else: + alert_config = self.create_alert_config(matches[0]) artifacts = [] - for mapping in self.rule.get('hive_observable_data_mapping', []): - for observable_type, match_data_key in mapping.items(): - try: - match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) - rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) - data_keys = match_data_keys + rule_data_keys - context_keys = list(context['match'].keys()) + list(context['rule'].keys()) - if all([True if k in context_keys else False for k in data_keys]): - artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) - except KeyError: - raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) + for match in matches: + artifacts += self.create_artifacts(match) + if 'related_events' in match: + for related_event in match['related_events']: + artifacts += self.create_artifacts(related_event) - alert_config = { - 'artifacts': artifacts, - 'sourceRef': str(uuid.uuid4())[0:6], - 'title': '{rule[index]}_{rule[name]}'.format(**context) - } - alert_config.update(self.rule.get('hive_alert_config', {})) - - for alert_config_field, alert_config_value in alert_config.items(): - if alert_config_field == 'customFields': - custom_fields = CustomFieldHelper() - for cf_key, cf_value in alert_config_value.items(): - try: - func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) - except AttributeError: - raise Exception('unsupported custom field type {}'.format(cf_value['type'])) - value = cf_value['value'].format(**context) - func(cf_key, value) - alert_config[alert_config_field] = custom_fields.build() - elif isinstance(alert_config_value, str): - alert_config[alert_config_field] = alert_config_value.format(**context) - elif isinstance(alert_config_value, (list, tuple)): - formatted_list = [] - for element in alert_config_value: - try: - formatted_list.append(element.format(**context)) - except (AttributeError, KeyError, IndexError): - formatted_list.append(element) - alert_config[alert_config_field] = formatted_list - - alert = Alert(**alert_config) - response = api.create_alert(alert) - - if response.status_code != 201: - raise Exception('alert not successfully created in TheHive\n{}'.format(response.text)) + alert_config['artifacts'] = artifacts + alert_config['title'] = self.create_title(matches) + alert_config['description'] = self.create_alert_body(matches) + self.send_to_thehive(alert_config) def get_info(self): return { 'type': 'hivealerter', 'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '') - } + } diff --git a/salt/elastalert/init.sls b/salt/elastalert/init.sls index 999bbbd91..599043d58 100644 --- a/salt/elastalert/init.sls +++ b/salt/elastalert/init.sls @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} {% if grains['role'] == 'so-master' %} {% set esalert = salt['pillar.get']('master:elastalert', '1') %} @@ -20,7 +21,7 @@ {% set esport = salt['pillar.get']('master:es_port', '') %} -{% elif grains['role'] == 'so-eval' %} +{% elif grains['role'] in ['so-eval','so-mastersearch'] %} {% set esalert = salt['pillar.get']('master:elastalert', '1') %} {% set esip = salt['pillar.get']('master:mainip', '') %} @@ -109,15 +110,9 @@ elastaconf: - group: 939 - template: jinja -so-elastalertimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-elastalert:HH1.1.1 - so-elastalert: docker_container.running: - - require: - - so-elastalertimage - - image: docker.io/soshybridhunter/so-elastalert:HH1.1.1 + - image: {{ MASTER }}:5000/soshybridhunter/so-elastalert:{{ VERSION }} - hostname: elastalert - name: so-elastalert - user: elastalert diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 6036d5da8..a97a2ae0f 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -1,4 +1,4 @@ -# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,26 +12,29 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} +{% if FEATURES %} + {% set FEATURES = "-features" %} +{% else %} + {% set FEATURES = '' %} +{% endif %} + {% if grains['role'] == 'so-master' %} {% set esclustername = salt['pillar.get']('master:esclustername', '') %} {% set esheap = salt['pillar.get']('master:esheap', '') %} -{% set freq = salt['pillar.get']('master:freq', '0') %} -{% set dstats = salt['pillar.get']('master:dstats', '0') %} -{% elif grains['role'] == 'so-eval' %} +{% elif grains['role'] in ['so-eval','so-mastersearch'] %} {% set esclustername = salt['pillar.get']('master:esclustername', '') %} {% set esheap = salt['pillar.get']('master:esheap', '') %} -{% set freq = salt['pillar.get']('master:freq', '0') %} -{% set dstats = salt['pillar.get']('master:dstats', '0') %} -{% elif grains['role'] == 'so-node' %} +{% elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {% set esclustername = salt['pillar.get']('node:esclustername', '') %} {% set esheap = salt['pillar.get']('node:esheap', '') %} -{% set freq = salt['pillar.get']('node:freq', '0') %} -{% set dstats = salt['pillar.get']('node:dstats', '0') %} {% endif %} @@ -104,15 +107,9 @@ eslogdir: - group: 939 - makedirs: True -so-elasticsearchimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-elasticsearch:HH1.1.0 - so-elasticsearch: docker_container.running: - - require: - - so-elasticsearchimage - - image: docker.io/soshybridhunter/so-elasticsearch:HH1.1.0 + - image: {{ MASTER }}:5000/soshybridhunter/so-elasticsearch:{{ VERSION }}{{ FEATURES }} - hostname: elasticsearch - name: so-elasticsearch - user: elasticsearch @@ -146,91 +143,3 @@ so-elasticsearch-pipelines-file: so-elasticsearch-pipelines: cmd.run: - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines {{ esclustername }} - -# Tell the main cluster I am here -#curl -XPUT http://\$ELASTICSEARCH_HOST:\$ELASTICSEARCH_PORT/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"$HOSTNAME": {"skip_unavailable": "true", "seeds": ["$DOCKER_INTERFACE:$REVERSE_PORT"]}}}}}' - -# See if Freqserver is enabled -{% if freq == 1 %} - -# Create the user -fservergroup: - group.present: - - name: freqserver - - gid: 935 - -# Add ES user -freqserver: - user.present: - - uid: 935 - - gid: 935 - - home: /opt/so/conf/freqserver - - createhome: False - -# Create the log directory -freqlogdir: - file.directory: - - name: /opt/so/log/freq_server - - user: 935 - - group: 935 - - makedirs: True - -so-freqimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-freqserver:HH1.0.3 - -so-freq: - docker_container.running: - - require: - - so-freqimage - - image: docker.io/soshybridhunter/so-freqserver:HH1.0.3 - - hostname: freqserver - - name: so-freqserver - - user: freqserver - - binds: - - /opt/so/log/freq_server:/var/log/freq_server:rw - - -{% endif %} - -{% if dstats == 1 %} - -# Create the group -dstatsgroup: - group.present: - - name: domainstats - - gid: 936 - -# Add user -domainstats: - user.present: - - uid: 936 - - gid: 936 - - home: /opt/so/conf/domainstats - - createhome: False - -# Create the log directory -dstatslogdir: - file.directory: - - name: /opt/so/log/domainstats - - user: 936 - - group: 939 - - makedirs: True - -so-domainstatsimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-domainstats:HH1.0.3 - -so-domainstats: - docker_container.running: - - require: - - so-domainstatsimage - - image: docker.io/soshybridhunter/so-domainstats:HH1.0.3 - - hostname: domainstats - - name: so-domainstats - - user: domainstats - - binds: - - /opt/so/log/domainstats:/var/log/domain_stats - - -{% endif %} diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 0da9b68bc..2eb2092f4 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -1,8 +1,15 @@ +{%- if grains.role == 'so-heavynode' %} +{%- set MASTER = salt['pillar.get']('sensor:mainip' '') %} +{%- else %} {%- set MASTER = grains['master'] %} +{%- endif %} + + {%- set HOSTNAME = salt['grains.get']('host', '') %} {%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %} {%- set WAZUHENABLED = salt['pillar.get']('static:wazuh_enabled', '1') %} {%- set FLEETENABLED = salt['pillar.get']('static:fleet_enabled', '1') %} +{%- set STRELKAENABLED = salt['pillar.get']('static:strelka_enabled', '1') %} name: {{ HOSTNAME }} @@ -66,12 +73,12 @@ filebeat.modules: # List of prospectors to fetch data. filebeat.prospectors: #------------------------------ Log prospector -------------------------------- -{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" %} +{%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" %} {%- if BROVER != 'SURICATA' %} {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '') %} - type: log paths: - - /nsm/bro/logs/current/{{ LOGNAME }}.log + - /nsm/zeek/logs/current/{{ LOGNAME }}.log fields: type: bro_{{ LOGNAME }} fields_under_root: true @@ -126,6 +133,19 @@ filebeat.prospectors: clean_removed: false close_removed: false +{%- endif %} + +{%- if STRELKAENABLED == '1' %} + + - type: log + paths: + - /opt/so/log/strelka/strelka.log + fields: + type: strelka + fields_under_root: true + clean_removed: false + close_removed: false + {%- endif %} #----------------------------- Logstash output --------------------------------- output.logstash: diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index fce1c6b38..671530cd1 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -1,5 +1,4 @@ # Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC - # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -12,31 +11,33 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{%- set MASTER = grains['master'] %} -{%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} - -# Filebeat Setup +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} +{% set MASTERIP = salt['pillar.get']('static:masterip', '') %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} +{% if FEATURES %} + {% set FEATURES = "-features" %} +{% else %} + {% set FEATURES = '' %} +{% endif %} filebeatetcdir: file.directory: - name: /opt/so/conf/filebeat/etc - user: 939 - group: 939 - makedirs: True - filebeatlogdir: file.directory: - name: /opt/so/log/filebeat - user: 939 - group: 939 - makedirs: True - filebeatpkidir: file.directory: - name: /opt/so/conf/filebeat/etc/pki - user: 939 - group: 939 - makedirs: True - # This needs to be owned by root filebeatconfsync: file.managed: @@ -45,44 +46,22 @@ filebeatconfsync: - user: 0 - group: 0 - template: jinja - -#filebeatcrt: -# file.managed: -# - name: /opt/so/conf/filebeat/etc/pki/filebeat.crt -# - source: salt://filebeat/files/filebeat.crt - -#filebeatkey: -# file.managed: -# - name: /opt/so/conf/filebeat/etc/pki/filebeat.key -# - source: salt://filebeat/files/filebeat.key - -so-filebeatimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-filebeat:HH1.1.1 - so-filebeat: docker_container.running: - - require: - - so-filebeatimage - - image: docker.io/soshybridhunter/so-filebeat:HH1.1.1 + - image: {{ MASTER }}:5000/soshybridhunter/so-filebeat:{{ VERSION }}{{ FEATURES }} - hostname: so-filebeat - user: root - extra_hosts: {{ MASTER }}:{{ MASTERIP }} - binds: - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - - /nsm/bro:/nsm/bro:ro + - /nsm/zeek:/nsm/zeek:ro - /opt/so/log/suricata:/suricata:ro - - /opt/so/wazuh/logs/alerts/:/wazuh/alerts:ro - - /opt/so/wazuh/logs/archives/:/wazuh/archives:ro + - /opt/so/wazuh/logs/alerts:/wazuh/alerts:ro + - /opt/so/wazuh/logs/archives:/wazuh/archives:ro - /opt/so/log/fleet/:/osquery/logs:ro -{%- if grains['role'] == 'so-master' %} - - /etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - - /etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro -{%- else %} - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro -{%- endif %} - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro - watch: - file: /opt/so/conf/filebeat/etc/filebeat.yml diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 8a6d41f0f..657ff7814 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -1,7 +1,7 @@ # Firewall Magic for the grid -{%- if grains['role'] in ['so-eval','so-master','so-helix'] %} +{%- if grains['role'] in ['so-eval','so-master','so-helix','so-mastersearch'] %} {%- set ip = salt['pillar.get']('static:masterip', '') %} -{%- elif grains['role'] == 'so-node' %} +{%- elif grains['role'] == 'so-node' or grains['role'] == 'so-heavynode' %} {%- set ip = salt['pillar.get']('node:mainip', '') %} {%- elif grains['role'] == 'so-sensor' %} {%- set ip = salt['pillar.get']('sensor:mainip', '') %} @@ -131,7 +131,7 @@ enable_wazuh_manager_1514_udp_{{ip}}: - save: True # Rules if you are a Master -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix'%} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %} #This should be more granular iptables_allow_master_docker: iptables.insert: @@ -359,6 +359,17 @@ enable_minion_osquery_8080_{{ip}}: - position: 1 - save: True +enable_minion_wazuh_55000_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 55000 + - position: 1 + - save: True + {% endfor %} # Allow Forward Nodes to send their beats traffic @@ -410,9 +421,9 @@ enable_forwardnode_sensoroni_9822_{{ip}}: {% endfor %} -{% for ip in pillar.get('storage_nodes') %} +{% for ip in pillar.get('search_nodes') %} -enable_storagenode_redis_6379_{{ip}}: +enable_searchnode_redis_6379_{{ip}}: iptables.insert: - table: filter - chain: DOCKER-USER @@ -423,7 +434,7 @@ enable_storagenode_redis_6379_{{ip}}: - position: 1 - save: True -enable_storagenode_ES_9300_{{ip}}: +enable_searchnode_ES_9300_{{ip}}: iptables.insert: - table: filter - chain: DOCKER-USER @@ -578,13 +589,13 @@ enable_standard_analyst_443_{{ip}}: {% endfor %} -# Rules for storage nodes connecting to master +# Rules for search nodes connecting to master {% endif %} -# Rules if you are a Storage Node -{% if grains['role'] == 'so-node' %} +# Rules if you are a Node +{% if 'node' in grains['role'] %} #This should be more granular iptables_allow_docker: @@ -655,3 +666,39 @@ iptables_drop_all_the_things: - chain: LOGGING - jump: DROP - save: True + +{% if grains['role'] == 'so-heavynode' %} +# Allow Redis +enable_heavynode_redis_6379_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 6379 + - position: 1 + - save: True + +enable_forwardnode_beats_5044_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 5044 + - position: 1 + - save: True + +enable_forwardnode_beats_5644_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 5644 + - position: 1 + - save: True +{% endif %} diff --git a/salt/fleet/init.sls b/salt/fleet/init.sls index 917ee541e..a27ad9eb6 100644 --- a/salt/fleet/init.sls +++ b/salt/fleet/init.sls @@ -1,6 +1,8 @@ {%- set MYSQLPASS = salt['pillar.get']('auth:mysql', 'iwonttellyou') %} {%- set FLEETPASS = salt['pillar.get']('auth:fleet', 'bazinga') -%} {%- set MASTERIP = salt['pillar.get']('static:masterip', '') -%} +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} # Fleet Setup fleetcdir: @@ -59,15 +61,9 @@ fleetdbpriv: - user: fleetdbuser - host: 172.17.0.0/255.255.0.0 -so-fleetimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-fleet:HH1.1.3 - so-fleet: docker_container.running: - - require: - - so-fleetimage - - image: docker.io/soshybridhunter/so-fleet:HH1.1.3 + - image: {{ MASTER }}:5000/soshybridhunter/so-fleet:{{ VERSION }} - hostname: so-fleet - port_bindings: - 0.0.0.0:8080:8080 diff --git a/salt/fleet/so-fleet-setup.sh b/salt/fleet/so-fleet-setup.sh index 32bbddbe7..cd082ff03 100644 --- a/salt/fleet/so-fleet-setup.sh +++ b/salt/fleet/so-fleet-setup.sh @@ -1,3 +1,5 @@ +#!/bin/bash + #so-fleet-setup.sh $MasterIP $FleetEmail if [ ! "$(docker ps -q -f name=so-fleet)" ]; then @@ -16,7 +18,7 @@ docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/Windows/o docker exec so-fleet fleetctl apply -f /packs/hh/hhdefault.yml docker exec so-fleet /bin/sh -c 'for pack in /packs/palantir/Fleet/Endpoints/packs/*.yaml; do fleetctl apply -f "$pack"; done' -esecret=$(sudo docker exec so-fleet fleetctl get enroll-secret) +esecret=$(docker exec so-fleet fleetctl get enroll-secret) #Concat fleet.crt & ca.crt - this is required for launcher connectivity cat /etc/pki/fleet.crt /etc/pki/ca.crt > /etc/pki/launcher.crt diff --git a/salt/freqserver/init.sls b/salt/freqserver/init.sls new file mode 100644 index 000000000..783d11b6a --- /dev/null +++ b/salt/freqserver/init.sls @@ -0,0 +1,52 @@ +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Create the user +fservergroup: + group.present: + - name: freqserver + - gid: 935 + +# Add ES user +freqserver: + user.present: + - uid: 935 + - gid: 935 + - home: /opt/so/conf/freqserver + - createhome: False + +# Create the log directory +freqlogdir: + file.directory: + - name: /opt/so/log/freq_server + - user: 935 + - group: 935 + - makedirs: True + +so-freqimage: + cmd.run: + - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-freqserver:HH1.0.3 + +so-freq: + docker_container.running: + - require: + - so-freqimage + - image: docker.io/soshybridhunter/so-freqserver:HH1.0.3 + - hostname: freqserver + - name: so-freqserver + - user: freqserver + - binds: + - /opt/so/log/freq_server:/var/log/freq_server:rw + diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 73b29b501..4b3b5b919 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -1,4 +1,6 @@ {% set MASTERIP = salt['pillar.get']('master:mainip', '') %} +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} hiveconfdir: file.directory: - name: /opt/so/conf/hive/etc @@ -53,15 +55,9 @@ hiveesdata: - user: 939 - group: 939 -so-thehive-esimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-thehive-es:HH1.1.1 - so-thehive-es: docker_container.running: - - require: - - so-thehive-esimage - - image: docker.io/soshybridhunter/so-thehive-es:HH1.1.1 + - image: {{ MASTER }}:5000/soshybridhunter/so-thehive-es:{{ VERSION }} - hostname: so-thehive-es - name: so-thehive-es - user: 939 @@ -87,16 +83,9 @@ so-thehive-es: - 0.0.0.0:9500:9500 # Install Cortex - -so-corteximage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-thehive-cortex:HH1.1.3 - so-cortex: docker_container.running: - - require: - - so-corteximage - - image: docker.io/soshybridhunter/so-thehive-cortex:HH1.1.3 + - image: {{ MASTER }}:5000/soshybridhunter/so-thehive-cortex:{{ VERSION }} - hostname: so-cortex - name: so-cortex - user: 939 @@ -107,19 +96,13 @@ so-cortex: cortexscript: cmd.script: - - source: salt://hive/thehive/scripts/cortex_init.sh + - source: salt://hive/thehive/scripts/cortex_init - cwd: /opt/so - template: jinja -so-thehiveimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-thehive:HH1.1.1 - so-thehive: docker_container.running: - - require: - - so-thehiveimage - - image: docker.io/soshybridhunter/so-thehive:HH1.1.1 + - image: {{ MASTER }}:5000/soshybridhunter/so-thehive:{{ VERSION }} - environment: - ELASTICSEARCH_HOST={{ MASTERIP }} - hostname: so-thehive @@ -132,6 +115,6 @@ so-thehive: hivescript: cmd.script: - - source: salt://hive/thehive/scripts/hive_init.sh + - source: salt://hive/thehive/scripts/hive_init - cwd: /opt/so - template: jinja diff --git a/salt/hive/thehive/scripts/cortex_init.sh b/salt/hive/thehive/scripts/cortex_init similarity index 98% rename from salt/hive/thehive/scripts/cortex_init.sh rename to salt/hive/thehive/scripts/cortex_init index 506b14be5..786039bf1 100644 --- a/salt/hive/thehive/scripts/cortex_init.sh +++ b/salt/hive/thehive/scripts/cortex_init @@ -1,5 +1,5 @@ #!/bin/bash -{%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} +{% set MASTERIP = salt['pillar.get']('static:masterip', '') %} {%- set CORTEXUSER = salt['pillar.get']('static:cortexuser', '') %} {%- set CORTEXPASSWORD = salt['pillar.get']('static:cortexpassword', '') %} {%- set CORTEXKEY = salt['pillar.get']('static:cortexkey', '') %} diff --git a/salt/hive/thehive/scripts/hive_init b/salt/hive/thehive/scripts/hive_init new file mode 100755 index 000000000..03b43e74f --- /dev/null +++ b/salt/hive/thehive/scripts/hive_init @@ -0,0 +1,64 @@ +#!/bin/bash +{% set MASTERIP = salt['pillar.get']('static:masterip', '') %} +{%- set HIVEUSER = salt['pillar.get']('static:hiveuser', '') %} +{%- set HIVEPASSWORD = salt['pillar.get']('static:hivepassword', '') %} +{%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') %} + +hive_init(){ + sleep 60 + HIVE_IP="{{MASTERIP}}" + HIVE_USER="{{HIVEUSER}}" + HIVE_PASSWORD="{{HIVEPASSWORD}}" + HIVE_KEY="{{HIVEKEY}}" + SOCTOPUS_CONFIG="/opt/so/saltstack/salt/soctopus/files/SOCtopus.conf" + + echo -n "Waiting for TheHive..." + COUNT=0 + HIVE_CONNECTED="no" + while [[ "$COUNT" -le 240 ]]; do + curl --output /dev/null --silent --head --fail -k "https://$HIVE_IP:/thehive" + if [ $? -eq 0 ]; then + HIVE_CONNECTED="yes" + echo "connected!" + break + else + ((COUNT+=1)) + sleep 1 + echo -n "." + fi + done + + if [ "$HIVE_CONNECTED" == "yes" ]; then + + # Migrate DB + curl -v -k -XPOST "https://$HIVE_IP:/thehive/api/maintenance/migrate" + + # Create intial TheHive user + curl -v -k "https://$HIVE_IP/thehive/api/user" -H "Content-Type: application/json" -d "{\"login\" : \"$HIVE_USER\",\"name\" : \"$HIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$HIVE_PASSWORD\", \"key\": \"$HIVE_KEY\"}" + + # Pre-load custom fields + # + # reputation + curl -v -k "https://$HIVE_IP/thehive/api/list/custom_fields" -H "Authorization: Bearer $HIVE_KEY" -H "Content-Type: application/json" -d "{\"value\":{\"name\": \"reputation\", \"reference\": \"reputation\", \"description\": \"This field provides an overall reputation status for an address/domain.\", \"type\": \"string\", \"options\": []}}" + + + touch /opt/so/state/thehive.txt + else + echo "We experienced an issue connecting to TheHive!" + fi +} + +if [ -f /opt/so/state/thehive.txt ]; then + exit 0 +else + rm -f garbage_file + while ! wget -O garbage_file {{MASTERIP}}:9500 2>/dev/null + do + echo "Waiting for Elasticsearch..." + rm -f garbage_file + sleep 1 + done + rm -f garbage_file + sleep 5 + hive_init +fi diff --git a/salt/hive/thehive/scripts/hive_init.sh b/salt/hive/thehive/scripts/hive_init.sh deleted file mode 100755 index 2215d4e44..000000000 --- a/salt/hive/thehive/scripts/hive_init.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -{%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} -{%- set HIVEUSER = salt['pillar.get']('static:hiveuser', '') %} -{%- set HIVEPASSWORD = salt['pillar.get']('static:hivepassword', '') %} -{%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') %} - -hive_init(){ - sleep 60 - HIVE_IP="{{MASTERIP}}" - HIVE_USER="{{HIVEUSER}}" - HIVE_PASSWORD="{{HIVEPASSWORD}}" - SOCTOPUS_CONFIG="/opt/so/saltstack/salt/soctopus/files/SOCtopus.conf" - - # Migrate DB - curl -v -k -XPOST "https://$HIVE_IP:/thehive/api/maintenance/migrate" - - # Generate unique ID for apikey - HIVE_KEY="{{HIVEKEY}}" - - # Create intial TheHive user - curl -v -k "https://$HIVE_IP/thehive/api/user" -H "Content-Type: application/json" -d "{\"login\" : \"$HIVE_USER\",\"name\" : \"$HIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$HIVE_PASSWORD\", \"key\": \"$HIVE_KEY\"}" - - # Pre-load custom fields - # - # reputation - curl -v -k "https://$HIVE_IP/thehive/api/list/custom_fields" -H "Authorization: Bearer $HIVE_KEY" -H "Content-Type: application/json" -d "{\"value\":{\"name\": \"reputation\", \"reference\": \"reputation\", \"description\": \"This field provides an overall reputation status for an address/domain.\", \"type\": \"string\", \"options\": []}}" - - - # Update SOCtopus config with apikey value - #sed -i "s/hive_key = .*/hive_key = $HIVE_KEY/" $SOCTOPUS_CONFIG - - # Check for correct authentication - #curl -v -k -H "Authorization: Bearer $HIVE_KEY" "https://$HIVE_IP/thehive/api/user/$USER" - - touch /opt/so/state/thehive.txt - -} - -if [ -f /opt/so/state/thehive.txt ]; then - exit 0 -else - rm -f garbage_file - while ! wget -O garbage_file {{MASTERIP}}:9500 2>/dev/null - do - echo "Waiting for Elasticsearch..." - rm -f garbage_file - sleep 1 - done - rm -f garbage_file - sleep 5 - hive_init -fi diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls index 9ec6f53f7..969215559 100644 --- a/salt/idstools/init.sls +++ b/salt/idstools/init.sls @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} # IDSTools Setup idstoolsdir: file.directory: @@ -61,15 +62,9 @@ ruleslink: - name: /opt/so/saltstack/salt/suricata/rules - target: /opt/so/rules/nids -so-idstoolsimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-idstools:HH1.1.0 - so-idstools: docker_container.running: - - require: - - so-idstoolsimage - - image: docker.io/soshybridhunter/so-idstools:HH1.1.0 + - image: {{ MASTER }}:5000/soshybridhunter/so-idstools:{{ VERSION }} - hostname: so-idstools - user: socore - binds: diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 0d6262600..d2cf30c1a 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -1,4 +1,11 @@ -{% set master = salt['grains.get']('master') %} +{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} +{% set MASTER = salt['grains.get']('master') %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} +{% if FEATURES %} + {% set FEATURES = "-features" %} +{% else %} + {% set FEATURES = '' %} +{% endif %} # Add ES Group kibanasearchgroup: @@ -52,25 +59,17 @@ synckibanacustom: - user: 932 - group: 939 -# File.Recurse for custom saved dashboards - -so-kibanaimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-kibana:HH1.1.1 - # Start the kibana docker so-kibana: docker_container.running: - - require: - - so-kibanaimage - - image: docker.io/soshybridhunter/so-kibana:HH1.1.1 + - image: {{ MASTER }}:5000/soshybridhunter/so-kibana:{{ VERSION }}{{ FEATURES }} - hostname: kibana - user: kibana - environment: - KIBANA_DEFAULTAPPID=dashboard/94b52620-342a-11e7-9d52-4f090484f59e - - ELASTICSEARCH_HOST={{ master }} + - ELASTICSEARCH_HOST={{ MASTER }} - ELASTICSEARCH_PORT=9200 - - MASTER={{ master }} + - MASTER={{ MASTER }} - binds: - /opt/so/conf/kibana/etc:/usr/share/kibana/config:rw - /opt/so/log/kibana:/var/log/kibana:rw @@ -78,11 +77,3 @@ so-kibana: - /sys/fs/cgroup:/sys/fs/cgroup:ro - port_bindings: - 0.0.0.0:5601:5601 - -# Keep the setting correct -#KibanaHappy: -# cmd.script: -# - shell: /bin/bash -# - runas: socore -# - source: salt://kibana/bin/keepkibanahappy.sh -# - template: jinja diff --git a/salt/logstash/conf/conf.enabled.txt.storage b/salt/logstash/conf/conf.enabled.txt.search similarity index 100% rename from salt/logstash/conf/conf.enabled.txt.storage rename to salt/logstash/conf/conf.enabled.txt.search diff --git a/salt/logstash/conf/conf.enabled.txt.so-mastersearch b/salt/logstash/conf/conf.enabled.txt.so-mastersearch new file mode 100644 index 000000000..6464496fa --- /dev/null +++ b/salt/logstash/conf/conf.enabled.txt.so-mastersearch @@ -0,0 +1,18 @@ +# This is where can specify which LogStash configs get loaded. +# +# The custom folder on the master gets automatically synced to each logstash +# node. +# +# To enable a custom configuration see the following example and uncomment: +# /usr/share/logstash/pipeline.custom/1234_input_custom.conf +## +# All of the defaults are loaded. +/usr/share/logstash/pipeline.so/0000_input_syslogng.conf +/usr/share/logstash/pipeline.so/0001_input_json.conf +/usr/share/logstash/pipeline.so/0002_input_windows_json.conf +/usr/share/logstash/pipeline.so/0003_input_syslog.conf +/usr/share/logstash/pipeline.so/0005_input_suricata.conf +#/usr/share/logstash/pipeline.dynamic/0006_input_beats.conf +/usr/share/logstash/pipeline.dynamic/0010_input_hhbeats.conf +/usr/share/logstash/pipeline.so/0007_input_import.conf +/usr/share/logstash/pipeline.dynamic/9999_output_redis.conf diff --git a/salt/logstash/conf/pipelines/6000_bro.conf b/salt/logstash/conf/pipelines/6000_bro.conf new file mode 100644 index 000000000..4ba3d3989 --- /dev/null +++ b/salt/logstash/conf/pipelines/6000_bro.conf @@ -0,0 +1,228 @@ +# Original Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Updated by: Doug Burks +# Last Update: 2/10/2018 +# +filter { + if "bro" in [tags] { + + # Bro logs have a high quality timestamp, so let's copy that to @timestamp. + # Before we do, let's copy the existing logstash @timestamp to timestamp. + mutate { + add_field => { "logstash_timestamp" => "%{@timestamp}" } + } + mutate { + convert => { "logstash_timestamp" => "string" } + } + mutate { + convert => { "timestamp" => "string" } + } + # New Bro JSON logs use ISO8601 timestamps. + # Old Bro TSV logs use UNIX timestamps. + date { + match => [ "timestamp", "ISO8601", "UNIX" ] + } + mutate { + rename => { "logstash_timestamp" => "timestamp" } + } + + if [duration] == "-" { + mutate { + replace => [ "duration", "0" ] + } + } + if [original_bytes] == "-" { + mutate { + replace => [ "original_bytes", "0" ] + } + } + # If MissedBytes is unspecified set it to zero so it is an integer + if [missed_bytes] == "-" { + mutate { + replace => [ "missed_bytes", "0" ] + } + } + # If OriginalIPBytes is unspecified set it to zero so it is an integer + if [original_ip_bytes] == "-" { + mutate { + replace => [ "original_ip_bytes", "0" ] + } + } + # If RespondBytes is unspecified set it to zero so it is an integer + if [respond_bytes] == "-" { + mutate { + replace => [ "respond_bytes", "0" ] + } + } + # If RespondIPBytes is unspecified set it to zero so it is an integer + if [respond_ip_bytes] == "-" { + mutate { + replace => [ "respond_ip_bytes", "0" ] + } + } + if [request_body_length] == "-" { + mutate { + replace => [ "request_body_length", "0" ] + } + } + if [response_body_length] == "-" { + mutate { + replace => [ "response_body_length", "0" ] + } + } + if [source_port] == "-" { + mutate { + remove_field => ["source_port"] + } + } + if [destination_port] == "-" { + mutate { + remove_field => ["destination_port"] + } + } + if [virtual_host] == "-" { + mutate { + remove_field => ["virtual_host"] + } + } + if [x_originating_ip] == "-" { + mutate { + remove_field => ["x_originating_ip"] + } + } + if [basic_constraints_path_length] == "-" { + mutate { + remove_field => ["basic_constraints_path_length"] + } + } + if [data_channel_source_ip] == "-" { + mutate { + remove_field => ["data_channel_source_ip"] + } + } + if [data_channel_destination_ip] == "-" { + mutate { + remove_field => ["data_channel_destination_ip"] + } + } + if [desktop_width] == "-" { + mutate { + remove_field => ["desktop_width"] + } + } + if [desktop_height] == "-" { + mutate { + remove_field => ["desktop_height"] + } + } + if [height] == "-" { + mutate { + remove_field => ["height"] + } + } + + + # I renamed conn_uids to uid so that it is easy to pivot to all things tied to a connection + mutate { + rename => [ "connection_uids", "uid" ] + } + # If total_bytes is set to "-" change it to 0 so it is an integer + if [total_bytes] == "-" { + mutate { + replace => [ "total_bytes", "0" ] + } + } + # If seen_bytes is set to "-" change it to 0 so it is an integer + if [seen_bytes] == "-" { + mutate { + replace => [ "seen_bytes", "0" ] + } + } + # If missing_bytes is set to "-" change it to 0 so it is an integer + if [missing_bytes] == "-" { + mutate { + replace => [ "missing_bytes", "0" ] + } + } + # If overflow_bytes is set to "-" change it to 0 so it is an integer + if [overflow_bytes] == "-" { + mutate { + replace => [ "overflow_bytes", "0" ] + } + } + if [dcc_file_size] == "-" { + mutate { + replace => [ "dcc_file_size", "0" ] + } + } + if [authentication_attempts] == "-" { + mutate { + replace => [ "authentication_attempts", "0" ] + } + } + if [file_size] == "-" { + mutate { + replace => [ "file_size", "0" ] + } + } + if [original_ip_bytes] == "-" { + mutate { + replace => [ "original_ip_bytes", "0" ] + } + } + + # I recommend changing the field types below to integer or floats so searches can do greater than or less than + # and also so math functions can be ran against them + mutate { + convert => [ "bound_port", "integer" ] + convert => [ "data_channel_destination_port", "integer" ] + convert => [ "destination_port", "integer" ] + convert => [ "depth", "integer" ] + #convert => [ "duration", "float" ] + convert => [ "info_code", "integer" ] + convert => [ "missed_bytes", "integer" ] + convert => [ "missing_bytes", "integer" ] + convert => [ "n", "integer" ] + convert => [ "original_bytes", "integer" ] + convert => [ "original_packets", "integer" ] + convert => [ "original_ip_bytes", "integer" ] + convert => [ "overflow_bytes", "integer" ] + convert => [ "p", "integer" ] + convert => [ "query_class", "integer" ] + convert => [ "query_type", "integer" ] + convert => [ "rcode", "integer" ] + convert => [ "request_body_length", "integer" ] + convert => [ "request_port", "integer" ] + convert => [ "respond_bytes", "integer" ] + convert => [ "respond_packets", "integer" ] + convert => [ "respond_ip_bytes", "integer" ] + convert => [ "response_body_length", "integer" ] + convert => [ "seen_bytes", "integer" ] + convert => [ "source_port", "integer" ] + convert => [ "status_code", "integer" ] + #convert => [ "suppress_for", "float" ] + convert => [ "total_bytes", "integer" ] + convert => [ "trans_depth", "integer" ] + convert => [ "transaction_id", "integer" ] + # convert the following boolean to text for now + convert => [ "local_respond", "string" ] + convert => [ "tc", "string" ] + convert => [ "is_orig", "string" ] + convert => [ "local_orig", "string" ] + lowercase => [ "query" ] + #remove_field => [ "timestamp" ] + } + + # Combine OriginalBytes and RespondBytes and save the value to total_bytes + if [original_bytes] { + if [respond_bytes] { + ruby { + code => "event.set('total_bytes', event.get('original_bytes') + event.get('respond_bytes'))" + } + } + } + mutate { + #add_tag => [ "conf_file_6000"] + } + } +} diff --git a/salt/logstash/conf/pipelines/6001_bro_import.conf b/salt/logstash/conf/pipelines/6001_bro_import.conf new file mode 100644 index 000000000..34c43f6ae --- /dev/null +++ b/salt/logstash/conf/pipelines/6001_bro_import.conf @@ -0,0 +1,16 @@ +# Updated by: Doug Burks +# Last Update: 2/10/2018 +# +filter { + if "import" in [tags] and "bro" in [tags] { + + # we're setting timestamp in 6000 now + #date { + # match => [ "timestamp", "UNIX" ] + #} + + mutate { + #add_tag => [ "conf_file_6001"] + } + } +} diff --git a/salt/logstash/conf/pipelines/8000_postprocess_bro_cleanup.conf b/salt/logstash/conf/pipelines/8000_postprocess_bro_cleanup.conf new file mode 100644 index 000000000..3998df8a4 --- /dev/null +++ b/salt/logstash/conf/pipelines/8000_postprocess_bro_cleanup.conf @@ -0,0 +1,17 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +filter { + if "bro" in [tags] { + if "_grokparsefailure" not in [tags] and "_csvparsefailure" not in [tags] and "_jsonparsefailure" not in [tags] { + #mutate { + # remove_field => [ "message" ] + #} + } + mutate { + #add_tag => [ "conf_file_8000"] + } + } +} diff --git a/salt/logstash/conf/pipelines/8006_postprocess_dns.conf b/salt/logstash/conf/pipelines/8006_postprocess_dns.conf new file mode 100644 index 000000000..a1520e6dc --- /dev/null +++ b/salt/logstash/conf/pipelines/8006_postprocess_dns.conf @@ -0,0 +1,47 @@ +# Original Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Updated by: Doug Burks +# Last Update: 5/13/2017 + +filter { + if [type] == "bro_dns" or "dns" in [tags] { + # Used for whois lookups - can create log loop + if [query] =~ "^whois\." { + drop { } + } + # REPLACE test.int with your internal domain + if [query] and [query] !~ "\.test\.int$" { + mutate { + lowercase => [ "query" ] + } + if [query_type_name] != "NB" and [query_type_name] != "TKEY" and [query_type_name] != "NBSTAT" and [query_type_name] != "PTR" { + tld { + source => "query" + } + ruby { + code => "event.set('query_length', event.get('query').length)" + } + mutate { + rename => { "[SubLog][sessionid]" => "sub_session_id" } + rename => { "[tld][domain]" => "highest_registered_domain" } + rename => { "[tld][trd]" => "subdomain" } + rename => { "[tld][tld]" => "top_level_domain" } + rename => { "[tld][sld]" => "parent_domain" } + } + if [parent_domain] { + ruby { + code => "event.set('parent_domain_length', event.get('parent_domain').length)" + } + } + if [subdomain] { + ruby { + code => "event.set('subdomain_length', event.get('subdomain').length)" + } + } + } + } + mutate { + #add_tag => [ "conf_file_8006"] + } + } +} diff --git a/salt/logstash/conf/pipelines/eval/0800_input_eval.conf b/salt/logstash/conf/pipelines/eval/0800_input_eval.conf new file mode 100644 index 000000000..d3fd00029 --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/0800_input_eval.conf @@ -0,0 +1,204 @@ +# Updated by: Mike Reeves +# Last Update: 11/1/2018 + +input { + file { + path => "/suricata/eve.json" + type => "ids" + add_field => { "engine" => "suricata" } + } + file { + path => "/nsm/zeek/logs/current/conn*.log" + type => "bro_conn" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/dce_rpc*.log" + type => "bro_dce_rpc" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/dhcp*.log" + type => "bro_dhcp" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/dnp3*.log" + type => "bro_dnp3" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/dns*.log" + type => "bro_dns" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/dpd*.log" + type => "bro_dpd" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/files*.log" + type => "bro_files" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/ftp*.log" + type => "bro_ftp" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/http*.log" + type => "bro_http" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/intel*.log" + type => "bro_intel" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/irc*.log" + type => "bro_irc" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/kerberos*.log" + type => "bro_kerberos" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/modbus*.log" + type => "bro_modbus" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/mysql*.log" + type => "bro_mysql" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/notice*.log" + type => "bro_notice" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/ntlm*.log" + type => "bro_ntlm" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/pe*.log" + type => "bro_pe" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/radius*.log" + type => "bro_radius" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/rdp*.log" + type => "bro_rdp" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/rfb*.log" + type => "bro_rfb" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/signatures*.log" + type => "bro_signatures" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/sip*.log" + type => "bro_sip" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/smb_files*.log" + type => "bro_smb_files" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/smb_mapping*.log" + type => "bro_smb_mapping" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/smtp*.log" + type => "bro_smtp" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/snmp*.log" + type => "bro_snmp" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/socks*.log" + type => "bro_socks" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/software*.log" + type => "bro_software" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/ssh*.log" + type => "bro_ssh" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/ssl*.log" + type => "bro_ssl" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/syslog*.log" + type => "bro_syslog" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/tunnel*.log" + type => "bro_tunnels" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/weird*.log" + type => "bro_weird" + tags => ["bro"] + } + file { + path => "/nsm/zeek/logs/current/x509*.log" + type => "bro_x509" + tags => ["bro"] + } + file { + path => "/wazuh/alerts/alerts.json" + type => "ossec" + } + file { + path => "/wazuh/archives/archive.json" + type => "ossec_archive" + } + file { + path => "/osquery/logs/result.log" + type => "osquery" + tags => ["osquery"] + } + file { + path => "/strelka/strelka.log" + type => "strelka" + } +} +filter { + if "import" in [tags] { + mutate { + #add_tag => [ "conf_file_0007"] + } + } +} diff --git a/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf b/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf new file mode 100644 index 000000000..d098eb11a --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf @@ -0,0 +1,13 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +filter { + ruby { + code => "event.set('task_start', Time.now.to_f)" + } + mutate { + #add_tag => [ "conf_file_1000"] + } +} diff --git a/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf b/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf new file mode 100644 index 000000000..84bce8802 --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf @@ -0,0 +1,33 @@ +# Updated by: Doug Burks and Wes Lambert +# Last Update: 10/30/2018 + +filter { + if "syslogng" in [tags] { + mutate { + rename => { "MESSAGE" => "message" } + rename => { "PROGRAM" => "type" } + rename => { "FACILITY" => "syslog-facility" } + rename => { "FILE_NAME" => "syslog-file_name" } + rename => { "HOST" => "syslog-host" } + rename => { "HOST_FROM" => "syslog-host_from" } + rename => { "LEGACY_MSGHDR" => "syslog-legacy_msghdr" } + rename => { "PID" => "syslog-pid" } + rename => { "PRIORITY" => "syslog-priority" } + rename => { "SOURCEIP" => "syslog-sourceip" } + rename => { "TAGS" => "syslog-tags" } + lowercase => [ "syslog-host_from" ] + remove_field => [ "ISODATE" ] + remove_field => [ "SEQNUM" ] + #add_tag => [ "conf_file_1001"] + } + if "bro_" in [type] { + mutate { + add_tag => [ "bro" ] + } + } else if [type] !~ /ossec.*|snort/ and "firewall" not in [tags] { + mutate { + add_tag => [ "syslog" ] + } + } + } +} diff --git a/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf b/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf new file mode 100644 index 000000000..ea7c677da --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf @@ -0,0 +1,18 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +filter { + if "json" in [tags]{ + json { + source => "message" + } + mutate { + remove_tag => [ "json" ] + } + mutate { + #add_tag => [ "conf_file_1002"] + } + } +} diff --git a/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf b/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf new file mode 100644 index 000000000..243abcc15 --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf @@ -0,0 +1,19 @@ +filter { + if "syslog" in [tags] { + if [host] == "172.16.1.1" { + mutate { + add_field => { "type" => "fortinet" } + add_tag => [ "firewall" ] + } + } + if [host] == "10.0.0.101" { + mutate { + add_field => { "type" => "brocade" } + add_tag => [ "switch" ] + } + } + mutate { + #add_tag => [ "conf_file_1004"] + } + } +} diff --git a/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf b/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf new file mode 100644 index 000000000..2f893cf7a --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf @@ -0,0 +1,140 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolutions.com +# Last Update: 12/9/2016 +# This conf file is based on accepting logs for DHCP. It is currently based on Windows DHCP only. +filter { + if [type] == "dhcp" { + mutate { + add_field => { "Hostname" => "%{host}" } + } + mutate { + strip => "message" + } + # This is the initial parsing of the log + grok { + # Server 2008+ + match => { "message" => "%{DATA:id},%{DATE_US:date},(?