diff --git a/pillar/top.sls b/pillar/top.sls index 41d3265f0..60cface84 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -10,6 +10,7 @@ base: - sensoroni.adv_sensoroni - telegraf.soc_telegraf - telegraf.adv_telegraf + - influxdb.token - node_data.ips '* and not *_eval and not *_import': @@ -27,6 +28,8 @@ base: - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.index_templates + - elasticsearch.soc_elasticsearch + - elasticsearch.adv_elasticsearch '*_manager': - logstash diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 0f748e6d6..2e54a5e9f 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -411,11 +411,9 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} - {% if TRUE_CLUSTER %} searchnodes: portgroups: - {{ portgroups.elasticsearch_node }} - {% endif %} self: portgroups: - {{ portgroups.syslog}} @@ -469,6 +467,8 @@ role: self: portgroups: - {{ portgroups.syslog}} + - {{ portgroups.elasticsearch_node }} + - {{ portgroups.elasticsearch_rest }} strelka_frontend: portgroups: - {{ portgroups.strelka_frontend }} diff --git a/salt/influxdb/curl.config.jinja b/salt/influxdb/curl.config.jinja index 9f636e851..d994b05d9 100644 --- a/salt/influxdb/curl.config.jinja +++ b/salt/influxdb/curl.config.jinja @@ -1 +1 @@ -header = "Authorization: Token {{ salt['pillar.get']('secrets:influx_token') }}" \ No newline at end of file +header = "Authorization: Token {{ salt['pillar.get']('influxdb:token') }}" diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index bd894a6d4..b6190fdfd 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -6,7 +6,7 @@ {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-eval', 'so-import'] %} {% set PASSWORD = salt['pillar.get']('secrets:influx_pass') %} -{% set TOKEN = salt['pillar.get']('secrets:influx_token') %} +{% set TOKEN = salt['pillar.get']('influxdb:token') %} include: - salt.minion diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index e16f047e4..85db938cc 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -1,7 +1,7 @@ {% import_yaml 'soc/defaults.yaml' as SOCDEFAULTS %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER -%} -{% set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %} +{% set INFLUXDB_TOKEN = salt['pillar.get']('influxdb:token') %} {% import_text 'influxdb/metrics_link.txt' as METRICS_LINK %} {% for module, application_url in GLOBALS.application_urls.items() %} diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 3fe2f4567..196828b77 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -7,19 +7,19 @@ soc: icon: fa-crosshairs target: links: - - '/#/hunt?q="{value|escape}" | groupby event.module event.dataset' + - '/#/hunt?q="{value|escape}" | groupby event.module* event.dataset' - name: actionCorrelate description: actionCorrelateHelp icon: fab fa-searchengin target: '' links: - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module event.dataset' - - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module event.dataset' - - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module event.dataset' - - '/#/hunt?q="{:log.id.fuid}" | groupby event.module event.dataset' - - '/#/hunt?q="{:log.id.uid}" | groupby event.module event.dataset' - - '/#/hunt?q="{:network.community_id}" | groupby event.module event.dataset' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:log.id.uid}") | groupby event.module* event.dataset' + - '/#/hunt?q=("{:log.id.fuid}" OR "{:network.community_id}") | groupby event.module* event.dataset' + - '/#/hunt?q=("{:log.id.uid}" OR "{:network.community_id}") | groupby event.module* event.dataset' + - '/#/hunt?q="{:log.id.fuid}" | groupby event.module* event.dataset' + - '/#/hunt?q="{:log.id.uid}" | groupby event.module* event.dataset' + - '/#/hunt?q="{:network.community_id}" | groupby event.module* event.dataset' - name: actionPcap description: actionPcapHelp icon: fa-stream @@ -560,13 +560,13 @@ soc: - destination.geo.country_iso_code - user.name - source.ip - ':sysmon:': + ':windows.sysmon_operational:': - soc_timestamp - - event.dataset + - event.action - process.executable - user.name - file.target - - dns.query.name + - dns.question.name - winlog.event_data.TargetObject '::network_connection': - soc_timestamp @@ -1116,12 +1116,12 @@ soc: enabled: true queries: - name: Default Query - description: Show all events grouped by the origin host + description: Show all events grouped by the observer host query: '* | groupby observer.name' showSubtitle: true - name: Log Type description: Show all events grouped by module and dataset - query: '* | groupby event.module event.dataset' + query: '* | groupby event.module* event.dataset' showSubtitle: true - name: SOC Auth description: Users authenticated to SOC grouped by IP address and identity @@ -1145,11 +1145,11 @@ soc: showSubtitle: true - name: Sysmon Events description: Show all Sysmon logs grouped by event type - query: 'event.module:sysmon | groupby event.dataset' + query: 'event.dataset: windows.sysmon_operational | groupby event.action' showSubtitle: true - name: Sysmon Usernames description: Show all Sysmon logs grouped by username - query: 'event.module:sysmon | groupby event.dataset, user.name.keyword' + query: 'event.dataset: windows.sysmon_operational | groupby event.action, user.name.keyword' showSubtitle: true - name: Strelka description: Show all Strelka logs grouped by file type @@ -1380,7 +1380,7 @@ soc: queries: - name: Overview description: Overview of all events - query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module | groupby event.dataset | groupby event.module | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module* | groupby event.dataset | groupby event.module* | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: SOC Auth description: SOC (Security Onion Console) authentication logs query: 'event.module:kratos AND event.dataset:audit AND msg:authenticated | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent' @@ -1389,28 +1389,31 @@ soc: query: '_index: "*:elastalert*" | groupby rule_name | groupby alert_info.type' - name: Alerts description: Overview of all alerts - query: 'event.dataset:alert | groupby event.module | groupby rule.name | groupby event.severity | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + query: 'event.dataset:alert | groupby event.module* | groupby rule.name | groupby event.severity | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: NIDS Alerts description: NIDS (Network Intrusion Detection System) alerts query: 'event.category:network AND event.dataset:alert | groupby rule.category | groupby -sankey source.ip destination.ip | groupby rule.name | groupby rule.uuid | groupby rule.gid | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: Sysmon Overview description: Overview of all Sysmon data types - query: 'event.module:sysmon | groupby -sankey event.dataset winlog.computer_name | groupby -sankey winlog.computer_name user.name | groupby winlog.computer_name | groupby event.dataset | groupby user.name | groupby dns.query.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.target | groupby source.ip | groupby destination.ip | groupby destination.port' - - name: Sysmon Registry - description: Registry changes captured by Sysmon - query: '(event.dataset:registry_create_delete OR event.dataset:registry_value_set OR event.dataset:registry_key_value_rename) | groupby -sankey event.dataset winlog.computer_name | groupby winlog.computer_name | groupby event.dataset | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.executable winlog.event_data.TargetObject' - - name: Sysmon DNS - description: DNS queries captured by Sysmon - query: 'event.dataset:dns_query | groupby -sankey winlog.computer_name dns.query.name | groupby winlog.computer_name | groupby process.executable | groupby dns.query.name | groupby dns.answers.name' - - name: Sysmon Process - description: Process activity captured by Sysmon - query: '(event.dataset:process_creation OR event.dataset:process_terminated OR event.dataset:process_access) | groupby -sankey winlog.computer_name user.name | groupby winlog.computer_name | groupby user.name | groupby event.dataset | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' - - name: Sysmon File - description: File activity captured by Sysmon - query: 'event.module:sysmon AND event.dataset:file_* | groupby -sankey winlog.computer_name process.executable | groupby winlog.computer_name | groupby event.dataset | groupby file.target | groupby process.executable' - - name: Sysmon Network - description: Network activity captured by Sysmon - query: 'event.dataset:network_connection | groupby -sankey winlog.computer_name destination.ip destination.port | groupby winlog.computer_name | groupby user.name | groupby process.executable | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' + query: 'event.dataset:windows.sysmon_operational | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby host.name | groupby event.category event.action | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby file.name | groupby source.ip | groupby destination.ip | groupby destination.port' + - name: Host Overview + description: Overview of all host data types + query: '((event.category:registry OR event.category:host OR event.category:process OR event.category:driver OR event.category:configuration) OR (event.category:file AND _exists_:process.executable) OR (event.category:network AND _exists_:host.name)) | groupby event.dataset* event.category* event.action* | groupby event.type | groupby host.name | groupby user.name | groupby file.name | groupby process.executable' + - name: Host Registry Changes + description: Windows Registry changes + query: 'event.category: registry | groupby -sankey event.action host.name | groupby event.dataset event.action | groupby host.name | groupby process.executable | groupby registry.path | groupby process.executable registry.path' + - name: Host DNS & Process Mappings + description: DNS queries mapped to originating processes + query: 'event.category: network AND _exists_:process.executable AND (_exists_:dns.question.name OR _exists_:dns.answers.data) | groupby -sankey host.name dns.question.name | groupby event.dataset event.type | groupby host.name | groupby process.executable | groupby dns.question.name | groupby dns.answers.data' + - name: Host Process Activity + description: Process activity captured on an endpoint + query: 'event.category:process | groupby -sankey host.name user.name* | groupby event.dataset event.action | groupby host.name | groupby user.name | groupby process.working_directory | groupby process.executable | groupby process.command_line | groupby process.parent.executable | groupby process.parent.command_line | groupby -sankey process.parent.executable process.executable' + - name: Host File Activity + description: File activity captured on an endpoint + query: 'event.category: file AND _exists_:process.executable | groupby -sankey host.name process.executable | groupby host.name | groupby event.dataset event.action event.type | groupby file.name | groupby process.executable' + - name: Host Network & Process Mappings + description: Network activity mapped to originating processes + query: 'event.category: network AND _exists_:process.executable | groupby -sankey event.action host.name | groupby -sankey host.name user.name | groupby event.dataset* event.type* event.action* | groupby host.name | groupby user.name | groupby dns.question.name | groupby process.executable | groupby winlog.event_data.TargetObject | groupby process.name | groupby source.ip | groupby destination.ip | groupby destination.port' - name: Strelka description: Strelka file analysis query: 'event.module:strelka | groupby file.mime_type | groupby -sankey file.mime_type file.source | groupby file.source | groupby file.name' @@ -1614,7 +1617,7 @@ soc: - acknowledged queries: - name: 'Group By Name, Module' - query: '* | groupby rule.name event.module event.severity_label' + query: '* | groupby rule.name event.module* event.severity_label' - name: 'Group By Sensor, Source IP/Port, Destination IP/Port, Name' query: '* | groupby observer.name source.ip source.port destination.ip destination.port rule.name network.community_id event.severity_label' - name: 'Group By Source IP, Name' diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 61843da5f..f2a89baf4 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -2,7 +2,7 @@ {%- set INFLUXDBHOST = GLOBALS.influxdb_host %} {%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} -{%- set TOKEN = salt['pillar.get']('secrets:influx_token', '') %} +{%- set TOKEN = salt['pillar.get']('influxdb:token', '') %} {%- set NODEIP = GLOBALS.node_ip %} {%- set UNIQUEID = salt['pillar.get']('sensor:uniqueid', '') %} {%- set ZEEK_ENABLED = salt['pillar.get']('zeek:enabled', True) %} diff --git a/salt/top.sls b/salt/top.sls index 889c95f99..a07e16013 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -147,6 +147,7 @@ base: - schedule - soctopus - playbook + - elastic-fleet - docker_clean '*_standalone and G@saltversion:{{saltversion}}': @@ -197,8 +198,8 @@ base: - schedule - soctopus - playbook - - docker_clean - elastic-fleet + - docker_clean '*_searchnode and G@saltversion:{{saltversion}}': - match: compound @@ -256,6 +257,7 @@ base: - schedule - soctopus - playbook + - elastic-fleet - docker_clean '*_heavynode and G@saltversion:{{saltversion}}': @@ -315,8 +317,8 @@ base: - suricata - zeek - schedule - - docker_clean - elastic-fleet + - docker_clean '*_receiver and G@saltversion:{{saltversion}}': - match: compound diff --git a/salt/vars/heavynode.map.jinja b/salt/vars/heavynode.map.jinja new file mode 100644 index 000000000..ccc241884 --- /dev/null +++ b/salt/vars/heavynode.map.jinja @@ -0,0 +1,17 @@ +{% from 'vars/elasticsearch.map.jinja' import ELASTICSEARCH_GLOBALS %} +{% from 'vars/logstash.map.jinja' import LOGSTASH_GLOBALS %} +{% from 'vars/sensor.map.jinja' import SENSOR_GLOBALS %} + +{% set ROLE_GLOBALS = {} %} + +{% set HEAVYNODE_GLOBALS = + [ + SENSOR_GLOBALS, + ELASTICSEARCH_GLOBALS, + LOGSTASH_GLOBALS + ] +%} + +{% for sg in HEAVYNODE_GLOBALS %} +{% do salt['defaults.merge'](ROLE_GLOBALS, sg, merge_lists=False, in_place=True) %} +{% endfor %} diff --git a/setup/so-functions b/setup/so-functions index b3c0b6126..5432ff0ec 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -897,6 +897,7 @@ create_manager_pillars() { kratos_pillar soc_pillar idh_pillar + influxdb_pillar } @@ -1536,6 +1537,9 @@ influxdb_pillar() { title "Create the influxdb pillar file" touch $adv_influxdb_pillar_file touch $influxdb_pillar_file + printf '%s\n'\ + "influxdb:"\ + " token: $INFLUXTOKEN" > $local_salt_dir/pillar/influxdb/token.sls } make_some_dirs() { @@ -1711,7 +1715,7 @@ process_installtype() { elif [ "$install_type" = 'SEARCHNODE' ]; then is_searchnode=true elif [ "$install_type" = 'HEAVYNODE' ]; then - is_heavy=true + is_heavynode=true elif [ "$install_type" = 'FLEET' ]; then is_fleet=true elif [ "$install_type" = 'IDH' ]; then @@ -2077,8 +2081,7 @@ secrets_pillar(){ " playbook_admin: $PLAYBOOKADMINPASS"\ " playbook_automation: $PLAYBOOKAUTOMATIONPASS"\ " playbook_automation_api_key: "\ - " influx_pass: $INFLUXPASS"\ - " influx_token: $INFLUXTOKEN" > $local_salt_dir/pillar/secrets.sls + " influx_pass: $INFLUXPASS" > $local_salt_dir/pillar/secrets.sls fi } diff --git a/setup/so-setup b/setup/so-setup index cff432c13..814fc6e79 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -658,5 +658,5 @@ if ! [[ -f $install_opt_file ]]; then verify_setup fi - # Need to make sure the latest install is located on the web server of the manager to check the versions and donwload the code if required + # Need to make sure the latest install is located on the web server of the manager to check the versions and download the code if required fi