diff --git a/pillar/top.sls b/pillar/top.sls index 60cface84..0c4c11957 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -51,6 +51,8 @@ base: - adv_global - manager.soc_manager - manager.adv_manager + - idstools.soc_idstools + - idstools.adv_idstools - soc.soc_soc - soc.adv_soc - kratos.soc_kratos @@ -90,6 +92,9 @@ base: - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - manager.soc_manager + - manager.adv_manager + - idstools.soc_idstools + - idstools.adv_idstools - soc.soc_soc - kratos.soc_kratos - kratos.adv_kratos @@ -120,6 +125,8 @@ base: - secrets - healthcheck.standalone - soc_global + - idstools.soc_idstools + - idstools.adv_idstools - kratos.soc_kratos - kratos.adv_kratos - redis.soc_redis @@ -129,6 +136,7 @@ base: - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - manager.soc_manager + - manager.adv_manager - soc.soc_soc - backup.soc_backup - backup.adv_backup @@ -195,6 +203,7 @@ base: - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - manager.soc_manager + - manager.adv_manager - soc.soc_soc - soc_global - adv_global diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 3548a7f0d..a837950e4 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -34,7 +34,7 @@ 'influxdb', 'soc', 'kratos', - 'elastic-fleet', + 'elasticfleet', 'firewall', 'idstools', 'suricata.manager', @@ -105,7 +105,7 @@ 'schedule', 'tcpreplay', 'docker_clean', - 'elastic-fleet' + 'elasticfleet' ], 'so-manager': [ 'salt.master', @@ -118,7 +118,7 @@ 'influxdb', 'soc', 'kratos', - 'elastic-fleet', + 'elasticfleet', 'firewall', 'idstools', 'suricata.manager', @@ -137,7 +137,7 @@ 'influxdb', 'soc', 'kratos', - 'elastic-fleet', + 'elasticfleet', 'firewall', 'manager', 'idstools', @@ -166,7 +166,7 @@ 'influxdb', 'soc', 'kratos', - 'elastic-fleet', + 'elasticfleet', 'firewall', 'idstools', 'suricata.manager', diff --git a/salt/backup/config_backup.sls b/salt/backup/config_backup.sls index b4eeccfc1..20616f780 100644 --- a/salt/backup/config_backup.sls +++ b/salt/backup/config_backup.sls @@ -19,6 +19,7 @@ config_backup_script: - source: salt://backup/tools/sbin/so-config-backup.jinja - defaults: BACKUPLOCATIONS: {{ BACKUP_MERGED.locations }} + DESTINATION: {{ BACKUP_MERGED.destination }} # Add config backup so_config_backup: diff --git a/salt/backup/defaults.yaml b/salt/backup/defaults.yaml index 9b8d5909d..1aae64910 100644 --- a/salt/backup/defaults.yaml +++ b/salt/backup/defaults.yaml @@ -3,4 +3,5 @@ backup: - /opt/so/saltstack/local - /etc/pki - /etc/salt - - /opt/so/conf/kratos + - /nsm/kratos + destination: "/nsm/backup" \ No newline at end of file diff --git a/salt/backup/soc_backup.yaml b/salt/backup/soc_backup.yaml new file mode 100644 index 000000000..bedecb1ca --- /dev/null +++ b/salt/backup/soc_backup.yaml @@ -0,0 +1,10 @@ +backup: + locations: + description: List of locations to back up to the destination. + helpLink: backup.html + global: True + destination: + description: Directory to store the configuration backups in. + helpLink: backup.html + global: True + \ No newline at end of file diff --git a/salt/backup/tools/sbin/so-config-backup.jinja b/salt/backup/tools/sbin/so-config-backup.jinja index c0e24cd80..23e407653 100755 --- a/salt/backup/tools/sbin/so-config-backup.jinja +++ b/salt/backup/tools/sbin/so-config-backup.jinja @@ -8,7 +8,8 @@ . /usr/sbin/so-common TODAY=$(date '+%Y_%m_%d') -BACKUPFILE="/nsm/backup/so-config-backup-$TODAY.tar" +BACKUPDIR={{ DESTINATION }} +BACKUPFILE="$BACKUPDIR/so-config-backup-$TODAY.tar" MAXBACKUPS=7 # Create backup dir if it does not exist diff --git a/salt/common/tools/sbin/so-elastic-agent-gen-installers b/salt/common/tools/sbin/so-elastic-agent-gen-installers index 131292dab..128f894e4 100755 --- a/salt/common/tools/sbin/so-elastic-agent-gen-installers +++ b/salt/common/tools/sbin/so-elastic-agent-gen-installers @@ -24,11 +24,11 @@ mkdir -p /tmp/elastic-agent-workspace for OS in "${CONTAINERGOOS[@]}" do printf "\n\nGenerating $OS Installer..." - cp /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz + cp /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz docker run -e CGO_ENABLED=0 -e GOOS=$OS \ --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \ --mount type=bind,source=/tmp/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ - --mount type=bind,source=/opt/so/saltstack/local/salt/elastic-fleet/files/so_agent-installers/,target=/output/ \ + --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_$OS printf "\n $OS Installer Generated..." done diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index 4e60bf9ad..b87ede0fe 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -17,7 +17,9 @@ SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http: # Disable certain Features from showing up in the Kibana UI echo -echo "Setting up default Security Onion package policies for Elastic Agent..." +echo "Disable certain Features from showing up in the Kibana UI" +so-kibana-space-defaults +echo # Suricata logs echo @@ -103,3 +105,9 @@ echo echo "Setting up Redis package policy..." curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }' echo + +# IDH logs +echo +echo "Setting up IDH package policy..." +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"policy_id":"so-grid-nodes","package":{"name":"log","version":"1.1.1"},"id":"idh-logs","name":"idh-logs","namespace":"so","description":"IDH integration","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/nsm/idh/opencanary.log"],"data_stream.dataset":"idh","custom":"pipeline: common","processors": "\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true\n- drop_fields:\n when:\n equals:\n logtype: \"1001\"\n fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n ignore_missing: true\n- rename:\n fields:\n - from: \"src_host\"\n to: \"source.ip\"\n - from: \"src_port\"\n to: \"source.port\"\n - from: \"dst_host\"\n to: \"destination.host\"\n - from: \"dst_port\"\n to: \"destination.port\"\n ignore_missing: true\n- convert:\n fields:\n - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n ignore_missing: true\n- drop_fields:\n fields: '\''[\"prospector\", \"input\", \"offset\", \"beat\"]'\''\n- add_fields:\n target: event\n fields:\n category: host\n module: opencanary","tags":[]}}}}}}' +echo \ No newline at end of file diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 9c2d60eca..8334490d5 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -91,19 +91,19 @@ printf '%s\n'\ "" >> "$global_pillar_file" # Call Elastic-Fleet Salt State -salt-call state.apply elastic-fleet queue=True +salt-call state.apply elasticfleet queue=True # Load Elastic Fleet integrations /usr/sbin/so-elastic-fleet-integration-policy-load # Temp -wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-darwin-x86_64.tar.gz -wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-linux-x86_64.tar.gz -wget -P /opt/so/saltstack/default/salt/elastic-fleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-windows-x86_64.tar.gz +wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-darwin-x86_64.tar.gz +wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-linux-x86_64.tar.gz +wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.4.1/so-elastic-agent-8.4.1-windows-x86_64.tar.gz #git clone -b 2.4-so-elastic-agent https://github.com/Security-Onion-Solutions/securityonion-image.git #cd securityonion-image/so-elastic-agent-builder #docker build -t so-elastic-agent-builder . so-elastic-agent-gen-installers -salt-call state.apply elastic-fleet.install_agent_grid queue=True +salt-call state.apply elasticfleet.install_agent_grid queue=True diff --git a/salt/common/tools/sbin/so-kibana-space-defaults b/salt/common/tools/sbin/so-kibana-space-defaults index 9175a36bc..430054e06 100755 --- a/salt/common/tools/sbin/so-kibana-space-defaults +++ b/salt/common/tools/sbin/so-kibana-space-defaults @@ -13,6 +13,6 @@ echo "Setting up default Space:" {% if HIGHLANDER %} curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch"]} ' >> /opt/so/log/kibana/misc.log {% else %} -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet","fleetv2","securitySolutionCases"]} ' >> /opt/so/log/kibana/misc.log +curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","securitySolutionCases"]} ' >> /opt/so/log/kibana/misc.log {% endif %} echo diff --git a/salt/elastic-fleet/files/so_agent-installers/readme b/salt/elasticfleet/files/so_agent-installers/readme similarity index 100% rename from salt/elastic-fleet/files/so_agent-installers/readme rename to salt/elasticfleet/files/so_agent-installers/readme diff --git a/salt/elastic-fleet/init.sls b/salt/elasticfleet/init.sls similarity index 100% rename from salt/elastic-fleet/init.sls rename to salt/elasticfleet/init.sls diff --git a/salt/elastic-fleet/install_agent_grid.sls b/salt/elasticfleet/install_agent_grid.sls similarity index 86% rename from salt/elastic-fleet/install_agent_grid.sls rename to salt/elasticfleet/install_agent_grid.sls index 2f848ac2e..c4c389cea 100644 --- a/salt/elastic-fleet/install_agent_grid.sls +++ b/salt/elasticfleet/install_agent_grid.sls @@ -9,7 +9,7 @@ run_installer: cmd.script: - - name: salt://elastic-fleet/files/so_agent-installers/so-elastic-agent_linux + - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux - args: -token={{ GRIDNODETOKEN }} {% endif %} diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml new file mode 100644 index 000000000..0e111feca --- /dev/null +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -0,0 +1,18 @@ +elasticfleet: + server: + endpoints_enrollment: + description: Endpoint enrollment key. + global: True + helpLink: elastic-fleet.html + es_token: + description: Elastic auth token. + global: True + helpLink: elastic-fleet.html + grid_enrollment: + description: Grid enrollment key. + global: True + helpLink: elastic-fleet.html + url: + description: Agent connection URL. + global: True + helpLink: elastic-fleet.html \ No newline at end of file diff --git a/salt/idh/init.sls b/salt/idh/init.sls index 2cf22c358..d1ba5ce33 100644 --- a/salt/idh/init.sls +++ b/salt/idh/init.sls @@ -74,6 +74,8 @@ so-idh: - file: opencanary_config - require: - file: opencanary_config + - extra_hosts: + - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} append_so-idh_so-status.conf: file.append: diff --git a/salt/idstools/defaults.yaml b/salt/idstools/defaults.yaml new file mode 100644 index 000000000..e937ebc2d --- /dev/null +++ b/salt/idstools/defaults.yaml @@ -0,0 +1,9 @@ +idstools: + config: + urls: [] + ruleset: ETOPEN + oinkcode: "" + sids: + enabled: [] + disabled: [] + modify: [] \ No newline at end of file diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf index 2b1a8cae1..fad421243 100644 --- a/salt/idstools/etc/rulecat.conf +++ b/salt/idstools/etc/rulecat.conf @@ -1,17 +1,14 @@ -{%- set URLS = salt['pillar.get']('idstools:config:urls') -%} -{%- set RULESET = salt['pillar.get']('idstools:config:ruleset') -%} -{%- set OINKCODE = salt['pillar.get']('idstools:config:oinkcode', '' ) -%} -{%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') -%} -{%- set MANAGERIP = salt['pillar.get']('global:managerip', '') -%} -{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %} -{%- if ISAIRGAP is sameas true -%} +{%- from 'vars/globals.map.jinja' import GLOBALS %} +{%- import_yaml 'idstools/defaults.yaml' as IDSTOOLSDEFAULTS %} +{%- set IDSTOOLSMERGED = salt['pillar.get']('idstools', IDSTOOLSDEFAULTS.idstools, merge=True) %} +{%- if GLOBALS.airgap is sameas true -%} --merged=/opt/so/rules/nids/all.rules --local=/opt/so/rules/nids/local.rules -{%- if ENGINE == "SURICATA" %} +{%- if GLOBAL.md_engine == "SURICATA" %} --local=/opt/so/rules/nids/sorules/extraction.rules --local=/opt/so/rules/nids/sorules/filters.rules -{%- endif %} ---url=http://{{ MANAGERIP }}:7788/rules/emerging-all.rules +{%- endif %} +--url=http://{{ GLOBALS.manager }}:7788/rules/emerging-all.rules --disable=/opt/so/idstools/etc/disable.conf --enable=/opt/so/idstools/etc/enable.conf --modify=/opt/so/idstools/etc/modify.conf @@ -19,23 +16,23 @@ --suricata-version=6.0 --merged=/opt/so/rules/nids/all.rules --local=/opt/so/rules/nids/local.rules -{%- if ENGINE == "SURICATA" %} +{%- if GLOBALS.md_engine == "SURICATA" %} --local=/opt/so/rules/nids/sorules/extraction.rules --local=/opt/so/rules/nids/sorules/filters.rules -{%- endif %} +{%- endif %} --disable=/opt/so/idstools/etc/disable.conf --enable=/opt/so/idstools/etc/enable.conf --modify=/opt/so/idstools/etc/modify.conf - {%- if RULESET == 'ETOPEN' %} +{%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} --etopen - {%- elif RULESET == 'ETPRO' %} ---etpro={{ OINKCODE }} - {%- elif RULESET == 'TALOS' %} ---url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ OINKCODE }} - {%- endif %} +{%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} +--etpro={{ IDSTOOLSMERGED.config.oinkcode }} +{%- elif IDSTOOLSMERGED.config.ruleset == 'TALOS' %} +--url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }} +{%- endif %} {%- endif %} -{%- if URLS != None %} -{%- for URL in URLS %} +{%- if IDSTOOLSMERGED.config.urls | length > 0 %} +{%- for URL in IDSTOOLSMERGED.config.urls %} --url={{ URL }} -{%- endfor %} +{%- endfor %} {%- endif %} \ No newline at end of file diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index e3a13c2f2..52d018354 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -95,7 +95,7 @@ so-nginx: - /opt/so/log/nginx/:/var/log/nginx:rw - /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/run:rw - - /opt/so/saltstack/local/salt/elastic-fleet/files/so_agent-installers/:/opt/socore/html/packages + - /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/:/opt/socore/html/packages {% if grains.role in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone', 'so-import'] %} - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro diff --git a/salt/repo/client/rocky.sls b/salt/repo/client/rocky.sls index a99d3f422..405bba7f6 100644 --- a/salt/repo/client/rocky.sls +++ b/salt/repo/client/rocky.sls @@ -16,13 +16,13 @@ file.absent: - name: {{ REPOPATH }}{{ file }} - onchanges_in: - - cmd: cleanyum + - cmd: cleandnf {% endfor %} {% endif %} -cleanyum: +cleandnf: cmd.run: - - name: 'yum clean all' + - name: 'dnf clean all' - onchanges: - so_repo diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index bf31fff27..4a521f12c 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -210,19 +210,19 @@ chownilogstashelasticfleetp8: # Create Symlinks to the keys so I can distribute it to all the things elasticfleetdircerts: file.directory: - - name: /opt/so/saltstack/local/salt/elastic-fleet/files/certs + - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs - makedirs: True efkeylink: file.symlink: - - name: /opt/so/saltstack/local/salt/elastic-fleet/files/certs/elasticfleet.p8 + - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet.p8 - target: /etc/pki/elasticfleet.p8 - user: socore - group: socore efcrtlink: file.symlink: - - name: /opt/so/saltstack/local/salt/elastic-fleet/files/certs/elasticfleet.crt + - name: /opt/so/saltstack/local/salt/elasticfleet/files/certs/elasticfleet.crt - target: /etc/pki/elasticfleet.crt - user: socore - group: socore diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml index 2c3b88513..bcef0fd9b 100644 --- a/salt/strelka/defaults.yaml +++ b/salt/strelka/defaults.yaml @@ -526,10 +526,9 @@ strelka: response: log: "/var/log/strelka/strelka.log" manager: - coordinator: - addr: 'HOST:6380' - db: 0 - + coordinator: + addr: 'HOST:6380' + db: 0 rules: enabled: True repos: @@ -552,3 +551,7 @@ strelka: - gen_susp_xor.yar - gen_webshells_ext_vars.yar - configured_vulns_ext_vars.yar + filecheck: + historypath: '/nsm/strelka/history/' + strelkapath: '/nsm/strelka/unprocessed/' + logfile: '/opt/so/log/strelka/filecheck.log' diff --git a/salt/strelka/filecheck/filecheck.yaml.jinja b/salt/strelka/filecheck/filecheck.yaml.jinja index 95c5abab2..1f5453f93 100644 --- a/salt/strelka/filecheck/filecheck.yaml.jinja +++ b/salt/strelka/filecheck/filecheck.yaml.jinja @@ -1 +1,2 @@ -{{ FILECHECKCONFIG | yaml(false) }} +filecheck: + {{ FILECHECKCONFIG | yaml(false) | indent(width=2) }} diff --git a/salt/strelka/filecheck/map.jinja b/salt/strelka/filecheck/map.jinja deleted file mode 100644 index 670136b45..000000000 --- a/salt/strelka/filecheck/map.jinja +++ /dev/null @@ -1,12 +0,0 @@ -{% from 'vars/globals.map.jinja' import GLOBALS %} -{% import_yaml 'strelka/filecheck/defaults.yaml' as FILECHECKDEFAULTS %} - -{% if GLOBALS.md_engine == "SURICATA" %} -{% set extract_path = '/nsm/suricata/extracted' %} -{% set filecheck_runas = 'suricata' %} -{% else %} -{% set extract_path = '/nsm/zeek/extracted/complete' %} -{% set filecheck_runas = 'socore' %} -{% endif %} - -{% do FILECHECKDEFAULTS.filecheck.update({'extract_path': extract_path}) %} diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index f8b8262b0..bbb2bcaf6 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -9,8 +9,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'strelka/map.jinja' import STRELKAMERGED %} -{% from 'strelka/filecheck/map.jinja' import FILECHECKDEFAULTS %} -{% from 'strelka/filecheck/map.jinja' import filecheck_runas %} +{% from 'strelka/map.jinja' import filecheck_runas %} # Strelka config strelkaconfdir: @@ -99,7 +98,7 @@ manager_config: - defaults: MANAGERCONFIG: {{ STRELKAMERGED.config.manager }} -{% if STRELKAMERGED.rules.enabled %} +{% if STRELKAMERGED.rules.enabled %} strelkarules: file.recurse: @@ -109,7 +108,7 @@ strelkarules: - group: 939 - clean: True -{% if grains['role'] in GLOBALS.manager_roles %} +{% if grains['role'] in GLOBALS.manager_roles %} strelkarepos: file.managed: - name: /opt/so/conf/strelka/repos.txt @@ -118,8 +117,8 @@ strelkarepos: - defaults: STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} -{% endif %} -{% endif %} +{% endif %} +{% endif %} strelkadatadir: file.directory: @@ -185,7 +184,7 @@ filecheck_conf: - source: salt://strelka/filecheck/filecheck.yaml.jinja - template: jinja - defaults: - FILECHECKCONFIG: {{ FILECHECKDEFAULTS }} + FILECHECKCONFIG: {{ STRELKAMERGED.filecheck }} filecheck_script: file.managed: diff --git a/salt/strelka/map.jinja b/salt/strelka/map.jinja index bf0a29a17..5df15aa59 100644 --- a/salt/strelka/map.jinja +++ b/salt/strelka/map.jinja @@ -17,4 +17,14 @@ {% set manager_coordinator_port = STRELKADEFAULTS.strelka.config.manager.coordinator.addr.split(':')[1] %} {% do STRELKADEFAULTS.strelka.config.manager.coordinator.update({'addr': HOST ~ ':' ~ manager_coordinator_port}) %} +{% if GLOBALS.md_engine == "SURICATA" %} +{% set extract_path = '/nsm/suricata/extracted' %} +{% set filecheck_runas = 'suricata' %} +{% else %} +{% set extract_path = '/nsm/zeek/extracted/complete' %} +{% set filecheck_runas = 'socore' %} +{% endif %} + +{% do STRELKADEFAULTS.strelka.filecheck.update({'extract_path': extract_path}) %} + {% set STRELKAMERGED = salt['pillar.get']('strelka', STRELKADEFAULTS.strelka, merge=True) %} diff --git a/salt/strelka/soc_strelka.yaml b/salt/strelka/soc_strelka.yaml new file mode 100644 index 000000000..dbe949817 --- /dev/null +++ b/salt/strelka/soc_strelka.yaml @@ -0,0 +1,589 @@ +strelka: + config: + backend: + backend: + logging_cfg: + description: Path to the Python logging configuration. + readonly: True + global: False + helpLink: strelka.html + advanced: True + limits: + max_files: + description: Number of files the backend will process before shutting down. + readonly: False + global: False + helpLink: strelka.html + time_to_live: + description: Amount of time (in seconds) that the backend will run before shutting down (0 to disable). + readonly: False + global: False + helpLink: strelka.html + max_depth: + description: Maximum depth that extracted files will be processed by the backend. + readonly: False + global: False + helpLink: strelka.html + distribution: + description: Amount of time (in seconds) that a single file can be distributed to all scanners. + readonly: False + global: False + helpLink: strelka.html + scanner: + description: Amount of time (in seconds) that a scanner can spend scanning a file (can be overridden per scanner). + readonly: False + global: False + helpLink: strelka.html + coordinator: + addr: + description: Network address of the coordinator. + readonly: False + global: False + helpLink: strelka.html + advanced: True + db: + description: Redis database of the coordinator. + readonly: False + global: False + helpLink: strelka.html + advanced: True + tasting: + mime_db: + description: Location of the MIME database used to taste files. + readonly: True + global: False + helpLink: strelka.html + advanced: True + yara_rules: + description: Location of the directory of YARA files that contains rules used to taste files. + readonly: True + global: False + helpLink: strelka.html + advanced: True + scanners: + 'ScanBase64': &scannerOptions + description: Configuration options for this scanner. + readonly: False + global: False + helpLink: strelka.html + advanced: True + type: json + multiline: True + 'ScanBatch': *scannerOptions + 'ScanBzip2': *scannerOptions + 'ScanDocx': *scannerOptions + 'ScanElf': *scannerOptions + 'ScanEmail': *scannerOptions + 'ScanEntropy': *scannerOptions + 'ScanExiftool': *scannerOptions + 'ScanGif': *scannerOptions + 'ScanGzip': *scannerOptions + 'ScanHash': *scannerOptions + 'ScanHeader': *scannerOptions + 'ScanHtml': *scannerOptions + 'ScanIni': *scannerOptions + 'ScanJarManifest': *scannerOptions + 'ScanJavascript': *scannerOptions + 'ScanJpeg': *scannerOptions + 'ScanJson': *scannerOptions + 'ScanLibarchive': *scannerOptions + 'ScanLzma': *scannerOptions + 'ScanMacho': *scannerOptions + 'ScanOcr': *scannerOptions + 'ScanOle': *scannerOptions + 'ScanPdf': *scannerOptions + 'ScanPe': *scannerOptions + 'ScanPgp': *scannerOptions + 'ScanPhp': *scannerOptions + 'ScanPkcs7': *scannerOptions + 'ScanPlist': *scannerOptions + 'ScanRar': *scannerOptions + 'ScanRpm': *scannerOptions + 'ScanRtf': *scannerOptions + 'ScanRuby': *scannerOptions + 'ScanSwf': *scannerOptions + 'ScanTar': *scannerOptions + 'ScanTnef': *scannerOptions + 'ScanUpx': *scannerOptions + 'ScanUrl': *scannerOptions + 'ScanVb': *scannerOptions + 'ScanVba': *scannerOptions + 'ScanX509': *scannerOptions + 'ScanXml': *scannerOptions + 'ScanYara': *scannerOptions + 'ScanZip': *scannerOptions + 'ScanZlib': *scannerOptions + logging: + version: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + formatters: + simple: + format: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + datefmt: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + handlers: + console: + class: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + formatter: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + stream: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + root: + level: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + handlers: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + loggers: + OpenSSL: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + bs4: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + bz2: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + chardet: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + docx: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + elftools: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + email: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + entropy: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + esprima: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + gzip: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + hashlib: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + json: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + libarchive: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + lxml: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + lzma: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + macholibre: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + olefile: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + oletools: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + pdfminer: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + pefile: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + pgpdump: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + pygments: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + pylzma: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + rarfile: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + requests: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + rpmfile: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + ssdeep: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + tarfile: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + tnefparse: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + yara: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + zipfile: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + zlib: + propagate: + description: This is an advanced option for Strelka logging. + readonly: False + global: False + helpLink: strelka.html + advanced: True + passwords: + description: Passwords that will be stored in the password_file used in scanner options. + readonly: False + global: False + helpLink: strelka.html + multiline: True + filestream: + conn: + server: + description: Network address of the frontend server. + readonly: False + global: False + helpLink: strelka.html + advanced: True + cert: + description: Local path to the frontend SSL server certificate. + readonly: False + global: False + helpLink: strelka.html + advanced: True + timeout: + dial: + description: Amount of time to wait for the client to dial the server. + readonly: False + global: False + helpLink: strelka.html + advanced: True + file: + description: Amount of time to wait for an individual file to complete a scan. + readonly: False + global: False + helpLink: strelka.html + advanced: True + throughput: + concurrency: + description: Number of concurrent requests to make. + readonly: False + global: False + helpLink: strelka.html + advanced: True + chunk: + description: Size of file chunks that will be sent to the frontend server. + readonly: False + global: False + helpLink: strelka.html + advanced: True + delay: + description: Artificial sleep between the submission of each chunk. + readonly: False + global: False + helpLink: strelka.html + advanced: True + files: + patterns: + description: List of glob patterns that determine which files will be sent for scanning. + readonly: False + global: False + helpLink: strelka.html + advanced: True + delete: + description: Boolean that determines if files should be deleted after being sent for scanning. + readonly: False + global: False + helpLink: strelka.html + advanced: True + gatekeeper: + description: Boolean that determines if events should be pulled from the temporary event cache. + readonly: False + global: False + helpLink: strelka.html + advanced: True + processed: + description: Directory where files will be moved after being submitted for scanning. + readonly: False + global: False + helpLink: strelka.html + advanced: True + response: + report: + description: Frequency at which the frontend reports the number of files processed. + readonly: False + global: False + helpLink: strelka.html + advanced: True + delta: + description: Time value that determines how much time must pass since a file was last modified before it is sent for scanning. + readonly: False + global: False + helpLink: strelka.html + advanced: True + staging: + description: Directory where files are staged before being sent to the cluster. + readonly: False + global: False + helpLink: strelka.html + advanced: True + frontend: + server: + description: Network address of the frontend server. + readonly: False + global: False + helpLink: strelka.html + advanced: True + coordinator: + addr: + description: Network address of the coordinator. + readonly: False + global: False + helpLink: strelka.html + advanced: True + db: + description: Redis database of the coordinator. + readonly: False + global: False + helpLink: strelka.html + advanced: True + gatekeeper: + addr: + description: Network address of the gatekeeper. + readonly: False + global: False + helpLink: strelka.html + advanced: True + db: + description: Redis database of the gatekeeper. + readonly: False + global: False + helpLink: strelka.html + advanced: True + ttl: + description: Time-to-live for events added to the gatekeeper. + readonly: False + global: False + helpLink: strelka.html + advanced: True + response: + log: + description: Location where worker scan results are logged to. + readonly: False + global: False + helpLink: strelka.html + advanced: True + manager: + coordinator: + addr: + description: Network address of the coordinator. + readonly: False + global: False + helpLink: strelka.html + advanced: True + db: + description: Redis database of the coordinator. + readonly: False + global: False + helpLink: strelka.html + advanced: True + rules: + enabled: + description: Boolean that determines if yara rules sync from the Salt manager to the backend nodes. + readonly: False + global: False + helpLink: strelka.html + advanced: False + repos: + description: List of repos for so-yara-update to use to download rules. + readonly: False + global: False + helpLink: strelka.html + advanced: False + excluded: + description: List of rules to exclude so-yara-update from download and propagating to backend nodes. + readonly: False + global: False + helpLink: strelka.html + advanced: False + filecheck: + historypath: + description: The path for previously scanned files. + readonly: True + global: False + helpLink: strelka.html + advanced: True + strelkapath: + description: The path for unprocessed files. + readonly: True + global: False + helpLink: strelka.html + advanced: True + logfile: + description: The path for the filecheck log. + readonly: False + global: False + helpLink: strelka.html + advanced: True + diff --git a/salt/top.sls b/salt/top.sls index a07e16013..2c6ad266f 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -59,7 +59,7 @@ base: {%- endif %} - schedule - docker_clean - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid '*_eval and G@saltversion:{{saltversion}}': - match: compound @@ -147,7 +147,7 @@ base: - schedule - soctopus - playbook - - elastic-fleet + - elasticfleet - docker_clean '*_standalone and G@saltversion:{{saltversion}}': @@ -198,7 +198,7 @@ base: - schedule - soctopus - playbook - - elastic-fleet + - elasticfleet - docker_clean '*_searchnode and G@saltversion:{{saltversion}}': @@ -215,7 +215,7 @@ base: - logstash {%- endif %} - schedule - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid - docker_clean '*_managersearch and G@saltversion:{{saltversion}}': @@ -257,7 +257,7 @@ base: - schedule - soctopus - playbook - - elastic-fleet + - elasticfleet - docker_clean '*_heavynode and G@saltversion:{{saltversion}}': @@ -286,7 +286,7 @@ base: - zeek {%- endif %} - schedule - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid - docker_clean '*_import and G@saltversion:{{saltversion}}': @@ -317,7 +317,7 @@ base: - suricata - zeek - schedule - - elastic-fleet + - elasticfleet - docker_clean '*_receiver and G@saltversion:{{saltversion}}': @@ -333,7 +333,7 @@ base: - redis {%- endif %} - schedule - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid - docker_clean '*_idh and G@saltversion:{{saltversion}}': @@ -343,7 +343,7 @@ base: - telegraf - firewall - schedule - - elastic-fleet.install_agent_grid + - elasticfleet.install_agent_grid - docker_clean - idh diff --git a/setup/so-functions b/setup/so-functions index 5432ff0ec..96c357c74 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -139,12 +139,12 @@ check_admin_pass() { check_manager_connection() { # See if you can curl the manager. If not you can either try again or continue info "Checking manager connectivity" - man_test_err=$(curl -k -L -sS https://$MSRVIP/repo --connect-timeout 5 2>&1) + man_test_err=$(curl -s $MSRVIP:4505 --connect-timeout 5 2>&1) local ret=$? - if [[ $ret != 0 ]]; then - error "Could not reach $MSRV" + if [[ $ret != 1 ]]; then + info "Could not reach $MSRV" whiptail_manager_unreachable fi } @@ -1357,18 +1357,6 @@ ls_heapsize() { idstools_pillar() { title "Ading IDSTOOLS pillar options" touch $adv_idstools_pillar_file - printf '%s\n'\ - "idstools:"\ - " config:"\ - " ruleset: '$RULESETUP'"\ - " oinkcode: '$OINKCODE'"\ - " urls: []"\ - " sids:"\ - " enabled: []"\ - " disabled: []"\ - " modify: []"\ - "" > "$idstools_pillar_file" - } soc_pillar() { @@ -1498,9 +1486,6 @@ create_strelka_pillar() { backup_pillar() { title "Create the backup pillar file" touch $adv_backup_pillar_file - printf '%s\n'\ - "backup:"\ - " locations: []" > "$backup_pillar_file" } soctopus_pillar() { @@ -1998,7 +1983,7 @@ repo_sync_local() { echo "gpgcheck=1" >> /root/repodownload.conf echo "gpgkey=https://repo.securityonion.net/file/securityonion-repo/keys/securityonion.pub" >> /root/repodownload.conf - dnf repolist + logCmd "dnf repolist" # Make sure we can get to the sig repo logCmd "curl --retry 5 --retry-delay 60 -A 'gridinstall/$SOVERSION/$OS/$(uname -r)/1' https://sigs.securityonion.net/checkup --output /tmp/checkup" logCmd "dnf reposync --norepopath -g --delete -m -c /root/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" @@ -2396,7 +2381,7 @@ update_sudoers() { update_packages() { if [[ $is_rocky ]]; then logCmd "dnf repolist" - logCmd "dnf -y update --exclude=salt*,wazuh*,docker*,containerd*" + logCmd "dnf -y update --allowerasing --exclude=salt*,wazuh*,docker*,containerd*" else retry 150 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1 retry 150 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1 diff --git a/setup/so-whiptail b/setup/so-whiptail index 6123ea97d..331c27be3 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -974,9 +974,10 @@ whiptail_manager_unreachable() { read -r -d '' msg <<- EOM Setup is unable to access the manager at this time. - Run the following on the manger: + Run the following on the manager: so-firewall-minion --role=$install_type --ip=$MAINIP + Would you like to retry? EOM whiptail --title "$whiptail_title" --yesno "$msg" 20 75 @@ -1143,11 +1144,13 @@ whiptail_reinstall() { read -r -d '' message <<- EOM Setup has detected a previous install. Continuing the install will remove the previous install configuration. + Selecting continue is a destructive action. + Would you like to continue? EOM whiptail --title "$whiptail_title" \ - --yesno "$message" 11 75 \ + --yesno "$message" 13 75 \ --yes-button "Continue" --no-button "Exit" --defaultno local exitstatus=$? @@ -1268,27 +1271,54 @@ whiptail_setup_complete() { [ -n "$TESTING" ] && return - if [[ -n "$REDIRECTIT" && $is_manager = true ]]; then + + if [[ $waitforstate ]]; then + # Manager-type Nodes - Install Summary if [[ -n $ALLOW_CIDR ]]; then local sentence_prefix="Access" else local sentence_prefix="Run so-allow to access" fi - local accessMessage="\n${sentence_prefix} the web interface at: https://${REDIRECTIT}\n" - elif [[ $is_idh ]]; then - local accessMessage="\nSSH for this node has been moved to TCP/2222, accessible only from the Manager node.\n" + + read -r -d '' message <<- EOM + ${install_type} setup is now complete! + + ${sentence_prefix} the Security Onion Console (SOC) web interface by navigating to: + https://${REDIRECTIT} + + Then login with the following username and password. + + SOC Username: ${WEBUSER} + SOC Password: Use the password that was entered during setup + + Press TAB and then the ENTER key to exit this screen. + EOM + whiptail --title "$whiptail_title" --msgbox "$message" 24 75 --scrolltext else - local accessMessage="" + if [[ $is_idh ]]; then + local accessMessage="\nSSH for this node has been moved to TCP/2222, accessible only from the Manager node.\n" + else + local accessMessage="" + fi + MINIONFINGERPRINT=$(salt-call --local key.finger --out=newline_values_only) + read -r -d '' message <<- EOM + ${install_type} initialization is now complete! + + To finish configuration, open the Security Onion Console web interface + and navigate to Administration -> Grid Members. + + Then find this node in the Pending Members list, + click the Review button, and then click the Accept button. + + Node Hostname: $HOSTNAME + Node Fingerprint: + $MINIONFINGERPRINT + $accessMessage + Press TAB and then the ENTER key to exit this screen. + EOM + + whiptail --title "$whiptail_title" --msgbox "$message" 24 75 --scrolltext fi - - - read -r -d '' message <<- EOM - Finished ${install_type} installation. - $accessMessage - Press the Enter key to exit setup. - EOM - - whiptail --title "$whiptail_title" --msgbox "$message" 12 75 } whiptail_setup_failed() {