From 16d94781967a96caf3afda24783ce6ee2c38ff5d Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 10 Mar 2023 16:54:47 -0500 Subject: [PATCH 01/23] Add index lifecycle management policy definitions for default Elastic Agent data streams --- salt/elasticsearch/defaults.yaml | 209 +++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index a0c431881..c4098e08c 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -84,6 +84,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -119,6 +138,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -154,6 +192,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -189,6 +246,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -224,6 +300,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -259,6 +354,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -294,6 +408,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -329,6 +462,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -364,6 +516,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -399,6 +570,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent @@ -434,6 +624,25 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + policy: + phases: + hot: + min_age: 0ms + actions: + set_priority: + priority: 100 + rollover: + max_age: 30d + max_primary_shard_size: 50gb + cold: + min_age: 30d + actions: + set_priority: + priority: 0 + delete: + min_age: 365d + actions: + delete: {} _meta: package: name: elastic_agent From 58343e39fa869cf5471a34e218a0bb2fd41c3d7c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 10 Mar 2023 17:32:14 -0500 Subject: [PATCH 02/23] 2.4 strelka --- .../files/so-yara-update.jinja} | 39 +- salt/manager/init.sls | 15 + salt/strelka/defaults.yaml | 554 +++++++++++++++++- salt/strelka/filecheck/defaults.yaml | 4 + salt/strelka/filecheck/filecheck.yaml | 10 - salt/strelka/filecheck/filecheck.yaml.jinja | 1 + salt/strelka/filecheck/map.jinja | 12 + salt/strelka/files/backend/backend.yaml | 420 ------------- salt/strelka/files/backend/backend.yaml.jinja | 1 + salt/strelka/files/backend/logging.yaml | 78 --- salt/strelka/files/backend/logging.yaml.jinja | 1 + salt/strelka/files/backend/passwords.dat | 2 - .../strelka/files/backend/passwords.dat.jinja | 1 + salt/strelka/files/filestream/filestream.yaml | 26 - .../files/filestream/filestream.yaml.jinja | 1 + salt/strelka/files/frontend/frontend.yaml | 16 - .../files/frontend/frontend.yaml.jinja | 1 + salt/strelka/files/manager/manager.yaml | 9 - salt/strelka/files/manager/manager.yaml.jinja | 1 + salt/strelka/init.sls | 98 +++- salt/strelka/map.jinja | 20 + 21 files changed, 706 insertions(+), 604 deletions(-) rename salt/{common/tools/sbin/so-yara-update => manager/files/so-yara-update.jinja} (70%) create mode 100644 salt/strelka/filecheck/defaults.yaml delete mode 100644 salt/strelka/filecheck/filecheck.yaml create mode 100644 salt/strelka/filecheck/filecheck.yaml.jinja create mode 100644 salt/strelka/filecheck/map.jinja delete mode 100644 salt/strelka/files/backend/backend.yaml create mode 100644 salt/strelka/files/backend/backend.yaml.jinja delete mode 100644 salt/strelka/files/backend/logging.yaml create mode 100644 salt/strelka/files/backend/logging.yaml.jinja delete mode 100644 salt/strelka/files/backend/passwords.dat create mode 100644 salt/strelka/files/backend/passwords.dat.jinja delete mode 100644 salt/strelka/files/filestream/filestream.yaml create mode 100644 salt/strelka/files/filestream/filestream.yaml.jinja delete mode 100644 salt/strelka/files/frontend/frontend.yaml create mode 100644 salt/strelka/files/frontend/frontend.yaml.jinja delete mode 100644 salt/strelka/files/manager/manager.yaml create mode 100644 salt/strelka/files/manager/manager.yaml.jinja create mode 100644 salt/strelka/map.jinja diff --git a/salt/common/tools/sbin/so-yara-update b/salt/manager/files/so-yara-update.jinja similarity index 70% rename from salt/common/tools/sbin/so-yara-update rename to salt/manager/files/so-yara-update.jinja index b4e83a172..ea07f72e4 100755 --- a/salt/common/tools/sbin/so-yara-update +++ b/salt/manager/files/so-yara-update.jinja @@ -5,14 +5,15 @@ # Elastic License 2.0. -{%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %} - echo "Starting to check for yara rule updates at $(date)..." output_dir="/opt/so/saltstack/default/salt/strelka/rules" mkdir -p $output_dir repos="$output_dir/repos.txt" newcounter=0 +excludedcounter=0 +excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) + {% if ISAIRGAP is sameas true %} @@ -20,22 +21,29 @@ echo "Airgap mode enabled." clone_dir="/nsm/repo/rules/strelka" repo_name="signature-base" -mkdir -p /opt/so/saltstack/default/salt/strelka/rules/signature-base +[ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name +mkdir -p mkdir -p $output_dir/$repo_name # Ensure a copy of the license is available for the rules [ -f $clone_dir/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name # Copy over rules for i in $(find $clone_dir/yara -name "*.yar*"); do rule_name=$(echo $i | awk -F '/' '{print $NF}') - echo "Adding rule: $rule_name..." - cp $i $output_dir/$repo_name - ((newcounter++)) + if [[ ! "${excluded_rules}" =~ ${rule_name} ]]; then + echo "Adding rule: $rule_name..." + cp $i $output_dir/$repo_name + ((newcounter++)) + else + echo "Excluding rule: $rule_name..." + ((excludedcounter++)) + fi done echo "Done!" -if [ "$newcounter" -gt 0 ];then +if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then echo "$newcounter rules added." + echo "$excludedcounter rule(s) excluded." fi {% else %} @@ -60,9 +68,15 @@ if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then # Copy over rules for i in $(find $clone_dir/$repo_name -name "*.yar*"); do rule_name=$(echo $i | awk -F '/' '{print $NF}') - echo "Adding rule: $rule_name..." - cp $i $output_dir/$repo_name - ((newcounter++)) + + if [[ ! "${excluded_rules}" =~ ${rule_name} ]]; then + echo "Adding rule: $rule_name..." + cp $i $output_dir/$repo_name + ((newcounter++)) + else + echo "Excluding rule: $rule_name..." + ((excludedcounter++)) + fi done rm -rf $clone_dir/$repo_name fi @@ -70,8 +84,9 @@ if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then echo "Done!" - if [ "$newcounter" -gt 0 ];then - echo "$newcounter rules added." + if [ "$newcounter" -gt 0 ] || [ "$excludedcounter" -gt 0 ];then + echo "$newcounter rule(s) added." + echo "$excludedcounter rule(s) excluded." fi else diff --git a/salt/manager/init.sls b/salt/manager/init.sls index c1062e8ae..5f2b0005a 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -5,6 +5,9 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'strelka/map.jinja' import STRELKAMERGED %} + include: - salt.minion - kibana.secrets @@ -20,6 +23,18 @@ socore_own_saltstack: - user - group +yara_update_script: + file.managed: + - name: /usr/sbin/so-yara-update + - source: salt://manager/files/so-yara-update.jinja + - user: root + - group: root + - mode: 755 + - template: jinja + - defaults: + ISAIRGAP: {{ GLOBALS.airgap }} + EXCLUDEDRULES: {{ STRELKAMERGED.excluded_rules }} + strelka_yara_update: cron.present: - user: root diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml index 792431dc6..12f0edda3 100644 --- a/salt/strelka/defaults.yaml +++ b/salt/strelka/defaults.yaml @@ -1,5 +1,557 @@ strelka: - ignore: + config: + backend: + backend: + logging_cfg: '/etc/strelka/logging.yaml' + limits: + max_files: 0 + time_to_live: 0 + max_depth: 15 + distribution: 600 + scanner: 150 + coordinator: + addr: 'HOST:6380' + db: 0 + tasting: + mime_db: '' + yara_rules: '/etc/strelka/taste/' + scanners: + 'ScanBase64': + - positive: + filename: '^base64_' + priority: 5 + 'ScanBatch': + - positive: + flavors: + - 'text/x-msdos-batch' + - 'batch_file' + priority: 5 + 'ScanBzip2': + - positive: + flavors: + - 'application/x-bzip2' + - 'bzip2_file' + priority: 5 + 'ScanDocx': + - positive: + flavors: + - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + priority: 5 + options: + extract_text: False + 'ScanElf': + - positive: + flavors: + - 'application/x-object' + - 'application/x-executable' + - 'application/x-sharedlib' + - 'application/x-coredump' + - 'elf_file' + priority: 5 + 'ScanEmail': + - positive: + flavors: + - 'application/vnd.ms-outlook' + - 'message/rfc822' + - 'email_file' + priority: 5 + 'ScanEntropy': + - positive: + flavors: + - '*' + priority: 5 + 'ScanExiftool': + - positive: + flavors: + - 'application/msword' + - 'application/vnd.openxmlformats-officedocument' + - 'application/vnd.openxmlformats-officedocument.presentationml.presentation' + - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + - 'olecf_file' + - 'ooxml_file' + - 'audio/mpeg' + - 'mp3_file' + - 'mhtml_file' + - 'application/pdf' + - 'pdf_file' + - 'text/rtf' + - 'rtf_file' + - 'wordml_file' + - 'application/x-dosexec' + - 'mz_file' + - 'application/x-object' + - 'application/x-executable' + - 'application/x-sharedlib' + - 'application/x-coredump' + - 'elf_file' + - 'lnk_file' + - 'application/x-mach-binary' + - 'macho_file' + - 'image/gif' + - 'gif_file' + - 'image/jpeg' + - 'jpeg_file' + - 'image/png' + - 'png_file' + - 'image/tiff' + - 'type_is_tiff' + - 'image/x-ms-bmp' + - 'bmp_file' + - 'application/x-shockwave-flash' + - 'fws_file' + - 'psd_file' + - 'video/mp4' + - 'video/quicktime' + - 'video/x-msvideo' + - 'avi_file' + - 'video/x-ms-wmv' + - 'wmv_file' + priority: 5 + options: + tmp_directory: '/dev/shm/' + 'ScanGif': + - positive: + flavors: + - 'image/gif' + - 'gif_file' + priority: 5 + 'ScanGzip': + - positive: + flavors: + - 'application/gzip' + - 'application/x-gzip' + - 'gzip_file' + priority: 5 + 'ScanHash': + - positive: + flavors: + - '*' + priority: 5 + 'ScanHeader': + - positive: + flavors: + - '*' + priority: 5 + options: + length: 50 + 'ScanHtml': + - positive: + flavors: + - 'hta_file' + - 'text/html' + - 'html_file' + priority: 5 + options: + parser: "html5lib" + 'ScanIni': + - positive: + filename: '(\.([Cc][Ff][Gg]|[Ii][Nn][Ii])|PROJECT)$' + flavors: + - 'ini_file' + priority: 5 + 'ScanJarManifest': + - positive: + flavors: + - 'jar_manifest_file' + priority: 5 + 'ScanJavascript': + - negative: + flavors: + - 'text/html' + - 'html_file' + positive: + flavors: + - 'javascript_file' + - 'text/javascript' + priority: 5 + options: + beautify: True + 'ScanJpeg': + - positive: + flavors: + - 'image/jpeg' + - 'jpeg_file' + priority: 5 + 'ScanJson': + - positive: + flavors: + - 'application/json' + - 'json_file' + priority: 5 + 'ScanLibarchive': + - positive: + flavors: + - 'application/vnd.ms-cab-compressed' + - 'cab_file' + - 'application/x-7z-compressed' + - '_7zip_file' + - 'application/x-cpio' + - 'cpio_file' + - 'application/x-xar' + - 'xar_file' + - 'arj_file' + - 'iso_file' + - 'application/x-debian-package' + - 'debian_package_file' + priority: 5 + options: + limit: 1000 + 'ScanLzma': + - positive: + flavors: + - 'application/x-lzma' + - 'lzma_file' + - 'application/x-xz' + - 'xz_file' + priority: 5 + 'ScanMacho': + - positive: + flavors: + - 'application/x-mach-binary' + - 'macho_file' + priority: 5 + options: + tmp_directory: '/dev/shm/' + 'ScanOcr': + - positive: + flavors: + - 'image/jpeg' + - 'jpeg_file' + - 'image/png' + - 'png_file' + - 'image/tiff' + - 'type_is_tiff' + - 'image/x-ms-bmp' + - 'bmp_file' + priority: 5 + options: + extract_text: False + tmp_directory: '/dev/shm/' + 'ScanOle': + - positive: + flavors: + - 'application/CDFV2' + - 'application/msword' + - 'olecf_file' + priority: 5 + 'ScanPdf': + - positive: + flavors: + - 'application/pdf' + - 'pdf_file' + priority: 5 + options: + extract_text: False + limit: 2000 + 'ScanPe': + - positive: + flavors: + - 'application/x-dosexec' + - 'mz_file' + priority: 5 + 'ScanPgp': + - positive: + flavors: + - 'application/pgp-keys' + - 'pgp_file' + priority: 5 + 'ScanPhp': + - positive: + flavors: + - 'text/x-php' + - 'php_file' + priority: 5 + 'ScanPkcs7': + - positive: + flavors: + - 'pkcs7_file' + priority: 5 + options: + tmp_directory: '/dev/shm/' + 'ScanPlist': + - positive: + flavors: + - 'bplist_file' + - 'plist_file' + priority: 5 + options: + keys: + - 'KeepAlive' + - 'Label' + - 'NetworkState' + - 'Program' + - 'ProgramArguments' + - 'RunAtLoad' + - 'StartInterval' + 'ScanRar': + - positive: + flavors: + - 'application/x-rar' + - 'rar_file' + priority: 5 + options: + limit: 1000 + 'ScanRpm': + - positive: + flavors: + - 'application/x-rpm' + - 'rpm_file' + priority: 5 + options: + tmp_directory: '/dev/shm/' + 'ScanRtf': + - positive: + flavors: + - 'text/rtf' + - 'rtf_file' + priority: 5 + options: + limit: 1000 + 'ScanRuby': + - positive: + flavors: + - 'text/x-ruby' + priority: 5 + 'ScanSwf': + - positive: + flavors: + - 'application/x-shockwave-flash' + - 'fws_file' + - 'cws_file' + - 'zws_file' + priority: 5 + 'ScanTar': + - positive: + flavors: + - 'application/x-tar' + - 'tar_file' + priority: 5 + options: + limit: 1000 + 'ScanTnef': + - positive: + flavors: + - 'application/vnd.ms-tnef' + - 'tnef_file' + priority: 5 + 'ScanUpx': + - positive: + flavors: + - 'upx_file' + priority: 5 + options: + tmp_directory: '/dev/shm/' + 'ScanUrl': + - negative: + flavors: + - 'javascript_file' + positive: + flavors: + - 'text/plain' + priority: 5 + 'ScanVb': + - positive: + flavors: + - 'vb_file' + - 'vbscript' + priority: 5 + 'ScanVba': + - positive: + flavors: + - 'mhtml_file' + - 'application/msword' + - 'olecf_file' + - 'wordml_file' + priority: 5 + options: + analyze_macros: True + 'ScanX509': + - positive: + flavors: + - 'x509_der_file' + priority: 5 + options: + type: 'der' + - positive: + flavors: + - 'x509_pem_file' + priority: 5 + options: + type: 'pem' + 'ScanXml': + - positive: + flavors: + - 'application/xml' + - 'text/xml' + - 'xml_file' + - 'mso_file' + - 'soap_file' + priority: 5 + 'ScanYara': + - positive: + flavors: + - '*' + priority: 5 + options: + location: '/etc/yara/' + 'ScanZip': + - positive: + flavors: + - 'application/java-archive' + - 'application/zip' + - 'zip_file' + - 'application/vnd.openxmlformats-officedocument' + - 'application/vnd.openxmlformats-officedocument.presentationml.presentation' + - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + - 'ooxml_file' + priority: 5 + options: + limit: 1000 + password_file: '/etc/strelka/passwords.dat' + 'ScanZlib': + - positive: + flavors: + - 'application/zlib' + - 'zlib_file' + priority: 5 + logging: + version: 1 + formatters: + simple: + format: '%(asctime)s - [%(levelname)s] %(name)s [%(module)s.%(funcName)s]: %(message)s' + datefmt: '%Y-%m-%d %H:%M:%S' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: DEBUG + handlers: [console] + loggers: + OpenSSL: + propagate: 0 + bs4: + propagate: 0 + bz2: + propagate: 0 + chardet: + propagate: 0 + docx: + propagate: 0 + elftools: + propagate: 0 + email: + propagate: 0 + entropy: + propagate: 0 + esprima: + propagate: 0 + gzip: + propagate: 0 + hashlib: + propagate: 0 + json: + propagate: 0 + libarchive: + propagate: 0 + lxml: + propagate: 0 + lzma: + propagate: 0 + macholibre: + propagate: 0 + olefile: + propagate: 0 + oletools: + propagate: 0 + pdfminer: + propagate: 0 + pefile: + propagate: 0 + pgpdump: + propagate: 0 + pygments: + propagate: 0 + pylzma: + propagate: 0 + rarfile: + propagate: 0 + requests: + propagate: 0 + rpmfile: + propagate: 0 + ssdeep: + propagate: 0 + tarfile: + propagate: 0 + tnefparse: + propagate: 0 + yara: + propagate: 0 + zipfile: + propagate: 0 + zlib: + propagate: 0 + passwords: + - infected + - password + filestream: + conn: + server: 'HOST:57314' + cert: '' + timeout: + dial: 5s + file: 1m + throughput: + concurrency: 8 + chunk: 32768 + delay: 0s + files: + patterns: + - '/nsm/strelka/unprocessed/*' + delete: false + gatekeeper: true + processed: '/nsm/strelka/processed' + response: + report: 5s + delta: 5s + staging: '/nsm/strelka/staging' + frontend: + server: ":57314" + coordinator: + addr: 'HOST:6380' + db: 0 + gatekeeper: + addr: 'HOST:6381' + db: 0 + ttl: 1h + response: + log: "/var/log/strelka/strelka.log" + manager: + coordinator: + addr: 'HOST:6380' + db: 0 + + + + + + + + + + + + + + + + + + excluded_rules: - apt_flame2_orchestrator.yar - apt_tetris.yar - gen_susp_js_obfuscatorio.yar diff --git a/salt/strelka/filecheck/defaults.yaml b/salt/strelka/filecheck/defaults.yaml new file mode 100644 index 000000000..6f45954d6 --- /dev/null +++ b/salt/strelka/filecheck/defaults.yaml @@ -0,0 +1,4 @@ +filecheck: + historypath: '/nsm/strelka/history/' + strelkapath: '/nsm/strelka/unprocessed/' + logfile: '/opt/so/log/strelka/filecheck.log' diff --git a/salt/strelka/filecheck/filecheck.yaml b/salt/strelka/filecheck/filecheck.yaml deleted file mode 100644 index 1c156fc3d..000000000 --- a/salt/strelka/filecheck/filecheck.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{%- set ENGINE = salt['pillar.get']('global:mdengine', '') %} -filecheck: - {%- if ENGINE == "SURICATA" %} - extract_path: '/nsm/suricata/extracted' - {%- else %} - extract_path: '/nsm/zeek/extracted/complete' - {%- endif %} - historypath: '/nsm/strelka/history/' - strelkapath: '/nsm/strelka/unprocessed/' - logfile: '/opt/so/log/strelka/filecheck.log' diff --git a/salt/strelka/filecheck/filecheck.yaml.jinja b/salt/strelka/filecheck/filecheck.yaml.jinja new file mode 100644 index 000000000..95c5abab2 --- /dev/null +++ b/salt/strelka/filecheck/filecheck.yaml.jinja @@ -0,0 +1 @@ +{{ FILECHECKCONFIG | yaml(false) }} diff --git a/salt/strelka/filecheck/map.jinja b/salt/strelka/filecheck/map.jinja new file mode 100644 index 000000000..670136b45 --- /dev/null +++ b/salt/strelka/filecheck/map.jinja @@ -0,0 +1,12 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'strelka/filecheck/defaults.yaml' as FILECHECKDEFAULTS %} + +{% if GLOBALS.md_engine == "SURICATA" %} +{% set extract_path = '/nsm/suricata/extracted' %} +{% set filecheck_runas = 'suricata' %} +{% else %} +{% set extract_path = '/nsm/zeek/extracted/complete' %} +{% set filecheck_runas = 'socore' %} +{% endif %} + +{% do FILECHECKDEFAULTS.filecheck.update({'extract_path': extract_path}) %} diff --git a/salt/strelka/files/backend/backend.yaml b/salt/strelka/files/backend/backend.yaml deleted file mode 100644 index db6ce0560..000000000 --- a/salt/strelka/files/backend/backend.yaml +++ /dev/null @@ -1,420 +0,0 @@ -{%- if grains.role in ['so-sensor', 'so-heavynode'] -%} - {%- set mainint = salt['pillar.get']('host:mainint') %} - {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} -{%- else %} - {%- set ip = salt['pillar.get']('global:managerip') %} -{%- endif -%} -logging_cfg: '/etc/strelka/logging.yaml' -limits: - max_files: 0 - time_to_live: 0 - max_depth: 15 - distribution: 600 - scanner: 150 -coordinator: - addr: '{{ ip }}:6380' - db: 0 -tasting: - mime_db: null - yara_rules: '/etc/strelka/taste/' -scanners: - 'ScanBase64': - - positive: - filename: '^base64_' - priority: 5 - 'ScanBatch': - - positive: - flavors: - - 'text/x-msdos-batch' - - 'batch_file' - priority: 5 - 'ScanBzip2': - - positive: - flavors: - - 'application/x-bzip2' - - 'bzip2_file' - priority: 5 - 'ScanDocx': - - positive: - flavors: - - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' - priority: 5 - options: - extract_text: False - 'ScanElf': - - positive: - flavors: - - 'application/x-object' - - 'application/x-executable' - - 'application/x-sharedlib' - - 'application/x-coredump' - - 'elf_file' - priority: 5 - 'ScanEmail': - - positive: - flavors: - - 'application/vnd.ms-outlook' - - 'message/rfc822' - - 'email_file' - priority: 5 - 'ScanEntropy': - - positive: - flavors: - - '*' - priority: 5 - 'ScanExiftool': - - positive: - flavors: - - 'application/msword' - - 'application/vnd.openxmlformats-officedocument' - - 'application/vnd.openxmlformats-officedocument.presentationml.presentation' - - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' - - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' - - 'olecf_file' - - 'ooxml_file' - - 'audio/mpeg' - - 'mp3_file' - - 'mhtml_file' - - 'application/pdf' - - 'pdf_file' - - 'text/rtf' - - 'rtf_file' - - 'wordml_file' - - 'application/x-dosexec' - - 'mz_file' - - 'application/x-object' - - 'application/x-executable' - - 'application/x-sharedlib' - - 'application/x-coredump' - - 'elf_file' - - 'lnk_file' - - 'application/x-mach-binary' - - 'macho_file' - - 'image/gif' - - 'gif_file' - - 'image/jpeg' - - 'jpeg_file' - - 'image/png' - - 'png_file' - - 'image/tiff' - - 'type_is_tiff' - - 'image/x-ms-bmp' - - 'bmp_file' - - 'application/x-shockwave-flash' - - 'fws_file' - - 'psd_file' - - 'video/mp4' - - 'video/quicktime' - - 'video/x-msvideo' - - 'avi_file' - - 'video/x-ms-wmv' - - 'wmv_file' - priority: 5 - options: - tmp_directory: '/dev/shm/' - 'ScanGif': - - positive: - flavors: - - 'image/gif' - - 'gif_file' - priority: 5 - 'ScanGzip': - - positive: - flavors: - - 'application/gzip' - - 'application/x-gzip' - - 'gzip_file' - priority: 5 - 'ScanHash': - - positive: - flavors: - - '*' - priority: 5 - 'ScanHeader': - - positive: - flavors: - - '*' - priority: 5 - options: - length: 50 - 'ScanHtml': - - positive: - flavors: - - 'hta_file' - - 'text/html' - - 'html_file' - priority: 5 - options: - parser: "html5lib" - 'ScanIni': - - positive: - filename: '(\.([Cc][Ff][Gg]|[Ii][Nn][Ii])|PROJECT)$' - flavors: - - 'ini_file' - priority: 5 - 'ScanJarManifest': - - positive: - flavors: - - 'jar_manifest_file' - priority: 5 - 'ScanJavascript': - - negative: - flavors: - - 'text/html' - - 'html_file' - positive: - flavors: - - 'javascript_file' - - 'text/javascript' - priority: 5 - options: - beautify: True - 'ScanJpeg': - - positive: - flavors: - - 'image/jpeg' - - 'jpeg_file' - priority: 5 - 'ScanJson': - - positive: - flavors: - - 'application/json' - - 'json_file' - priority: 5 - 'ScanLibarchive': - - positive: - flavors: - - 'application/vnd.ms-cab-compressed' - - 'cab_file' - - 'application/x-7z-compressed' - - '_7zip_file' - - 'application/x-cpio' - - 'cpio_file' - - 'application/x-xar' - - 'xar_file' - - 'arj_file' - - 'iso_file' - - 'application/x-debian-package' - - 'debian_package_file' - priority: 5 - options: - limit: 1000 - 'ScanLzma': - - positive: - flavors: - - 'application/x-lzma' - - 'lzma_file' - - 'application/x-xz' - - 'xz_file' - priority: 5 - 'ScanMacho': - - positive: - flavors: - - 'application/x-mach-binary' - - 'macho_file' - priority: 5 - options: - tmp_directory: '/dev/shm/' - 'ScanOcr': - - positive: - flavors: - - 'image/jpeg' - - 'jpeg_file' - - 'image/png' - - 'png_file' - - 'image/tiff' - - 'type_is_tiff' - - 'image/x-ms-bmp' - - 'bmp_file' - priority: 5 - options: - extract_text: False - tmp_directory: '/dev/shm/' - 'ScanOle': - - positive: - flavors: - - 'application/CDFV2' - - 'application/msword' - - 'olecf_file' - priority: 5 - 'ScanPdf': - - positive: - flavors: - - 'application/pdf' - - 'pdf_file' - priority: 5 - options: - extract_text: False - limit: 2000 - 'ScanPe': - - positive: - flavors: - - 'application/x-dosexec' - - 'mz_file' - priority: 5 - 'ScanPgp': - - positive: - flavors: - - 'application/pgp-keys' - - 'pgp_file' - priority: 5 - 'ScanPhp': - - positive: - flavors: - - 'text/x-php' - - 'php_file' - priority: 5 - 'ScanPkcs7': - - positive: - flavors: - - 'pkcs7_file' - priority: 5 - options: - tmp_directory: '/dev/shm/' - 'ScanPlist': - - positive: - flavors: - - 'bplist_file' - - 'plist_file' - priority: 5 - options: - keys: - - 'KeepAlive' - - 'Label' - - 'NetworkState' - - 'Program' - - 'ProgramArguments' - - 'RunAtLoad' - - 'StartInterval' - 'ScanRar': - - positive: - flavors: - - 'application/x-rar' - - 'rar_file' - priority: 5 - options: - limit: 1000 - 'ScanRpm': - - positive: - flavors: - - 'application/x-rpm' - - 'rpm_file' - priority: 5 - options: - tmp_directory: '/dev/shm/' - 'ScanRtf': - - positive: - flavors: - - 'text/rtf' - - 'rtf_file' - priority: 5 - options: - limit: 1000 - 'ScanRuby': - - positive: - flavors: - - 'text/x-ruby' - priority: 5 - 'ScanSwf': - - positive: - flavors: - - 'application/x-shockwave-flash' - - 'fws_file' - - 'cws_file' - - 'zws_file' - priority: 5 - 'ScanTar': - - positive: - flavors: - - 'application/x-tar' - - 'tar_file' - priority: 5 - options: - limit: 1000 - 'ScanTnef': - - positive: - flavors: - - 'application/vnd.ms-tnef' - - 'tnef_file' - priority: 5 - 'ScanUpx': - - positive: - flavors: - - 'upx_file' - priority: 5 - options: - tmp_directory: '/dev/shm/' - 'ScanUrl': - - negative: - flavors: - - 'javascript_file' - positive: - flavors: - - 'text/plain' - priority: 5 - 'ScanVb': - - positive: - flavors: - - 'vb_file' - - 'vbscript' - priority: 5 - 'ScanVba': - - positive: - flavors: - - 'mhtml_file' - - 'application/msword' - - 'olecf_file' - - 'wordml_file' - priority: 5 - options: - analyze_macros: True - 'ScanX509': - - positive: - flavors: - - 'x509_der_file' - priority: 5 - options: - type: 'der' - - positive: - flavors: - - 'x509_pem_file' - priority: 5 - options: - type: 'pem' - 'ScanXml': - - positive: - flavors: - - 'application/xml' - - 'text/xml' - - 'xml_file' - - 'mso_file' - - 'soap_file' - priority: 5 - 'ScanYara': - - positive: - flavors: - - '*' - priority: 5 - options: - location: '/etc/yara/' - 'ScanZip': - - positive: - flavors: - - 'application/java-archive' - - 'application/zip' - - 'zip_file' - - 'application/vnd.openxmlformats-officedocument' - - 'application/vnd.openxmlformats-officedocument.presentationml.presentation' - - 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' - - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' - - 'ooxml_file' - priority: 5 - options: - limit: 1000 - password_file: '/etc/strelka/passwords.dat' - 'ScanZlib': - - positive: - flavors: - - 'application/zlib' - - 'zlib_file' - priority: 5 diff --git a/salt/strelka/files/backend/backend.yaml.jinja b/salt/strelka/files/backend/backend.yaml.jinja new file mode 100644 index 000000000..151cff550 --- /dev/null +++ b/salt/strelka/files/backend/backend.yaml.jinja @@ -0,0 +1 @@ +{{ BACKENDCONFIG | yaml(false) }} diff --git a/salt/strelka/files/backend/logging.yaml b/salt/strelka/files/backend/logging.yaml deleted file mode 100644 index b21d3c396..000000000 --- a/salt/strelka/files/backend/logging.yaml +++ /dev/null @@ -1,78 +0,0 @@ -version: 1 -formatters: - simple: - format: '%(asctime)s - [%(levelname)s] %(name)s [%(module)s.%(funcName)s]: %(message)s' - datefmt: '%Y-%m-%d %H:%M:%S' -handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout -root: - level: DEBUG - handlers: [console] -loggers: - OpenSSL: - propagate: 0 - bs4: - propagate: 0 - bz2: - propagate: 0 - chardet: - propagate: 0 - docx: - propagate: 0 - elftools: - propagate: 0 - email: - propagate: 0 - entropy: - propagate: 0 - esprima: - propagate: 0 - gzip: - propagate: 0 - hashlib: - propagate: 0 - json: - propagate: 0 - libarchive: - propagate: 0 - lxml: - propagate: 0 - lzma: - propagate: 0 - macholibre: - propagate: 0 - olefile: - propagate: 0 - oletools: - propagate: 0 - pdfminer: - propagate: 0 - pefile: - propagate: 0 - pgpdump: - propagate: 0 - pygments: - propagate: 0 - pylzma: - propagate: 0 - rarfile: - propagate: 0 - requests: - propagate: 0 - rpmfile: - propagate: 0 - ssdeep: - propagate: 0 - tarfile: - propagate: 0 - tnefparse: - propagate: 0 - yara: - propagate: 0 - zipfile: - propagate: 0 - zlib: - propagate: 0 diff --git a/salt/strelka/files/backend/logging.yaml.jinja b/salt/strelka/files/backend/logging.yaml.jinja new file mode 100644 index 000000000..f3915e9f1 --- /dev/null +++ b/salt/strelka/files/backend/logging.yaml.jinja @@ -0,0 +1 @@ +{{ LOGGINGCONFIG | yaml(false) }} diff --git a/salt/strelka/files/backend/passwords.dat b/salt/strelka/files/backend/passwords.dat deleted file mode 100644 index e9541f540..000000000 --- a/salt/strelka/files/backend/passwords.dat +++ /dev/null @@ -1,2 +0,0 @@ -infected -password diff --git a/salt/strelka/files/backend/passwords.dat.jinja b/salt/strelka/files/backend/passwords.dat.jinja new file mode 100644 index 000000000..45ac9c6e0 --- /dev/null +++ b/salt/strelka/files/backend/passwords.dat.jinja @@ -0,0 +1 @@ +{{ PASSWORDS | join('\n') }} diff --git a/salt/strelka/files/filestream/filestream.yaml b/salt/strelka/files/filestream/filestream.yaml deleted file mode 100644 index 57ef65127..000000000 --- a/salt/strelka/files/filestream/filestream.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains.role in ['so-sensor', 'so-heavynode'] -%} - {%- set mainint = salt['pillar.get']('host:mainint') %} - {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} -{%- else %} - {%- set ip = salt['pillar.get']('global:managerip') %} -{%- endif -%} -conn: - server: '{{ ip }}:57314' - cert: '' - timeout: - dial: 5s - file: 1m -throughput: - concurrency: 8 - chunk: 32768 - delay: 0s -files: - patterns: - - '/nsm/strelka/unprocessed/*' - delete: false - gatekeeper: true - processed: '/nsm/strelka/processed' -response: - report: 5s -delta: 5s -staging: '/nsm/strelka/staging' diff --git a/salt/strelka/files/filestream/filestream.yaml.jinja b/salt/strelka/files/filestream/filestream.yaml.jinja new file mode 100644 index 000000000..dc435fd9c --- /dev/null +++ b/salt/strelka/files/filestream/filestream.yaml.jinja @@ -0,0 +1 @@ +{{ FILESTREAMCONFIG | yaml(false) }} diff --git a/salt/strelka/files/frontend/frontend.yaml b/salt/strelka/files/frontend/frontend.yaml deleted file mode 100644 index 137966c8e..000000000 --- a/salt/strelka/files/frontend/frontend.yaml +++ /dev/null @@ -1,16 +0,0 @@ -{%- if grains.role in ['so-sensor', 'so-heavynode'] -%} - {%- set mainint = salt['pillar.get']('host:mainint') %} - {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} -{%- else %} - {%- set ip = salt['pillar.get']('global:managerip') %} -{%- endif -%} -server: ":57314" -coordinator: - addr: '{{ ip }}:6380' - db: 0 -gatekeeper: - addr: '{{ ip }}:6381' - db: 0 - ttl: 1h -response: - log: "/var/log/strelka/strelka.log" diff --git a/salt/strelka/files/frontend/frontend.yaml.jinja b/salt/strelka/files/frontend/frontend.yaml.jinja new file mode 100644 index 000000000..4cb281736 --- /dev/null +++ b/salt/strelka/files/frontend/frontend.yaml.jinja @@ -0,0 +1 @@ +{{ FRONTENDCONFIG | yaml(false) }} diff --git a/salt/strelka/files/manager/manager.yaml b/salt/strelka/files/manager/manager.yaml deleted file mode 100644 index bd15b6423..000000000 --- a/salt/strelka/files/manager/manager.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{%- if grains.role in ['so-sensor', 'so-heavynode'] -%} - {%- set mainint = salt['pillar.get']('host:mainint') %} - {%- set ip = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %} -{%- else %} - {%- set ip = salt['pillar.get']('global:managerip') %} -{%- endif -%} -coordinator: - addr: '{{ ip }}:6380' - db: 0 diff --git a/salt/strelka/files/manager/manager.yaml.jinja b/salt/strelka/files/manager/manager.yaml.jinja new file mode 100644 index 000000000..c91c2e8c8 --- /dev/null +++ b/salt/strelka/files/manager/manager.yaml.jinja @@ -0,0 +1 @@ +{{ MANAGERCONFIG | yaml(false) }} diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index d29053229..bec22c1fa 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -8,15 +8,10 @@ {% from 'docker/docker.map.jinja' import DOCKER %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %} -{% import_yaml 'strelka/defaults.yaml' as strelka_config with context %} -{% set IGNORELIST = salt['pillar.get']('strelka:ignore', strelka_config.strelka.ignore, merge=True, merge_nested_lists=True) %} -{% set ENGINE = salt['pillar.get']('global:mdengine', '') %} -{% if ENGINE == "SURICATA" %} - {% set filecheck_runas = 'suricata' %} -{% else %} - {% set filecheck_runas = 'socore' %} -{% endif %} +{% from 'strelka/map.jinja' import STRELKAMERGED %} +{% from 'strelka/filecheck/map.jinja' import FILECHECKDEFAULTS %} +{% from 'strelka/filecheck/map.jinja' import filecheck_runas %} # Strelka config strelkaconfdir: @@ -33,14 +28,65 @@ strelkarulesdir: - group: 939 - makedirs: True -# Sync dynamic config to conf dir -strelkasync: - file.recurse: - - name: /opt/so/conf/strelka/ - - source: salt://strelka/files +backend_backend_config: + file.managed: + - name: /opt/so/conf/strelka/backend/backend.yaml + - source: salt://strelka/files/backend/backend.yaml.jinja + - template: jinja - user: 939 - group: 939 + - defaults: + BACKENDCONFIG: {{ STRELKAMERGED.config.backend.backend }} + +backend_logging_config: + file.managed: + - name: /opt/so/conf/strelka/backend/logging.yaml + - source: salt://strelka/files/backend/logging.yaml.jinja - template: jinja + - user: 939 + - group: 939 + - defaults: + LOGGINGCONFIG: {{ STRELKAMERGED.config.backend.logging }} + +backend_passwords: + file.managed: + - name: /opt/so/conf/strelka/backend/passwords.dat + - source: salt://strelka/files/backend/passwords.dat.jinja + - template: jinja + - user: 939 + - group: 939 + - defaults: + PASSWORDS: {{ STRELKAMERGED.config.backend.passwords }} + +filestream_config: + file.managed: + - name: /opt/so/conf/strelka/filestream/filestream.yaml + - source: salt://strelka/files/filestream/filestream.yaml.jinja + - template: jinja + - user: 939 + - group: 939 + - defaults: + FILESTREAMCONFIG: {{ STRELKAMERGED.config.filestream }} + +frontend_config: + file.managed: + - name: /opt/so/conf/strelka/frontend/frontend.yaml + - source: salt://strelka/files/frontend/frontend.yaml.jinja + - template: jinja + - user: 939 + - group: 939 + - defaults: + FRONTENDCONFIG: {{ STRELKAMERGED.config.frontend }} + +manager_config: + file.managed: + - name: /opt/so/conf/strelka/manager/manager.yaml + - source: salt://strelka/files/manager/manager.yaml.jinja + - template: jinja + - user: 939 + - group: 939 + - defaults: + MANAGERCONFIG: {{ STRELKAMERGED.config.manager }} {% if STRELKA_RULES == 1 %} @@ -51,16 +97,6 @@ strelkarules: - user: 939 - group: 939 - clean: True - - exclude_pat: - {% for IGNOREDRULE in IGNORELIST %} - - {{ IGNOREDRULE }} - {% endfor %} - - {% for IGNOREDRULE in IGNORELIST %} -remove_rule_{{ IGNOREDRULE }}: - file.absent: - - name: /opt/so/conf/strelka/rules/signature-base/{{ IGNOREDRULE }} - {% endfor %} {% if grains['role'] in GLOBALS.manager_roles %} strelkarepos: @@ -133,8 +169,10 @@ filecheck_history: filecheck_conf: file.managed: - name: /opt/so/conf/strelka/filecheck.yaml - - source: salt://strelka/filecheck/filecheck.yaml + - source: salt://strelka/filecheck/filecheck.yaml.jinja - template: jinja + - defaults: + FILECHECKCONFIG: {{ FILECHECKDEFAULTS }} filecheck_script: file.managed: @@ -173,7 +211,7 @@ strelka_coordinator: - ipv4_address: {{ DOCKER.containers['so-strelka-coordinator'].ip }} - entrypoint: redis-server --save "" --appendonly no - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} + - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }} - port_bindings: {% for BINDING in DOCKER.containers['so-strelka-coordinator'].port_bindings %} - {{ BINDING }} @@ -193,7 +231,7 @@ strelka_gatekeeper: - ipv4_address: {{ DOCKER.containers['so-strelka-gatekeeper'].ip }} - entrypoint: redis-server --save "" --appendonly no --maxmemory-policy allkeys-lru - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} + - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }} - port_bindings: {% for BINDING in DOCKER.containers['so-strelka-gatekeeper'].port_bindings %} - {{ BINDING }} @@ -217,7 +255,7 @@ strelka_frontend: - ipv4_address: {{ DOCKER.containers['so-strelka-frontend'].ip }} - command: strelka-frontend - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} + - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }} - port_bindings: {% for BINDING in DOCKER.containers['so-strelka-frontend'].port_bindings %} - {{ BINDING }} @@ -240,7 +278,7 @@ strelka_backend: - ipv4_address: {{ DOCKER.containers['so-strelka-backend'].ip }} - command: strelka-backend - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} + - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }} - restart_policy: on-failure append_so-strelka-backend_so-status.conf: @@ -259,7 +297,7 @@ strelka_manager: - ipv4_address: {{ DOCKER.containers['so-strelka-manager'].ip }} - command: strelka-manager - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} + - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }} append_so-strelka-manager_so-status.conf: file.append: @@ -278,7 +316,7 @@ strelka_filestream: - ipv4_address: {{ DOCKER.containers['so-strelka-filestream'].ip }} - command: strelka-filestream - extra_hosts: - - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }} + - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }} append_so-strelka-filestream_so-status.conf: file.append: diff --git a/salt/strelka/map.jinja b/salt/strelka/map.jinja new file mode 100644 index 000000000..bf0a29a17 --- /dev/null +++ b/salt/strelka/map.jinja @@ -0,0 +1,20 @@ +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'strelka/defaults.yaml' as STRELKADEFAULTS %} +{% set HOST = GLOBALS.hostname %} + +{% set backend_coordinator_port = STRELKADEFAULTS.strelka.config.backend.backend.coordinator.addr.split(':')[1] %} +{% do STRELKADEFAULTS.strelka.config.backend.backend.coordinator.update({'addr': HOST ~ ':' ~ backend_coordinator_port}) %} + +{% set filestream_conn_port = STRELKADEFAULTS.strelka.config.filestream.conn.server.split(':')[1] %} +{% do STRELKADEFAULTS.strelka.config.filestream.conn.update({'server': HOST ~ ':' ~ filestream_conn_port}) %} + +{% set frontend_coordinator_port = STRELKADEFAULTS.strelka.config.frontend.coordinator.addr.split(':')[1] %} +{% do STRELKADEFAULTS.strelka.config.frontend.coordinator.update({'addr': HOST ~ ':' ~ frontend_coordinator_port}) %} + +{% set frontend_gatekeeper_port = STRELKADEFAULTS.strelka.config.frontend.gatekeeper.addr.split(':')[1] %} +{% do STRELKADEFAULTS.strelka.config.frontend.gatekeeper.update({'addr': HOST ~ ':' ~ frontend_gatekeeper_port}) %} + +{% set manager_coordinator_port = STRELKADEFAULTS.strelka.config.manager.coordinator.addr.split(':')[1] %} +{% do STRELKADEFAULTS.strelka.config.manager.coordinator.update({'addr': HOST ~ ':' ~ manager_coordinator_port}) %} + +{% set STRELKAMERGED = salt['pillar.get']('strelka', STRELKADEFAULTS.strelka, merge=True) %} From e105e56facbb5c9639c1da1fb30b26ec27a14073 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 13 Mar 2023 13:27:02 +0000 Subject: [PATCH 03/23] Move data stream configuration outside of ILM policy definition --- salt/elasticsearch/defaults.yaml | 57 ++++++++++++++------------------ 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index c4098e08c..d47125972 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -84,6 +84,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -108,9 +111,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.auditbeat: index_sorting: False index_template: @@ -138,6 +138,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -162,9 +165,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.cloudbeat: index_sorting: False index_template: @@ -216,9 +216,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.endpoint_security: index_sorting: False index_template: @@ -246,6 +243,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -270,9 +270,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.filebeat: index_sorting: False index_template: @@ -324,9 +321,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.fleet_server: index_sorting: False index_template: @@ -354,6 +348,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -378,9 +375,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.heartbeat: index_sorting: False index_template: @@ -432,9 +426,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent: index_sorting: False index_template: @@ -462,6 +453,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -486,9 +480,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.metricbeat: index_sorting: False index_template: @@ -516,6 +507,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -540,9 +534,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.osquerybeat: index_sorting: False index_template: @@ -570,6 +561,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -594,9 +588,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-logs-elastic_agent.packetbeat: index_sorting: False index_template: @@ -624,6 +615,9 @@ elasticsearch: - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 200 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: hot: @@ -648,9 +642,6 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - data_stream: - hidden: false - allow_custom_routing: false so-aws: warm: 7 close: 30 From b3a2680847f4222caa290051859fbd716fea3f63 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 13 Mar 2023 11:41:36 -0400 Subject: [PATCH 04/23] auto-apply firewall rules --- salt/common/tools/sbin/so-firewall | 2 ++ salt/common/tools/sbin/so-firewall-minion | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/common/tools/sbin/so-firewall b/salt/common/tools/sbin/so-firewall index 69808c709..16dcdf729 100755 --- a/salt/common/tools/sbin/so-firewall +++ b/salt/common/tools/sbin/so-firewall @@ -97,6 +97,8 @@ echo "$IP" >> $local_salt_dir/hostgroups/$ROLE if [ "$APPLY" = "true" ]; then echo "Applying the firewall rules" salt-call state.apply firewall queue=True + echo "Firewall rules have been applied... Review logs further if there were errors." + echo "" else echo "Firewall rules will be applied next salt run" fi diff --git a/salt/common/tools/sbin/so-firewall-minion b/salt/common/tools/sbin/so-firewall-minion index e796035f9..19ea26864 100755 --- a/salt/common/tools/sbin/so-firewall-minion +++ b/salt/common/tools/sbin/so-firewall-minion @@ -54,25 +54,25 @@ fi 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT') so-firewall --role=manager --ip="$IP" so-firewall --role=sensors --ip="$IP" - so-firewall --apply --role=searchnodes --ip="$IP" + so-firewall --apply=true --role=searchnodes --ip="$IP" ;; 'SENSOR' | 'SEARCHNODE' | 'HEAVYNODE' | 'IDH' | 'RECEIVER') case "$ROLE" in 'SENSOR') - so-firewall --apply --role=sensors --ip="$IP" + so-firewall --apply=true --role=sensors --ip="$IP" ;; 'SEARCHNODE') - so-firewall --apply --role=searchnodes --ip="$IP" + so-firewall --apply=true --role=searchnodes --ip="$IP" ;; 'HEAVYNODE') so-firewall --role=sensors --ip="$IP" - so-firewall --apply --role=heavynodes --ip="$IP" + so-firewall --apply=true --role=heavynodes --ip="$IP" ;; 'IDH') - so-firewall --apply --role=sensors --ip="$IP" + so-firewall --apply=true --role=sensors --ip="$IP" ;; 'RECEIVER') - so-firewall --apply --role=receivers --ip="$IP" + so-firewall --apply=true --role=receivers --ip="$IP" ;; esac ;; From f7be4ba31c48d7b808f9d31b4fa79c5ba09e5f61 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 13 Mar 2023 14:07:17 -0400 Subject: [PATCH 05/23] Remove host field from NIDS logs --- salt/elasticsearch/files/ingest/common.nids | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/common.nids b/salt/elasticsearch/files/ingest/common.nids index df6af7a85..53a3f7b79 100644 --- a/salt/elasticsearch/files/ingest/common.nids +++ b/salt/elasticsearch/files/ingest/common.nids @@ -11,7 +11,7 @@ { "set": { "if": "ctx.rule.severity == 3", "field": "event.severity", "value": 1, "override": true } }, { "set": { "if": "ctx.rule.severity == 2", "field": "event.severity", "value": 2, "override": true } }, { "set": { "if": "ctx.rule.severity == 1", "field": "event.severity", "value": 3, "override": true } }, - { "remove": { "field": ["rule_type", "rest_of_rulename"], "ignore_failure": true } }, + { "remove": { "field": ["rule_type", "rest_of_rulename", "host"], "ignore_failure": true } }, { "pipeline": { "name": "common" } } ] } From 9d4e1cc1499dd6b957bee814b650bb48882857af Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 13 Mar 2023 16:48:21 -0400 Subject: [PATCH 06/23] jinja for strelka --- salt/manager/files/so-yara-update.jinja | 2 +- salt/manager/init.sls | 2 +- salt/strelka/defaults.yaml | 58 ++++++++++--------------- salt/strelka/init.sls | 13 ++++-- salt/strelka/repos.txt.jinja | 2 + salt/strelka/rules/ignore.txt | 4 -- salt/strelka/rules/repos.txt | 1 - salt/strelka/rules/repos.txt.jinja | 4 -- 8 files changed, 36 insertions(+), 50 deletions(-) create mode 100644 salt/strelka/repos.txt.jinja delete mode 100644 salt/strelka/rules/ignore.txt delete mode 100644 salt/strelka/rules/repos.txt delete mode 100644 salt/strelka/rules/repos.txt.jinja diff --git a/salt/manager/files/so-yara-update.jinja b/salt/manager/files/so-yara-update.jinja index ea07f72e4..beaa97ab6 100755 --- a/salt/manager/files/so-yara-update.jinja +++ b/salt/manager/files/so-yara-update.jinja @@ -9,7 +9,7 @@ echo "Starting to check for yara rule updates at $(date)..." output_dir="/opt/so/saltstack/default/salt/strelka/rules" mkdir -p $output_dir -repos="$output_dir/repos.txt" +repos="/opt/so/conf/strelka/repos.txt" newcounter=0 excludedcounter=0 excluded_rules=({{ EXCLUDEDRULES | join(' ') }}) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 5f2b0005a..a360fb2c5 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -33,7 +33,7 @@ yara_update_script: - template: jinja - defaults: ISAIRGAP: {{ GLOBALS.airgap }} - EXCLUDEDRULES: {{ STRELKAMERGED.excluded_rules }} + EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }} strelka_yara_update: cron.present: diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml index 12f0edda3..cdd75a22d 100644 --- a/salt/strelka/defaults.yaml +++ b/salt/strelka/defaults.yaml @@ -13,7 +13,7 @@ strelka: addr: 'HOST:6380' db: 0 tasting: - mime_db: '' + mime_db: null yara_rules: '/etc/strelka/taste/' scanners: 'ScanBase64': @@ -535,37 +535,25 @@ strelka: addr: 'HOST:6380' db: 0 - - - - - - - - - - - - - - - - - excluded_rules: - - apt_flame2_orchestrator.yar - - apt_tetris.yar - - gen_susp_js_obfuscatorio.yar - - gen_webshells.yar - - generic_anomalies.yar - - general_cloaking.yar - - thor_inverse_matches.yar - - yara_mixed_ext_vars.yar - - apt_apt27_hyperbro.yar - - apt_turla_gazer.yar - - gen_google_anomaly.yar - - gen_icon_anomalies.yar - - gen_nvidia_leaked_cert.yar - - gen_sign_anomalies.yar - - gen_susp_xor.yar - - gen_webshells_ext_vars.yar - - configured_vulns_ext_vars.yar + rules: + enabled: True + repos: + - https://github.com/Neo23x0/signature-base + excluded: + - apt_flame2_orchestrator.yar + - apt_tetris.yar + - gen_susp_js_obfuscatorio.yar + - gen_webshells.yar + - generic_anomalies.yar + - general_cloaking.yar + - thor_inverse_matches.yar + - yara_mixed_ext_vars.yar + - apt_apt27_hyperbro.yar + - apt_turla_gazer.yar + - gen_google_anomaly.yar + - gen_icon_anomalies.yar + - gen_nvidia_leaked_cert.yar + - gen_sign_anomalies.yar + - gen_susp_xor.yar + - gen_webshells_ext_vars.yar + - configured_vulns_ext_vars.yar diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index bec22c1fa..bded9ca70 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -7,7 +7,6 @@ {% if sls in allowed_states %} {% from 'docker/docker.map.jinja' import DOCKER %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %} {% from 'strelka/map.jinja' import STRELKAMERGED %} {% from 'strelka/filecheck/map.jinja' import FILECHECKDEFAULTS %} @@ -35,6 +34,7 @@ backend_backend_config: - template: jinja - user: 939 - group: 939 + - makedirs: True - defaults: BACKENDCONFIG: {{ STRELKAMERGED.config.backend.backend }} @@ -65,6 +65,7 @@ filestream_config: - template: jinja - user: 939 - group: 939 + - makedirs: True - defaults: FILESTREAMCONFIG: {{ STRELKAMERGED.config.filestream }} @@ -75,6 +76,7 @@ frontend_config: - template: jinja - user: 939 - group: 939 + - makedirs: True - defaults: FRONTENDCONFIG: {{ STRELKAMERGED.config.frontend }} @@ -85,10 +87,11 @@ manager_config: - template: jinja - user: 939 - group: 939 + - makedirs: True - defaults: MANAGERCONFIG: {{ STRELKAMERGED.config.manager }} -{% if STRELKA_RULES == 1 %} +{% if STRELKAMERGED.rules.enabled %} strelkarules: file.recurse: @@ -101,9 +104,11 @@ strelkarules: {% if grains['role'] in GLOBALS.manager_roles %} strelkarepos: file.managed: - - name: /opt/so/saltstack/default/salt/strelka/rules/repos.txt - - source: salt://strelka/rules/repos.txt.jinja + - name: /opt/so/conf/strelka/repos.txt + - source: salt://strelka/repos.txt.jinja - template: jinja + - defaults: + STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} {% endif %} {% endif %} diff --git a/salt/strelka/repos.txt.jinja b/salt/strelka/repos.txt.jinja new file mode 100644 index 000000000..043a02203 --- /dev/null +++ b/salt/strelka/repos.txt.jinja @@ -0,0 +1,2 @@ +# DO NOT EDIT THIS FILE! Strelka YARA rule repos are stored here from the strelka:rules:repos pillar section +{{ STRELKAREPOS | join('\n') }} diff --git a/salt/strelka/rules/ignore.txt b/salt/strelka/rules/ignore.txt deleted file mode 100644 index a803f8c28..000000000 --- a/salt/strelka/rules/ignore.txt +++ /dev/null @@ -1,4 +0,0 @@ -generic_anomalies.yar -general_cloaking.yar -thor_inverse_matches.yar -yara_mixed_ext_vars.yar diff --git a/salt/strelka/rules/repos.txt b/salt/strelka/rules/repos.txt deleted file mode 100644 index e26687ea9..000000000 --- a/salt/strelka/rules/repos.txt +++ /dev/null @@ -1 +0,0 @@ -https://github.com/Neo23x0/signature-base diff --git a/salt/strelka/rules/repos.txt.jinja b/salt/strelka/rules/repos.txt.jinja deleted file mode 100644 index 7d449f18d..000000000 --- a/salt/strelka/rules/repos.txt.jinja +++ /dev/null @@ -1,4 +0,0 @@ -# DO NOT EDIT THIS FILE! Strelka YARA rule repos are stored here from the strelka.repos pillar section -{%- for repo in salt['pillar.get']('strelka:repos', {}) %} -{{ repo }} -{%- endfor %} From 8d395dc465911918c3a2633bffb58199b524e7fa Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 13 Mar 2023 20:54:13 +0000 Subject: [PATCH 07/23] Add Elastic Agent default data stream backing indices for management by Curator --- salt/curator/defaults.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml index 237a50c81..958dd99ef 100644 --- a/salt/curator/defaults.yaml +++ b/salt/curator/defaults.yaml @@ -15,6 +15,27 @@ elasticsearch: logs-zeek-so: close: 30 delete: 365 + logs-elastic_agent-metricbeat-default: + close: 30 + delete: 365 + logs-elastic_agent-osquerybeat-default: + close: 30 + delete: 365 + logs-elastic_agent-fleet_server-default: + close: 30 + delete: 365 + logs-elastic_agent-filebeat-default: + close: 30 + delete: 365 + logs-elastic_agent-default: + close: 30 + delete: 365 + logs-system-auth-default: + close: 30 + delete: 365 + logs-system-syslog-default: + close: 30 + delete: 365 so-beats: close: 30 delete: 365 From efc58324999253c158915e710154206ffc671988 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 13 Mar 2023 20:54:38 +0000 Subject: [PATCH 08/23] Add Elastic Agent default log action files --- .../logs-elastic_agent-default-close.yaml | 27 +++++++++++++++++++ .../logs-elastic_agent-default-delete.yaml | 27 +++++++++++++++++++ ...-elastic_agent-filebeat-default-close.yaml | 27 +++++++++++++++++++ ...elastic_agent-filebeat-default-delete.yaml | 27 +++++++++++++++++++ ...stic_agent-fleet_server-default-close.yaml | 27 +++++++++++++++++++ ...tic_agent-fleet_server-default-delete.yaml | 27 +++++++++++++++++++ ...lastic_agent-metricbeat-default-close.yaml | 27 +++++++++++++++++++ ...astic_agent-metricbeat-default-delete.yaml | 27 +++++++++++++++++++ ...astic_agent-osquerybeat-default-close.yaml | 27 +++++++++++++++++++ ...stic_agent-osquerybeat-default-delete.yaml | 27 +++++++++++++++++++ ...logs-elastic_agent-osquerybeat-delete.yaml | 27 +++++++++++++++++++ .../logs-system-auth-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-auth-default-delete.yaml | 27 +++++++++++++++++++ .../action/logs-system-auth-syslog-close.yaml | 27 +++++++++++++++++++ .../logs-system-syslog-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-syslog-default-delete.yaml | 27 +++++++++++++++++++ 16 files changed, 432 insertions(+) create mode 100644 salt/curator/files/action/logs-elastic_agent-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml create mode 100644 salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml create mode 100644 salt/curator/files/action/logs-system-auth-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-auth-default-delete.yaml create mode 100644 salt/curator/files/action/logs-system-auth-syslog-close.yaml create mode 100644 salt/curator/files/action/logs-system-syslog-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-syslog-default-delete.yaml diff --git a/salt/curator/files/action/logs-elastic_agent-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-default-close.yaml new file mode 100644 index 000000000..ef03e4ba2 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent default indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-default-delete.yaml new file mode 100644 index 000000000..dee51c758 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent default indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml new file mode 100644 index 000000000..9277b25fd --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent.filebeat-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Filebeat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.filebeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml new file mode 100644 index 000000000..dfa51f260 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-filebeat-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-filebeat-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Filebeat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.filebeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml new file mode 100644 index 000000000..6bc2026b9 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-fleet_server-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Fleet Server indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.fleet_server-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml new file mode 100644 index 000000000..6fa775ba8 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-fleet_server-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-fleet_server-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.fleet_server-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml new file mode 100644 index 000000000..a4e38cd8e --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-metricbeat-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Metricbeat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.metricbeat-default-.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml new file mode 100644 index 000000000..b42e42c83 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs--elastic_agent-metricbeat-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Metricbeat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.metricbeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml new file mode 100644 index 000000000..9243d8cfb --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-osquerybeat-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Osquerybeat indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.osquerybeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml new file mode 100644 index 000000000..bce3b7e63 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-osquerybeat-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-osquerybeat-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Osquerybeat indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-elastic_agent.osquerybeat-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml b/salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml new file mode 100644 index 000000000..b46a5fc73 --- /dev/null +++ b/salt/curator/files/action/logs-elastic_agent-osquerybeat-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-auth-default-close.yaml b/salt/curator/files/action/logs-system-auth-default-close.yaml new file mode 100644 index 000000000..7c04a0ca9 --- /dev/null +++ b/salt/curator/files/action/logs-system-auth-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system.auth-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system auth indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.auth-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-auth-default-delete.yaml b/salt/curator/files/action/logs-system-auth-default-delete.yaml new file mode 100644 index 000000000..d14d560f3 --- /dev/null +++ b/salt/curator/files/action/logs-system-auth-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system.auth-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system auth indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.auth-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-auth-syslog-close.yaml b/salt/curator/files/action/logs-system-auth-syslog-close.yaml new file mode 100644 index 000000000..52ddb5eb5 --- /dev/null +++ b/salt/curator/files/action/logs-system-auth-syslog-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-import-so'].close %} +actions: + 1: + action: close + description: >- + Close import indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-syslog-default-close.yaml b/salt/curator/files/action/logs-system-syslog-default-close.yaml new file mode 100644 index 000000000..a9a697a66 --- /dev/null +++ b/salt/curator/files/action/logs-system-syslog-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system.syslog-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system syslog indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.syslog-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-syslog-default-delete.yaml b/salt/curator/files/action/logs-system-syslog-default-delete.yaml new file mode 100644 index 000000000..b46a5fc73 --- /dev/null +++ b/salt/curator/files/action/logs-system-syslog-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete import indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-import-so.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + From d5bb223235c6ac48cd69691a0b36419ea20cfb70 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:10:52 -0400 Subject: [PATCH 09/23] Fix system syslog delete file configuration --- .../files/action/logs-system-syslog-default-delete.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/curator/files/action/logs-system-syslog-default-delete.yaml b/salt/curator/files/action/logs-system-syslog-default-delete.yaml index b46a5fc73..36e079408 100644 --- a/salt/curator/files/action/logs-system-syslog-default-delete.yaml +++ b/salt/curator/files/action/logs-system-syslog-default-delete.yaml @@ -3,19 +3,19 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-import-so'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-system.syslog-default'].delete %} actions: 1: action: delete_indices description: >- - Delete import indices when older than {{ DELETE_DAYS }} days. + Delete Elastic Agent system syslog indices when older than {{ DELETE_DAYS }} days. options: ignore_empty_list: True disable_action: False filters: - filtertype: pattern kind: regex - value: '^(.ds-logs-import-so.*)$' + value: '^(.ds-logs-system.syslog-default.*)$' - filtertype: age source: name direction: older From c2701f1835372a75ed5ccb3fbca41561679fba3f Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:24:12 -0400 Subject: [PATCH 10/23] Fix system syslog default key value --- .../curator/files/action/logs-system-syslog-default-delete.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-system-syslog-default-delete.yaml b/salt/curator/files/action/logs-system-syslog-default-delete.yaml index 36e079408..1a7d217e9 100644 --- a/salt/curator/files/action/logs-system-syslog-default-delete.yaml +++ b/salt/curator/files/action/logs-system-syslog-default-delete.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-system.syslog-default'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-system-syslog-default'].delete %} actions: 1: action: delete_indices From 8ade7b85fc450efbd9cb28ee5264b7ccd76213e7 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:24:40 -0400 Subject: [PATCH 11/23] Fix system syslog default key value --- salt/curator/files/action/logs-system-syslog-default-close.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-system-syslog-default-close.yaml b/salt/curator/files/action/logs-system-syslog-default-close.yaml index a9a697a66..3c9482b40 100644 --- a/salt/curator/files/action/logs-system-syslog-default-close.yaml +++ b/salt/curator/files/action/logs-system-syslog-default-close.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set cur_close_days = CURATORMERGED['logs-system.syslog-default'].close %} +{%- set cur_close_days = CURATORMERGED['logs-system-syslog-default'].close %} actions: 1: action: close From 785f100132bf6fc21010da55fad47450b1d8b666 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:25:33 -0400 Subject: [PATCH 12/23] Fix system auth default key value --- salt/curator/files/action/logs-system-auth-default-close.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-system-auth-default-close.yaml b/salt/curator/files/action/logs-system-auth-default-close.yaml index 7c04a0ca9..af9843b35 100644 --- a/salt/curator/files/action/logs-system-auth-default-close.yaml +++ b/salt/curator/files/action/logs-system-auth-default-close.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set cur_close_days = CURATORMERGED['logs-system.auth-default'].close %} +{%- set cur_close_days = CURATORMERGED['logs-system-auth-default'].close %} actions: 1: action: close From bab40de58d7becd7e71059cc01fa5933ac36bf32 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:26:05 -0400 Subject: [PATCH 13/23] Fix system auth default key value --- salt/curator/files/action/logs-system-auth-default-delete.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-system-auth-default-delete.yaml b/salt/curator/files/action/logs-system-auth-default-delete.yaml index d14d560f3..9a1cc6a9a 100644 --- a/salt/curator/files/action/logs-system-auth-default-delete.yaml +++ b/salt/curator/files/action/logs-system-auth-default-delete.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-system.auth-default'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-system-auth-default'].delete %} actions: 1: action: delete_indices From f4112b30c0402bdca6a5711a48bff4c88f4e1473 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:27:06 -0400 Subject: [PATCH 14/23] Fix index reference for system auth default --- salt/curator/files/action/logs-system-auth-syslog-close.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/curator/files/action/logs-system-auth-syslog-close.yaml b/salt/curator/files/action/logs-system-auth-syslog-close.yaml index 52ddb5eb5..f71ffacb5 100644 --- a/salt/curator/files/action/logs-system-auth-syslog-close.yaml +++ b/salt/curator/files/action/logs-system-auth-syslog-close.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set cur_close_days = CURATORMERGED['logs-import-so'].close %} +{%- set cur_close_days = CURATORMERGED['logs-system-auth-default'].close %} actions: 1: action: close @@ -17,7 +17,7 @@ actions: filters: - filtertype: pattern kind: regex - value: '^(.ds-logs-import-so.*)$' + value: '^(.ds-logs-system.auth-default.*)$' - filtertype: age source: name direction: older From 486de12ca5eaee9ecbb9c43dbdab7f73db18a476 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 13 Mar 2023 17:27:52 -0400 Subject: [PATCH 15/23] Delete logs-system-auth-syslog-close.yaml --- .../action/logs-system-auth-syslog-close.yaml | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 salt/curator/files/action/logs-system-auth-syslog-close.yaml diff --git a/salt/curator/files/action/logs-system-auth-syslog-close.yaml b/salt/curator/files/action/logs-system-auth-syslog-close.yaml deleted file mode 100644 index f71ffacb5..000000000 --- a/salt/curator/files/action/logs-system-auth-syslog-close.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -{%- set cur_close_days = CURATORMERGED['logs-system-auth-default'].close %} -actions: - 1: - action: close - description: >- - Close import indices older than {{cur_close_days}} days. - options: - delete_aliases: False - timeout_override: - continue_if_exception: False - disable_action: False - filters: - - filtertype: pattern - kind: regex - value: '^(.ds-logs-system.auth-default.*)$' - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: {{cur_close_days}} - exclude: From 412e5c0402745ee6c287f476c8613ac37c54c64c Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 14 Mar 2023 13:46:08 +0000 Subject: [PATCH 16/23] Add more Elastic Agent Curator action files --- ...logs-system-application-default-close.yaml | 27 +++++++++++++++++++ ...ogs-system-application-default-delete.yaml | 27 +++++++++++++++++++ .../logs-system-security-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-security-default-delete.yaml | 27 +++++++++++++++++++ .../logs-system-system-default-close.yaml | 27 +++++++++++++++++++ .../logs-system-system-default-delete.yaml | 27 +++++++++++++++++++ ...logs-windows-powershell-default-close.yaml | 27 +++++++++++++++++++ ...ogs-windows-powershell-default-delete.yaml | 27 +++++++++++++++++++ ...dows-sysmon_operational-default-close.yaml | 27 +++++++++++++++++++ ...ows-sysmon_operational-default-delete.yaml | 27 +++++++++++++++++++ 10 files changed, 270 insertions(+) create mode 100644 salt/curator/files/action/logs-system-application-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-application-default-delete.yaml create mode 100644 salt/curator/files/action/logs-system-security-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-security-default-delete.yaml create mode 100644 salt/curator/files/action/logs-system-system-default-close.yaml create mode 100644 salt/curator/files/action/logs-system-system-default-delete.yaml create mode 100644 salt/curator/files/action/logs-windows-powershell-default-close.yaml create mode 100644 salt/curator/files/action/logs-windows-powershell-default-delete.yaml create mode 100644 salt/curator/files/action/logs-windows-sysmon_operational-default-close.yaml create mode 100644 salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml diff --git a/salt/curator/files/action/logs-system-application-default-close.yaml b/salt/curator/files/action/logs-system-application-default-close.yaml new file mode 100644 index 000000000..76d01ecb4 --- /dev/null +++ b/salt/curator/files/action/logs-system-application-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system-application-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system application indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.application-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-application-default-delete.yaml b/salt/curator/files/action/logs-system-application-default-delete.yaml new file mode 100644 index 000000000..b15c06fcb --- /dev/null +++ b/salt/curator/files/action/logs-system-application-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system-application-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system application indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.application-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-security-default-close.yaml b/salt/curator/files/action/logs-system-security-default-close.yaml new file mode 100644 index 000000000..9a8cab35c --- /dev/null +++ b/salt/curator/files/action/logs-system-security-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system-security-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system security indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.security-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-security-default-delete.yaml b/salt/curator/files/action/logs-system-security-default-delete.yaml new file mode 100644 index 000000000..0bac45aeb --- /dev/null +++ b/salt/curator/files/action/logs-system-security-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system-security-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system security indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.security-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-system-system-default-close.yaml b/salt/curator/files/action/logs-system-system-default-close.yaml new file mode 100644 index 000000000..284d6e219 --- /dev/null +++ b/salt/curator/files/action/logs-system-system-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-system-system-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent system system indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.system-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-system-system-default-delete.yaml b/salt/curator/files/action/logs-system-system-default-delete.yaml new file mode 100644 index 000000000..4701d0492 --- /dev/null +++ b/salt/curator/files/action/logs-system-system-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system-system-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system system indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.system-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-windows-powershell-default-close.yaml b/salt/curator/files/action/logs-windows-powershell-default-close.yaml new file mode 100644 index 000000000..7c3cebab3 --- /dev/null +++ b/salt/curator/files/action/logs-windows-powershell-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-windows-powershell-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Windows Powershell indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-windows.powershell-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-windows-powershell-default-delete.yaml b/salt/curator/files/action/logs-windows-powershell-default-delete.yaml new file mode 100644 index 000000000..447f8102b --- /dev/null +++ b/salt/curator/files/action/logs-windows-powershell-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-windows-powershell-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent Windows Powershell indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-windows.powershell-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + diff --git a/salt/curator/files/action/logs-windows-sysmon_operational-default-close.yaml b/salt/curator/files/action/logs-windows-sysmon_operational-default-close.yaml new file mode 100644 index 000000000..ae98b8939 --- /dev/null +++ b/salt/curator/files/action/logs-windows-sysmon_operational-default-close.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set cur_close_days = CURATORMERGED['logs-windows-sysmon_operational-default'].close %} +actions: + 1: + action: close + description: >- + Close Elastic Agent Windows Sysmon operational indices older than {{cur_close_days}} days. + options: + delete_aliases: False + timeout_override: + continue_if_exception: False + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-windows.sysmon_operational-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{cur_close_days}} + exclude: diff --git a/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml b/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml new file mode 100644 index 000000000..9a1cc6a9a --- /dev/null +++ b/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml @@ -0,0 +1,27 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{%- set DELETE_DAYS = CURATORMERGED['logs-system-auth-default'].delete %} +actions: + 1: + action: delete_indices + description: >- + Delete Elastic Agent system auth indices when older than {{ DELETE_DAYS }} days. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: regex + value: '^(.ds-logs-system.auth-default.*)$' + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: {{ DELETE_DAYS }} + exclude: + + From f0d4c16b2ba35a5a7f990b4f4d2e51fdc091ebbd Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 14 Mar 2023 13:49:13 +0000 Subject: [PATCH 17/23] Add more Elastic Agent index keys for Curator --- salt/curator/defaults.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/curator/defaults.yaml b/salt/curator/defaults.yaml index 958dd99ef..e1333c3a6 100644 --- a/salt/curator/defaults.yaml +++ b/salt/curator/defaults.yaml @@ -33,9 +33,24 @@ elasticsearch: logs-system-auth-default: close: 30 delete: 365 + logs-system-application-default: + close: 30 + delete: 365 + logs-system-security-default: + close: 30 + delete: 365 + logs-system-system-default: + close: 30 + delete: 365 logs-system-syslog-default: close: 30 delete: 365 + logs-windows-powershell-default: + close: 30 + delete: 365 + logs-windows-sysmon_operational-default: + close: 30 + delete: 365 so-beats: close: 30 delete: 365 From 766e6a79745671dc0cffad8d7c7f92d3071326fc Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 14 Mar 2023 13:51:49 +0000 Subject: [PATCH 18/23] Add 'logs-windows-sysmon_operational-delete' for Windows Sysmon operational indices --- .../logs-windows-sysmon_operational-default-delete.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml b/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml index 9a1cc6a9a..a1413bc1c 100644 --- a/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml +++ b/salt/curator/files/action/logs-windows-sysmon_operational-default-delete.yaml @@ -3,19 +3,19 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs-system-auth-default'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-windows-sysmon_operational-default'].delete %} actions: 1: action: delete_indices description: >- - Delete Elastic Agent system auth indices when older than {{ DELETE_DAYS }} days. + Delete Elastic Agent Windows Sysmon operational indices when older than {{ DELETE_DAYS }} days. options: ignore_empty_list: True disable_action: False filters: - filtertype: pattern kind: regex - value: '^(.ds-logs-system.auth-default.*)$' + value: '^(.ds-logs-windows.sysmon_operational-default.*)$' - filtertype: age source: name direction: older From 7c39938e14b8d8c87484d46ad5890fbf5ffff2b8 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 14 Mar 2023 10:48:50 -0400 Subject: [PATCH 19/23] Change 'elastic_agent.filebeat' to 'elastic_agent-filebeat' --- .../files/action/logs-elastic_agent-filebeat-default-close.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml index 9277b25fd..1157f94b2 100644 --- a/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml +++ b/salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set cur_close_days = CURATORMERGED['logs-elastic_agent.filebeat-default'].close %} +{%- set cur_close_days = CURATORMERGED['logs-elastic_agent-filebeat-default'].close %} actions: 1: action: close From 8eba3426be104d34dc73247aa2e0ede293cda78e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 14 Mar 2023 10:51:50 -0400 Subject: [PATCH 20/23] Remove extra dash for 'logs-elastic_agent-metricbeat-default' key --- .../action/logs-elastic_agent-metricbeat-default-delete.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml index b42e42c83..c69e1130a 100644 --- a/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml +++ b/salt/curator/files/action/logs-elastic_agent-metricbeat-default-delete.yaml @@ -3,7 +3,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set DELETE_DAYS = CURATORMERGED['logs--elastic_agent-metricbeat-default'].delete %} +{%- set DELETE_DAYS = CURATORMERGED['logs-elastic_agent-metricbeat-default'].delete %} actions: 1: action: delete_indices From b38d5df68407b2ed38a64e4a0a272951a3012a8d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 14 Mar 2023 13:25:51 -0400 Subject: [PATCH 21/23] set default mime_db --- salt/strelka/defaults.yaml | 2 +- salt/strelka/init.sls | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml index cdd75a22d..8060f520d 100644 --- a/salt/strelka/defaults.yaml +++ b/salt/strelka/defaults.yaml @@ -13,7 +13,7 @@ strelka: addr: 'HOST:6380' db: 0 tasting: - mime_db: null + mime_db: '/usr/lib/file/magic.mgc' yara_rules: '/etc/strelka/taste/' scanners: 'ScanBase64': diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index bded9ca70..80b43a017 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -58,6 +58,14 @@ backend_passwords: - defaults: PASSWORDS: {{ STRELKAMERGED.config.backend.passwords }} +backend_taste: + file.managed: + - name: /opt/so/conf/strelka/backend/taste/taste.yara + - source: salt://strelka/files/backend/taste/taste.yara + - makedirs: True + - user: 939 + - group: 939 + filestream_config: file.managed: - name: /opt/so/conf/strelka/filestream/filestream.yaml From 7cf4e6b03b92a5c08c4833b96a94ed79a78f3728 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 14 Mar 2023 13:59:31 -0400 Subject: [PATCH 22/23] add rules dir, change so-yar-update to save to local/salt/strelka/rules --- salt/manager/files/so-yara-update.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/files/so-yara-update.jinja b/salt/manager/files/so-yara-update.jinja index beaa97ab6..d11ba1a76 100755 --- a/salt/manager/files/so-yara-update.jinja +++ b/salt/manager/files/so-yara-update.jinja @@ -7,7 +7,7 @@ echo "Starting to check for yara rule updates at $(date)..." -output_dir="/opt/so/saltstack/default/salt/strelka/rules" +output_dir="/opt/so/saltstack/local/salt/strelka/rules" mkdir -p $output_dir repos="/opt/so/conf/strelka/repos.txt" newcounter=0 From f9b8c78d74cd0686280412211757a741bc1ba5d3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 14 Mar 2023 14:43:13 -0400 Subject: [PATCH 23/23] move repos to rules dir --- salt/strelka/init.sls | 2 +- salt/strelka/{ => rules}/repos.txt.jinja | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename salt/strelka/{ => rules}/repos.txt.jinja (100%) diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls index 80b43a017..f8b8262b0 100644 --- a/salt/strelka/init.sls +++ b/salt/strelka/init.sls @@ -113,7 +113,7 @@ strelkarules: strelkarepos: file.managed: - name: /opt/so/conf/strelka/repos.txt - - source: salt://strelka/repos.txt.jinja + - source: salt://strelka/rules/repos.txt.jinja - template: jinja - defaults: STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} diff --git a/salt/strelka/repos.txt.jinja b/salt/strelka/rules/repos.txt.jinja similarity index 100% rename from salt/strelka/repos.txt.jinja rename to salt/strelka/rules/repos.txt.jinja