From a89508f1ae63cf80d156fbfd136af23dd4cbde2f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 31 Jul 2023 15:17:24 -0400 Subject: [PATCH 001/350] Heavy Node fixes --- salt/elasticagent/enabled.sls | 3 + .../files/elastic-agent.yml.jinja | 349 +++++++++++++++++- .../grid-nodes_heavy/elasticsearch-logs.json | 106 ------ .../grid-nodes_heavy/kratos-logs.json | 29 -- .../grid-nodes_heavy/osquery-grid-nodes.json | 2 +- .../grid-nodes_heavy/redis-logs.json | 76 ---- .../grid-nodes_heavy/soc-auth-sync-logs.json | 29 -- .../grid-nodes_heavy/soc-salt-relay-logs.json | 29 -- .../grid-nodes_heavy/soc-sensoroni-logs.json | 29 -- .../grid-nodes_heavy/soc-server-logs.json | 29 -- .../grid-nodes_heavy/system-grid-nodes.json | 2 +- 11 files changed, 336 insertions(+), 347 deletions(-) delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls index 4c00920ac..b133d94ab 100644 --- a/salt/elasticagent/enabled.sls +++ b/salt/elasticagent/enabled.sls @@ -35,6 +35,7 @@ so-elastic-agent: - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro - /nsm:/nsm:ro + - /opt/so/log:/opt/so/log:ro {% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} - {{ BIND }} @@ -47,6 +48,8 @@ so-elastic-agent: - {{ XTRAENV }} {% endfor %} {% endif %} + - require: + - file: create-elastic-agent-config - watch: - file: create-elastic-agent-config diff --git a/salt/elasticagent/files/elastic-agent.yml.jinja b/salt/elasticagent/files/elastic-agent.yml.jinja index 2d32a3b17..92aacfa44 100644 --- a/salt/elasticagent/files/elastic-agent.yml.jinja +++ b/salt/elasticagent/files/elastic-agent.yml.jinja @@ -3,7 +3,7 @@ {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} id: aea1ba80-1065-11ee-a369-97538913b6a9 -revision: 2 +revision: 1 outputs: default: type: elasticsearch @@ -22,56 +22,369 @@ agent: metrics: false features: {} inputs: - - id: logfile-logs-80ffa884-2cfc-459a-964a-34df25714d85 - name: suricata-logs - revision: 1 + - id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62 + name: import-evtx-logs + revision: 2 type: logfile use_output: default meta: package: name: log - version: + version: data_stream: namespace: so - package_policy_id: 80ffa884-2cfc-459a-964a-34df25714d85 + package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62 streams: - - id: logfile-log.log-80ffa884-2cfc-459a-964a-34df25714d85 + - id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62 + data_stream: + dataset: import + paths: + - /nsm/import/*/evtx/*.json + processors: + - dissect: + field: log.file.path + tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}' + target_prefix: '' + - decode_json_fields: + fields: + - message + target: '' + - drop_fields: + ignore_missing: true + fields: + - host + - add_fields: + fields: + dataset: system.security + type: logs + namespace: default + target: data_stream + - add_fields: + fields: + dataset: system.security + module: system + imported: true + target: event + - then: + - add_fields: + fields: + dataset: windows.sysmon_operational + target: data_stream + - add_fields: + fields: + dataset: windows.sysmon_operational + module: windows + imported: true + target: event + if: + equals: + winlog.channel: Microsoft-Windows-Sysmon/Operational + - then: + - add_fields: + fields: + dataset: system.application + target: data_stream + - add_fields: + fields: + dataset: system.application + target: event + if: + equals: + winlog.channel: Application + - then: + - add_fields: + fields: + dataset: system.system + target: data_stream + - add_fields: + fields: + dataset: system.system + target: event + if: + equals: + winlog.channel: System + - then: + - add_fields: + fields: + dataset: windows.powershell_operational + target: data_stream + - add_fields: + fields: + dataset: windows.powershell_operational + module: windows + target: event + if: + equals: + winlog.channel: Microsoft-Windows-PowerShell/Operational + tags: + - import + - id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0 + name: redis-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: redis + version: + data_stream: + namespace: default + package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0 + streams: + - id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0 + data_stream: + dataset: redis.log + type: logs + exclude_files: + - .gz$ + paths: + - /opt/so/log/redis/redis.log + tags: + - redis-log + exclude_lines: + - '^\s+[\-`(''.|_]' + - id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8 + name: import-suricata-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8 + streams: + - id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8 + data_stream: + dataset: import + pipeline: suricata.common + paths: + - /nsm/import/*/suricata/eve*.json + processors: + - add_fields: + fields: + module: suricata + imported: true + category: network + target: event + - dissect: + field: log.file.path + tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}' + target_prefix: '' + - id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d + name: soc-server-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d + streams: + - id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d + data_stream: + dataset: soc + pipeline: common + paths: + - /opt/so/log/soc/sensoroni-server.log + processors: + - decode_json_fields: + add_error_key: true + process_array: true + max_depth: 2 + fields: + - message + target: soc + - add_fields: + fields: + module: soc + dataset_temp: server + category: host + target: event + - rename: + ignore_missing: true + fields: + - from: soc.fields.sourceIp + to: source.ip + - from: soc.fields.status + to: http.response.status_code + - from: soc.fields.method + to: http.request.method + - from: soc.fields.path + to: url.path + - from: soc.message + to: event.action + - from: soc.level + to: log.level + tags: + - so-soc + - id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073 + name: soc-sensoroni-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073 + streams: + - id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073 + data_stream: + dataset: soc + pipeline: common + paths: + - /opt/so/log/sensoroni/sensoroni.log + processors: + - decode_json_fields: + add_error_key: true + process_array: true + max_depth: 2 + fields: + - message + target: sensoroni + - add_fields: + fields: + module: soc + dataset_temp: sensoroni + category: host + target: event + - rename: + ignore_missing: true + fields: + - from: sensoroni.fields.sourceIp + to: source.ip + - from: sensoroni.fields.status + to: http.response.status_code + - from: sensoroni.fields.method + to: http.request.method + - from: sensoroni.fields.path + to: url.path + - from: sensoroni.message + to: event.action + - from: sensoroni.level + to: log.level + - id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515 + name: soc-salt-relay-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515 + streams: + - id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515 + data_stream: + dataset: soc + pipeline: common + paths: + - /opt/so/log/soc/salt-relay.log + processors: + - dissect: + field: message + tokenizer: '%{soc.ts} | %{event.action}' + target_prefix: '' + - add_fields: + fields: + module: soc + dataset_temp: salt_relay + category: host + target: event + tags: + - so-soc + - id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0 + name: soc-auth-sync-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0 + streams: + - id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0 + data_stream: + dataset: soc + pipeline: common + paths: + - /opt/so/log/soc/sync.log + processors: + - dissect: + field: message + tokenizer: '%{event.action}' + target_prefix: '' + - add_fields: + fields: + module: soc + dataset_temp: auth_sync + category: host + target: event + tags: + - so-soc + - id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253 + name: suricata-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253 + streams: + - id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253 data_stream: dataset: suricata + pipeline: suricata.common paths: - /nsm/suricata/eve*.json processors: - add_fields: - target: event fields: - category: network module: suricata - pipeline: suricata.common - - id: logfile-logs-90103ac4-f6bd-4a4a-b596-952c332390fc + category: network + target: event + - id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327 name: strelka-logs - revision: 1 + revision: 2 type: logfile use_output: default meta: package: name: log - version: + version: data_stream: namespace: so - package_policy_id: 90103ac4-f6bd-4a4a-b596-952c332390fc + package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327 streams: - - id: logfile-log.log-90103ac4-f6bd-4a4a-b596-952c332390fc + - id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327 data_stream: dataset: strelka + pipeline: strelka.file paths: - /nsm/strelka/log/strelka.log processors: - add_fields: - target: event fields: - category: file module: strelka - pipeline: strelka.file + category: file + target: event - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d name: zeek-logs revision: 1 diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json deleted file mode 100644 index 711602775..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "package": { - "name": "elasticsearch", - "version": "" - }, - "name": "elasticsearch-logs", - "namespace": "default", - "description": "Elasticsearch Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "elasticsearch-logfile": { - "enabled": true, - "streams": { - "elasticsearch.audit": { - "enabled": false, - "vars": { - "paths": [ - "/var/log/elasticsearch/*_audit.json" - ] - } - }, - "elasticsearch.deprecation": { - "enabled": false, - "vars": { - "paths": [ - "/var/log/elasticsearch/*_deprecation.json" - ] - } - }, - "elasticsearch.gc": { - "enabled": false, - "vars": { - "paths": [ - "/var/log/elasticsearch/gc.log.[0-9]*", - "/var/log/elasticsearch/gc.log" - ] - } - }, - "elasticsearch.server": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/elasticsearch/*.log" - ] - } - }, - "elasticsearch.slowlog": { - "enabled": false, - "vars": { - "paths": [ - "/var/log/elasticsearch/*_index_search_slowlog.json", - "/var/log/elasticsearch/*_index_indexing_slowlog.json" - ] - } - } - } - }, - "elasticsearch-elasticsearch/metrics": { - "enabled": false, - "vars": { - "hosts": [ - "http://localhost:9200" - ], - "scope": "node" - }, - "streams": { - "elasticsearch.stack_monitoring.ccr": { - "enabled": false - }, - "elasticsearch.stack_monitoring.cluster_stats": { - "enabled": false - }, - "elasticsearch.stack_monitoring.enrich": { - "enabled": false - }, - "elasticsearch.stack_monitoring.index": { - "enabled": false - }, - "elasticsearch.stack_monitoring.index_recovery": { - "enabled": false, - "vars": { - "active.only": true - } - }, - "elasticsearch.stack_monitoring.index_summary": { - "enabled": false - }, - "elasticsearch.stack_monitoring.ml_job": { - "enabled": false - }, - "elasticsearch.stack_monitoring.node": { - "enabled": false - }, - "elasticsearch.stack_monitoring.node_stats": { - "enabled": false - }, - "elasticsearch.stack_monitoring.pending_tasks": { - "enabled": false - }, - "elasticsearch.stack_monitoring.shard": { - "enabled": false - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json deleted file mode 100644 index c9e4183de..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "kratos-logs", - "namespace": "so", - "description": "Kratos logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/kratos/kratos.log" - ], - "data_stream.dataset": "kratos", - "tags": ["so-kratos"], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", - "custom": "pipeline: kratos" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json index d0281c111..b1454d4bd 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json @@ -3,7 +3,7 @@ "name": "osquery_manager", "version": "" }, - "name": "osquery-grid-nodes", + "name": "osquery-grid-nodes_heavy", "namespace": "default", "policy_id": "so-grid-nodes_heavy", "inputs": { diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json deleted file mode 100644 index cddcedfd8..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "package": { - "name": "redis", - "version": "" - }, - "name": "redis-logs", - "namespace": "default", - "description": "Redis logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "redis-logfile": { - "enabled": true, - "streams": { - "redis.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/redis/redis.log" - ], - "tags": [ - "redis-log" - ], - "preserve_original_event": false - } - } - } - }, - "redis-redis": { - "enabled": false, - "streams": { - "redis.slowlog": { - "enabled": false, - "vars": { - "hosts": [ - "127.0.0.1:6379" - ], - "password": "" - } - } - } - }, - "redis-redis/metrics": { - "enabled": false, - "vars": { - "hosts": [ - "127.0.0.1:6379" - ], - "idle_timeout": "20s", - "maxconn": 10, - "network": "tcp", - "password": "" - }, - "streams": { - "redis.info": { - "enabled": false, - "vars": { - "period": "10s" - } - }, - "redis.key": { - "enabled": false, - "vars": { - "key.patterns": "- limit: 20\n pattern: *\n", - "period": "10s" - } - }, - "redis.keyspace": { - "enabled": false, - "vars": { - "period": "10s" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json deleted file mode 100644 index 2004c8c5d..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "soc-auth-sync-logs", - "namespace": "so", - "description": "Security Onion - Elastic Auth Sync - Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/soc/sync.log" - ], - "data_stream.dataset": "soc", - "tags": ["so-soc"], - "processors": "- dissect:\n tokenizer: \"%{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: auth_sync", - "custom": "pipeline: common" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json deleted file mode 100644 index b1b6098c1..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "soc-salt-relay-logs", - "namespace": "so", - "description": "Security Onion - Salt Relay - Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/soc/salt-relay.log" - ], - "data_stream.dataset": "soc", - "tags": ["so-soc"], - "processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay", - "custom": "pipeline: common" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json deleted file mode 100644 index 5954e5052..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "soc-sensoroni-logs", - "namespace": "so", - "description": "Security Onion - Sensoroni - Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/sensoroni/sensoroni.log" - ], - "data_stream.dataset": "soc", - "tags": [], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"sensoroni\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: sensoroni\n- rename:\n fields:\n - from: \"sensoroni.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"sensoroni.fields.status\"\n to: \"http.response.status_code\"\n - from: \"sensoroni.fields.method\"\n to: \"http.request.method\"\n - from: \"sensoroni.fields.path\"\n to: \"url.path\"\n - from: \"sensoroni.message\"\n to: \"event.action\"\n - from: \"sensoroni.level\"\n to: \"log.level\"\n ignore_missing: true", - "custom": "pipeline: common" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json deleted file mode 100644 index 89e26563a..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "soc-server-logs", - "namespace": "so", - "description": "Security Onion Console Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/soc/sensoroni-server.log" - ], - "data_stream.dataset": "soc", - "tags": ["so-soc"], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: server\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", - "custom": "pipeline: common" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json index 31d30d4e0..3df514f0b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json @@ -4,7 +4,7 @@ "name": "system", "version": "" }, - "name": "system-grid-nodes", + "name": "system-grid-nodes_heavy", "namespace": "default", "inputs": { "system-logfile": { From b6dd347eb8ba085b9452b705aa860fe88f89e8d0 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 31 Jul 2023 15:22:29 -0400 Subject: [PATCH 002/350] Heavy Node add manager --- salt/logstash/enabled.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index a88e97b19..cd9d6dd7e 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -9,6 +9,11 @@ {% from 'docker/docker.map.jinja' import DOCKER %} {% from 'logstash/map.jinja' import LOGSTASH_MERGED %} {% from 'logstash/map.jinja' import REDIS_NODES %} +{# we append the manager here so that it is added to extra_hosts so the heavynode can resolve it #} +{# we cannont append in the logstash/map.jinja because then it would be added to the 0900_input_redis.conf #} +{% if GLOBALS.role == 'so-heavynode' %} +{% do REDIS_NODES.append({GLOBALS.manager:GLOBALS.manager_ip}) %} +{% endif %} {% set lsheap = LOGSTASH_MERGED.settings.lsheap %} include: From 6a55a8e5c08c0cbfd3f7fae2ec2c3fda12eece82 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 22:17:22 -0400 Subject: [PATCH 003/350] Elastic 8.2.2 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 9b69eb781..a2dedd324 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.1","id": "8.7.1","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.8.2","id": "8.8.2","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 44c926ba8d0672a6545fdf31f596a7e5797bf8a2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 22:18:07 -0400 Subject: [PATCH 004/350] Elastic 8.8.2 --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index e65955178..159a69e68 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.8.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 5dd5f9fc1caa8c613226faf801c6f7f83796eedc Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 22:18:43 -0400 Subject: [PATCH 005/350] Elastic 8.8.2 --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 53c8664d2..f9459587d 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -ELASTIC_AGENT_TARBALL_VERSION="8.7.1" +ELASTIC_AGENT_TARBALL_VERSION="8.8.2" DEFAULT_SALT_DIR=/opt/so/saltstack/default DOC_BASE_URL="https://docs.securityonion.net/en/2.4" From 29b64eadd42306852873047bd883900b558ea958 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 1 Aug 2023 02:20:22 +0000 Subject: [PATCH 006/350] Change log.log to log.logs --- .../grid-nodes_general/import-zeek-logs.json | 2 +- .../integrations-dynamic/grid-nodes_general/zeek-logs.json | 2 +- .../files/integrations/grid-nodes_general/idh-logs.json | 2 +- .../files/integrations/grid-nodes_general/import-evtx-logs.json | 2 +- .../integrations/grid-nodes_general/import-suricata-logs.json | 2 +- .../files/integrations/grid-nodes_general/kratos-logs.json | 2 +- .../integrations/grid-nodes_general/soc-auth-sync-logs.json | 2 +- .../integrations/grid-nodes_general/soc-salt-relay-logs.json | 2 +- .../integrations/grid-nodes_general/soc-sensoroni-logs.json | 2 +- .../files/integrations/grid-nodes_general/soc-server-logs.json | 2 +- .../files/integrations/grid-nodes_general/strelka-logs.json | 2 +- .../files/integrations/grid-nodes_general/suricata-logs.json | 2 +- .../files/integrations/grid-nodes_heavy/kratos-logs.json | 2 +- .../files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json | 2 +- .../integrations/grid-nodes_heavy/soc-salt-relay-logs.json | 2 +- .../files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json | 2 +- .../files/integrations/grid-nodes_heavy/soc-server-logs.json | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index 4c22f0446..0979f98b6 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -13,7 +13,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 2cec88bf2..32bff857b 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -14,7 +14,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index 32055112a..29cc1a879 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index d9f8daeb9..178b6ed53 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -12,7 +12,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index f17ee33d1..3b8cffcc1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json index c342b57bd..b1fb71077 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index 84e9ae94d..3aa740881 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index 07bd89b89..840f36f6b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index bee14ebf5..60ee95f45 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index 285d79148..b789adc1d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 6f6beca99..089b5d4f8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index 7ff43c3a8..a9d857b24 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json index c9e4183de..684cfd59b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json index 2004c8c5d..e031fe08c 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json index b1b6098c1..1c8399bca 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json index 5954e5052..a5e4b6217 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json index 89e26563a..f36a00c37 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ From 48d9c14563fe44e2c28a978140a1944cfe73e1cc Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 1 Aug 2023 02:20:43 +0000 Subject: [PATCH 007/350] Enable log package by default --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 46d496955..3d806d63f 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -32,4 +32,5 @@ elasticfleet: - fim - github - google_workspace + - log - 1password From 9d59e4250f39b56023a87cd0c5d39fb5a67a9311 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 22:23:54 -0400 Subject: [PATCH 008/350] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 59aa62c1f..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.5 +2.4.0-foxtrot From f84b0a3219d3f2046f48138b39d310afaef4937a Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 23:16:46 -0400 Subject: [PATCH 009/350] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f..59aa62c1f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.5 From 527a6ba454e26f48bec1af1abd409019e1075f2d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 23:52:38 -0400 Subject: [PATCH 010/350] Use asterisk when searching 'msg' since it is now a keyword --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 53db2c838..cb7d400a0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1140,7 +1140,7 @@ soc: showSubtitle: true - name: SOC - Auth description: Users authenticated to SOC grouped by IP address and identity - query: 'event.dataset:kratos.audit AND msg:authenticated | groupby http_request.headers.x-real-ip identity_id' + query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip identity_id' showSubtitle: true - name: SOC - App description: Logs generated by the Security Onion Console (SOC) server and modules @@ -1405,7 +1405,7 @@ soc: query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module* | groupby event.dataset | groupby event.module* | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: SOC Auth description: SOC (Security Onion Console) authentication logs - query: 'event.dataset:kratos.audit AND msg:authenticated | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent' + query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent' - name: Elastalerts description: Elastalert logs query: '_index: "*:elastalert*" | groupby rule_name | groupby alert_info.type' From 2875a7a2e5163fae947e58e354154c8c64fa5366 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Aug 2023 09:48:44 -0400 Subject: [PATCH 011/350] Sensor NIC offload --- salt/sensor/files/99-so-checksum-offload-disable | 12 ++++++++++++ salt/sensor/init.sls | 11 +++++++++++ salt/top.sls | 5 +++++ 3 files changed, 28 insertions(+) create mode 100755 salt/sensor/files/99-so-checksum-offload-disable create mode 100644 salt/sensor/init.sls diff --git a/salt/sensor/files/99-so-checksum-offload-disable b/salt/sensor/files/99-so-checksum-offload-disable new file mode 100755 index 000000000..fdce54f5e --- /dev/null +++ b/salt/sensor/files/99-so-checksum-offload-disable @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +init_monitor $MNIC diff --git a/salt/sensor/init.sls b/salt/sensor/init.sls new file mode 100644 index 000000000..34133e488 --- /dev/null +++ b/salt/sensor/init.sls @@ -0,0 +1,11 @@ +offload_script: + file.managed: + - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable + - source: salt://sensor/files/99-so-checksum-offload-disable + - mode: 755 + +execute_checksum: + cmd.run: + - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable + - onchanges: + - file: offload_script \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index e53895324..bc51c2db1 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -36,6 +36,7 @@ base: '*_sensor and G@saltversion:{{saltversion}}': - match: compound + - sensor - ssl - sensoroni - telegraf @@ -52,6 +53,7 @@ base: '*_eval and G@saltversion:{{saltversion}}': - match: compound - salt.master + - sensor - ca - ssl - registry @@ -118,6 +120,7 @@ base: '*_standalone and G@saltversion:{{saltversion}}': - match: compound - salt.master + - sensor - ca - ssl - registry @@ -196,6 +199,7 @@ base: '*_heavynode and G@saltversion:{{saltversion}}': - match: compound + - sensor - ssl - sensoroni - nginx @@ -216,6 +220,7 @@ base: '*_import and G@saltversion:{{saltversion}}': - match: compound - salt.master + - sensor - ca - ssl - registry From 87a5d20ac968f811338556d71d66edcf066eb9dd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Aug 2023 10:03:59 -0400 Subject: [PATCH 012/350] Sensor NIC offload --- salt/sensor/files/99-so-checksum-offload-disable | 4 +++- salt/sensor/init.sls | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/sensor/files/99-so-checksum-offload-disable b/salt/sensor/files/99-so-checksum-offload-disable index fdce54f5e..72f7838db 100755 --- a/salt/sensor/files/99-so-checksum-offload-disable +++ b/salt/sensor/files/99-so-checksum-offload-disable @@ -9,4 +9,6 @@ . /usr/sbin/so-common -init_monitor $MNIC +{% set MNIC = salt['pillar.get']('sensor:interface') %} + +init_monitor {{ MNIC }} diff --git a/salt/sensor/init.sls b/salt/sensor/init.sls index 34133e488..53cd808c6 100644 --- a/salt/sensor/init.sls +++ b/salt/sensor/init.sls @@ -3,6 +3,7 @@ offload_script: - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable - source: salt://sensor/files/99-so-checksum-offload-disable - mode: 755 + - template: jinja execute_checksum: cmd.run: From b6579d7d45474c229316cfa1653bc86565e3e725 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Aug 2023 10:13:44 -0400 Subject: [PATCH 013/350] Sensor NIC offload --- salt/common/tools/sbin/so-common | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index f9459587d..0581c09c6 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -225,12 +225,15 @@ init_monitor() { if [[ $MONITORNIC == "bond0" ]]; then BIFACES=$(lookup_bond_interfaces) + for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload; do + ethtool -K "$MONITORNIC" "$i" off; + done else BIFACES=$MONITORNIC fi for DEVICE_IFACE in $BIFACES; do - for i in rx tx sg tso ufo gso gro lro; do + for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload; do ethtool -K "$DEVICE_IFACE" "$i" off; done ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on From 4adaddf13f2e5b42dc16362d4bc24726277ad5bf Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 1 Aug 2023 10:14:59 -0400 Subject: [PATCH 014/350] Move syslog to the INPUT chain where needed --- salt/firewall/defaults.yaml | 79 +++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 20b966e48..3095c052e 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -198,9 +198,6 @@ firewall: portgroups: - redis - elasticsearch_node - self: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -218,9 +215,6 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - syslog: - portgroups: - - syslog analyst: portgroups: - nginx @@ -255,6 +249,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -425,12 +425,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - self: - portgroups: - - syslog - syslog: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -497,6 +491,12 @@ firewall: receiver: portgroups: - salt_manager + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -588,9 +588,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - self: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -608,9 +605,6 @@ firewall: endgame: portgroups: - endgame - syslog: - portgroups: - - syslog analyst: portgroups: - nginx @@ -660,6 +654,12 @@ firewall: receiver: portgroups: - salt_manager + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -760,9 +760,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - self: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -783,9 +780,6 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - syslog: - portgroups: - - syslog analyst: portgroups: - nginx @@ -838,6 +832,12 @@ firewall: receiver: portgroups: - salt_manager + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -884,9 +884,6 @@ firewall: searchnode: portgroups: - elasticsearch_node - self: - portgroups: - - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -918,6 +915,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -942,9 +945,6 @@ firewall: chain: DOCKER-USER: hostgroups: - self: - portgroups: - - syslog strelka_frontend: portgroups: - strelka_frontend @@ -979,6 +979,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -1030,6 +1036,9 @@ firewall: strelka_frontend: portgroups: - strelka_frontend + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -1189,11 +1198,7 @@ firewall: self: portgroups: - redis - - syslog - beats_5644 - syslog: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -1234,6 +1239,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: From f35f42c83d58c4aabee7ca7c7a48a8d16b344c97 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Aug 2023 10:23:45 -0400 Subject: [PATCH 015/350] Sensor NIC offload --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 0581c09c6..3c79110b3 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -225,7 +225,7 @@ init_monitor() { if [[ $MONITORNIC == "bond0" ]]; then BIFACES=$(lookup_bond_interfaces) - for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload; do + for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do ethtool -K "$MONITORNIC" "$i" off; done else @@ -233,7 +233,7 @@ init_monitor() { fi for DEVICE_IFACE in $BIFACES; do - for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload; do + for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do ethtool -K "$DEVICE_IFACE" "$i" off; done ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on From 3fa0a98830682de91c80c0eaa862bfd0fa5516a1 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 1 Aug 2023 12:45:09 -0400 Subject: [PATCH 016/350] Update verbiage and links in soc_sensor.yaml --- salt/sensor/soc_sensor.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/sensor/soc_sensor.yaml b/salt/sensor/soc_sensor.yaml index 0774e9bcf..9ab0c236e 100644 --- a/salt/sensor/soc_sensor.yaml +++ b/salt/sensor/soc_sensor.yaml @@ -1,9 +1,9 @@ sensor: interface: description: Main sensor monitoring interface. - helpLink: sensor.html + helpLink: network.html readonly: True mtu: - description: Main IP address of the grid host. - helpLink: host.html + description: Maximum Transmission Unit (MTU) of the sensor monitoring interface. + helpLink: network.html readonly: True From 968fee3488eee120dcf1fc1e403539e09e93c459 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 1 Aug 2023 13:10:41 -0400 Subject: [PATCH 017/350] Regen Agent Installers when Fleet URLs change --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 6 ++++++ .../tools/sbin_jinja/so-elastic-fleet-urls-update | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 2a19dcbd9..d7d6458c9 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -11,6 +11,12 @@ . /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common +LOG="/opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log" + +# Check to see if we are already running +NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers") +[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING gen installers script processes running...exiting." >>$LOG && exit 0 + for i in {1..30} do ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 24c5dabed..4a744665a 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -62,7 +62,7 @@ fi NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}") NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') -# Compare the current & new list of URLs - if different, update the Fleet Server URLs +# Compare the current & new list of URLs - if different, update the Fleet Server URLs & regenerate the agent installer if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then printf "\nHashes match - no update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" @@ -71,4 +71,5 @@ else printf "\nHashes don't match - update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" update_fleet_urls + /sbin/so-elastic-agent-gen-installers & fi From 2d13bf1a61441f43ee14cfc33e495a32249e3d7c Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 1 Aug 2023 14:40:12 -0400 Subject: [PATCH 018/350] Present logs to the host --- salt/elasticagent/enabled.sls | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls index b133d94ab..bff4cee6b 100644 --- a/salt/elasticagent/enabled.sls +++ b/salt/elasticagent/enabled.sls @@ -33,6 +33,7 @@ so-elastic-agent: {% endif %} - binds: - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro + - /opt/so/log/elastic-agent:/usr/share/elastic-agent/logs - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro - /nsm:/nsm:ro - /opt/so/log:/opt/so/log:ro @@ -40,7 +41,8 @@ so-elastic-agent: {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} - {{ BIND }} {% endfor %} - {% endif %} + {% endif %} + - LOGS_PATH=logs - environment: - FLEET_CA=/etc/pki/tls/certs/intca.crt {% if DOCKER.containers['so-elastic-agent'].extra_env %} From 1cbf60825d0f47bc0a7831840fdb7ef6f8bb4d9d Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 1 Aug 2023 14:40:52 -0400 Subject: [PATCH 019/350] Add log dir --- salt/elasticagent/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/elasticagent/config.sls b/salt/elasticagent/config.sls index 8b24f3b22..b0b4321fa 100644 --- a/salt/elasticagent/config.sls +++ b/salt/elasticagent/config.sls @@ -28,6 +28,13 @@ elasticagentconfdir: - group: 939 - makedirs: True +elasticagentlogdir: + file.directory: + - name: /opt/so/log/elastic-agent + - user: 949 + - group: 939 + - makedirs: True + elasticagent_sbin_jinja: file.recurse: - name: /usr/sbin From 4e2eb86b36e4fc2c999bbb0957618f5b78ebda56 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 1 Aug 2023 20:11:51 +0000 Subject: [PATCH 020/350] Move LOGS_PATH to environment vars --- salt/elasticagent/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls index bff4cee6b..67d7b975d 100644 --- a/salt/elasticagent/enabled.sls +++ b/salt/elasticagent/enabled.sls @@ -42,9 +42,9 @@ so-elastic-agent: - {{ BIND }} {% endfor %} {% endif %} - - LOGS_PATH=logs - environment: - FLEET_CA=/etc/pki/tls/certs/intca.crt + - LOGS_PATH=logs {% if DOCKER.containers['so-elastic-agent'].extra_env %} {% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %} - {{ XTRAENV }} From 44b086a02864415010764d5afe5bae25a4e87461 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 1 Aug 2023 20:13:50 +0000 Subject: [PATCH 021/350] Change path --- salt/elasticagent/config.sls | 2 +- salt/elasticagent/enabled.sls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticagent/config.sls b/salt/elasticagent/config.sls index b0b4321fa..b54186fab 100644 --- a/salt/elasticagent/config.sls +++ b/salt/elasticagent/config.sls @@ -30,7 +30,7 @@ elasticagentconfdir: elasticagentlogdir: file.directory: - - name: /opt/so/log/elastic-agent + - name: /opt/so/log/elasticagent - user: 949 - group: 939 - makedirs: True diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls index 67d7b975d..963b8549b 100644 --- a/salt/elasticagent/enabled.sls +++ b/salt/elasticagent/enabled.sls @@ -33,7 +33,7 @@ so-elastic-agent: {% endif %} - binds: - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro - - /opt/so/log/elastic-agent:/usr/share/elastic-agent/logs + - /opt/so/log/elasticagent:/usr/share/elastic-agent/logs - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro - /nsm:/nsm:ro - /opt/so/log:/opt/so/log:ro From 0e047cffad7d39ed0d3cde192e110c60ffde7242 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 1 Aug 2023 20:14:53 +0000 Subject: [PATCH 022/350] Add to logrotate --- salt/logrotate/defaults.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 311a344b3..4d6a688e4 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -90,6 +90,26 @@ logrotate: - extension .log - dateext - dateyesterday + /opt/so/log/elasticagent/*_x_log: + - daily + - rotate 14 + - missingok + - copytruncate + - compress + - create + - extension .log + - dateext + - dateyesterday + /opt/so/log/elasticagent/*_x_ndjson: + - daily + - rotate 14 + - missingok + - copytruncate + - compress + - create + - extension .ndjson + - dateext + - dateyesterday /opt/so/log/elasticfleet/*_x_log: - daily - rotate 14 From 7037fc52f805623825f3bee9794bd2dab820ed3a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 1 Aug 2023 16:21:06 -0400 Subject: [PATCH 023/350] sync all modules before running states --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index ce0aa83f7..20a1168c9 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -661,6 +661,7 @@ if ! [[ -f $install_opt_file ]]; then logCmd "salt-call state.show_top" sleep 2 # Debug RSA Key format errors logCmd "salt-key -ya $MINION_ID" + logCmd "salt-call saltutil.sync_all" logCmd "salt-call state.apply common.packages" logCmd "salt-call state.apply common" From 8b3a38f5733aa1ca8920d8c5be33fa3b86c1d91c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 1 Aug 2023 16:30:24 -0400 Subject: [PATCH 024/350] resolve login page flicker --- salt/nginx/etc/nginx.conf | 4 +++- setup/so-verify | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 52e3d6d3d..05da0b5d8 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -296,7 +296,9 @@ http { error_page 429 = @error429; location @error401 { - add_header Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400"; + if ($request_uri ~* ^/(?!(^/api/.*))) { + add_header Set-Cookie "AUTH_REDIRECT=$request_uri;Path=/;Max-Age=14400"; + } return 302 /auth/self-service/login/browser; } diff --git a/setup/so-verify b/setup/so-verify index 918610732..07d24d114 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -51,6 +51,7 @@ log_has_errors() { grep -vE "/nsm/rules/sigma*" | \ grep -vE "/nsm/rules/yara*" | \ grep -vE "Failed to restart snapd" | \ + grep -vE "Login Failed Details" | \ grep -vE "Running scope as unit" &> "$error_log" if [[ $? -eq 0 ]]; then From 23414599eed535d95ec2a4ba8946b461a7c3644a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 1 Aug 2023 16:53:26 -0400 Subject: [PATCH 025/350] use simple json (w/o template) to resolve sluggishness --- salt/common/tools/sbin/so-status | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-status b/salt/common/tools/sbin/so-status index 4a12d71b4..f4abd8aa3 100755 --- a/salt/common/tools/sbin/so-status +++ b/salt/common/tools/sbin/so-status @@ -103,7 +103,7 @@ def output(options, console, code, data): def check_container_status(options, console): code = 0 cli = "docker" - proc = subprocess.run([cli, 'ps', '--format', '{{json .}}'], stdout=subprocess.PIPE, encoding="utf-8") + proc = subprocess.run([cli, 'ps', '--format', 'json'], stdout=subprocess.PIPE, encoding="utf-8") if proc.returncode != 0: fail("Container system error; unable to obtain container process statuses") From 0d5ed2e8359e30642bb6081e070f83e3f526d68a Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 2 Aug 2023 13:21:03 +0000 Subject: [PATCH 026/350] Set version for Elastic Defend and enable updates --- .../endpoints-initial/elastic-defend-endpoints.json | 4 ++-- .../tools/sbin/so-elastic-fleet-integration-policy-load | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json index 7d7f5bb35..6ffb6418e 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "" + "version": "8.8.0" }, "enabled": true, "policy_id": "endpoints-initial", @@ -25,4 +25,4 @@ } } }] -} \ No newline at end of file +} diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 49bfb69ac..501aafbda 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -15,10 +15,8 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n" elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION" if [ -n "$INTEGRATION_ID" ]; then - if [ "$NAME" != "elastic-defend-endpoints" ]; then - printf "\n\nIntegration $NAME exists - Updating integration\n" - elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" - fi + printf "\n\nIntegration $NAME exists - Updating integration\n" + elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION" else printf "\n\nIntegration does not exist - Creating integration\n" elastic_fleet_integration_create "@$INTEGRATION" From e6940190274bf438e6b1bf33b04cb933bb4675d8 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 2 Aug 2023 13:50:14 +0000 Subject: [PATCH 027/350] Add package list --- .../tools/sbin/so-elastic-fleet-package-list | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list b/salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list new file mode 100755 index 000000000..7e68c6e83 --- /dev/null +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-elastic-fleet-common + +# Let's snag a cookie from Kibana +SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}') + +# List configured package policies +curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages" -H 'kbn-xsrf: true' | jq + +echo From b520c1abb777a479df05e7e033edfa7b57b37d77 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 10:36:40 -0400 Subject: [PATCH 028/350] Allow multiple Custom Fleet FQDN --- salt/elasticfleet/defaults.yaml | 3 ++- salt/elasticfleet/soc_elasticfleet.yaml | 2 +- .../sbin_jinja/so-elastic-fleet-outputs-update | 14 +++++++++----- .../sbin_jinja/so-elastic-fleet-urls-update | 16 ++++++++++------ 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 3d806d63f..62a1302c1 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -2,7 +2,8 @@ elasticfleet: enabled: False config: server: - custom_fqdn: '' + custom_fqdn: + - '' enable_auto_configuration: True endpoints_enrollment: '' es_token: '' diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 9b918f0ac..772e68181 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -12,7 +12,7 @@ elasticfleet: config: server: custom_fqdn: - description: Custom FQDN for Agents to connect to. + description: Custom FQDN for Agents to connect to. One per line. global: True helpLink: elastic-fleet.html advanced: True diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 042084d84..400a6224f 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -2,7 +2,6 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} . /usr/sbin/so-common @@ -41,10 +40,15 @@ else NEW_LIST=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055") fi -{% if CUSTOMFQDN != "" %} -# Add Custom Hostname to list -NEW_LIST+=("{{ CUSTOMFQDN }}:5055") -{% endif %} +# Query for FQDN entries & add them to the list +CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +if [ -n "$CUSTOMFQDNLIST" ]; then + readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST + for CUSTOMNAME in "${CUSTOMFQDN[@]}" + do + NEW_LIST+=("$CUSTOMNAME:5055") + done +fi # Query for the current Grid Nodes that are running Logstash LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 4a744665a..52727780d 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -2,7 +2,6 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} . /usr/sbin/so-common @@ -41,10 +40,15 @@ else NEW_LIST=("https://{{ GLOBALS.url_base }}:8220" "https://{{ GLOBALS.hostname }}:8220") fi -{% if CUSTOMFQDN != "" %} -# Add Custom Hostname to list -NEW_LIST+=("https://{{ CUSTOMFQDN }}:8220") -{% endif %} +# Query for FQDN entries & add them to the list +CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +if [ -n "$CUSTOMFQDNLIST" ]; then + readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST + for CUSTOMNAME in "${CUSTOMFQDN[@]}" + do + NEW_LIST+=("https://$CUSTOMNAME:8220") + done +fi # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') @@ -71,5 +75,5 @@ else printf "\nHashes don't match - update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" update_fleet_urls - /sbin/so-elastic-agent-gen-installers & + /sbin/so-elastic-agent-gen-installers >> /opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log & fi From 407cb2a537f0c19e170e0905d495760fa5fe9ae6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 2 Aug 2023 10:56:41 -0400 Subject: [PATCH 029/350] force portgroups added to hostgroups in roles to be list of strings --- salt/firewall/soc_firewall.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index d1db56a0b..0011a245e 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -191,6 +191,7 @@ firewall: description: Portgroups to add access to the docker containers for this role. advanced: True multiline: True + forcedType: "[]string" helpLink: firewall.html sensor: portgroups: *portgroupsdocker @@ -241,6 +242,7 @@ firewall: description: Portgroups to add access to the host. advanced: True multiline: True + forcedType: "[]string" helpLink: firewall.html dockernet: portgroups: *portgroupshost From 5630b353c4106928f9a7e9debc2d636fd7471243 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 2 Aug 2023 11:20:51 -0400 Subject: [PATCH 030/350] change how pgrep finds salt-master PID --- salt/common/packages.sls | 2 ++ salt/manager/tools/sbin/soup | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 0fc067245..9cbfd08bb 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -17,6 +17,7 @@ commonpkgs: - netcat-openbsd - sqlite3 - libssl-dev + - procps - python3-dateutil - python3-docker - python3-packaging @@ -70,6 +71,7 @@ commonpkgs: - net-tools - nmap-ncat - openssl + - procps - python3-dnf-plugin-versionlock - python3-docker - python3-m2crypto diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 31f1d0fea..582e4502b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -436,7 +436,7 @@ stop_salt_master() { echo "" echo "Storing salt-master pid." - MASTERPID=$(pgrep salt-master | head -1) + MASTERPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-master MainProcess') echo "Found salt-master PID $MASTERPID" systemctl_func "stop" "salt-master" timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option." From 98731210003a80cac470db809f665081b963b00f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 2 Aug 2023 12:54:31 -0400 Subject: [PATCH 031/350] change pgrep for salt-minion PID --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 582e4502b..71f3f7a2a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -455,7 +455,7 @@ stop_salt_minion() { set -e echo "Storing salt-minion pid." - MINIONPID=$(pgrep salt-minion | head -1) + MINIONPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion' | head -1) echo "Found salt-minion PID $MINIONPID" systemctl_func "stop" "salt-minion" From f6c620455556a1edad4b0dbb398a976f591fa424 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 2 Aug 2023 13:05:24 -0400 Subject: [PATCH 032/350] procps to procps-ng --- salt/common/packages.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 9cbfd08bb..5f4a348e7 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -71,7 +71,7 @@ commonpkgs: - net-tools - nmap-ncat - openssl - - procps + - procps-ng - python3-dnf-plugin-versionlock - python3-docker - python3-m2crypto From ac28f90af3bd66a6f443711fa3be61c8ef4d9f92 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Aug 2023 13:15:11 -0400 Subject: [PATCH 033/350] Remove override --- salt/elasticsearch/files/ingest/filterlog | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/filterlog b/salt/elasticsearch/files/ingest/filterlog index fb197c706..850c15d99 100644 --- a/salt/elasticsearch/files/ingest/filterlog +++ b/salt/elasticsearch/files/ingest/filterlog @@ -49,7 +49,6 @@ "on_failure" : [ {"set" : {"field" : "error.message","value" : "{{ _ingest.on_failure_message }}"}}] } }, - { "set": { "field": "_index", "value": "so-firewall", "override": true } }, { "set": { "if": "ctx.network?.transport_id == '0'", "field": "network.transport", "value": "icmp", "override": true } }, { "community_id": {} }, { "set": { "field": "module", "value": "pfsense", "override": true } }, From f1023510524d5c46a5ebca8acf6cf2293faa6026 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 2 Aug 2023 13:25:44 -0400 Subject: [PATCH 034/350] Add event --- salt/elasticsearch/files/ingest/filterlog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/filterlog b/salt/elasticsearch/files/ingest/filterlog index 850c15d99..52d83dd0a 100644 --- a/salt/elasticsearch/files/ingest/filterlog +++ b/salt/elasticsearch/files/ingest/filterlog @@ -51,8 +51,8 @@ }, { "set": { "if": "ctx.network?.transport_id == '0'", "field": "network.transport", "value": "icmp", "override": true } }, { "community_id": {} }, - { "set": { "field": "module", "value": "pfsense", "override": true } }, - { "set": { "field": "dataset", "value": "firewall", "override": true } }, + { "set": { "field": "event.module", "value": "pfsense", "override": true } }, + { "set": { "field": "event.dataset", "value": "firewall", "override": true } }, { "set": { "field": "category", "value": "network", "override": true } }, { "remove": { "field": ["real_message", "ip_sub_msg", "firewall.sub_message"], "ignore_failure": true } } ] From c17b324108a1ba353b92f6a5cd89d17c2ca18654 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 2 Aug 2023 14:04:19 -0400 Subject: [PATCH 035/350] dont count adv_ sls files for number of minions in deployment --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 71f3f7a2a..0a1c9237d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -859,7 +859,7 @@ main() { set +e echo "Checking the number of minions." - NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l) + NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | grep -v adv_ | wc -l) if [[ $UPGRADESALT -eq 1 ]] && [[ $NUM_MINIONS -gt 1 ]]; then if [[ $is_airgap -eq 0 ]]; then echo "" From 64776936cc4e50d21e623f874e0ba599adc12b78 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 2 Aug 2023 14:09:43 -0400 Subject: [PATCH 036/350] no longer need so-user migrate in 2.4 --- salt/manager/tools/sbin/soup | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0a1c9237d..1b0fb1478 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -875,9 +875,6 @@ main() { echo "Checking sudoers file." check_sudoers - echo "Checking for necessary user migrations." - so-user migrate - systemctl_func "start" "$cron_service_name" if [[ -n $lsl_msg ]]; then From aab55c8cf6d76b3a81e68db7e6f85d864c957ce8 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 15:09:26 -0400 Subject: [PATCH 037/350] Regen Agent Installers --- salt/manager/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 1b0fb1478..85f5b45f4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -419,7 +419,8 @@ post_to_2.4.4() { } post_to_2.4.5() { - echo "Nothing to apply" + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers POSTVERSION=2.4.5 } From 8036df4b203d2998f26201a69acdb9c786ba165f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Aug 2023 15:10:31 -0400 Subject: [PATCH 038/350] ensure suri rules are synced for import installs --- setup/so-setup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 20a1168c9..ccc9f6f2f 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -695,9 +695,11 @@ if ! [[ -f $install_opt_file ]]; then logCmd "so-rule-update" title "Downloading YARA rules" logCmd "su socore -c '/usr/sbin/so-yara-download'" - if [[ $monints ]]; then + if [[ $monints || $is_import ]]; then title "Restarting Suricata to pick up the new rules" logCmd "so-suricata-restart" + fi + if [[ $monints ]]; then title "Restarting Strelka to use new rules" logCmd "so-strelka-restart" fi From f153c1125d9dba74b5358c298936fbd0b873c2f8 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 15:23:18 -0400 Subject: [PATCH 039/350] Allow multiple Custom Fleet FQDN --- salt/elasticfleet/defaults.yaml | 3 +-- salt/elasticfleet/enabled.sls | 13 ++++++++++--- salt/elasticfleet/soc_elasticfleet.yaml | 1 + 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 62a1302c1..0ae7a5176 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -2,8 +2,7 @@ elasticfleet: enabled: False config: server: - custom_fqdn: - - '' + custom_fqdn: [] enable_auto_configuration: True endpoints_enrollment: '' es_token: '' diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 025a87e14..bb6410f2c 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -15,6 +15,7 @@ include: - elasticfleet.config - elasticfleet.sostatus + - ssl # If enabled, automatically update Fleet Logstash Outputs {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %} @@ -61,11 +62,14 @@ so-elastic-fleet: - {{ BINDING }} {% endfor %} - binds: - - /etc/pki:/etc/pki:ro + - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro + - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro + - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro {% if GLOBALS.os_family == 'Debian' %} - - /etc/ssl:/etc/ssl:ro + - /etc/ssl/elasticfleet-server.crt:/etc/ssl/elasticfleet-server.crt:ro + - /etc/ssl/elasticfleet-server.key:/etc/ssl/elasticfleet-server.key:ro + - /etc/ssl/tls/certs/intca.crt:/etc/ssl/tls/certs/intca.crt:ro {% endif %} - #- /opt/so/conf/elastic-fleet/state:/usr/share/elastic-agent/state:rw - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs {% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %} @@ -93,6 +97,9 @@ so-elastic-fleet: - {{ XTRAENV }} {% endfor %} {% endif %} + - watch: + - x509: etc_elasticfleet_key + - x509: etc_elasticfleet_crt {% endif %} {% if GLOBALS.role != "so-fleet" %} diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index 772e68181..af660358a 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -16,6 +16,7 @@ elasticfleet: global: True helpLink: elastic-fleet.html advanced: True + forcedType: "[]string" enable_auto_configuration: description: Enable auto-configuration of Logstash Outputs & Fleet Host URLs. global: True From eb512d9aa27c1f8f7db7ede491bdc743d899bf88 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 2 Aug 2023 16:21:23 -0400 Subject: [PATCH 040/350] add mono-devel --- salt/desktop/packages.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 401be0cd6..30d2f96e5 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -295,6 +295,7 @@ desktop_packages: - mesa-vulkan-drivers - microcode_ctl - mobile-broadband-provider-info + - mono-devel - mpfr - mpg123-libs - mtdev From 435da77388d2d268166811194652342c915dff24 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 2 Aug 2023 16:53:45 -0400 Subject: [PATCH 041/350] add gtk2 --- salt/desktop/packages.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 30d2f96e5..3b0d4c8ba 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -181,6 +181,7 @@ desktop_packages: - gstreamer1-plugins-good-gtk - gstreamer1-plugins-ugly-free - gtk-update-icon-cache + - gtk2 - gtk3 - gtk4 - gtkmm30 From ab28cee7cf3041ac6276b120956e8d117a1323b4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 17:45:37 -0400 Subject: [PATCH 042/350] Allow multiple Custom Fleet FQDN --- salt/ssl/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 97e971b83..15c29791f 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -7,7 +7,7 @@ {% if sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set CUSTOMFQDN = salt['pillar.get']('elasticfleet:config:server:custom_fqdn') %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} {% set global_ca_text = [] %} {% set global_ca_server = [] %} @@ -154,7 +154,7 @@ etc_elasticfleet_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-server.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -211,7 +211,7 @@ etc_elasticfleet_logstash_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }} {% if CUSTOMFQDN != "" %},DNS:{{ CUSTOMFQDN }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 1c8a8c460c90572cbeea725a88a60dc358c5b5f9 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 17:53:29 -0400 Subject: [PATCH 043/350] Restart logstash when certs change --- salt/logstash/enabled.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index cd9d6dd7e..a33080f8d 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -22,6 +22,7 @@ include: {% endif %} - logstash.config - logstash.sostatus + - ssl so-logstash: docker_container.running: @@ -90,6 +91,8 @@ so-logstash: {% endfor %} {% endif %} - watch: + - x509: etc_elasticfleet_logstash_key + - x509: etc_elasticfleet_logstash_crt - file: lsetcsync {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} - file: ls_pipeline_{{assigned_pipeline}} From 3054b8dcb9fa452ca25e6cd936999f3ff4e41727 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Aug 2023 18:57:46 -0400 Subject: [PATCH 044/350] refactor elastic-agent download for soup ctrl+c anomalies --- salt/common/tools/sbin/so-common | 64 +++++++++++++++++++++++++++++++- salt/manager/tools/sbin/soup | 18 ++++----- setup/so-functions | 20 +--------- 3 files changed, 74 insertions(+), 28 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 3c79110b3..702c73c8c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -5,7 +5,16 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +# Elastic agent is not managed by salt. Because of this we must store this base information in a +# script that accompanies the soup system. Since so-common is one of those special soup files, +# and since this same logic is required during installation, it's included in this file. ELASTIC_AGENT_TARBALL_VERSION="8.8.2" +ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" +ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" +ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" +ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" +ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent + DEFAULT_SALT_DIR=/opt/so/saltstack/default DOC_BASE_URL="https://docs.securityonion.net/en/2.4" @@ -161,6 +170,37 @@ disable_fastestmirror() { sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf } +download_and_verify() { + source_url=$1 + source_md5_url=$2 + dest_file=$3 + md5_file=$4 + expand_dir=$5 + + if [[ -n "$expand_dir" ]]; then + mkdir -p "$expand_dir" + fi + + if ! verify_md5_checksum "$dest_file" "$md5_file"; then + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_url' --output '$dest_file'" "" "" + retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_md5_url' --output '$md5_file'" "" "" + + SOURCEHASH=$(md5sum "$dest_file" | awk '{ print $1 }') + HASH=$(cat "$md5_file") + + if verify_md5_checksum "$dest_file" "$md5_file"; then + echo "Source file and checksum are good." + else + echo "Unable to download and verify the source file and checksum." + return 1 + fi + fi + + if [[ -n "$expand_dir" ]]; then + tar -xf "$dest_file" -C "$expand_dir" + fi +} + elastic_license() { read -r -d '' message <<- EOM @@ -211,7 +251,7 @@ gpg_rpm_import() { echo "Imported $RPMKEY" done elif [[ $is_rpm ]]; then - info "Importing the security onion GPG key" + echo "Importing the security onion GPG key" rpm --import ../salt/repo/client/files/oracle/keys/securityonion.pub fi } @@ -470,6 +510,11 @@ has_uppercase() { || return 1 } +update_elastic_agent() { + echo "Checking if Elastic Agent update is necessary..." + download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR" +} + valid_cidr() { # Verify there is a backslash in the string echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1 @@ -623,6 +668,23 @@ valid_username() { echo "$user" | grep -qP '^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$' && return 0 || return 1 } +verify_md5_checksum() { + data_file=$1 + md5_file=${2:-${data_file}.md5} + + if [[ ! -f "$dest_file" || ! -f "$md5_file" ]]; then + return 2 + fi + + SOURCEHASH=$(md5sum "$data_file" | awk '{ print $1 }') + HASH=$(cat "$md5_file") + + if [[ "$HASH" == "$SOURCEHASH" ]]; then + return 0 + fi + return 1 +} + wait_for_web_response() { url=$1 expected=$2 diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 85f5b45f4..bd41bdcf2 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -467,21 +467,21 @@ stop_salt_minion() { up_to_2.4.3() { - echo "Nothing to do for 2.4.3" - ## - INSTALLEDVERSION=2.4.3 + echo "Nothing to do for 2.4.3" + + INSTALLEDVERSION=2.4.3 } up_to_2.4.4() { - echo "Nothing to do for 2.4.4" - ## - INSTALLEDVERSION=2.4.4 + echo "Nothing to do for 2.4.4" + + INSTALLEDVERSION=2.4.4 } up_to_2.4.5() { - echo "Nothing to do for 2.4.5" - ## - INSTALLEDVERSION=2.4.5 + update_elastic_agent + + INSTALLEDVERSION=2.4.5 } verify_upgradespace() { diff --git a/setup/so-functions b/setup/so-functions index d46c42e0e..c8da13043 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1014,25 +1014,9 @@ detect_os() { } download_elastic_agent_artifacts() { - agentArchive=/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz - agentMd5=/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5 - beatsDir=/nsm/elastic-fleet/artifacts/beats/elastic-agent - logCmd "mkdir -p $beatsDir" - if [[ ! -f "$agentArchive" ]]; then - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz --output $agentArchive" "" "" - retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5 --output $agentMd5" "" "" - - SOURCEHASH=$(md5sum $agentArchive | awk '{ print $1 }') - HASH=$(cat $agentMd5) - - if [[ "$HASH" == "$SOURCEHASH" ]]; then - info "Elastic Agent source hash is good." - else - info "Unable to download the Elastic Agent source files." - fail_setup - fi + if ! update_elastic_agent 2>&1 | tee -a "$setup_log"; then + fail_setup fi - logCmd "tar -xf $agentArchive -C $beatsDir" } installer_progress_loop() { From 5414b0756c71a23cb61c4d639a258f3749d5a030 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 2 Aug 2023 19:25:07 -0400 Subject: [PATCH 045/350] remove unused vars --- salt/common/tools/sbin/so-common | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 702c73c8c..a76aab1f1 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -184,9 +184,6 @@ download_and_verify() { if ! verify_md5_checksum "$dest_file" "$md5_file"; then retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_url' --output '$dest_file'" "" "" retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_md5_url' --output '$md5_file'" "" "" - - SOURCEHASH=$(md5sum "$dest_file" | awk '{ print $1 }') - HASH=$(cat "$md5_file") if verify_md5_checksum "$dest_file" "$md5_file"; then echo "Source file and checksum are good." From 1bc7bbc76efe8dcead09867a34358f64ea71e2c5 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 2 Aug 2023 20:02:37 -0400 Subject: [PATCH 046/350] Refactor custom_fqdn --- salt/ssl/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 15c29791f..063172e00 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -154,7 +154,7 @@ etc_elasticfleet_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-server.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -211,7 +211,7 @@ etc_elasticfleet_logstash_crt: - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn[0] != "" %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 3368789b43f78c6fa3616151d94de4ee99c46a66 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 08:49:45 -0400 Subject: [PATCH 047/350] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 59aa62c1f..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.5 +2.4.0-foxtrot From 3847863b3d471e69e591e5d1c9c9d26fce569f51 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 08:51:23 -0400 Subject: [PATCH 048/350] Add time shift --- salt/common/tools/sbin_jinja/so-import-evtx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index fec7223b8..5920f58c1 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -44,6 +44,10 @@ while [[ $# -gt 0 ]]; do --quiet) quiet=1 ;; + --shift) + SHIFTDATE=$1 + shift + ;; -*) echo "Encountered unexpected parameter: $param" usage @@ -68,8 +72,10 @@ function status { function evtx2es() { EVTX=$1 HASH=$2 + SHIFTDATE=$3 docker run --rm \ + -e "SHIFTTS=$SHIFTDATE" \ -v "$EVTX:/tmp/data.evtx" \ -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \ -v "/nsm/import/evtx-end_newest:/tmp/newest" \ @@ -113,7 +119,9 @@ echo $END_NEWEST > /nsm/import/evtx-end_newest for EVTX in $INPUT_FILES; do EVTX=$(/usr/bin/realpath "$EVTX") status "Processing Import: ${EVTX}" - + if ! [ -z "$SHIFTDATE" ]; then + status "- timeshifting logs to end date of $SHIFTDATE" + fi # generate a unique hash to assist with dedupe checks HASH=$(md5sum "${EVTX}" | awk '{ print $1 }') HASH_DIR=/nsm/import/${HASH} @@ -136,7 +144,7 @@ for EVTX in $INPUT_FILES; do # import evtx and write them to import ingest pipeline status "- importing logs to Elasticsearch..." - evtx2es "${EVTX}" $HASH + evtx2es "${EVTX}" $HASH "$SHIFTDATE" if [[ $? -ne 0 ]]; then INVALID_EVTXS_COUNT=$((INVALID_EVTXS_COUNT + 1)) status "- WARNING: This evtx file may not have fully imported successfully" @@ -222,4 +230,4 @@ if [[ $json -eq 1 ]]; then }''' fi -exit $RESULT \ No newline at end of file +exit $RESULT From cf2233bbb6702c4e4da396ade6449373493a933b Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 08:54:54 -0400 Subject: [PATCH 049/350] Add help information for time shift --- salt/common/tools/sbin_jinja/so-import-evtx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index 5920f58c1..dff2133cf 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -27,6 +27,8 @@ Imports one or more evtx files into Security Onion. The evtx files will be analy Options: --json Outputs summary in JSON format. Implies --quiet. --quiet Silences progress information to stdout. + --shift Adds a time shift. Accepts a single argument that is intended to be the date of the last record, and shifts the dates of the previous records accordingly. + Ex. sudo so-import-evtx --shift 2023-08-01T01:01:01.00000Z example.evtx EOF } From d4389d5057dbef48a4965296b38fbc15978baa85 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 3 Aug 2023 11:56:48 -0400 Subject: [PATCH 050/350] ensure AIRGAP is lowercase and check for true --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 31f1d0fea..20517f58d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -179,8 +179,8 @@ update_registry() { check_airgap() { # See if this is an airgap install - AIRGAP=$(cat /opt/so/saltstack/local/pillar/global/soc_global.sls | grep airgap: | awk '{print $2}') - if [[ "$AIRGAP" == "True" ]]; then + AIRGAP=$(cat /opt/so/saltstack/local/pillar/global/soc_global.sls | grep airgap: | awk '{print $2}' | tr '[:upper:]' '[:lower:]') + if [[ "$AIRGAP" == "true" ]]; then is_airgap=0 UPDATE_DIR=/tmp/soagupdate/SecurityOnion AGDOCKER=/tmp/soagupdate/docker From 80598d7f8d26530f4ce55ea097f6fab526c94131 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 3 Aug 2023 14:36:47 -0400 Subject: [PATCH 051/350] Update soup for airgap --- salt/manager/tools/sbin/soup | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f47c1d5e2..0cea456f4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -184,7 +184,7 @@ check_airgap() { is_airgap=0 UPDATE_DIR=/tmp/soagupdate/SecurityOnion AGDOCKER=/tmp/soagupdate/docker - AGREPO=/tmp/soagupdate/Packages + AGREPO=/tmp/soagupdate/minimal/Packages else is_airgap=1 fi @@ -402,9 +402,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.2 ]] && post_to_2.4.3 [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4 - [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 - - + [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 true } @@ -479,11 +477,22 @@ up_to_2.4.4() { } up_to_2.4.5() { - update_elastic_agent + determine_elastic_agent_upgrade INSTALLEDVERSION=2.4.5 } +determine_elastic_agent_upgrade() { + if [[ $is_airgap -eq 0 ]]; then + update_elastic_agent_airgap + else + update_elastic_agent +} + +update_elastic_agent_airgap() { + rsync -av /tmp/soagupdate/fleet/* /nsm/elastic-fleet/artifacts/ +} + verify_upgradespace() { CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//') if [ "$CURRENTSPACE" -lt "10" ]; then @@ -521,6 +530,7 @@ update_centos_repo() { echo "Syncing new updates to /nsm/repo" rsync -av $AGREPO/* /nsm/repo/ echo "Creating repo" + dnf -y install yum-utils createrepo createrepo /nsm/repo } From 9172e10dbabfb7d2217f054da5c89a6a0a4ba541 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 3 Aug 2023 14:47:53 -0400 Subject: [PATCH 052/350] check if there are files in yum.repos.d before trying to move them --- setup/so-functions | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index c8da13043..567584a2f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1881,7 +1881,9 @@ securityonion_repo() { if [[ $is_oracle ]]; then logCmd "dnf -v clean all" logCmd "mkdir -vp /root/oldrepos" - logCmd "mv -v /etc/yum.repos.d/* /root/oldrepos/" + if [ -n "$(ls -A /etc/yum.repos.d/ 2>/dev/null)" ]; then + logCmd "mv -v /etc/yum.repos.d/* /root/oldrepos/" + fi if [[ $is_desktop_iso ]]; then gpg_rpm_import if [[ ! $is_airgap ]]; then From d40a8927c3c184aa716b37452726175a4d86dedf Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 3 Aug 2023 14:51:43 -0400 Subject: [PATCH 053/350] install salt version specified in master.defaults.yaml for desktop --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 567584a2f..0f73a11a6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -85,12 +85,13 @@ analyze_system() { desktop_salt_local() { + SALTVERSION=$(egrep 'version: [0-9]{4}' ../salt/salt/master.defaults.yaml | sed 's/^.*version: //') # Install everything using local salt # Set the repo securityonion_repo gpg_rpm_import # Install salt - logCmd "yum -y install salt-minion-3004.1 httpd-tools python3 python36-docker python36-dateutil python36-m2crypto python36-mysql python36-packaging python36-lxml yum-utils device-mapper-persistent-data lvm2 openssl jq" + logCmd "yum -y install salt-minion-$SALTVERSION httpd-tools python3 python36-docker python36-dateutil python36-m2crypto python36-mysql python36-packaging python36-lxml yum-utils device-mapper-persistent-data lvm2 openssl jq" logCmd "yum -y update --exclude=salt*" logCmd "salt-call state.apply desktop --local --file-root=../salt/ -l info" From 27b70cbf6891d021981d5c798332f602f8612b25 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 15:21:20 -0400 Subject: [PATCH 054/350] Use jinja instead --- .../tools/sbin_jinja/so-elastic-fleet-urls-update | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 52727780d..c484fa704 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -2,6 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} . /usr/sbin/so-common @@ -41,7 +42,8 @@ else fi # Query for FQDN entries & add them to the list -CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} +CUSTOMFQDNLIST=({{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}) if [ -n "$CUSTOMFQDNLIST" ]; then readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST for CUSTOMNAME in "${CUSTOMFQDN[@]}" @@ -49,6 +51,7 @@ if [ -n "$CUSTOMFQDNLIST" ]; then NEW_LIST+=("https://$CUSTOMNAME:8220") done fi +{% endif %} # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') From e78fcbc6cbc2fa4362e45e378eebfcc80a0d8fc9 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 15:25:11 -0400 Subject: [PATCH 055/350] Refactor for Jinja instead --- .../tools/sbin_jinja/so-elastic-fleet-outputs-update | 7 +++++-- .../tools/sbin_jinja/so-elastic-fleet-urls-update | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 400a6224f..17c867c07 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -2,6 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} . /usr/sbin/so-common @@ -41,14 +42,16 @@ else fi # Query for FQDN entries & add them to the list -CUSTOMFQDNLIST=$( salt-call --out=json pillar.get elasticfleet:config:server:custom_fqdn | jq -r '.local | .[]') +{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} +CUSTOMFQDNLIST=({{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}) if [ -n "$CUSTOMFQDNLIST" ]; then readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST for CUSTOMNAME in "${CUSTOMFQDN[@]}" do - NEW_LIST+=("$CUSTOMNAME:5055") + NEW_LIST+=("https://$CUSTOMNAME:8220") done fi +{% endif %} # Query for the current Grid Nodes that are running Logstash LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local') diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index c484fa704..7d29fe080 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -2,7 +2,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} . /usr/sbin/so-common From d4fbf7d6a694288aa6bb9168dd2245eee7011d17 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 3 Aug 2023 15:26:43 -0400 Subject: [PATCH 056/350] convert to gnome classic --- salt/desktop/scripts/convert-gnome-classic.sh | 0 salt/desktop/xwindows.sls | 4 ++++ 2 files changed, 4 insertions(+) create mode 100644 salt/desktop/scripts/convert-gnome-classic.sh diff --git a/salt/desktop/scripts/convert-gnome-classic.sh b/salt/desktop/scripts/convert-gnome-classic.sh new file mode 100644 index 000000000..e69de29bb diff --git a/salt/desktop/xwindows.sls b/salt/desktop/xwindows.sls index ea0c7df4f..ebb7ecb9f 100644 --- a/salt/desktop/xwindows.sls +++ b/salt/desktop/xwindows.sls @@ -14,6 +14,10 @@ graphical_target: - require: - desktop_packages +convert_gnome_classic: + cmd.script: + - name: salt://desktop/scripts/convert-gnome-classic.sh + {% else %} desktop_xwindows_os_fail: From 9319c3f2e1c7757abcd2fc68b6d1a2e7713a0b39 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 3 Aug 2023 15:27:24 -0400 Subject: [PATCH 057/350] Update soup for airgap --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0cea456f4..cede5c438 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -487,6 +487,7 @@ determine_elastic_agent_upgrade() { update_elastic_agent_airgap else update_elastic_agent + fi } update_elastic_agent_airgap() { From 15b8e1a753902b906e9a73f274bd8fb46760e120 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 3 Aug 2023 15:37:26 -0400 Subject: [PATCH 058/350] add convert-gnome-classic.sh --- salt/desktop/scripts/convert-gnome-classic.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/desktop/scripts/convert-gnome-classic.sh b/salt/desktop/scripts/convert-gnome-classic.sh index e69de29bb..e69a43b2d 100644 --- a/salt/desktop/scripts/convert-gnome-classic.sh +++ b/salt/desktop/scripts/convert-gnome-classic.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "Setting default session to gnome-classic" +cp /usr/share/accountsservice/user-templates/standard /etc/accountsservice/user-templates/ +sed -i 's|Session=gnome|Session=gnome-classic|g' /etc/accountsservice/user-templates/standard From 3e4136e641c27dbd0f2a08cd870550c640d65a3f Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 15:56:05 -0400 Subject: [PATCH 059/350] Update help text --- salt/common/tools/sbin_jinja/so-import-evtx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index dff2133cf..59a13612c 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -28,7 +28,7 @@ Options: --json Outputs summary in JSON format. Implies --quiet. --quiet Silences progress information to stdout. --shift Adds a time shift. Accepts a single argument that is intended to be the date of the last record, and shifts the dates of the previous records accordingly. - Ex. sudo so-import-evtx --shift 2023-08-01T01:01:01.00000Z example.evtx + Ex. sudo so-import-evtx --shift "2023-08-01 01:01:01" example.evtx EOF } From ca6276b922c86b0df31a25670b6872a6b6523b5e Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 3 Aug 2023 15:58:33 -0400 Subject: [PATCH 060/350] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f..59aa62c1f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.5 From 6b5343f582d5651a1b7ad1bcb403f106796af630 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 16:25:02 -0400 Subject: [PATCH 061/350] Update for 8.8.2 --- .../endpoints-initial/elastic-defend-endpoints.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json index 6ffb6418e..8ab4f748e 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/elastic-defend-endpoints.json @@ -9,13 +9,12 @@ }, "enabled": true, "policy_id": "endpoints-initial", - "vars": {}, "inputs": [{ - "type": "endpoint", + "type": "ENDPOINT_INTEGRATION_CONFIG", "enabled": true, "streams": [], "config": { - "integration_config": { + "_config": { "value": { "type": "endpoint", "endpointConfig": { From 2caca92082ef54c71ad617b469ca699bdbcf6418 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 3 Aug 2023 17:11:43 -0400 Subject: [PATCH 062/350] Raid refactor + yara and rule proxy --- salt/common/tools/sbin_jinja/so-raid-status | 116 ++++++++---------- salt/idstools/tools/sbin_jinja/so-rule-update | 16 ++- .../manager/tools/sbin_jinja/so-yara-download | 3 +- 3 files changed, 62 insertions(+), 73 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-raid-status b/salt/common/tools/sbin_jinja/so-raid-status index c5ac5fac6..0249f4ccd 100755 --- a/salt/common/tools/sbin_jinja/so-raid-status +++ b/salt/common/tools/sbin_jinja/so-raid-status @@ -1,7 +1,7 @@ #!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -9,25 +9,26 @@ . /usr/sbin/so-common -appliance_check() { - {%- if salt['grains.get']('sosmodel', '') %} - APPLIANCE=1 - {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %} - exit 0 - {%- endif %} - DUDEYOUGOTADELL=$(dmidecode |grep Dell) - if [[ -n $DUDEYOUGOTADELL ]]; then - APPTYPE=dell - else - APPTYPE=sm - fi - mkdir -p /opt/so/log/raid - - {%- else %} - echo "This is not an appliance" - exit 0 - {%- endif %} -} +{%- if salt['grains.get']('sosmodel', '') %} +{%- set model = salt['grains.get']('sosmodel') %} +model={{ model }} +# Don't need cloud images to use this +if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then + exit 0 +fi +{%- else %} +echo "This is not an appliance" +exit 0 +{%- endif %} +if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then + is_bossraid=true +fi +if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then + is_swraid=true +fi +if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then + is_hwraid=true +fi check_nsm_raid() { PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl) @@ -49,61 +50,44 @@ check_nsm_raid() { check_boss_raid() { MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional) - if [[ -n $DUDEYOUGOTADELL ]]; then - if [[ -n $MVCLI ]]; then - BOSSRAID=0 - else - BOSSRAID=1 - fi + if [[ -n $MVCLI ]]; then + BOSSRAID=0 + else + BOSSRAID=1 fi } check_software_raid() { - if [[ -n $DUDEYOUGOTADELL ]]; then - SWRC=$(grep "_" /proc/mdstat) - - if [[ -n $SWRC ]]; then - # RAID is failed in some way - SWRAID=1 - else - SWRAID=0 - fi + SWRC=$(grep "_" /proc/mdstat) + if [[ -n $SWRC ]]; then + # RAID is failed in some way + SWRAID=1 + else + SWRAID=0 fi } -# This script checks raid status if you use SO appliances +# Set everything to 0 +SWRAID=0 +BOSSRAID=0 +HWRAID=0 -# See if this is an appliance - -appliance_check -check_nsm_raid -check_boss_raid -{%- if salt['grains.get']('sosmodel', '') %} -{%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %} -check_software_raid -{%- endif %} -{%- endif %} - -if [[ -n $SWRAID ]]; then - if [[ $SWRAID == '0' && $BOSSRAID == '0' ]]; then - RAIDSTATUS=0 - else - RAIDSTATUS=1 - fi -elif [[ -n $DUDEYOUGOTADELL ]]; then - if [[ $BOSSRAID == '0' && $HWRAID == '0' ]]; then - RAIDSTATUS=0 - else - RAIDSTATUS=1 - fi -elif [[ "$APPTYPE" == 'sm' ]]; then - if [[ -n "$HWRAID" ]]; then - RAIDSTATUS=0 - else - RAIDSTATUS=1 - fi +if [[ $is_hwraid ]]; then + check_nsm_raid +fi +if [[ $is_bossraid ]]; then + check_boss_raid +fi +if [[ $is_swraid ]]; then + check_software_raid fi -echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log +sum=$(($SWRAID + $BOSSRAID + $HWRAID)) +if [[ $sum == "0" ]]; then + RAIDSTATUS=0 +else + RAIDSTATUS=1 +fi +echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log \ No newline at end of file diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update index 3e4b382e6..504831f9f 100755 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ b/salt/idstools/tools/sbin_jinja/so-rule-update @@ -3,17 +3,21 @@ {%- from 'vars/globals.map.jinja' import GLOBALS %} {%- from 'idstools/map.jinja' import IDSTOOLSMERGED %} -{%- set proxy = salt['pillar.get']('manager:proxy') %} + +{%- set proxy = salt['pillar.get']('manager:proxy') %} +{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} + +# Download the rules from the internet +{%- if proxy %} +export http_proxy={{ proxy }} +export https_proxy={{ proxy }} +export no_proxy="{{ noproxy }}" +{%- endif %} mkdir -p /nsm/rules/suricata chown -R socore:socore /nsm/rules/suricata # Download the rules from the internet {%- if GLOBALS.airgap != 'True' %} -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy=salt['pillar.get']('manager:no_proxy') -{%- endif %} {%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force {%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} diff --git a/salt/manager/tools/sbin_jinja/so-yara-download b/salt/manager/tools/sbin_jinja/so-yara-download index e9b991b6c..aa9576253 100644 --- a/salt/manager/tools/sbin_jinja/so-yara-download +++ b/salt/manager/tools/sbin_jinja/so-yara-download @@ -3,12 +3,13 @@ NOROOT=1 . /usr/sbin/so-common {%- set proxy = salt['pillar.get']('manager:proxy') %} +{%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} # Download the rules from the internet {%- if proxy %} export http_proxy={{ proxy }} export https_proxy={{ proxy }} -export no_proxy=salt['pillar.get']('manager:no_proxy') +export no_proxy="{{ noproxy }}" {%- endif %} repos="/opt/so/conf/strelka/repos.txt" From 2472d6a7279e025e6714925dac83bb7c9f9eca42 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 3 Aug 2023 18:52:29 -0400 Subject: [PATCH 063/350] Don't watch certs on search nodes --- salt/logstash/enabled.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index a33080f8d..731ad4ca3 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -91,8 +91,10 @@ so-logstash: {% endfor %} {% endif %} - watch: + {% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-import', 'so-fleet', 'so-receiver'] %} - x509: etc_elasticfleet_logstash_key - x509: etc_elasticfleet_logstash_crt + {% endif %} - file: lsetcsync {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} - file: ls_pipeline_{{assigned_pipeline}} From 593cdbd06001f3492423c1c590b4e5ac3d5ae92d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 08:50:06 -0400 Subject: [PATCH 064/350] add rules for idh to connect to managers, change idh from sensor to idh in so-firewall-minion --- salt/firewall/defaults.yaml | 33 ++++++++++++++++++++++ salt/manager/tools/sbin/so-firewall-minion | 4 +-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 3095c052e..48074b0be 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -383,6 +383,17 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + idh: + portgroups: + - docker_registry + - influxdb + - sensoroni + - yum + - beats_5044 + - beats_5644 + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update sensor: portgroups: - beats_5044 @@ -548,6 +559,17 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + idh: + portgroups: + - docker_registry + - influxdb + - sensoroni + - yum + - beats_5044 + - beats_5644 + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update sensor: portgroups: - beats_5044 @@ -723,6 +745,17 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + idh: + portgroups: + - docker_registry + - influxdb + - sensoroni + - yum + - beats_5044 + - beats_5644 + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update sensor: portgroups: - docker_registry diff --git a/salt/manager/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion index 4834f0e41..7b0ddab90 100755 --- a/salt/manager/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -74,9 +74,9 @@ fi so-firewall includehost heavynode "$IP" --apply ;; 'IDH') - so-firewall includehost sensor "$IP" --apply + so-firewall includehost idh "$IP" --apply ;; 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; - esac \ No newline at end of file + esac From 682289ef23736b687cd271a503c58200143c4c9f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:01:09 -0400 Subject: [PATCH 065/350] add sensoroni ports where missing --- salt/firewall/defaults.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 48074b0be..125bf0f08 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -370,6 +370,7 @@ firewall: - elastic_agent_data - elastic_agent_update - localrules + - sensoroni fleet: portgroups: - elasticsearch_rest @@ -404,6 +405,7 @@ firewall: - yum - docker_registry - influxdb + - sensoroni searchnode: portgroups: - redis @@ -416,6 +418,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni heavynode: portgroups: - redis @@ -428,6 +431,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni receiver: portgroups: - yum @@ -436,6 +440,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni beats_endpoint: portgroups: - beats_5044 @@ -546,6 +551,7 @@ firewall: - elastic_agent_data - elastic_agent_update - localrules + - sensoroni fleet: portgroups: - elasticsearch_rest @@ -580,6 +586,7 @@ firewall: - yum - docker_registry - influxdb + - sensoroni searchnode: portgroups: - redis @@ -591,6 +598,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni heavynode: portgroups: - redis @@ -602,6 +610,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni receiver: portgroups: - yum @@ -610,6 +619,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni beats_endpoint: portgroups: - beats_5044 @@ -793,6 +803,7 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + - sensoroni beats_endpoint: portgroups: - beats_5044 From dd1fa51eb5bb2dc916401af541cc961c531497f2 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 4 Aug 2023 09:03:17 -0400 Subject: [PATCH 066/350] Generate community_id for defend endpoint logs --- salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 index 0c317ae48..45583a464 100644 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 @@ -78,6 +78,7 @@ { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, + {"community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } ], "on_failure": [ From 78950ebfbb39d21ec3917b9bb3819c420b0935cc Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 4 Aug 2023 09:16:58 -0400 Subject: [PATCH 067/350] Update so-whiptail --- setup/so-whiptail | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 4e9ccea60..c55e2db8f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -1012,9 +1012,9 @@ whiptail_manager_unreachable() { local msg read -r -d '' msg <<- EOM - Setup is unable to access the manager at this time. + Setup is unable to access the manager. This most likely means that you need to allow this machine to connect through the manager's firewall. - Run the following on the manager: + You can either go to SOC --> Administration --> Configuration and choose the correct firewall option from the list OR you can run the following command on the manager: sudo so-firewall-minion --role=$install_type --ip=$MAINIP From a51acfc314004e9c2f066fe387a85f34a92ab7da Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:17:22 -0400 Subject: [PATCH 068/350] rename analyst to workstation for fw rules. allow workstation to connect to salt_manager port on managers --- salt/firewall/defaults.yaml | 31 ++++++++++++++-------- salt/firewall/soc_firewall.yaml | 2 +- salt/manager/tools/sbin/so-firewall-minion | 3 +++ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 125bf0f08..0d32d57ca 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -1,6 +1,5 @@ firewall: hostgroups: - analyst: [] anywhere: - 0.0.0.0/0 beats_endpoint: [] @@ -26,6 +25,7 @@ firewall: standalone: [] strelka_frontend: [] syslog: [] + workstation: [] customhostgroup0: [] customhostgroup1: [] customhostgroup2: [] @@ -215,9 +215,9 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: @@ -458,9 +458,9 @@ firewall: endgame: portgroups: - endgame - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: @@ -507,6 +507,9 @@ firewall: receiver: portgroups: - salt_manager + workstation: + portgroups: + - salt_manager self: portgroups: - syslog @@ -637,9 +640,9 @@ firewall: endgame: portgroups: - endgame - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: @@ -686,6 +689,9 @@ firewall: receiver: portgroups: - salt_manager + workstation: + portgroups: + - salt_manager self: portgroups: - syslog @@ -824,9 +830,9 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: @@ -876,6 +882,9 @@ firewall: receiver: portgroups: - salt_manager + workstation: + portgroups: + - salt_manager self: portgroups: - syslog @@ -1169,9 +1178,9 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - analyst: + workstation: portgroups: - - nginx + - yum customhostgroup0: portgroups: [] customhostgroup1: diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 0011a245e..78c0ebc73 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -1,6 +1,6 @@ firewall: hostgroups: - analyst: &hostgroupsettings + workstation: &hostgroupsettings description: List of IP or CIDR blocks to allow access to this hostgroup. forcedType: "[]string" helplink: firewall.html diff --git a/salt/manager/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion index 7b0ddab90..d3bbb3eeb 100755 --- a/salt/manager/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -79,4 +79,7 @@ fi 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; + 'WORKSTATION') + so-firewall includehost workstation "$IP" --apply + ;; esac From 726ec7235000959622e9af7df4f5a80dc6aa1fb3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:22:59 -0400 Subject: [PATCH 069/350] allow idh to connect to salt_manager ports on managres --- salt/firewall/defaults.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 0d32d57ca..ff776d309 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -492,6 +492,9 @@ firewall: fleet: portgroups: - salt_manager + idh: + portgroups: + - salt_manager localhost: portgroups: - all @@ -674,6 +677,9 @@ firewall: fleet: portgroups: - salt_manager + idh: + portgroups: + - salt_manager localhost: portgroups: - all @@ -863,7 +869,10 @@ firewall: - all fleet: portgroups: - - salt_manager + - salt_manager + idh: + portgroups: + - salt_manager localhost: portgroups: - all From 0f52530d0760cf67cbda82ee81d18b220fe3cc17 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:37:58 -0400 Subject: [PATCH 070/350] soc_firewall.yaml update adding idh and rename analyst to workstation --- salt/firewall/soc_firewall.yaml | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 78c0ebc73..27c52e123 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -213,7 +213,7 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -338,7 +338,9 @@ firewall: DOCKER-USER: hostgroups: manager: - portgroups: *portgroupsdocker + portgroups: *portgroupsdocker + idh: + portgroups: *portgroupsdocker sensor: portgroups: *portgroupsdocker searchnode: @@ -359,7 +361,7 @@ firewall: portgroups: *portgroupsdocker endgame: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -389,12 +391,16 @@ firewall: portgroups: *portgroupshost localhost: portgroups: *portgroupshost + idh: + portgroups: *portgroupshost sensor: portgroups: *portgroupshost searchnode: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost + workstation: + portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost customhostgroup1: @@ -422,6 +428,8 @@ firewall: hostgroups: managersearch: portgroups: *portgroupsdocker + idh: + portgroups: *portgroupsdocker sensor: portgroups: *portgroupsdocker searchnode: @@ -442,7 +450,7 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -472,12 +480,16 @@ firewall: portgroups: *portgroupshost localhost: portgroups: *portgroupshost + idh: + portgroups: *portgroupshost sensor: portgroups: *portgroupshost searchnode: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost + workstation: + portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost customhostgroup1: @@ -509,6 +521,8 @@ firewall: portgroups: *portgroupsdocker fleet: portgroups: *portgroupsdocker + idh: + portgroups: *portgroupsdocker sensor: portgroups: *portgroupsdocker searchnode: @@ -531,7 +545,7 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -565,12 +579,16 @@ firewall: portgroups: *portgroupshost standalone: portgroups: *portgroupshost + idh: + portgroups: *portgroupshost sensor: portgroups: *portgroupshost searchnode: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost + workstation: + portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost customhostgroup1: @@ -793,7 +811,7 @@ firewall: portgroups: *portgroupsdocker elastic_agent_endpoint: portgroups: *portgroupsdocker - analyst: + workstation: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker From 014aeffb2af91889bc182a8dd4cbf215ceef820f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 09:56:33 -0400 Subject: [PATCH 071/350] add analyst back --- salt/firewall/defaults.yaml | 17 +++++++++++++++-- salt/firewall/soc_firewall.yaml | 13 ++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index ff776d309..9b8325a34 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -1,5 +1,6 @@ firewall: hostgroups: + analyst: [] anywhere: - 0.0.0.0/0 beats_endpoint: [] @@ -215,9 +216,9 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - workstation: + analyst: portgroups: - - yum + - nginx customhostgroup0: portgroups: [] customhostgroup1: @@ -441,6 +442,9 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni + analyst: + portgroups: + - nginx beats_endpoint: portgroups: - beats_5044 @@ -626,6 +630,9 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni + analyst: + portgroups: + - nginx beats_endpoint: portgroups: - beats_5044 @@ -816,6 +823,9 @@ firewall: - elastic_agent_data - elastic_agent_update - sensoroni + analyst: + portgroups: + - nginx beats_endpoint: portgroups: - beats_5044 @@ -1187,6 +1197,9 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + analyst: + portgroups: + - nginx workstation: portgroups: - yum diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 27c52e123..8f8dbb69d 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -1,6 +1,6 @@ firewall: hostgroups: - workstation: &hostgroupsettings + analyst: &hostgroupsettings description: List of IP or CIDR blocks to allow access to this hostgroup. forcedType: "[]string" helplink: firewall.html @@ -45,6 +45,7 @@ firewall: standalone: *hostgroupsettings strelka_frontend: *hostgroupsettings syslog: *hostgroupsettings + workstation: *hostgroupsettings customhostgroup0: &customhostgroupsettings description: List of IP or CIDR blocks to allow to this hostgroup. forcedType: "[]string" @@ -213,6 +214,8 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: @@ -361,6 +364,8 @@ firewall: portgroups: *portgroupsdocker endgame: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: @@ -450,6 +455,8 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: @@ -545,6 +552,8 @@ firewall: portgroups: *portgroupsdocker syslog: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: @@ -811,6 +820,8 @@ firewall: portgroups: *portgroupsdocker elastic_agent_endpoint: portgroups: *portgroupsdocker + analyst: + portgroups: *portgroupsdocker workstation: portgroups: *portgroupsdocker customhostgroup0: From 209da766ba1c8af4822fabb4f1848f275a792595 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 4 Aug 2023 12:16:14 -0400 Subject: [PATCH 072/350] Update soup to rotate log file --- salt/manager/tools/sbin/soup | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index cede5c438..e32936c90 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -972,6 +972,11 @@ while getopts ":b:f:y" opt; do done shift $((OPTIND - 1)) +if [ -f $SOUP_LOG ]; then + CURRENT_TIME=$(date +%Y%m%d.%H%M%S) + mv $SOUP_LOG $SOUP_LOG.$INSTALLEDVERSION.CURRENT_TIME +fi + if [[ -z $UNATTENDED ]]; then cat << EOF From 63373710b4e107311e50e1e93b4a02fb57fca004 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 4 Aug 2023 12:26:36 -0400 Subject: [PATCH 073/350] Update soup to rotate log file --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index e32936c90..d31ee997b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -974,7 +974,7 @@ shift $((OPTIND - 1)) if [ -f $SOUP_LOG ]; then CURRENT_TIME=$(date +%Y%m%d.%H%M%S) - mv $SOUP_LOG $SOUP_LOG.$INSTALLEDVERSION.CURRENT_TIME + mv $SOUP_LOG $SOUP_LOG.$INSTALLEDVERSION.$CURRENT_TIME fi if [[ -z $UNATTENDED ]]; then From 36747cf940566bb6aaccf50a7e5b3dad094f4197 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 13:52:01 -0400 Subject: [PATCH 074/350] add networkminer to desktop.packages --- salt/desktop/packages.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 3b0d4c8ba..5c0121e7b 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -349,6 +349,7 @@ desktop_packages: - snappy - sound-theme-freedesktop - soundtouch + - securityonion-networkminer - speech-dispatcher - speech-dispatcher-espeak-ng - speex From 0ba1e7521a551fa5f3d5b85dba7651e54325619b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 15:36:44 -0400 Subject: [PATCH 075/350] set default session for preexisting users --- salt/desktop/files/session.jinja | 7 +++++++ salt/desktop/xwindows.sls | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 salt/desktop/files/session.jinja diff --git a/salt/desktop/files/session.jinja b/salt/desktop/files/session.jinja new file mode 100644 index 000000000..823e62f2d --- /dev/null +++ b/salt/desktop/files/session.jinja @@ -0,0 +1,7 @@ +# This file is managed by Salt in the desktop.xwindows state +# It will not be overwritten if it already exists + +[User] +Session=gnome-classic +Icon=/home/{{USERNAME}}/.face +SystemAccount=false diff --git a/salt/desktop/xwindows.sls b/salt/desktop/xwindows.sls index ebb7ecb9f..792724eb4 100644 --- a/salt/desktop/xwindows.sls +++ b/salt/desktop/xwindows.sls @@ -18,6 +18,23 @@ convert_gnome_classic: cmd.script: - name: salt://desktop/scripts/convert-gnome-classic.sh +{% for username in salt['file.find'](path='/home/',mindepth=1,maxdepth=1,type='d') %} +{% set username = username.split('/')[2] %} +{% if username != 'zeek' %} +{% if not salt['file.file_exists']('/var/lib/AccountsService/users/' ~ username) %} + +{{username}}_session: + file.managed: + - name: /var/lib/AccountsService/users/{{username}} + - source: salt://desktop/files/session.jinja + - template: jinja + - defaults: + USERNAME: {{username}} + +{% endif %} +{% endif %} +{% endfor %} + {% else %} desktop_xwindows_os_fail: From 89c4f58296aa3ed2081b9a73c56bf88adf75b030 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 15:41:10 -0400 Subject: [PATCH 076/350] fix indents --- salt/desktop/xwindows.sls | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/desktop/xwindows.sls b/salt/desktop/xwindows.sls index 792724eb4..c7790f9f4 100644 --- a/salt/desktop/xwindows.sls +++ b/salt/desktop/xwindows.sls @@ -18,10 +18,10 @@ convert_gnome_classic: cmd.script: - name: salt://desktop/scripts/convert-gnome-classic.sh -{% for username in salt['file.find'](path='/home/',mindepth=1,maxdepth=1,type='d') %} -{% set username = username.split('/')[2] %} -{% if username != 'zeek' %} -{% if not salt['file.file_exists']('/var/lib/AccountsService/users/' ~ username) %} +{% for username in salt['file.find'](path='/home/',mindepth=1,maxdepth=1,type='d') %} +{% set username = username.split('/')[2] %} +{% if username != 'zeek' %} +{% if not salt['file.file_exists']('/var/lib/AccountsService/users/' ~ username) %} {{username}}_session: file.managed: @@ -31,9 +31,9 @@ convert_gnome_classic: - defaults: USERNAME: {{username}} +{% endif %} {% endif %} -{% endif %} -{% endfor %} +{% endfor %} {% else %} From 9d3744aa2567f6774cc5dd8d0328948dcfc03646 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 4 Aug 2023 16:05:28 -0400 Subject: [PATCH 077/350] Refactor to remove new line --- .../sbin_jinja/so-elastic-fleet-outputs-update | 14 ++++++-------- .../tools/sbin_jinja/so-elastic-fleet-urls-update | 14 ++++++-------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update index 17c867c07..b88b564ed 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update @@ -43,14 +43,12 @@ fi # Query for FQDN entries & add them to the list {% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} -CUSTOMFQDNLIST=({{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}) -if [ -n "$CUSTOMFQDNLIST" ]; then - readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST - for CUSTOMNAME in "${CUSTOMFQDN[@]}" - do - NEW_LIST+=("https://$CUSTOMNAME:8220") - done -fi +CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}') +readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST") +for CUSTOMNAME in "${CUSTOMFQDN[@]}" +do + NEW_LIST+=("$CUSTOMNAME:5055") +done {% endif %} # Query for the current Grid Nodes that are running Logstash diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update index 7d29fe080..31c7becca 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update @@ -43,14 +43,12 @@ fi # Query for FQDN entries & add them to the list {% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %} -CUSTOMFQDNLIST=({{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}) -if [ -n "$CUSTOMFQDNLIST" ]; then - readarray -t CUSTOMFQDN <<< $CUSTOMFQDNLIST - for CUSTOMNAME in "${CUSTOMFQDN[@]}" - do - NEW_LIST+=("https://$CUSTOMNAME:8220") - done -fi +CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}') +readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST") +for CUSTOMNAME in "${CUSTOMFQDN[@]}" +do + NEW_LIST+=("https://$CUSTOMNAME:8220") +done {% endif %} # Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes) From 9af2a731ca7152d98db820a52cb147b02c942fd4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 4 Aug 2023 16:29:30 -0400 Subject: [PATCH 078/350] fix count of WORKERS for zeekcaptureloss script for telegraf --- salt/telegraf/scripts/zeekcaptureloss.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index e0c8758f2..e254ada32 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -11,10 +11,15 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp # if this script isn't already running +{%- from 'zeek/config.map.jinja' import ZEEKMERGED %} if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then - WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} +{%- if ZEEKMERGED.config.node.pins %} + WORKERS={{ ZEEKMERGED.config.node.pins | length }} +{%- else %} + WORKERS={{ ZEEKMERGED.config.node.lb_procs }} +{%- endif %} ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then WORKERS=1 From ec81cbd70d8aa33986c5609d5dc92895784434f1 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sat, 5 Aug 2023 09:11:58 -0400 Subject: [PATCH 079/350] Revert yesterday's change to zeekcaptureloss.sh --- salt/telegraf/scripts/zeekcaptureloss.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index e254ada32..4389fd601 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -11,15 +11,9 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp # if this script isn't already running -{%- from 'zeek/config.map.jinja' import ZEEKMERGED %} if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then -{%- if ZEEKMERGED.config.node.pins %} - WORKERS={{ ZEEKMERGED.config.node.pins | length }} -{%- else %} - WORKERS={{ ZEEKMERGED.config.node.lb_procs }} -{%- endif %} ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then WORKERS=1 From 90102b1148047a445ba900d524d562ae6b75227b Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sat, 5 Aug 2023 09:23:27 -0400 Subject: [PATCH 080/350] Finish reverting yesterday's change to zeekcaptureloss.sh --- salt/telegraf/scripts/zeekcaptureloss.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 4389fd601..e0c8758f2 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -14,6 +14,7 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then + WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then WORKERS=1 From 3c5cd941c78b123835c3a473cc5ae9970b73e690 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 7 Aug 2023 08:45:30 -0400 Subject: [PATCH 081/350] Update DOWNLOAD_AND_VERIFY_ISO.md for 2.4.5 --- DOWNLOAD_AND_VERIFY_ISO.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index c1594b954..0ea6db8ed 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.4.4-20230728 ISO image built on 2023/07/28 +### 2.4.5-20230807 ISO image released on 2023/08/07 ### Download and Verify -2.4.4-20230728 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.4-20230728.iso - -MD5: F63E76245F3E745B5BDE9E6E647A7CB6 -SHA1: 6CE4E4A3399CD282D4F8592FB19D510388AB3EEA -SHA256: BF8FEB91B1D94B67C3D4A79D209B068F4A46FEC7C15EEF65B0FCE9851D7E6C9F +2.4.5-20230807 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso + +MD5: F83FD635025A3A65B380EAFCEB61A92E +SHA1: 5864D4CD520617E3328A3D956CAFCC378A8D2D08 +SHA256: D333BAE0DD198DFD80DF59375456D228A4E18A24EDCDB15852CD4CA3F92B69A7 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.4-20230728.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.4-20230728.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.4-20230728.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.4-20230728.iso.sig securityonion-2.4.4-20230728.iso +gpg --verify securityonion-2.4.5-20230807.iso.sig securityonion-2.4.5-20230807.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Tue 11 Jul 2023 06:23:37 PM EDT using RSA key ID FE507013 +gpg: Signature made Sat 05 Aug 2023 10:12:46 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. From 6e8f31e08373177b4b82b9d16f02664aaa916dbb Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 7 Aug 2023 08:59:24 -0400 Subject: [PATCH 082/350] Delete sigs --- sigs | Bin 566 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 sigs diff --git a/sigs b/sigs deleted file mode 100644 index 75a14e1a124888e706fa4e8a2cb8c950e0df7217..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%XJRr~nEH5PT3| zxBgIY6IR&||9=cs@>p+WKyMXEHkdmM9dKN0A_e16|Moe1B^c6!o-~gn=Kj$D8!zb`EjAq^qcU?a}o%Y zDg9=2aqmC*5q>UkyS%K!=yB%liislB5j5FN*Cpo}eO0mrT4n|YgLgq+bx%~B&xTUb ztNgVOS17l{E;!>9=IsB*%qV5K>b~dg>q`zAKq?LL>fdWJEYKH`Lh40-3ZSL1y(Has zLx)b4*VPCx;QTRT9YqP)L}1+sLSnsKZNkRsM0d-(lrnP?{6T8)n5B_CUG=Fx&}umS zr%NsOIn^)hUDLNq+P0kSbQ-W--|qGoc5@cZ;3Z89-8lHAc>pT3v2x8v_e_6vs!o4d zwD5%VtV5fj5`ddN&NgtSJI;;(QyqtKsuJU~M809}yP6#5E0=~sT|HDCt~1K7gPLf? z@C7|#7tJ91EY55yD`(Q8;N7@pBl0O?XV=j|qyCHbM#Jj5bTOQNtyAg9srzf(VL^Z* EkfHt+T>t<8 From 9ae32e2bd66168ac37fd924d5fe042a11fd1091d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 7 Aug 2023 09:02:52 -0400 Subject: [PATCH 083/350] create sigs directory and add sig for 2.4.5 --- sigs/securityonion-2.4.5-20230807.iso.sig | Bin 0 -> 566 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 sigs/securityonion-2.4.5-20230807.iso.sig diff --git a/sigs/securityonion-2.4.5-20230807.iso.sig b/sigs/securityonion-2.4.5-20230807.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..fdf914164d39f45c413fd96952298f9733355d32 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%Xou-T(>-5PT3| zxBgIY6W3V}|7ARLdu8Uk&=%aLd1eaANXl51uCg=jVr4eayWWo!3JlNr+(j+XW?8qFP5X6QaP5#PecYT1>OAuzC7>i4Rj@v&4=B(BpnOQWIA z2A@7#vZgCT;V|o&PF;@JY9=^AaOW{>4#zzg?c1Zas>6>-wS%Mmh$GYnudPa^Jr9zu zs(~Aip0%%-0^LpOJjMtMdyS2Z*t-J$)8Dg4?{H`YsPy}t9_h%qJa*UX0ZD)~OY#uM zBz!Z96O4cz441jxvb>Y-PM$%Un^Jl}ZJr_=mC~HS*YP7m zj}Z+_UTFqr6zSgYq7P~*1tW6Z)u7wkCF`;At%WU!r;6W6J?9EmiFg9>T`P=#eZmIC EYdvxXlK=n! literal 0 HcmV?d00001 From 37b98ba1889fcda67d030f40ee888f282954f1ff Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 7 Aug 2023 09:29:34 -0400 Subject: [PATCH 084/350] add spaces for proper rendering DOWNLOAD_AND_VERIFY_ISO.md --- DOWNLOAD_AND_VERIFY_ISO.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 0ea6db8ed..b9b3da297 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -7,9 +7,9 @@ 2.4.5-20230807 ISO image: https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso -MD5: F83FD635025A3A65B380EAFCEB61A92E -SHA1: 5864D4CD520617E3328A3D956CAFCC378A8D2D08 -SHA256: D333BAE0DD198DFD80DF59375456D228A4E18A24EDCDB15852CD4CA3F92B69A7 +MD5: F83FD635025A3A65B380EAFCEB61A92E +SHA1: 5864D4CD520617E3328A3D956CAFCC378A8D2D08 +SHA256: D333BAE0DD198DFD80DF59375456D228A4E18A24EDCDB15852CD4CA3F92B69A7 Signature for ISO image: https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig From 5278601e5d1cd05a613a2486e867c95d87936d8c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 7 Aug 2023 11:18:35 -0400 Subject: [PATCH 085/350] manage telegraf scripts with a defaults file assigned per node type --- salt/telegraf/config.sls | 15 ++--- salt/telegraf/defaults.yaml | 79 ++++++++++++++++++++++ salt/telegraf/enabled.sls | 5 +- salt/telegraf/etc/telegraf.conf | 114 +++----------------------------- salt/telegraf/map.jinja | 10 ++- salt/telegraf/soc_telegraf.yaml | 19 +++++- 6 files changed, 127 insertions(+), 115 deletions(-) diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index 1cc7ceed0..0711260b5 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -32,17 +32,16 @@ tgrafetsdir: - name: /opt/so/conf/telegraf/scripts - makedirs: True -tgrafsyncscripts: - file.recurse: - - name: /opt/so/conf/telegraf/scripts +{% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +tgraf_sync_script_{{script}}: + file.managed: + - name: /opt/so/conf/telegraf/scripts/{{script}} - user: root - group: 939 - - file_mode: 770 + - mode: 770 - template: jinja - - source: salt://telegraf/scripts - {% if GLOBALS.md_engine == 'SURICATA' %} - - exclude_pat: zeekcaptureloss.sh - {% endif %} + - source: salt://telegraf/scripts/{{script}} +{% endfor %} telegraf_sbin: file.recurse: diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 63d437763..36ef679f0 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -9,3 +9,82 @@ telegraf: flush_jitter: '0s' debug: 'false' quiet: 'false' + scripts: + eval: + - beatseps.sh + - checkfiles.sh + - influxdbsize.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + standalone: + - beatseps.sh + - checkfiles.sh + - eps.sh + - influxdbsize.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + manager: + - beatseps.sh + - influxdbsize.sh + - raid.sh + - redis.sh + - sostatus.sh + managersearch: + - beatseps.sh + - eps.sh + - influxdbsize.sh + - raid.sh + - redis.sh + - sostatus.sh + import: + - sostatus.sh + sensor: + - beatseps.sh + - checkfiles.sh + - oldpcap.sh + - raid.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + heavynode: + - beatseps.sh + - checkfiles.sh + - eps.sh + - oldpcap.sh + - raid.sh + - redis.sh + - sostatus.sh + - stenoloss.sh + - suriloss.sh + - zeekcaptureloss.sh + - zeekloss.sh + idh: + - sostatus.sh + searchnode: + - beatseps.sh + - eps.sh + - raid.sh + - sostatus.sh + receiver: + - beatseps.sh + - eps.sh + - raid.sh + - redis.sh + - sostatus.sh + fleet: + - sostatus.sh + desktop: [] diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls index 209c85fb0..598587e17 100644 --- a/salt/telegraf/enabled.sls +++ b/salt/telegraf/enabled.sls @@ -7,6 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'telegraf/map.jinja' import TELEGRAFMERGED %} include: @@ -67,8 +68,10 @@ so-telegraf: {% endif %} - watch: - file: tgrafconf - - file: tgrafsyncscripts - file: node_config + {% for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} + - file: tgraf_sync_script_{{script}} + {% endfor %} - require: - file: tgrafconf - file: node_config diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 1a6cdc311..45b1283e0 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -193,7 +193,7 @@ username = "{{ ES_USER }}" password = "{{ ES_PASS }}" insecure_skip_verify = true -{%- elif grains['role'] in ['so-searchnode', 'so-hotnode', 'so-warmnode'] %} +{%- elif grains['role'] in ['so-searchnode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] cluster_stats = false @@ -244,6 +244,8 @@ {%- endif %} # # Read metrics from one or more commands that can output to stdout +{%- if 'sostatus.sh' in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} +{%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('sostatus.sh') %} [[inputs.exec]] commands = [ "/scripts/sostatus.sh" @@ -251,122 +253,26 @@ data_format = "influx" timeout = "15s" interval = "60s" +{%- endif %} -# ## Commands array -{% if grains['role'] in ['so-manager'] %} +{%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %} [[inputs.exec]] commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" +{%- for script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %} + "/scripts/{{script}}"{% if not loop.last %},{% endif %} +{%- endfor %} ] data_format = "influx" ## Timeout for each command to complete. timeout = "15s" -{% elif grains['role'] in ['so-managersearch'] %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - ## Timeout for each command to complete. - timeout = "15s" -{% elif grains['role'] in ['so-searchnode', 'so-receiver'] %} -[[inputs.exec]] - commands = [ - "/scripts/eps.sh", - "/scripts/raid.sh", - {% if grains.role == 'so-receiver' %} - "/scripts/redis.sh", - {% endif %} - "/scripts/beatseps.sh" - ] - data_format = "influx" - ## Timeout for each command to complete. - timeout = "15s" -{% elif grains['role'] == 'so-sensor' %} -[[inputs.exec]] - commands = [ - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-heavynode' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-standalone' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/influxdbsize.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/eps.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% elif grains['role'] == 'so-eval' %} -[[inputs.exec]] - commands = [ - "/scripts/redis.sh", - "/scripts/stenoloss.sh", - "/scripts/suriloss.sh", - "/scripts/checkfiles.sh", - {%- if MDENGINE == 'ZEEK' and ZEEK_ENABLED %} - "/scripts/zeekloss.sh", - "/scripts/zeekcaptureloss.sh", - {%- endif %} - "/scripts/oldpcap.sh", - "/scripts/influxdbsize.sh", - "/scripts/raid.sh", - "/scripts/beatseps.sh" - ] - data_format = "influx" - timeout = "15s" -{% endif %} +{%- endif %} {%- if salt['pillar.get']('healthcheck:enabled', False) %} [[inputs.file]] files = ["/host/nsm/zeek/logs/zeek_restart.log"] data_format = "influx" {%- endif %} + [[inputs.file]] files = ["/etc/telegraf/node_config.json"] name_override = "node_config" diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index f1412d3ac..0f197a7b1 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -2,6 +2,14 @@ or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at https://securityonion.net/license; you may not use this file except in compliance with the Elastic License 2.0. #} - + +{% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'telegraf/defaults.yaml' as TELEGRAFDEFAULTS %} {% set TELEGRAFMERGED = salt['pillar.get']('telegraf', TELEGRAFDEFAULTS.telegraf, merge=True) %} +{% from 'zeek/config.map.jinja' import ZEEKMERGED %} + +{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} +{% if (GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode']) and (GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled) %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% endif %} diff --git a/salt/telegraf/soc_telegraf.yaml b/salt/telegraf/soc_telegraf.yaml index a688ea2a3..1550c66cb 100644 --- a/salt/telegraf/soc_telegraf.yaml +++ b/salt/telegraf/soc_telegraf.yaml @@ -42,4 +42,21 @@ telegraf: global: True advanced: True helpLink: telegraf.html - \ No newline at end of file + scripts: + eval: &telegrafscripts + description: List of input.exec scripts to run for this node type. The script must be present in salt/telegraf/scripts. + forcedType: "[]string" + multiline: True + advanced: True + helpLink: telegraf.html + standalone: *telegrafscripts + manager: *telegrafscripts + managersearch: *telegrafscripts + import: *telegrafscripts + sensor: *telegrafscripts + heavynode: *telegrafscripts + idh: *telegrafscripts + searchnode: *telegrafscripts + receiver: *telegrafscripts + fleet: *telegrafscripts + desktop: *telegrafscripts From 8611d1848c8b09ab6b263288da0e6717bbaf9fda Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 7 Aug 2023 15:55:53 -0400 Subject: [PATCH 086/350] Set as default --- .../tools/sbin_jinja/so-elastic-fleet-es-url-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 6acda746c..975a8aec1 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -13,7 +13,7 @@ fi function update_es_urls() { # Generate updated JSON payload - JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":false,"is_default_monitoring":false,"config_yaml":""}') + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') # Update Fleet Elasticsearch URLs curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" From b9d0d032238cd926961348087ab773632bf754a4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 7 Aug 2023 16:35:05 -0400 Subject: [PATCH 087/350] update version --- README.md | 4 ++-- VERSION | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aa3aa6ddf..19a560419 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## Security Onion 2.4 Release Candidate 2 (RC2) +## Security Onion 2.4 -Security Onion 2.4 Release Candidate 2 (RC2) is here! +Security Onion 2.4 is here! ## Screenshots diff --git a/VERSION b/VERSION index 59aa62c1f..b0f6bf0cd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.5 +2.4.10 From 609a2bf32e7085b18c0d039b0de51501800e468c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:27:03 -0400 Subject: [PATCH 088/350] only import ZEEKMERGED if a sensor type node --- salt/telegraf/map.jinja | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/salt/telegraf/map.jinja b/salt/telegraf/map.jinja index 0f197a7b1..e6d3460d6 100644 --- a/salt/telegraf/map.jinja +++ b/salt/telegraf/map.jinja @@ -6,10 +6,12 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'telegraf/defaults.yaml' as TELEGRAFDEFAULTS %} {% set TELEGRAFMERGED = salt['pillar.get']('telegraf', TELEGRAFDEFAULTS.telegraf, merge=True) %} -{% from 'zeek/config.map.jinja' import ZEEKMERGED %} -{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} -{% if (GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode']) and (GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled) %} -{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} -{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% if GLOBALS.role in ['so-eval', 'so-standalone', 'so-sensor', 'so-heavynode'] %} +{% from 'zeek/config.map.jinja' import ZEEKMERGED %} +{# if the md engine isn't zeek or zeek is disabled, dont run the zeek scripts for telegraf #} +{% if GLOBALS.md_engine != 'ZEEK' or not ZEEKMERGED.enabled %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekloss.sh') %} +{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('zeekcaptureloss.sh') %} +{% endif %} {% endif %} From 69553f9017b3ffe5ffa4ae776bf5006eddfec894 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:34:59 -0400 Subject: [PATCH 089/350] removes spaces from zeekcaptureloss script --- salt/telegraf/scripts/zeekcaptureloss.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index e254ada32..f2c3fcd2d 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -5,9 +5,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. - - - # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp # if this script isn't already running From a06040c035e6db035426fb001afc1887276cc55a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:37:37 -0400 Subject: [PATCH 090/350] add WORKERS calculation back to zeekcaptureloss script --- salt/telegraf/scripts/zeekcaptureloss.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index 6e1785237..ddb6cd128 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -11,7 +11,11 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then - WORKERS={{ salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) }} +{%- if ZEEKMERGED.config.node.pins %} + WORKERS={{ ZEEKMERGED.config.node.pins | length }} +{%- else %} + WORKERS={{ ZEEKMERGED.config.node.lb_procs }} +{%- endif %} ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log elif [ -d "/host/nsm/zeek/spool/zeeksa" ]; then WORKERS=1 From 673b45af092b6a655bf115edffc4d88ef0ba393a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 09:41:42 -0400 Subject: [PATCH 091/350] import ZEEKMERGED --- salt/telegraf/scripts/zeekcaptureloss.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/telegraf/scripts/zeekcaptureloss.sh b/salt/telegraf/scripts/zeekcaptureloss.sh index ddb6cd128..f2c3fcd2d 100644 --- a/salt/telegraf/scripts/zeekcaptureloss.sh +++ b/salt/telegraf/scripts/zeekcaptureloss.sh @@ -8,6 +8,7 @@ # This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp # if this script isn't already running +{%- from 'zeek/config.map.jinja' import ZEEKMERGED %} if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [ -d "/host/nsm/zeek/spool/logger" ]; then From b2e75e77e8a079ee7066f0ab867691ea9cfb496f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 13:50:19 -0400 Subject: [PATCH 092/350] add local.rules and filter.rules to suricata defaults. add extraction.rules, local.rules and filter.rules for suricata metadata --- salt/suricata/defaults.yaml | 3 ++- salt/suricata/suricata_mdengine.yaml | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 4651b7268..fd1b00929 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -416,7 +416,6 @@ suricata: enabled: "yes" filename: keyword_perf.log append: "yes" - prefilter: enabled: "yes" filename: prefilter_perf.log @@ -443,6 +442,8 @@ suricata: default-rule-path: /etc/suricata/rules rule-files: - all.rules + - local.rules + - filter.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index 1c3855501..c6844541f 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -70,3 +70,9 @@ suricata: - flow #- netflow #- metadata + profiling: + rule-files: + - all.rules + - extraction.rules + - local.rules + - filter.rules From aab89d2483822359cc235827f3f4486024d3b288 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 13:54:58 -0400 Subject: [PATCH 093/350] rule-files does not go under profiling --- salt/suricata/suricata_mdengine.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index c6844541f..80299dc5b 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -70,9 +70,8 @@ suricata: - flow #- netflow #- metadata - profiling: - rule-files: - - all.rules - - extraction.rules - - local.rules - - filter.rules + rule-files: + - all.rules + - extraction.rules + - local.rules + - filter.rules From 9118ac2b569e9fd3e3c994b24a8f6a4502d4331c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 13:59:43 -0400 Subject: [PATCH 094/350] filter.rules to filters.rules --- salt/suricata/defaults.yaml | 2 +- salt/suricata/suricata_mdengine.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index fd1b00929..4253794a8 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -443,7 +443,7 @@ suricata: rule-files: - all.rules - local.rules - - filter.rules + - filters.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index 80299dc5b..d1fb7c2c3 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -74,4 +74,4 @@ suricata: - all.rules - extraction.rules - local.rules - - filter.rules + - filters.rules From 20dedab4b283cf3d7345170abd7ee24dac9bfbc3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:03:06 -0400 Subject: [PATCH 095/350] remove previously add rules files --- salt/suricata/defaults.yaml | 2 -- salt/suricata/suricata_mdengine.yaml | 5 ----- 2 files changed, 7 deletions(-) diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml index 4253794a8..050efa8f8 100644 --- a/salt/suricata/defaults.yaml +++ b/salt/suricata/defaults.yaml @@ -442,8 +442,6 @@ suricata: default-rule-path: /etc/suricata/rules rule-files: - all.rules - - local.rules - - filters.rules classification-file: /etc/suricata/classification.config reference-config-file: /etc/suricata/reference.config threshold-file: /etc/suricata/threshold.conf diff --git a/salt/suricata/suricata_mdengine.yaml b/salt/suricata/suricata_mdengine.yaml index d1fb7c2c3..1c3855501 100644 --- a/salt/suricata/suricata_mdengine.yaml +++ b/salt/suricata/suricata_mdengine.yaml @@ -70,8 +70,3 @@ suricata: - flow #- netflow #- metadata - rule-files: - - all.rules - - extraction.rules - - local.rules - - filters.rules From 230f5868f9ab59ae235d970d5319dd89276bdaab Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:14:27 -0400 Subject: [PATCH 096/350] sync sorules --- salt/idstools/sync_files.sls | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index 64479e937..e8d5edda6 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -26,6 +26,13 @@ rulesdir: - group: 939 - makedirs: True +SOrulesdir: + file.directory: + - name: /opt/so/rules/nids/sorules + - user: 939 + - group: 939 + - makedirs: True + # Don't show changes because all.rules can be large synclocalnidsrules: file.recurse: @@ -35,3 +42,13 @@ synclocalnidsrules: - group: 939 - show_changes: False - include_pat: 'E@.rules' + +# Don't show changes because all.rules can be large +syncnidsSOrules: + file.recurse: + - name: /opt/so/rules/nids/sorules + - source: salt://idstools/sorules/ + - user: 939 + - group: 939 + - show_changes: False + - include_pat: 'E@.rules' From 5c704d7e5864dc16c37224d000c019b257315a5c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:20:44 -0400 Subject: [PATCH 097/350] run so-rule-update if idstools configs change --- salt/idstools/sync_files.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index e8d5edda6..cc6c45baa 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -19,6 +19,12 @@ idstoolsetcsync: - group: 939 - template: jinja +run_so-rule-update: + cmd.run: + - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' + - onchanges: + - file: idstoolsetcsync + rulesdir: file.directory: - name: /opt/so/rules/nids From 21c80e4953a4de0b0cfa58464808fc44e8665704 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 15:27:23 -0400 Subject: [PATCH 098/350] run so-rule-update after idstools container restart --- salt/idstools/enabled.sls | 8 ++++++++ salt/idstools/sync_files.sls | 6 ------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index bf5650773..966cb6786 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -69,6 +69,14 @@ so-rule-update: - minute: '1' - hour: '7' +run_so-rule-update: + cmd.run: + - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' + - require: + - docker_container: so-idstools + - onchanges: + - file: idstoolsetcsync + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index cc6c45baa..e8d5edda6 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -19,12 +19,6 @@ idstoolsetcsync: - group: 939 - template: jinja -run_so-rule-update: - cmd.run: - - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' - - onchanges: - - file: idstoolsetcsync - rulesdir: file.directory: - name: /opt/so/rules/nids From 3d4fd08547a32d713480e8e48f04e39fe6216182 Mon Sep 17 00:00:00 2001 From: bryant-treacle Date: Tue, 8 Aug 2023 15:28:06 -0400 Subject: [PATCH 099/350] Update defaults.yaml --- salt/soc/defaults.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index cb7d400a0..f97089e02 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -570,14 +570,13 @@ soc: - destination.geo.country_iso_code - user.name - source.ip - ':windows.sysmon_operational:': + '::sysmon_operational': - soc_timestamp - event.action - - process.executable + - winlog.computer_name - user.name - - file.target - - dns.question.name - - winlog.event_data.TargetObject + - process.executable + - process.pid '::network_connection': - soc_timestamp - source.ip From 036b81707b275f96f9ed13c0021be6ee765d690b Mon Sep 17 00:00:00 2001 From: bryant-treacle Date: Tue, 8 Aug 2023 16:10:54 -0400 Subject: [PATCH 100/350] Update defaults.yaml --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index f97089e02..49be076c0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -69,7 +69,7 @@ soc: - log.id.uid - network.community_id - event.dataset - ':kratos:kratos.audit': + ':kratos:audit': - soc_timestamp - http_request.headers.x-real-ip - identity_id From 2f74b69cc39f1ed577bc48ce0df5025f9fcd58bf Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 8 Aug 2023 16:27:11 -0400 Subject: [PATCH 101/350] Update soup for 2.4.10 --- salt/manager/tools/sbin/soup | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d31ee997b..f8221a4f4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -393,6 +393,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.2 ]] && up_to_2.4.3 [[ "$INSTALLEDVERSION" == 2.4.3 ]] && up_to_2.4.4 [[ "$INSTALLEDVERSION" == 2.4.4 ]] && up_to_2.4.5 + [[ "$INSTALLEDVERSION" == 2.4.5 ]] && up_to_2.4.10 true } @@ -403,6 +404,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.2 ]] && post_to_2.4.3 [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4 [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 + [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 true } @@ -422,6 +424,11 @@ post_to_2.4.5() { POSTVERSION=2.4.5 } +post_to_2.4.10() { + echo "Nothing to apply" + POSTVERSION=2.4.10 +} + stop_salt_master() { # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts set +e @@ -482,6 +489,12 @@ up_to_2.4.5() { INSTALLEDVERSION=2.4.5 } +up_to_2.4.10() { + echo "Nothing to do for 2.4.10" + + INSTALLEDVERSION=2.4.10 +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap From 2dbe6798498d3086298b79ca355940ed98da96ee Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:05:03 -0400 Subject: [PATCH 102/350] force restart of filecheck if the config changes --- salt/strelka/filestream/config.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index a215967ee..d4615b174 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -85,6 +85,7 @@ filecheck_restart: - success_retcodes: [0,1] - onchanges: - file: filecheck_script + - file: filecheck_conf filecheck_run: cron.present: From 6da2f117f215ee856fe6800ff91c94ff11cea168 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:25:08 -0400 Subject: [PATCH 103/350] change which user runs filecheck cron based on md engine --- salt/strelka/filestream/config.sls | 30 ++++++++++++++++++++++++++---- salt/strelka/map.jinja | 2 -- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index d4615b174..9c0ef1357 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -6,7 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'strelka/map.jinja' import STRELKAMERGED %} -{% from 'strelka/map.jinja' import filecheck_runas %} +{% from 'vars/globals.map.jinja' import GLOBALS %} include: - strelka.config @@ -87,11 +87,33 @@ filecheck_restart: - file: filecheck_script - file: filecheck_conf -filecheck_run: +{% if GLOBALS.md_engine == 'ZEEK' %} + +filecheck_run_socore: cron.present: - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' - - identifier: filecheck_run - - user: {{ filecheck_runas }} + - identifier: filecheck_run_socore + - user: socore + +remove_filecheck_run_suricata: + cron.absent: + - identifier: filecheck_run_suricata + - user: suricata + +{% elif GLOBALS.md_engine == 'SURICATA'%} + +filecheck_run_suricata: + cron.present: + - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' + - identifier: filecheck_run_suricata + - user: suricata + +remove_filecheck_run_socore: + cron.absent: + - identifier: filecheck_run_socore + - user: socore + +{% endif %} filcheck_history_clean: cron.present: diff --git a/salt/strelka/map.jinja b/salt/strelka/map.jinja index 646f7a746..387036248 100644 --- a/salt/strelka/map.jinja +++ b/salt/strelka/map.jinja @@ -24,10 +24,8 @@ {% if GLOBALS.md_engine == "SURICATA" %} {% set extract_path = '/nsm/suricata/extracted' %} -{% set filecheck_runas = 'suricata' %} {% else %} {% set extract_path = '/nsm/zeek/extracted/complete' %} -{% set filecheck_runas = 'socore' %} {% endif %} {% do STRELKADEFAULTS.strelka.filecheck.update({'extract_path': extract_path}) %} From 553b758c61e87909f704b9159c13d30022ea3ac4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:28:14 -0400 Subject: [PATCH 104/350] update cronjobs first, the kill filecheck --- salt/strelka/filestream/config.sls | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index 9c0ef1357..193241f32 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -78,15 +78,6 @@ filecheck_script: - group: 939 - mode: 755 -filecheck_restart: - cmd.run: - - name: pkill -f "python3 /opt/so/conf/strelka/filecheck" - - hide_output: True - - success_retcodes: [0,1] - - onchanges: - - file: filecheck_script - - file: filecheck_conf - {% if GLOBALS.md_engine == 'ZEEK' %} filecheck_run_socore: @@ -115,6 +106,15 @@ remove_filecheck_run_socore: {% endif %} +filecheck_restart: + cmd.run: + - name: pkill -f "python3 /opt/so/conf/strelka/filecheck" + - hide_output: True + - success_retcodes: [0,1] + - onchanges: + - file: filecheck_script + - file: filecheck_conf + filcheck_history_clean: cron.present: - name: '/usr/bin/find /nsm/strelka/history/ -type f -mtime +2 -exec rm {} + > /dev/null 2>&1' From 58fe25623b6ece773278c293727428059e8944fb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:48:34 -0400 Subject: [PATCH 105/350] ensure ownership of /opt/so/log/strelka/filecheck_stdout.log --- salt/strelka/filestream/config.sls | 7 +++++++ salt/strelka/map.jinja | 2 ++ 2 files changed, 9 insertions(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index 193241f32..c827ff5fb 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -7,6 +7,7 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'strelka/map.jinja' import STRELKAMERGED %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'strelka/map.jinja' import filecheck_runas %} include: - strelka.config @@ -78,6 +79,12 @@ filecheck_script: - group: 939 - mode: 755 +filecheck_stdout.log: + file.managed: + - name: /opt/so/log/strelka/filecheck_stdout.log + - user: {{ filecheck_runas }} + - group: {{ filecheck_runas }} + {% if GLOBALS.md_engine == 'ZEEK' %} filecheck_run_socore: diff --git a/salt/strelka/map.jinja b/salt/strelka/map.jinja index 387036248..646f7a746 100644 --- a/salt/strelka/map.jinja +++ b/salt/strelka/map.jinja @@ -24,8 +24,10 @@ {% if GLOBALS.md_engine == "SURICATA" %} {% set extract_path = '/nsm/suricata/extracted' %} +{% set filecheck_runas = 'suricata' %} {% else %} {% set extract_path = '/nsm/zeek/extracted/complete' %} +{% set filecheck_runas = 'socore' %} {% endif %} {% do STRELKADEFAULTS.strelka.filecheck.update({'extract_path': extract_path}) %} From 789fff561efdbccd10497b7cf66e496cfeb543f8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 8 Aug 2023 17:55:30 -0400 Subject: [PATCH 106/350] ensure ownership of /opt/so/log/strelka/filecheck.log --- salt/strelka/filestream/config.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index c827ff5fb..993a59650 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -79,6 +79,12 @@ filecheck_script: - group: 939 - mode: 755 +filecheck.log: + file.managed: + - name: /opt/so/log/strelka/filecheck.log + - user: {{ filecheck_runas }} + - group: {{ filecheck_runas }} + filecheck_stdout.log: file.managed: - name: /opt/so/log/strelka/filecheck_stdout.log From e1e535b009701a9adda2bdda06ceeeb52790746f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 8 Aug 2023 18:38:18 -0400 Subject: [PATCH 107/350] Retry if exit code is error --- salt/elasticfleet/enabled.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index bb6410f2c..096610af2 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -22,6 +22,7 @@ include: so-elastic-fleet-auto-configure-logstash-outputs: cmd.run: - name: /usr/sbin/so-elastic-fleet-outputs-update + - retry: True {% endif %} # If enabled, automatically update Fleet Server URLs & ES Connection @@ -29,6 +30,7 @@ so-elastic-fleet-auto-configure-logstash-outputs: so-elastic-fleet-auto-configure-server-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-urls-update + - retry: True {% endif %} # Automatically update Fleet Server Elasticsearch URLs @@ -36,6 +38,7 @@ so-elastic-fleet-auto-configure-server-urls: so-elastic-fleet-auto-configure-elasticsearch-urls: cmd.run: - name: /usr/sbin/so-elastic-fleet-es-url-update + - retry: True {% endif %} {% if SERVICETOKEN != '' %} From 00efc2f88f5bfdb28bf4dfb18df8855709486bfb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 07:31:31 -0400 Subject: [PATCH 108/350] rename workstation to desktop for firewall --- salt/firewall/defaults.yaml | 16 ++++++++-------- salt/firewall/soc_firewall.yaml | 18 +++++++++--------- salt/manager/tools/sbin/so-firewall-minion | 4 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 9b8325a34..347ddd4b0 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -26,7 +26,7 @@ firewall: standalone: [] strelka_frontend: [] syslog: [] - workstation: [] + desktop: [] customhostgroup0: [] customhostgroup1: [] customhostgroup2: [] @@ -462,7 +462,7 @@ firewall: endgame: portgroups: - endgame - workstation: + desktop: portgroups: - yum customhostgroup0: @@ -514,7 +514,7 @@ firewall: receiver: portgroups: - salt_manager - workstation: + desktop: portgroups: - salt_manager self: @@ -650,7 +650,7 @@ firewall: endgame: portgroups: - endgame - workstation: + desktop: portgroups: - yum customhostgroup0: @@ -702,7 +702,7 @@ firewall: receiver: portgroups: - salt_manager - workstation: + desktop: portgroups: - salt_manager self: @@ -846,7 +846,7 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - workstation: + desktop: portgroups: - yum customhostgroup0: @@ -901,7 +901,7 @@ firewall: receiver: portgroups: - salt_manager - workstation: + desktop: portgroups: - salt_manager self: @@ -1200,7 +1200,7 @@ firewall: analyst: portgroups: - nginx - workstation: + desktop: portgroups: - yum customhostgroup0: diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 8f8dbb69d..6ba5bea76 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -45,7 +45,7 @@ firewall: standalone: *hostgroupsettings strelka_frontend: *hostgroupsettings syslog: *hostgroupsettings - workstation: *hostgroupsettings + desktop: *hostgroupsettings customhostgroup0: &customhostgroupsettings description: List of IP or CIDR blocks to allow to this hostgroup. forcedType: "[]string" @@ -216,7 +216,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -366,7 +366,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -404,7 +404,7 @@ firewall: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost - workstation: + desktop: portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost @@ -457,7 +457,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -495,7 +495,7 @@ firewall: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost - workstation: + desktop: portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost @@ -554,7 +554,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker @@ -596,7 +596,7 @@ firewall: portgroups: *portgroupshost heavynode: portgroups: *portgroupshost - workstation: + desktop: portgroups: *portgroupshost customhostgroup0: portgroups: *portgroupshost @@ -822,7 +822,7 @@ firewall: portgroups: *portgroupsdocker analyst: portgroups: *portgroupsdocker - workstation: + desktop: portgroups: *portgroupsdocker customhostgroup0: portgroups: *portgroupsdocker diff --git a/salt/manager/tools/sbin/so-firewall-minion b/salt/manager/tools/sbin/so-firewall-minion index d3bbb3eeb..66a0afcea 100755 --- a/salt/manager/tools/sbin/so-firewall-minion +++ b/salt/manager/tools/sbin/so-firewall-minion @@ -79,7 +79,7 @@ fi 'RECEIVER') so-firewall includehost receiver "$IP" --apply ;; - 'WORKSTATION') - so-firewall includehost workstation "$IP" --apply + 'DESKTOP') + so-firewall includehost desktop "$IP" --apply ;; esac From 1440c7255994f7b262643a01f3ae540a53aa46c0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:06:51 -0400 Subject: [PATCH 109/350] changes for desktop referencing Rocky/CentOS to OEL --- salt/common/tools/sbin_jinja/so-desktop-install | 10 +++++----- salt/desktop/trusted-ca.sls | 2 +- salt/top.sls | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-desktop-install b/salt/common/tools/sbin_jinja/so-desktop-install index 448421f8e..2eb5c209f 100755 --- a/salt/common/tools/sbin_jinja/so-desktop-install +++ b/salt/common/tools/sbin_jinja/so-desktop-install @@ -6,8 +6,8 @@ # Elastic License 2.0. -{# we only want the script to install the desktop if it is Rocky -#} -{% if grains.os == 'Rocky' -%} +{# we only want the script to install the desktop if it is OEL -#} +{% if grains.os == 'OEL' -%} {# if this is a manager -#} {% if grains.master == grains.id.split('_')|first -%} @@ -80,12 +80,12 @@ echo "Since this is not a manager, the pillar values to enable Security Onion De {#- endif if this is a manager #} {% endif -%} -{#- if not Rocky #} +{#- if not OEL #} {%- else %} -echo "The Security Onion Desktop can only be installed on Rocky Linux. Please view the documentation at $doc_desktop_url." +echo "The Security Onion Desktop can only be installed on Oracle Linux. Please view the documentation at $doc_desktop_url." -{#- endif grains.os == Rocky #} +{#- endif grains.os == OEL #} {% endif -%} exit 0 diff --git a/salt/desktop/trusted-ca.sls b/salt/desktop/trusted-ca.sls index b9bde5ae5..87fc70ef9 100644 --- a/salt/desktop/trusted-ca.sls +++ b/salt/desktop/trusted-ca.sls @@ -31,6 +31,6 @@ update_ca_certs: desktop_trusted-ca_os_fail: test.fail_without_changes: - - comment: 'SO Desktop can only be installed on CentOS' + - comment: 'SO Desktop can only be installed on Oracle Linux' {% endif %} diff --git a/salt/top.sls b/salt/top.sls index bc51c2db1..2323731a1 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -277,10 +277,10 @@ base: - schedule - docker_clean - 'J@desktop:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:Rocky )': + 'J@desktop:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:OEL )': - match: compound - desktop - 'J@desktop:gui:enabled:^[Ff][Aa][Ll][Ss][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:Rocky )': + 'J@desktop:gui:enabled:^[Ff][Aa][Ll][Ss][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:OEL )': - match: compound - desktop.remove_gui From 4297d51a2db0b541f76f4f904bd0113cb94d7b89 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 08:14:52 -0400 Subject: [PATCH 110/350] Refactor for multiple agents --- .../sbin_jinja/so-elastic-agent-grid-upgrade | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade new file mode 100644 index 000000000..23bf304f2 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -0,0 +1,38 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. + +. /usr/sbin/so-common + +# Only run on Managers +if ! is_manager_node; then + printf "Not a Manager Node... Exiting" + exit 0 +fi + +# Get current list of Grid Node Agents that need to be upgraded +RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true") + +# Check to make sure that the server responded with good data - else, bail from script +CHECKSUM=$(jq -r '.statusSummary.online' <<< "$RAW_JSON") +if [ "$CHECKSUM" -lt 1 ]; then + printf "Failed to query for current Grid Agents...\n" + exit 1 +fi + +# Generate list of Node Agents that need updates +OUTDATED_LIST=$(jq -r '.items | map(.id) | (tojson)' <<< "$RAW_JSON") + +if [ "$OUTDATED_LIST" != '[]' ]; then + AGENTNUMBERS=$(jq -r '.total' <<< "$RAW_JSON") + printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic $ELASTIC_AGENT_TARBALL_VERSION...\n\n" + + # Generate updated JSON payload + JSON_STRING=$(jq -n --arg ELASTICVERSION $ELASTIC_AGENT_TARBALL_VERSION --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }') + + # Update Node Agents + curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +else + printf "No Agents need updates... Exiting\n\n" + exit 0 +fi \ No newline at end of file From 2d25e352d4fe201d442b82cf233d412a5ad6258d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:18:13 -0400 Subject: [PATCH 111/350] write to adv_ pillar file since that is where it would be stored from using the soc ui --- salt/common/tools/sbin_jinja/so-desktop-install | 2 +- salt/desktop/packages.sls | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-desktop-install b/salt/common/tools/sbin_jinja/so-desktop-install index 2eb5c209f..06385e810 100755 --- a/salt/common/tools/sbin_jinja/so-desktop-install +++ b/salt/common/tools/sbin_jinja/so-desktop-install @@ -13,7 +13,7 @@ source /usr/sbin/so-common doc_desktop_url="$DOC_BASE_URL/desktop.html" -pillar_file="/opt/so/saltstack/local/pillar/minions/{{grains.id}}.sls" +pillar_file="/opt/so/saltstack/local/pillar/minions/adv_{{grains.id}}.sls" if [ -f "$pillar_file" ]; then if ! grep -q "^desktop:$" "$pillar_file"; then diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 5c0121e7b..524c2c266 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -3,7 +3,6 @@ {# we only want this state to run it is CentOS #} {% if GLOBALS.os == 'OEL' %} - desktop_packages: pkg.installed: - pkgs: From e586d6b96755110a122323577c3b4395d5add033 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 08:30:19 -0400 Subject: [PATCH 112/350] Extract Elastic Agent tarball for airgap soup --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d31ee997b..b7abd05d8 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -492,6 +492,7 @@ determine_elastic_agent_upgrade() { update_elastic_agent_airgap() { rsync -av /tmp/soagupdate/fleet/* /nsm/elastic-fleet/artifacts/ + tar -xf "$ELASTIC_AGENT_FILE" -C "$ELASTIC_AGENT_EXPANSION_DIR" } verify_upgradespace() { From fe7a940082004c178995c18e1aad749ffd8e8331 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:31:54 -0400 Subject: [PATCH 113/350] add details for enabling in soc gui --- salt/common/tools/sbin_jinja/so-desktop-install | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-desktop-install b/salt/common/tools/sbin_jinja/so-desktop-install index 06385e810..bd3d9b373 100755 --- a/salt/common/tools/sbin_jinja/so-desktop-install +++ b/salt/common/tools/sbin_jinja/so-desktop-install @@ -65,7 +65,7 @@ if [ -f "$pillar_file" ]; then fi else # desktop is already added echo "The desktop pillar already exists in $pillar_file." - echo "To enable/disable the gui, set 'desktop:gui:enabled' to true or false in $pillar_file." + echo "To enable/disable the gui, set 'desktop:gui:enabled' to true or false in $pillar_file. Alternatively, this can be set in the SOC UI under advanced." echo "Additional documentation can be found at $doc_desktop_url." fi else # if the pillar file doesn't exist @@ -75,7 +75,12 @@ fi {#- if this is not a manager #} {% else -%} -echo "Since this is not a manager, the pillar values to enable Security Onion Desktop must be set manually. Please view the documentation at $doc_desktop_url." +echo "Since this is not a manager, the pillar values to enable Security Onion Desktop must be set manually. This can be enabled in the SOC UI under advanced by adding the following:" +echo "desktop:" +echo " gui:" +echo " enabled: true" +echo "" +echo "Please view the documentation at $doc_desktop_url." {#- endif if this is a manager #} {% endif -%} From 6413050f2e27fd004dfcfd7b2925b5133fa044e9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:39:46 -0400 Subject: [PATCH 114/350] set doc_desktop_url before jinja --- salt/common/tools/sbin_jinja/so-desktop-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-desktop-install b/salt/common/tools/sbin_jinja/so-desktop-install index bd3d9b373..6275bb3b6 100755 --- a/salt/common/tools/sbin_jinja/so-desktop-install +++ b/salt/common/tools/sbin_jinja/so-desktop-install @@ -5,14 +5,14 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +source /usr/sbin/so-common +doc_desktop_url="$DOC_BASE_URL/desktop.html" {# we only want the script to install the desktop if it is OEL -#} {% if grains.os == 'OEL' -%} {# if this is a manager -#} {% if grains.master == grains.id.split('_')|first -%} -source /usr/sbin/so-common -doc_desktop_url="$DOC_BASE_URL/desktop.html" pillar_file="/opt/so/saltstack/local/pillar/minions/adv_{{grains.id}}.sls" if [ -f "$pillar_file" ]; then From a443c654e58f507bade98d12812ca5969985ffc6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:48:00 -0400 Subject: [PATCH 115/350] fix desktop pillar in setup --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 0f73a11a6..d138d97df 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -117,7 +117,7 @@ desktop_pillar() { " mainint: '$MNIC'"\ "desktop:"\ " gui:"\ - " enabled: true" >> "$pillar_file"\ + " enabled: true"\ "sensoroni:"\ " config:"\ " node_description: '${NODE_DESCRIPTION//\'/''}'" > $pillar_file From 28dfdbf06dc6de143716e94fd9c3432799e1421f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 08:51:39 -0400 Subject: [PATCH 116/350] securityonion_desktop is just desktop --- salt/firewall/defaults.yaml | 1 - salt/firewall/soc_firewall.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 347ddd4b0..ff127c419 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -20,7 +20,6 @@ firewall: managersearch: [] receiver: [] searchnode: [] - securityonion_desktop: [] self: [] sensor: [] standalone: [] diff --git a/salt/firewall/soc_firewall.yaml b/salt/firewall/soc_firewall.yaml index 6ba5bea76..209484b6e 100644 --- a/salt/firewall/soc_firewall.yaml +++ b/salt/firewall/soc_firewall.yaml @@ -39,7 +39,6 @@ firewall: managersearch: *hostgroupsettings receiver: *hostgroupsettings searchnode: *hostgroupsettings - securityonion_desktop: *hostgroupsettings self: *ROhostgroupsettingsadv sensor: *hostgroupsettings standalone: *hostgroupsettings From bf78faa0f081c873d371fa0ec2cf22c970755add Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 10:43:34 -0400 Subject: [PATCH 117/350] Enable upgrade check during state run --- salt/elasticfleet/enabled.sls | 5 +++++ .../tools/sbin_jinja/so-elastic-agent-grid-upgrade | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 096610af2..82c7735db 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -109,6 +109,11 @@ so-elastic-fleet: so-elastic-fleet-integrations: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-policy-load + +so-elastic-agent-grid-upgrade: + cmd.run: + - name: /usr/sbin/so-elastic-agent-grid-upgrade + - retry: True {% endif %} delete_so-elastic-fleet_so-status.disabled: diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade index 23bf304f2..b1ca8c476 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade @@ -14,8 +14,8 @@ fi RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true") # Check to make sure that the server responded with good data - else, bail from script -CHECKSUM=$(jq -r '.statusSummary.online' <<< "$RAW_JSON") -if [ "$CHECKSUM" -lt 1 ]; then +CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON") +if [ "$CHECKSUM" -ne 1 ]; then printf "Failed to query for current Grid Agents...\n" exit 1 fi From 8844e305ab5da068c670061a45c0382f500ee40d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 11:18:47 -0400 Subject: [PATCH 118/350] use sensor.interface for suricata. make af-packet.interface ro in soc ui --- salt/suricata/map.jinja | 2 +- salt/suricata/soc_suricata.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 5576117cc..2d79c30fd 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -11,7 +11,7 @@ {# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #} {# we are limited to only one iterface #} {% load_yaml as afpacket %} -- interface: {{ SURICATAMERGED.config['af-packet'].interface }} +- interface: {{ GLOBALS.sensor.interface) }} cluster-id: {{ SURICATAMERGED.config['af-packet']['cluster-id'] }} cluster-type: {{ SURICATAMERGED.config['af-packet']['cluster-type'] }} defrag: {{ SURICATAMERGED.config['af-packet'].defrag }} diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index f13e89618..30f277c0a 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -14,7 +14,9 @@ suricata: config: af-packet: interface: - description: The network interface that Suricata will monitor. + description: The network interface that Suricata will monitor. This is set under sensor > interface. + advanced: True + readonly: True helpLink: suricata.html cluster-id: advanced: True From 30e3fbb41c86fafacad01849c502d2bd9cae1753 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 11:21:16 -0400 Subject: [PATCH 119/350] remove extra ) --- salt/suricata/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/map.jinja b/salt/suricata/map.jinja index 2d79c30fd..01d019de8 100644 --- a/salt/suricata/map.jinja +++ b/salt/suricata/map.jinja @@ -11,7 +11,7 @@ {# suricata.config.af-packet has to be rewritten here since we cant display '- interface' in the ui #} {# we are limited to only one iterface #} {% load_yaml as afpacket %} -- interface: {{ GLOBALS.sensor.interface) }} +- interface: {{ GLOBALS.sensor.interface }} cluster-id: {{ SURICATAMERGED.config['af-packet']['cluster-id'] }} cluster-type: {{ SURICATAMERGED.config['af-packet']['cluster-type'] }} defrag: {{ SURICATAMERGED.config['af-packet'].defrag }} From dfe916d7c8a996c5070fc89fb1ade0d957480bfd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 15:19:17 -0400 Subject: [PATCH 120/350] add annotation for so-logs index --- salt/elasticsearch/soc_elasticsearch.yaml | 109 ++++++++++++++++------ 1 file changed, 78 insertions(+), 31 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index da22268f6..2228eccf6 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -46,28 +46,26 @@ elasticsearch: description: Max number of boolean clauses per query. global: True helpLink: elasticsearch.html - index_settings: - so-elasticsearch: &indexSettings - warm: - description: Age (in days) of this index before it will move to warm storage, if warm nodes are present. Once moved, events on this index can take longer to fetch. - global: True - helpLink: elasticsearch.html - close: - description: Age (in days) of this index before it will be closed. Once closed, events on this index cannot be retrieved without first re-opening the index. - global: True - helpLink: elasticsearch.html - delete: - description: Age (in days) of this index before it will be deleted. Once deleted, events are permanently unrecoverable. - global: True - helpLink: elasticsearch.html + index_settings: + so-logs: &indexSettings index_sorting: description: Sorts the index by event time, at the cost of additional processing resource consumption. global: True helpLink: elasticsearch.html index_template: + index_patterns: + description: Patterns for matching multiple indices or tables. + forceType: "[]string" + multiline: True + global: True + helpLink: elasticsearch.html template: settings: index: + number_of_replicas: + description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + global: True + helpLink: elasticsearch.html mapping: total_fields: limit: @@ -75,17 +73,59 @@ elasticsearch: global: True helpLink: elasticsearch.html refresh_interval: - description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. - global: True - helpLink: elasticsearch.html + description: Seconds between index refreshes. Shorter intervals can cause query performance to suffer since this is a synchronous and resource-intensive operation. + global: True + helpLink: elasticsearch.html number_of_shards: - description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. + description: Number of shards required for this index. Using multiple shards increases fault tolerance, but also increases storage and network costs. + global: True + helpLink: elasticsearch.html + sort: + field: + description: The field to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html - number_of_replicas: - description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + order: + description: The order to sort by. Must set index_sorting to True. global: True helpLink: elasticsearch.html + mappings: + _meta: + package: + name: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed_by: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed: + description: Meta settings for the mapping. + forcedType: bool + global: True + helpLink: elasticsearch.html + composed_of: + description: The index template is composed of these component templates. + forcedType: "[]string" + global: True + helpLink: elasticsearch.html + priority: + description: The priority of the index template. + forcedType: int + global: True + helpLink: elasticsearch.html + data_stream: + hidden: + description: Hide the data stream. + forcedType: bool + global: True + helpLink: elasticsearch.html + allow_custom_routing: + description: Allow custom routing for the data stream. + forcedType: bool + global: True + helpLink: elasticsearch.html policy: phases: hot: @@ -97,6 +137,7 @@ elasticsearch: set_priority: priority: description: Priority of index. This is used for recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int global: True helpLink: elasticsearch.html rollover: @@ -117,20 +158,26 @@ elasticsearch: set_priority: priority: description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities. + forcedType: int global: True helpLink: elasticsearch.html delete: min_age: description: Minimum age of index. This determines when the index should be deleted. global: True - helpLink: elastic - so-endgame: *indexSettings - so-firewall: *indexSettings - so-import: *indexSettings - so-kibana: *indexSettings - so-logstash: *indexSettings - so-osquery: *indexSettings - so-redis: *indexSettings - so-strelka: *indexSettings - so-syslog: *indexSettings - so-zeek: *indexSettings + helpLink: elasticsearch.html + _meta: + package: + name: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed_by: + description: Meta settings for the mapping. + global: True + helpLink: elasticsearch.html + managed: + description: Meta settings for the mapping. + forcedType: bool + global: True + helpLink: elasticsearch.html From f9e272dd8f07613c748fee6540c88e4bea59b145 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 9 Aug 2023 16:09:23 -0400 Subject: [PATCH 121/350] add additional annotations for elasticsearch index settings --- salt/elasticsearch/soc_elasticsearch.yaml | 86 +++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 2228eccf6..89d347b42 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -181,3 +181,89 @@ elasticsearch: forcedType: bool global: True helpLink: elasticsearch.html + so-logs-system.auth: *indexSettings + so-logs-system.syslog: *indexSettings + so-logs-system.system: *indexSettings + so-logs-system.application: *indexSettings + so-logs-system.security: *indexSettings + so-logs-windows.forwarded: *indexSettings + so-logs-windows.powershell: *indexSettings + so-logs-windows.powershell_operational: *indexSettings + so-logs-windows.sysmon_operational: *indexSettings + so-logs-aws.cloudtrail: *indexSettings + so-logs-aws.cloudwatch_logs: *indexSettings + so-logs-aws.ec2_logs: *indexSettings + so-logs-aws.elb_logs: *indexSettings + so-logs-aws.firewall_logs: *indexSettings + so-logs-aws.route53_public_logs: *indexSettings + so-logs-aws.route53_resolver_logs: *indexSettings + so-logs-aws.s3access: *indexSettings + so-logs-aws.vpcflow: *indexSettings + so-logs-aws.waf: *indexSettings + so-logs-azure.activitylogs: *indexSettings + so-logs-azure.application_gateway: *indexSettings + so-logs-azure.auditlogs: *indexSettings + so-logs-azure.eventhub: *indexSettings + so-logs-azure.firewall_logs: *indexSettings + so-logs-azure.identity_protection: *indexSettings + so-logs-azure.platformlogs: *indexSettings + so-logs-azure.provisioning: *indexSettings + so-logs-azure.signinlogs: *indexSettings + so-logs-azure.springcloudlogs: *indexSettings + so-logs-cloudflare.audit: *indexSettings + so-logs-cloudflare.logpull: *indexSettings + so-logs-fim.event: *indexSettings + so-logs-github.audit: *indexSettings + so-logs-github.code_scanning: *indexSettings + so-logs-github.dependabot: *indexSettings + so-logs-github.issues: *indexSettings + so-logs-github.secret_scanning: *indexSettings + so-logs-google_workspace.access_transparency: *indexSettings + so-logs-google_workspace.admin: *indexSettings + so-logs-google_workspace.alert: *indexSettings + so-logs-google_workspace.context_aware_access: *indexSettings + so-logs-google_workspace.device: *indexSettings + so-logs-google_workspace.drive: *indexSettings + so-logs-google_workspace.gcp: *indexSettings + so-logs-google_workspace.group_enterprise: *indexSettings + so-logs-google_workspace.groups: *indexSettings + so-logs-google_workspace.login: *indexSettings + so-logs-google_workspace.rules: *indexSettings + so-logs-google_workspace.saml: *indexSettings + so-logs-google_workspace.token: *indexSettings + so-logs-google_workspace.user_accounts: *indexSettings + so-logs-1password.item_usages: *indexSettings + so-logs-1password.signin_attempts: *indexSettings + so-logs-osquery-manager-actions: *indexSettings + so-logs-osquery-manager-action.responses: *indexSettings + so-logs-elastic_agent.apm_server: *indexSettings + so-logs-elastic_agent.auditbeat: *indexSettings + so-logs-elastic_agent.cloudbeat: *indexSettings + so-logs-elastic_agent.endpoint_security: *indexSettings + so-logs-endpoint.alerts: *indexSettings + so-logs-endpoint.events.api: *indexSettings + so-logs-endpoint.events.file: *indexSettings + so-logs-endpoint.events.library: *indexSettings + so-logs-endpoint.events.network: *indexSettings + so-logs-endpoint.events.process: *indexSettings + so-logs-endpoint.events.registry: *indexSettings + so-logs-endpoint.events.security: *indexSettings + so-logs-elastic_agent.filebeat: *indexSettings + so-logs-elastic_agent.fleet_server: *indexSettings + so-logs-elastic_agent.heartbeat: *indexSettings + so-logs-elastic_agent: *indexSettings + so-logs-elastic_agent.metricbeat: *indexSettings + so-logs-elastic_agent.osquerybeat: *indexSettings + so-logs-elastic_agent.packetbeat: *indexSettings + so-case: *indexSettings + so-common: *indexSettings + so-endgame: *indexSettings + so-idh: *indexSettings + so-suricata: *indexSettings + so-import: *indexSettings + so-kratos: *indexSettings + so-logstash: *indexSettings + so-redis: *indexSettings + so-strelka: *indexSettings + so-syslog: *indexSettings + so-zeek: *indexSettings From e844cf11db85ba2a9dd51c07ad72e3a1e673a911 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 16:38:27 -0400 Subject: [PATCH 122/350] Move base_url to cert SAN --- salt/ssl/init.sls | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 063172e00..a25a7c270 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -153,8 +153,8 @@ etc_elasticfleet_crt: - ca_server: {{ ca_server }} - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-server.key - - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - CN: {{ GLOBALS.hostname }} + - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True @@ -210,8 +210,8 @@ etc_elasticfleet_logstash_crt: - ca_server: {{ ca_server }} - signing_policy: elasticfleet - private_key: /etc/pki/elasticfleet-logstash.key - - CN: {{ GLOBALS.url_base }} - - subjectAltName: DNS:{{ GLOBALS.hostname }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} + - CN: {{ GLOBALS.hostname }} + - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %} - days_remaining: 0 - days_valid: 820 - backup: True From 874dab75355479338e7dd3a06a1ff3b75615a86b Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 9 Aug 2023 19:02:53 -0400 Subject: [PATCH 123/350] Unset defaults --- .../tools/sbin_jinja/so-elastic-fleet-es-url-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 975a8aec1..3b0e01f43 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -13,7 +13,7 @@ fi function update_es_urls() { # Generate updated JSON payload - JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"config_yaml":""}') # Update Fleet Elasticsearch URLs curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" From 4d497022dbcd9730b2e0903d93a388bc48c7c564 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 10 Aug 2023 09:52:18 -0400 Subject: [PATCH 124/350] replace . with _x_ for soc ui compat --- salt/elasticsearch/defaults.yaml | 144 +++++++++--------- salt/elasticsearch/soc_elasticsearch.yaml | 144 +++++++++--------- salt/elasticsearch/template.map.jinja | 6 +- .../so-elasticsearch-ilm-policy-load | 3 +- 4 files changed, 149 insertions(+), 148 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 035079f54..579197040 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -113,7 +113,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-system.auth: + so-logs-system_x_auth: index_sorting: False index_template: index_patterns: @@ -132,7 +132,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.syslog: + so-logs-system_x_syslog: index_sorting: False index_template: index_patterns: @@ -151,7 +151,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.system: + so-logs-system_x_system: index_sorting: False index_template: index_patterns: @@ -170,7 +170,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.application: + so-logs-system_x_application: index_sorting: False index_template: index_patterns: @@ -189,7 +189,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-system.security: + so-logs-system_x_security: index_sorting: False index_template: index_patterns: @@ -208,7 +208,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.forwarded: + so-logs-windows_x_forwarded: index_sorting: False index_template: index_patterns: @@ -226,7 +226,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.powershell: + so-logs-windows_x_powershell: index_sorting: False index_template: index_patterns: @@ -244,7 +244,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.powershell_operational: + so-logs-windows_x_powershell_operational: index_sorting: False index_template: index_patterns: @@ -262,7 +262,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-windows.sysmon_operational: + so-logs-windows_x_sysmon_operational: index_sorting: False index_template: index_patterns: @@ -280,7 +280,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.cloudtrail: + so-logs-aws_x_cloudtrail: index_sorting: False index_template: index_patterns: @@ -298,7 +298,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.cloudwatch_logs: + so-logs-aws_x_cloudwatch_logs: index_sorting: False index_template: index_patterns: @@ -316,7 +316,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.ec2_logs: + so-logs-aws_x_ec2_logs: index_sorting: False index_template: index_patterns: @@ -334,7 +334,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.elb_logs: + so-logs-aws_x_elb_logs: index_sorting: False index_template: index_patterns: @@ -352,7 +352,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.firewall_logs: + so-logs-aws_x_firewall_logs: index_sorting: False index_template: index_patterns: @@ -370,7 +370,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.route53_public_logs: + so-logs-aws_x_route53_public_logs: index_sorting: False index_template: index_patterns: @@ -388,7 +388,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.route53_resolver_logs: + so-logs-aws_x_route53_resolver_logs: index_sorting: False index_template: index_patterns: @@ -406,7 +406,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.s3access: + so-logs-aws_x_s3access: index_sorting: False index_template: index_patterns: @@ -424,7 +424,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.vpcflow: + so-logs-aws_x_vpcflow: index_sorting: False index_template: index_patterns: @@ -442,7 +442,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-aws.waf: + so-logs-aws_x_waf: index_sorting: False index_template: index_patterns: @@ -460,7 +460,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.activitylogs: + so-logs-azure_x_activitylogs: index_sorting: False index_template: index_patterns: @@ -478,7 +478,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.application_gateway: + so-logs-azure_x_application_gateway: index_sorting: False index_template: index_patterns: @@ -496,7 +496,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.auditlogs: + so-logs-azure_x_auditlogs: index_sorting: False index_template: index_patterns: @@ -514,7 +514,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.eventhub: + so-logs-azure_x_eventhub: index_sorting: False index_template: index_patterns: @@ -532,7 +532,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.firewall_logs: + so-logs-azure_x_firewall_logs: index_sorting: False index_template: index_patterns: @@ -550,7 +550,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.identity_protection: + so-logs-azure_x_identity_protection: index_sorting: False index_template: index_patterns: @@ -568,7 +568,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.platformlogs: + so-logs-azure_x_platformlogs: index_sorting: False index_template: index_patterns: @@ -586,7 +586,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.provisioning: + so-logs-azure_x_provisioning: index_sorting: False index_template: index_patterns: @@ -604,7 +604,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.signinlogs: + so-logs-azure_x_signinlogs: index_sorting: False index_template: index_patterns: @@ -622,7 +622,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-azure.springcloudlogs: + so-logs-azure_x_springcloudlogs: index_sorting: False index_template: index_patterns: @@ -640,7 +640,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-cloudflare.audit: + so-logs-cloudflare_x_audit: index_sorting: False index_template: index_patterns: @@ -658,7 +658,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-cloudflare.logpull: + so-logs-cloudflare_x_logpull: index_sorting: False index_template: index_patterns: @@ -676,7 +676,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-fim.event: + so-logs-fim_x_event: index_sorting: False index_template: index_patterns: @@ -694,7 +694,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.audit: + so-logs-github_x_audit: index_sorting: False index_template: index_patterns: @@ -712,7 +712,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.code_scanning: + so-logs-github_x_code_scanning: index_sorting: False index_template: index_patterns: @@ -730,7 +730,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.dependabot: + so-logs-github_x_dependabot: index_sorting: False index_template: index_patterns: @@ -748,7 +748,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.issues: + so-logs-github_x_issues: index_sorting: False index_template: index_patterns: @@ -766,7 +766,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-github.secret_scanning: + so-logs-github_x_secret_scanning: index_sorting: False index_template: index_patterns: @@ -784,7 +784,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.access_transparency: + so-logs-google_workspace_x_access_transparency: index_sorting: False index_template: index_patterns: @@ -802,7 +802,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.admin: + so-logs-google_workspace_x_admin: index_sorting: False index_template: index_patterns: @@ -820,7 +820,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.alert: + so-logs-google_workspace_x_alert: index_sorting: False index_template: index_patterns: @@ -838,7 +838,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.context_aware_access: + so-logs-google_workspace_x_context_aware_access: index_sorting: False index_template: index_patterns: @@ -856,7 +856,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.device: + so-logs-google_workspace_x_device: index_sorting: False index_template: index_patterns: @@ -874,7 +874,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.drive: + so-logs-google_workspace_x_drive: index_sorting: False index_template: index_patterns: @@ -892,7 +892,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.gcp: + so-logs-google_workspace_x_gcp: index_sorting: False index_template: index_patterns: @@ -910,7 +910,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.group_enterprise: + so-logs-google_workspace_x_group_enterprise: index_sorting: False index_template: index_patterns: @@ -928,7 +928,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.groups: + so-logs-google_workspace_x_groups: index_sorting: False index_template: index_patterns: @@ -946,7 +946,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.login: + so-logs-google_workspace_x_login: index_sorting: False index_template: index_patterns: @@ -964,7 +964,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.rules: + so-logs-google_workspace_x_rules: index_sorting: False index_template: index_patterns: @@ -982,7 +982,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.saml: + so-logs-google_workspace_x_saml: index_sorting: False index_template: index_patterns: @@ -1000,7 +1000,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.token: + so-logs-google_workspace_x_token: index_sorting: False index_template: index_patterns: @@ -1018,7 +1018,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-google_workspace.user_accounts: + so-logs-google_workspace_x_user_accounts: index_sorting: False index_template: index_patterns: @@ -1036,7 +1036,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-1password.item_usages: + so-logs-1password_x_item_usages: index_sorting: False index_template: index_patterns: @@ -1054,7 +1054,7 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-1password.signin_attempts: + so-logs-1password_x_signin_attempts: index_sorting: False index_template: index_patterns: @@ -1089,7 +1089,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-osquery-manager-action.responses: + so-logs-osquery-manager-action_x_responses: index_sorting: False index_template: index_patterns: @@ -1106,7 +1106,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.apm_server: + so-logs-elastic_agent_x_apm_server: index_sorting: False index_template: index_patterns: @@ -1160,7 +1160,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.auditbeat: + so-logs-elastic_agent_x_auditbeat: index_sorting: False index_template: index_patterns: @@ -1214,7 +1214,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.cloudbeat: + so-logs-elastic_agent_x_cloudbeat: index_sorting: False index_template: index_patterns: @@ -1265,7 +1265,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.endpoint_security: + so-logs-elastic_agent_x_endpoint_security: index_sorting: False index_template: index_patterns: @@ -1314,7 +1314,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.alerts: + so-logs-endpoint_x_alerts: index_sorting: False index_template: index_patterns: @@ -1363,7 +1363,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.api: + so-logs-endpoint_x_events_x_api: index_sorting: False index_template: index_patterns: @@ -1412,7 +1412,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.file: + so-logs-endpoint_x_events_x_file: index_sorting: False index_template: index_patterns: @@ -1461,7 +1461,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.library: + so-logs-endpoint_x_events_x_library: index_sorting: False index_template: index_patterns: @@ -1510,7 +1510,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.network: + so-logs-endpoint_x_events_x_network: index_sorting: False index_template: index_patterns: @@ -1559,7 +1559,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.process: + so-logs-endpoint_x_events_x_process: index_sorting: False index_template: index_patterns: @@ -1608,7 +1608,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.registry: + so-logs-endpoint_x_events_x_registry: index_sorting: False index_template: index_patterns: @@ -1657,7 +1657,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-endpoint.events.security: + so-logs-endpoint_x_events_x_security: index_sorting: False index_template: index_patterns: @@ -1706,7 +1706,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.filebeat: + so-logs-elastic_agent_x_filebeat: index_sorting: False index_template: index_patterns: @@ -1755,7 +1755,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.fleet_server: + so-logs-elastic_agent_x_fleet_server: index_sorting: False index_template: index_patterns: @@ -1801,7 +1801,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.heartbeat: + so-logs-elastic_agent_x_heartbeat: index_sorting: False index_template: index_patterns: @@ -1907,7 +1907,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.metricbeat: + so-logs-elastic_agent_x_metricbeat: index_sorting: False index_template: index_patterns: @@ -1956,7 +1956,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.osquerybeat: + so-logs-elastic_agent_x_osquerybeat: index_sorting: False index_template: index_patterns: @@ -2005,7 +2005,7 @@ elasticsearch: name: elastic_agent managed_by: security_onion managed: true - so-logs-elastic_agent.packetbeat: + so-logs-elastic_agent_x_packetbeat: index_sorting: False index_template: index_patterns: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 89d347b42..889e9f6a4 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -181,80 +181,80 @@ elasticsearch: forcedType: bool global: True helpLink: elasticsearch.html - so-logs-system.auth: *indexSettings - so-logs-system.syslog: *indexSettings - so-logs-system.system: *indexSettings - so-logs-system.application: *indexSettings - so-logs-system.security: *indexSettings - so-logs-windows.forwarded: *indexSettings - so-logs-windows.powershell: *indexSettings - so-logs-windows.powershell_operational: *indexSettings - so-logs-windows.sysmon_operational: *indexSettings - so-logs-aws.cloudtrail: *indexSettings - so-logs-aws.cloudwatch_logs: *indexSettings - so-logs-aws.ec2_logs: *indexSettings - so-logs-aws.elb_logs: *indexSettings - so-logs-aws.firewall_logs: *indexSettings - so-logs-aws.route53_public_logs: *indexSettings - so-logs-aws.route53_resolver_logs: *indexSettings - so-logs-aws.s3access: *indexSettings - so-logs-aws.vpcflow: *indexSettings - so-logs-aws.waf: *indexSettings - so-logs-azure.activitylogs: *indexSettings - so-logs-azure.application_gateway: *indexSettings - so-logs-azure.auditlogs: *indexSettings - so-logs-azure.eventhub: *indexSettings - so-logs-azure.firewall_logs: *indexSettings - so-logs-azure.identity_protection: *indexSettings - so-logs-azure.platformlogs: *indexSettings - so-logs-azure.provisioning: *indexSettings - so-logs-azure.signinlogs: *indexSettings - so-logs-azure.springcloudlogs: *indexSettings - so-logs-cloudflare.audit: *indexSettings - so-logs-cloudflare.logpull: *indexSettings - so-logs-fim.event: *indexSettings - so-logs-github.audit: *indexSettings - so-logs-github.code_scanning: *indexSettings - so-logs-github.dependabot: *indexSettings - so-logs-github.issues: *indexSettings - so-logs-github.secret_scanning: *indexSettings - so-logs-google_workspace.access_transparency: *indexSettings - so-logs-google_workspace.admin: *indexSettings - so-logs-google_workspace.alert: *indexSettings - so-logs-google_workspace.context_aware_access: *indexSettings - so-logs-google_workspace.device: *indexSettings - so-logs-google_workspace.drive: *indexSettings - so-logs-google_workspace.gcp: *indexSettings - so-logs-google_workspace.group_enterprise: *indexSettings - so-logs-google_workspace.groups: *indexSettings - so-logs-google_workspace.login: *indexSettings - so-logs-google_workspace.rules: *indexSettings - so-logs-google_workspace.saml: *indexSettings - so-logs-google_workspace.token: *indexSettings - so-logs-google_workspace.user_accounts: *indexSettings - so-logs-1password.item_usages: *indexSettings - so-logs-1password.signin_attempts: *indexSettings + so-logs-system_x_auth: *indexSettings + so-logs-system_x_syslog: *indexSettings + so-logs-system_x_system: *indexSettings + so-logs-system_x_application: *indexSettings + so-logs-system_x_security: *indexSettings + so-logs-windows_x_forwarded: *indexSettings + so-logs-windows_x_powershell: *indexSettings + so-logs-windows_x_powershell_operational: *indexSettings + so-logs-windows_x_sysmon_operational: *indexSettings + so-logs-aws_x_cloudtrail: *indexSettings + so-logs-aws_x_cloudwatch_logs: *indexSettings + so-logs-aws_x_ec2_logs: *indexSettings + so-logs-aws_x_elb_logs: *indexSettings + so-logs-aws_x_firewall_logs: *indexSettings + so-logs-aws_x_route53_public_logs: *indexSettings + so-logs-aws_x_route53_resolver_logs: *indexSettings + so-logs-aws_x_s3access: *indexSettings + so-logs-aws_x_vpcflow: *indexSettings + so-logs-aws_x_waf: *indexSettings + so-logs-azure_x_activitylogs: *indexSettings + so-logs-azure_x_application_gateway: *indexSettings + so-logs-azure_x_auditlogs: *indexSettings + so-logs-azure_x_eventhub: *indexSettings + so-logs-azure_x_firewall_logs: *indexSettings + so-logs-azure_x_identity_protection: *indexSettings + so-logs-azure_x_platformlogs: *indexSettings + so-logs-azure_x_provisioning: *indexSettings + so-logs-azure_x_signinlogs: *indexSettings + so-logs-azure_x_springcloudlogs: *indexSettings + so-logs-cloudflare_x_audit: *indexSettings + so-logs-cloudflare_x_logpull: *indexSettings + so-logs-fim_x_event: *indexSettings + so-logs-github_x_audit: *indexSettings + so-logs-github_x_code_scanning: *indexSettings + so-logs-github_x_dependabot: *indexSettings + so-logs-github_x_issues: *indexSettings + so-logs-github_x_secret_scanning: *indexSettings + so-logs-google_workspace_x_access_transparency: *indexSettings + so-logs-google_workspace_x_admin: *indexSettings + so-logs-google_workspace_x_alert: *indexSettings + so-logs-google_workspace_x_context_aware_access: *indexSettings + so-logs-google_workspace_x_device: *indexSettings + so-logs-google_workspace_x_drive: *indexSettings + so-logs-google_workspace_x_gcp: *indexSettings + so-logs-google_workspace_x_group_enterprise: *indexSettings + so-logs-google_workspace_x_groups: *indexSettings + so-logs-google_workspace_x_login: *indexSettings + so-logs-google_workspace_x_rules: *indexSettings + so-logs-google_workspace_x_saml: *indexSettings + so-logs-google_workspace_x_token: *indexSettings + so-logs-google_workspace_x_user_accounts: *indexSettings + so-logs-1password_x_item_usages: *indexSettings + so-logs-1password_x_signin_attempts: *indexSettings so-logs-osquery-manager-actions: *indexSettings - so-logs-osquery-manager-action.responses: *indexSettings - so-logs-elastic_agent.apm_server: *indexSettings - so-logs-elastic_agent.auditbeat: *indexSettings - so-logs-elastic_agent.cloudbeat: *indexSettings - so-logs-elastic_agent.endpoint_security: *indexSettings - so-logs-endpoint.alerts: *indexSettings - so-logs-endpoint.events.api: *indexSettings - so-logs-endpoint.events.file: *indexSettings - so-logs-endpoint.events.library: *indexSettings - so-logs-endpoint.events.network: *indexSettings - so-logs-endpoint.events.process: *indexSettings - so-logs-endpoint.events.registry: *indexSettings - so-logs-endpoint.events.security: *indexSettings - so-logs-elastic_agent.filebeat: *indexSettings - so-logs-elastic_agent.fleet_server: *indexSettings - so-logs-elastic_agent.heartbeat: *indexSettings + so-logs-osquery-manager-action_x_responses: *indexSettings + so-logs-elastic_agent_x_apm_server: *indexSettings + so-logs-elastic_agent_x_auditbeat: *indexSettings + so-logs-elastic_agent_x_cloudbeat: *indexSettings + so-logs-elastic_agent_x_endpoint_security: *indexSettings + so-logs-endpoint_x_alerts: *indexSettings + so-logs-endpoint_x_events_x_api: *indexSettings + so-logs-endpoint_x_events_x_file: *indexSettings + so-logs-endpoint_x_events_x_library: *indexSettings + so-logs-endpoint_x_events_x_network: *indexSettings + so-logs-endpoint_x_events_x_process: *indexSettings + so-logs-endpoint_x_events_x_registry: *indexSettings + so-logs-endpoint_x_events_x_security: *indexSettings + so-logs-elastic_agent_x_filebeat: *indexSettings + so-logs-elastic_agent_x_fleet_server: *indexSettings + so-logs-elastic_agent_x_heartbeat: *indexSettings so-logs-elastic_agent: *indexSettings - so-logs-elastic_agent.metricbeat: *indexSettings - so-logs-elastic_agent.osquerybeat: *indexSettings - so-logs-elastic_agent.packetbeat: *indexSettings + so-logs-elastic_agent_x_metricbeat: *indexSettings + so-logs-elastic_agent_x_osquerybeat: *indexSettings + so-logs-elastic_agent_x_packetbeat: *indexSettings so-case: *indexSettings so-common: *indexSettings so-endgame: *indexSettings diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 49d86d187..5fe0ed303 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,9 +1,11 @@ {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} -{% for index, settings in ES_INDEX_SETTINGS.items() %} +{%- set ES_INDEX_SETTINGS_ORIG = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} +{% set ES_INDEX_SETTINGS = {} %} +{% for index, settings in ES_INDEX_SETTINGS_ORIG.items() %} {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endif %} + {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_ORIG[index]}) %} {% endfor %} diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load index afb8bdc67..b00fcbedf 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load @@ -6,8 +6,7 @@ . /usr/sbin/so-common -{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} -{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} +{%- from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} {%- for index, settings in ES_INDEX_SETTINGS.items() %} {%- if settings.policy is defined %} From e43900074a3b2d02ec147ce69da786f9d0d6e9d6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 10 Aug 2023 11:54:49 -0400 Subject: [PATCH 125/350] ensure only 1 instance of so-rule-update runs. execute the cmd at the end of state run --- salt/idstools/enabled.sls | 6 ++-- salt/idstools/tools/sbin_jinja/so-rule-update | 34 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index 966cb6786..3f5acda19 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -63,19 +63,21 @@ delete_so-idstools_so-status.disabled: so-rule-update: cron.present: - - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1 + - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download_cron.log 2>&1 - identifier: so-rule-update - user: root - minute: '1' - hour: '7' +# order this last to give so-idstools container time to be ready run_so-rule-update: cmd.run: - - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1' + - name: '/usr/sbin/so-rule-update > /opt/so/log/idstools/download_idstools_state.log 2>&1' - require: - docker_container: so-idstools - onchanges: - file: idstoolsetcsync + - order: last {% else %} diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update index 504831f9f..db110abc1 100755 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ b/salt/idstools/tools/sbin_jinja/so-rule-update @@ -1,5 +1,9 @@ #!/bin/bash -. /usr/sbin/so-common + +# if this script isn't already running +if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then + + . /usr/sbin/so-common {%- from 'vars/globals.map.jinja' import GLOBALS %} {%- from 'idstools/map.jinja' import IDSTOOLSMERGED %} @@ -9,28 +13,30 @@ # Download the rules from the internet {%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy="{{ noproxy }}" + export http_proxy={{ proxy }} + export https_proxy={{ proxy }} + export no_proxy="{{ noproxy }}" {%- endif %} -mkdir -p /nsm/rules/suricata -chown -R socore:socore /nsm/rules/suricata + mkdir -p /nsm/rules/suricata + chown -R socore:socore /nsm/rules/suricata # Download the rules from the internet {%- if GLOBALS.airgap != 'True' %} {%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} -docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force + docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force {%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} -docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} + docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --etpro={{ IDSTOOLSMERGED.config.oinkcode }} {%- elif IDSTOOLSMERGED.config.ruleset == 'TALOS' %} -docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }} + docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force --url=https://www.snort.org/rules/snortrules-snapshot-2983.tar.gz?oinkcode={{ IDSTOOLSMERGED.config.oinkcode }} {%- endif %} {%- endif %} -argstr="" -for arg in "$@"; do - argstr="${argstr} \"${arg}\"" -done + argstr="" + for arg in "$@"; do + argstr="${argstr} \"${arg}\"" + done -docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}" + docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}" + +fi From 4426437ad35e5b743bfb010edc8d511bc1f35270 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 10 Aug 2023 15:04:31 -0400 Subject: [PATCH 126/350] Update motd.md --- salt/soc/files/soc/motd.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index b5a4fac5f..cf22e863d 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -8,6 +8,10 @@ If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to To see all the latest features and fixes in this version of Security Onion, click the upper-right menu and then click the [What's New](/docs/release-notes.html) link. +## Enterprise Appliances + +Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonionsolutions.com/hardware/)! + ## Customize This Space Make this area your own by customizing the content in the [Config](/#/config?s=soc.files.soc.motd__md) interface. From caced64d1144d1f05797650a22caa979ebd2ac96 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 10 Aug 2023 16:10:39 -0400 Subject: [PATCH 127/350] set desktop background --- salt/desktop/files/00-background | 8 ++++++++ salt/desktop/xwindows.sls | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 salt/desktop/files/00-background diff --git a/salt/desktop/files/00-background b/salt/desktop/files/00-background new file mode 100644 index 000000000..6f76c6408 --- /dev/null +++ b/salt/desktop/files/00-background @@ -0,0 +1,8 @@ +# Specify the dconf path +[org/gnome/desktop/background] + +# Specify the path to the desktop background image file +picture-uri='file:///usr/local/share/backgrounds/so-wallpaper.jpg' + +# Specify one of the rendering options for the background image: +picture-options='zoom' diff --git a/salt/desktop/xwindows.sls b/salt/desktop/xwindows.sls index c7790f9f4..b18109d45 100644 --- a/salt/desktop/xwindows.sls +++ b/salt/desktop/xwindows.sls @@ -35,6 +35,23 @@ convert_gnome_classic: {% endif %} {% endfor %} +desktop_wallpaper: + file.managed: + - name: /usr/local/share/backgrounds/so-wallpaper.jpg + - source: salt://desktop/files/so-wallpaper.jpg + - makedirs: True + +set_wallpaper: + file.managed: + - name: /etc/dconf/db/local.d/00-background + - source: salt://desktop/files/00-background + +run_dconf_update: + cmd.run: + - name: 'dconf update' + - onchanges: + - file: set_wallpaper + {% else %} desktop_xwindows_os_fail: From 0d894b7f527e8e35949aa01b2530294e0b1fe63e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 10 Aug 2023 18:57:17 -0400 Subject: [PATCH 128/350] Upgrade integration packages --- salt/elasticfleet/defaults.yaml | 7 +++++++ .../tools/sbin/so-elastic-fleet-common | 5 +++++ .../so-elastic-fleet-integration-policy-load | 3 +++ .../sbin_jinja/so-elastic-fleet-package-upgrade | 17 +++++++++++++++++ 4 files changed, 32 insertions(+) create mode 100644 salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 0ae7a5176..c30d49dd4 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -29,8 +29,15 @@ elasticfleet: - azure - cloudflare - endpoint + - fleet_server - fim - github - google_workspace - log + - osquery_manager + - redis + - system + - tcp + - udp + - windows - 1password diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 73c36e5c8..197a111fb 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -56,6 +56,11 @@ elastic_fleet_package_version_check() { curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version' } +elastic_fleet_package_latest_version_check() { + PACKAGE=$1 + curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.latestVersion' +} + elastic_fleet_package_install() { PKGKEY=$1 curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PKGKEY" diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load index 501aafbda..ae0fbb6ba 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load @@ -9,6 +9,9 @@ RETURN_CODE=0 if [ ! -f /opt/so/state/eaintegrations.txt ]; then + # First, check for any package upgrades + /usr/sbin/so-elastic-fleet-package-upgrade + # Initial Endpoints for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json do diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade new file mode 100644 index 000000000..81eb01534 --- /dev/null +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -0,0 +1,17 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use +# this file except in compliance with the Elastic License 2.0. +{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %} +{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %} + +. /usr/sbin/so-elastic-fleet-common + +{%- for PACKAGE in SUPPORTED_PACKAGES %} +echo "Upgrading {{ PACKAGE }} package..." +VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}") +elastic_fleet_package_install "{{ PACKAGE }}-$VERSION" +echo +{%- endfor %} +echo From 1d83b2f2e644aa8a68b7982265f91ce12c98e60e Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 10 Aug 2023 19:51:12 -0400 Subject: [PATCH 129/350] Add elasticsearch integration --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index c30d49dd4..cb282aade 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -28,6 +28,7 @@ elasticfleet: - aws - azure - cloudflare + - elasticsearch - endpoint - fleet_server - fim From fdb2ca4167311f9044299a548337ecfcb4943351 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 11 Aug 2023 09:15:41 -0400 Subject: [PATCH 130/350] set SO desktop wallpaper for iso install --- setup/so-functions | 9 +++++++++ setup/so-setup | 2 ++ 2 files changed, 11 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index d138d97df..1a1eb1919 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2302,6 +2302,15 @@ set_default_log_size() { log_size_limit=$( echo "$disk_size_gb" "$percentage" | awk '{printf("%.0f", $1 * ($2/100))}') } +set_desktop_background() { + + logCmd "mkdir /usr/local/share/backgrounds" + logCmd "cp ../salt/desktop/files/so-wallpaper.jpg /usr/local/share/backgrounds/so-wallpaper.jpg" + logCmd "cp ../salt/desktop/files/00-background /etc/dconf/db/local.d/00-background" + logCmd "dconf update" + +} + set_hostname() { logCmd "hostnamectl set-hostname --static $HOSTNAME" diff --git a/setup/so-setup b/setup/so-setup index ccc9f6f2f..d048cc8bc 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -341,6 +341,8 @@ if [[ $is_desktop ]]; then securityonion_repo info "Enabling graphical interface and setting it to load at boot" systemctl set-default graphical.target + info "Setting desktop background" + set_desktop_background echo "Desktop Install Complete!" echo "" echo "Please reboot to start graphical interface." From 3f054031a0bc1e3ac5932d1c5437919a698dcf81 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 11 Aug 2023 13:32:22 -0400 Subject: [PATCH 131/350] Set default for import and eval only --- .../tools/sbin_jinja/so-elastic-fleet-es-url-update | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 3b0e01f43..481287eef 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -12,9 +12,13 @@ if ! is_manager_node; then fi function update_es_urls() { - # Generate updated JSON payload + + # Generate updated JSON payload +{% if grains.role not in ['so-import', 'so-eval'] %} JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"config_yaml":""}') - +{%- else %} + JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"so-manager_elasticsearch","type":"elasticsearch","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}') +{%- endif %} # Update Fleet Elasticsearch URLs curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" } From a5e60363cf41793de89026c6f55d40ab2ad8c7d7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 11 Aug 2023 13:38:16 -0400 Subject: [PATCH 132/350] add missing annotations to avoid soc crash --- salt/soc/soc_soc.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index e3d704e80..03fd47e80 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -45,9 +45,10 @@ soc: actions: description: A list of actions a user can take from the SOC UI against a hunt, alert, and other records. The action must be defined in JSON object format, and contain a "name" key and "links" key. The links is a list of URLs, where the most suitable URL in the list will be the selected URL when the user clicks the action. global: True + forcedType: "[]{}" eventFields: default: - description: The list of fields to show as columns in the Hunt/Dashboards event table, when no other specific mapping applies. Mappings are defined by the format ":event.module:event.dataset". + description: Event fields mappings are defined by the format ":event.module:event.dataset", so if you would like to customize which fields show for syslog events of originating from zeek you will find that entry in the left panel that looks like :zeek:syslog. This default entry is used for all events that do not match an existing mapping defined on the left side of this configuration screen. global: True advanced: True server: @@ -139,6 +140,7 @@ soc: description: List of available external tools visible in the SOC UI. Each tool is defined in JSON object notation, and must include the "name" key and "link" key, where the link is the tool's URL. global: True advanced: True + forcedType: "[]{}" hunt: &appSettings groupItemsPerPage: description: Default number of aggregations to show per page. Larger values consume more vertical area in the SOC UI. @@ -164,6 +166,12 @@ soc: queries: description: List of default queries to show in the query list. Each query is represented in JSON object notation, and must include the "name" key and "query" key. global: True + forcedType: "[]{}" + queryToggleFilters: + description: Customize togglable query filters that apply to all queries. Exclusive toggles will invert the filter if toggled off rather than omitting the filter from the query. + global: True + advanced: True + forcedType: "[]{}" alerts: *appSettings cases: *appSettings dashboards: *appSettings From 1fb3a595735fdbebf9eed6e8649b5a919d1f2f61 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 11 Aug 2023 13:41:58 -0400 Subject: [PATCH 133/350] add missing annotations to avoid soc crash --- salt/soc/soc_soc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 03fd47e80..b2ed893f6 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -48,7 +48,7 @@ soc: forcedType: "[]{}" eventFields: default: - description: Event fields mappings are defined by the format ":event.module:event.dataset", so if you would like to customize which fields show for syslog events of originating from zeek you will find that entry in the left panel that looks like :zeek:syslog. This default entry is used for all events that do not match an existing mapping defined on the left side of this configuration screen. + description: Event fields mappings are defined by the format ":event.module:event.dataset". For example, to customize which fields show for 'syslog' events originating from 'zeek', find the eventField item in the left panel that looks like ':zeek:syslog'. This 'default' entry is used for all events that do not match an existing mapping defined in the list to the left. global: True advanced: True server: From ce63e47fcdc5b134e9b94c048d35e0d86ee9eae4 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 11 Aug 2023 14:47:33 -0400 Subject: [PATCH 134/350] Enable forced update --- .../tools/sbin_jinja/so-elastic-fleet-es-url-update | 7 +++++++ salt/manager/tools/sbin/soup | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update index 481287eef..5d5b7e7e0 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update @@ -46,6 +46,13 @@ NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "$ NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}') # Compare the current & new list of URLs - if different, update the Fleet Elasticsearch URLs +if [ "$1" = "--force" ]; then + printf "\nUpdating List, since --force was specified.\n" + printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" + update_es_urls + exit 0 +fi + if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then printf "\nHashes match - no update needed.\n" printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n" diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index ebc9c8074..a41d60648 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -425,7 +425,8 @@ post_to_2.4.5() { } post_to_2.4.10() { - echo "Nothing to apply" + echo "Updating Elastic Fleet ES URLs...." + /sbin/so-elastic-fleet-es-url-update --force POSTVERSION=2.4.10 } From 00297cd864706036628bf607e580da1bc5164f07 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 11 Aug 2023 16:10:16 -0400 Subject: [PATCH 135/350] Move from post to pre --- salt/manager/tools/sbin/soup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a41d60648..8054fb14d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -425,8 +425,7 @@ post_to_2.4.5() { } post_to_2.4.10() { - echo "Updating Elastic Fleet ES URLs...." - /sbin/so-elastic-fleet-es-url-update --force + echo "Nothing to apply" POSTVERSION=2.4.10 } @@ -491,8 +490,8 @@ up_to_2.4.5() { } up_to_2.4.10() { - echo "Nothing to do for 2.4.10" - + echo "Updating Elastic Fleet ES URLs...." + /sbin/so-elastic-fleet-es-url-update --force INSTALLEDVERSION=2.4.10 } From f38b77892b4a5c94314694e11f7f1523728232c1 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 11 Aug 2023 17:14:48 -0400 Subject: [PATCH 136/350] Move back --- salt/manager/tools/sbin/soup | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8054fb14d..b242fd279 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -425,7 +425,8 @@ post_to_2.4.5() { } post_to_2.4.10() { - echo "Nothing to apply" + echo "Updating Elastic Fleet ES URLs...." + /sbin/so-elastic-fleet-es-url-update --force POSTVERSION=2.4.10 } @@ -490,8 +491,8 @@ up_to_2.4.5() { } up_to_2.4.10() { - echo "Updating Elastic Fleet ES URLs...." - /sbin/so-elastic-fleet-es-url-update --force + echo "Nothing to do for 2.4.10" + INSTALLEDVERSION=2.4.10 } From f3a58cd336175fb7c8b4781315a330e26057c0ee Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sun, 13 Aug 2023 16:46:32 -0400 Subject: [PATCH 137/350] soup should respect current indentation in soc_global.sls --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index b242fd279..af09cc9df 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -562,7 +562,7 @@ update_version() { echo "Updating the Security Onion version file." echo $NEWVERSION > /etc/soversion echo $HOTFIXVERSION > /etc/sohotfix - sed -i "/ soversion:/c\ soversion: $NEWVERSION" /opt/so/saltstack/local/pillar/global/soc_global.sls + sed -i "s/soversion:.*/soversion: $NEWVERSION/" /opt/so/saltstack/local/pillar/global/soc_global.sls } upgrade_check() { From 4106d1f69d624467b1b123a81a1a7ee3784636ea Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 14 Aug 2023 16:33:08 -0400 Subject: [PATCH 138/350] 2.4.10 --- DOWNLOAD_AND_VERIFY_ISO.md | 16 ++++++++-------- sigs/securityonion-2.4.10-20230815.iso.sig | Bin 0 -> 566 bytes 2 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 sigs/securityonion-2.4.10-20230815.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index b9b3da297..7fed2991c 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.4.5-20230807 ISO image released on 2023/08/07 +### 2.4.10-20230815 ISO image released on 2023/08/07 ### Download and Verify -2.4.5-20230807 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso +2.4.10-20230815 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso MD5: F83FD635025A3A65B380EAFCEB61A92E SHA1: 5864D4CD520617E3328A3D956CAFCC378A8D2D08 SHA256: D333BAE0DD198DFD80DF59375456D228A4E18A24EDCDB15852CD4CA3F92B69A7 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.5-20230807.iso.sig securityonion-2.4.5-20230807.iso +gpg --verify securityonion-2.4.10-20230815.iso.sig securityonion-2.4.10-20230815.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Sat 05 Aug 2023 10:12:46 AM EDT using RSA key ID FE507013 +gpg: Signature made Sun 13 Aug 2023 05:30:29 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.10-20230815.iso.sig b/sigs/securityonion-2.4.10-20230815.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..636dfe63b7dcce9057c6f73a795234f32821fd18 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%X}sbpQ$p5PT3| zxBgIY6Xz`t|9z8nQ?NY-QP9ZVJcc{@E~xDra#)inhc&W^+E5YB0bD^Bzk#8o&v9UB zxd(n%%3G*3*hRPWTldLgI9~ntBKzxDYiPlkWr~(0Jx4XeyrA~uYHcayLQyh<`6U0? z`#=nl;KmqW(T5QYQHAMp+t~zk{yAcJ8M9l2 zf2L=<>F}kZ!^Q1QANb2fn?Rd(9z}#a+tD;zwxczvdTxD@$YDHnX14+3t z=m|8_td=SDU~~}Ve&|n>{W(>gg&;1M%$>SZ5#_K#;IF{ONymNV_^X`^sxq#_CfwN5 zyh33;LJA;Y%VYqwn~=mVL(OmODVn|zJPyvXXza5!Z EXqbu^eE Date: Mon, 14 Aug 2023 16:34:32 -0400 Subject: [PATCH 139/350] 2.4.10 --- DOWNLOAD_AND_VERIFY_ISO.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 7fed2991c..980bb062f 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -7,9 +7,9 @@ 2.4.10-20230815 ISO image: https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso -MD5: F83FD635025A3A65B380EAFCEB61A92E -SHA1: 5864D4CD520617E3328A3D956CAFCC378A8D2D08 -SHA256: D333BAE0DD198DFD80DF59375456D228A4E18A24EDCDB15852CD4CA3F92B69A7 +MD5: 97AEC929FB1FC22F106C0C93E3476FAB +SHA1: 78AF37FD19FDC34BA324C1A661632D19D1F2284A +SHA256: D04BA45D1664FC3CF7EA2188CB7E570642F6390C3959B4AFBB8222A853859394 Signature for ISO image: https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig From ad9da07de1ef4ca7e307cec31dab3d615064d341 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 14 Aug 2023 16:51:24 -0400 Subject: [PATCH 140/350] Update DOWNLOAD_AND_VERIFY_ISO.md --- DOWNLOAD_AND_VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 980bb062f..816c4f827 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 2.4.10-20230815 ISO image released on 2023/08/07 +### 2.4.10-20230815 ISO image released on 2023/08/15 From 075ef5e02cf04df448c877905a6173708889d766 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 15 Aug 2023 07:27:48 -0400 Subject: [PATCH 141/350] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index b0f6bf0cd..a3ab5389f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.10 +2.4.20 From b22776dc5a380cae37f7fc4e256137e519f4005a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 15 Aug 2023 16:22:02 -0400 Subject: [PATCH 142/350] set timezone to etc/utc during setup --- setup/so-functions | 7 +++++++ setup/so-setup | 2 ++ 2 files changed, 9 insertions(+) diff --git a/setup/so-functions b/setup/so-functions index 1a1eb1919..fc0876248 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2372,6 +2372,13 @@ set_redirect() { ;; esac } + +set_timezone() { + + logCmd "timedatectl set-timezone Etc/UTC" + +} + so_add_user() { local username=$1 local uid=$2 diff --git a/setup/so-setup b/setup/so-setup index d048cc8bc..6bca72ab7 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -576,6 +576,7 @@ if ! [[ -f $install_opt_file ]]; then fi if [[ $waitforstate ]]; then + set_timezone touch /root/accept_changes make_some_dirs percentage=0 @@ -725,6 +726,7 @@ if ! [[ -f $install_opt_file ]]; then systemctl restart salt-minion verify_setup else + set_timezone touch /root/accept_changes mkdir -p /opt/so es_heapsize From 53d7d69135eeaace0505605f038a15efb6ad75ac Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 16 Aug 2023 08:46:24 -0400 Subject: [PATCH 143/350] update salt docs url in service file --- salt/salt/service/salt-minion.service.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/service/salt-minion.service.jinja b/salt/salt/service/salt-minion.service.jinja index c7bae0bc2..27c7a15b6 100644 --- a/salt/salt/service/salt-minion.service.jinja +++ b/salt/salt/service/salt-minion.service.jinja @@ -1,6 +1,6 @@ [Unit] Description=The Salt Minion -Documentation=man:salt-minion(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html +Documentation=man:salt-minion(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltproject.io/en/latest/contents.html After=network.target salt-master.service [Service] @@ -12,4 +12,4 @@ ExecStart=/usr/bin/salt-minion ExecStartPre=/bin/sleep {{ salt['pillar.get']('salt:minion:service_start_delay', service_start_delay) }} [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target From ab19fa9ece5ea3e903704a0bef07126e2bb1d586 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 16 Aug 2023 09:21:06 -0400 Subject: [PATCH 144/350] set salt log levels to info --- salt/salt/minion.sls | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 5e06a361f..43f7539f9 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -47,24 +47,24 @@ hold_salt_packages: {% endfor %} {% endif %} -remove_info_log_level_logfile: +remove_error_log_level_logfile: file.line: - name: /etc/salt/minion - - match: "log_level_logfile: info" + - match: "log_level_logfile: error" - mode: delete -remove_info_log_level: +remove_error_log_level: file.line: - name: /etc/salt/minion - - match: "log_level: info" + - match: "log_level: error" - mode: delete set_log_levels: file.append: - name: /etc/salt/minion - text: - - "log_level: error" - - "log_level_logfile: error" + - "log_level: info" + - "log_level_logfile: info" salt_minion_service_unit_file: file.managed: From 9bf7b9bda58cc3bae475727c66d462e7bf4e5f91 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 16 Aug 2023 10:02:47 -0400 Subject: [PATCH 145/350] set the timezone earlier in setup --- setup/so-setup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index 6bca72ab7..52da8dcb2 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -58,6 +58,7 @@ while [[ $# -gt 0 ]]; do esac done +set_timezone # Preserve old setup/error logs [ -f "$error_log" ] && mv "$error_log" "$error_log.$(date +%Y-%m-%dT%H:%M:%S)" [ -f "$setup_log" ] && mv "$setup_log" "$setup_log.$(date +%Y-%m-%dT%H:%M:%S)" @@ -576,7 +577,6 @@ if ! [[ -f $install_opt_file ]]; then fi if [[ $waitforstate ]]; then - set_timezone touch /root/accept_changes make_some_dirs percentage=0 @@ -726,7 +726,6 @@ if ! [[ -f $install_opt_file ]]; then systemctl restart salt-minion verify_setup else - set_timezone touch /root/accept_changes mkdir -p /opt/so es_heapsize From e84d624d2326456fcde6c4c8230f3a1d53f3812c Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 16 Aug 2023 20:10:20 +0000 Subject: [PATCH 146/350] Force package installation --- salt/elasticfleet/tools/sbin/so-elastic-fleet-common | 5 +++-- .../tools/sbin_jinja/so-elastic-fleet-package-upgrade | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common index 197a111fb..6ada43003 100755 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common +++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common @@ -62,8 +62,9 @@ elastic_fleet_package_latest_version_check() { } elastic_fleet_package_install() { - PKGKEY=$1 - curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PKGKEY" + PKG=$1 + VERSION=$2 + curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION" } elastic_fleet_package_is_installed() { diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index 81eb01534..2fb3f7798 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -11,7 +11,7 @@ {%- for PACKAGE in SUPPORTED_PACKAGES %} echo "Upgrading {{ PACKAGE }} package..." VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}") -elastic_fleet_package_install "{{ PACKAGE }}-$VERSION" +elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION" echo {%- endfor %} echo From 4887eb4957cd0c2409bc5472e657fa4115f3df2f Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 16 Aug 2023 22:31:14 -0400 Subject: [PATCH 147/350] Update so-elastic-fleet-package-load --- .../elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load index c1e14f64f..819d7ecff 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load @@ -11,7 +11,7 @@ {%- for PACKAGE in SUPPORTED_PACKAGES %} echo "Setting up {{ PACKAGE }} package..." VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") -elastic_fleet_package_install "{{ PACKAGE }}-$VERSION" +elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION" echo {%- endfor %} echo From 7971d9749ac5bc441a4aa35e76ea2d23688b0fd7 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 17 Aug 2023 14:08:48 +0000 Subject: [PATCH 148/350] Assign pipeline to import --- .../integrations/grid-nodes_general/import-evtx-logs.json | 4 ++-- salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index 178b6ed53..4887a1a01 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -20,8 +20,8 @@ ], "data_stream.dataset": "import", "custom": "", - "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n namespace: default\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows", - "tags": [ + "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-1.34.0\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-1.24.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-1.34.0\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-1.34.0\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-1.24.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import", + "tags": [ "import" ] } diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 index 45583a464..688000fb7 100644 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 @@ -78,7 +78,9 @@ { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, - {"community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, + { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, + { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, + { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } ], "on_failure": [ From 4363e71e80464bd5d50610ce367b9fdbe538b395 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 17 Aug 2023 10:51:59 -0400 Subject: [PATCH 149/350] Add soup for 2.4.20 --- salt/manager/tools/sbin/soup | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index af09cc9df..145260c97 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -394,6 +394,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.3 ]] && up_to_2.4.4 [[ "$INSTALLEDVERSION" == 2.4.4 ]] && up_to_2.4.5 [[ "$INSTALLEDVERSION" == 2.4.5 ]] && up_to_2.4.10 + [[ "$INSTALLEDVERSION" == 2.4.10 ]] && up_to_2.4.20 true } @@ -405,6 +406,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4 [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 + [[ "$POSTVERSION" == 2.4.10 ]] && post_to_2.4.20 true } @@ -430,6 +432,11 @@ post_to_2.4.10() { POSTVERSION=2.4.10 } +post_to_2.4.20() { + echo "Nothing to apply" + POSTVERSION=2.4.20 +} + stop_salt_master() { # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts set +e @@ -496,6 +503,12 @@ up_to_2.4.10() { INSTALLEDVERSION=2.4.10 } +up_to_2.4.20() { + echo "Nothing to do for 2.4.20" + + INSTALLEDVERSION=2.4.20 +} + determine_elastic_agent_upgrade() { if [[ $is_airgap -eq 0 ]]; then update_elastic_agent_airgap From 09dd3f529bc118b530deddfee8cde2e6e5b4723e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Aug 2023 13:45:51 -0400 Subject: [PATCH 150/350] force image pulls to go into soup log --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 145260c97..e5ef4178a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -803,7 +803,7 @@ main() { else update_registry set +e - update_docker_containers "soup" + update_docker_containers "soup" "" "" "$SOUP_LOG" set -e fi From fb3fee5d4bb288b9582a19b11de0c03be04a1919 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 17 Aug 2023 14:43:35 -0400 Subject: [PATCH 151/350] Update HOTFIX --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index d3f5a12fa..2fd1e16d4 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ - +20230821 From 9cba9d9ae05c44a1b9b0a7ccbacf13b45cafcc9d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 17 Aug 2023 15:00:01 -0400 Subject: [PATCH 152/350] allow to override number_of_replicas from one place in soc ui --- salt/elasticsearch/defaults.yaml | 6 ++++++ salt/elasticsearch/soc_elasticsearch.yaml | 11 ++++++++++ salt/elasticsearch/template.map.jinja | 25 +++++++++++++++++++---- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 579197040..5cb027fd2 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -58,6 +58,12 @@ elasticsearch: elasticsearch: deprecation: ERROR index_settings: + global_overrides: + index_template: + template: + settings: + index: + number_of_replicas: default_placeholder so-logs: index_sorting: False index_template: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 889e9f6a4..bed6939e1 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -47,6 +47,16 @@ elasticsearch: global: True helpLink: elasticsearch.html index_settings: + global_overrides: + index_template: + template: + settings: + index: + number_of_replicas: + description: Number of replicas required for all indicies. Multiple replicas protects against data loss, but also increases storage costs. This setting will be applied to all indicies. + forcedType: int + global: True + helpLink: elasticsearch.html so-logs: &indexSettings index_sorting: description: Sorts the index by event time, at the cost of additional processing resource consumption. @@ -64,6 +74,7 @@ elasticsearch: index: number_of_replicas: description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + forcedType: int global: True helpLink: elasticsearch.html mapping: diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 5fe0ed303..f92aa4c8f 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,11 +1,28 @@ -{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} -{%- set ES_INDEX_SETTINGS_ORIG = salt['pillar.get']('elasticsearch:index_settings', default=ELASTICSEARCHDEFAULTS.elasticsearch.index_settings, merge=True) %} -{% set ES_INDEX_SETTINGS = {} %} +{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} +{% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} + +{% set PILLAR_GLOBAL_OVERRIDES = {} %} +{% if salt['pillar.get']('elasticsearch:index_settings') is defined %} +{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings') %} +{% if ES_INDEX_PILLAR.global_overrides is defined %} +{% set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %} +{% endif %} +{% endif %} + +{% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} + +{% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %} {% for index, settings in ES_INDEX_SETTINGS_ORIG.items() %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% if settings.index_template is defined %} {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} {% do settings.index_template.template.settings.index.pop('sort') %} {% endif %} {% endif %} - {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_ORIG[index]}) %} +{% endfor %} + +{% set ES_INDEX_SETTINGS = {} %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} +{% for index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.keys() %} + {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} {% endfor %} From 4ac95447ebf38904f3669c10271cbfcaf8eb9795 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 17 Aug 2023 16:15:27 -0400 Subject: [PATCH 153/350] pop sort settings if index_sorting is false --- salt/elasticsearch/template.map.jinja | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index f92aa4c8f..f5a124a9a 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -12,17 +12,17 @@ {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %} {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %} -{% for index, settings in ES_INDEX_SETTINGS_ORIG.items() %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} - {% if settings.index_template is defined %} - {% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} - {% do settings.index_template.template.settings.index.pop('sort') %} - {% endif %} - {% endif %} +{% for index in ES_INDEX_SETTINGS_ORIG.keys() %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %} {% endfor %} {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} -{% for index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.keys() %} - {% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} +{% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} +{% if settings.index_template is defined %} +{% if not settings.get('index_sorting', False) | to_bool and settings.index_template.template.settings.index.sort is defined %} +{% do settings.index_template.template.settings.index.pop('sort') %} +{% endif %} +{% endif %} +{% do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %} {% endfor %} From 222352b4b3c116adb5038eb7482eeddd6ea5e383 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 17 Aug 2023 17:26:35 -0400 Subject: [PATCH 154/350] fix typo --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index bed6939e1..f269ec014 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -53,7 +53,7 @@ elasticsearch: settings: index: number_of_replicas: - description: Number of replicas required for all indicies. Multiple replicas protects against data loss, but also increases storage costs. This setting will be applied to all indicies. + description: Number of replicas required for all indices. Multiple replicas protects against data loss, but also increases storage costs. This setting will be applied to all indices. forcedType: int global: True helpLink: elasticsearch.html From e04ec1042a6a8cec477b8475416095e5506e4e8c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 18 Aug 2023 09:12:19 -0400 Subject: [PATCH 155/350] Update soup --- salt/manager/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index af09cc9df..857ce0775 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -660,15 +660,15 @@ verify_latest_update_script() { } # Keeping this block in case we need to do a hotfix that requires salt update -#apply_hotfix() { +apply_hotfix() { # if [[ "$INSTALLEDVERSION" == "2.3.90" ]] ; then # fix_wazuh # elif [[ "$INSTALLEDVERSION" == "2.3.110" ]] ; then # 2_3_10_hotfix_1 # else -# echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)" + echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)" # fi -#} +} #upgrade salt to 3004.1 From 8aeb4706e1cc7416248faf1d523682871747a69a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 18 Aug 2023 09:57:51 -0400 Subject: [PATCH 156/350] force soup docker output to log --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index af09cc9df..0277d373d 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -790,7 +790,7 @@ main() { else update_registry set +e - update_docker_containers "soup" + update_docker_containers "soup" "" "" "$SOUP_LOG" set -e fi From 0d4a49a0fff14cc42ceff84ef619b7c9e8bda4d7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 18 Aug 2023 15:34:36 -0400 Subject: [PATCH 157/350] Update so-setup --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index d048cc8bc..2b9a0fd01 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -577,6 +577,7 @@ if ! [[ -f $install_opt_file ]]; then if [[ $waitforstate ]]; then touch /root/accept_changes + touch /etc/sohotfix make_some_dirs percentage=0 es_heapsize From 421cfc46ad8d456a0b6f46044a64771a1b572956 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 18 Aug 2023 15:39:58 -0400 Subject: [PATCH 158/350] Update soup --- salt/manager/tools/sbin/soup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d0b8f4b22..0b4136065 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -569,6 +569,9 @@ upgrade_check() { # Let's make sure we actually need to update. NEWVERSION=$(cat $UPDATE_DIR/VERSION) HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX) + if [ ! -f /etc/sohotfix ]; then + touch /etc/sohotfix + fi [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix) if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then echo "Checking to see if there are hotfixes needed" From 6784bdcb5480f075a06dbad08baa471ebe44e920 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 20 Aug 2023 15:46:07 -0400 Subject: [PATCH 159/350] Fix certs for Rec & Heavy --- salt/ssl/init.sls | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index a25a7c270..4e48688f3 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -198,7 +198,7 @@ etc_elasticfleet_logstash_key: - new: True {% if salt['file.file_exists']('/etc/pki/elasticfleet-logstash.key') -%} - prereq: - - x509: etc_elasticfleet_crt + - x509: etc_elasticfleet_logstash_crt {%- endif %} - retry: attempts: 5 @@ -259,7 +259,7 @@ etc_elasticfleetlumberjack_key: - new: True {% if salt['file.file_exists']('/etc/pki/elasticfleet-lumberjack.key') -%} - prereq: - - x509: etc_elasticfleet_crt + - x509: etc_elasticfleetlumberjack_crt {%- endif %} - retry: attempts: 5 @@ -283,7 +283,7 @@ etc_elasticfleetlumberjack_crt: cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-lumberjack.key -topk8 -out /etc/pki/elasticfleet-lumberjack.p8 -nocrypt" - onchanges: - - x509: etc_elasticfleet_key + - x509: etc_elasticfleetlumberjack_key eflogstashlumberjackperms: file.managed: @@ -327,7 +327,7 @@ etc_elasticfleet_agent_key: - new: True {% if salt['file.file_exists']('/etc/pki/elasticfleet-agent.key') -%} - prereq: - - x509: etc_elasticfleet_crt + - x509: etc_elasticfleet_agent_crt {%- endif %} - retry: attempts: 5 @@ -350,7 +350,7 @@ etc_elasticfleet_agent_crt: cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-agent.key -topk8 -out /etc/pki/elasticfleet-agent.p8 -nocrypt" - onchanges: - - x509: etc_elasticfleet_key + - x509: etc_elasticfleet_agent_key efagentperms: file.managed: From e2fd371886fb7d24e49523d63b79db78fab5c8d7 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 21 Aug 2023 07:26:37 -0400 Subject: [PATCH 160/350] Fix certs on Rec and Heavy --- salt/ssl/init.sls | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index a25a7c270..4e48688f3 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -198,7 +198,7 @@ etc_elasticfleet_logstash_key: - new: True {% if salt['file.file_exists']('/etc/pki/elasticfleet-logstash.key') -%} - prereq: - - x509: etc_elasticfleet_crt + - x509: etc_elasticfleet_logstash_crt {%- endif %} - retry: attempts: 5 @@ -259,7 +259,7 @@ etc_elasticfleetlumberjack_key: - new: True {% if salt['file.file_exists']('/etc/pki/elasticfleet-lumberjack.key') -%} - prereq: - - x509: etc_elasticfleet_crt + - x509: etc_elasticfleetlumberjack_crt {%- endif %} - retry: attempts: 5 @@ -283,7 +283,7 @@ etc_elasticfleetlumberjack_crt: cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-lumberjack.key -topk8 -out /etc/pki/elasticfleet-lumberjack.p8 -nocrypt" - onchanges: - - x509: etc_elasticfleet_key + - x509: etc_elasticfleetlumberjack_key eflogstashlumberjackperms: file.managed: @@ -327,7 +327,7 @@ etc_elasticfleet_agent_key: - new: True {% if salt['file.file_exists']('/etc/pki/elasticfleet-agent.key') -%} - prereq: - - x509: etc_elasticfleet_crt + - x509: etc_elasticfleet_agent_crt {%- endif %} - retry: attempts: 5 @@ -350,7 +350,7 @@ etc_elasticfleet_agent_crt: cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-agent.key -topk8 -out /etc/pki/elasticfleet-agent.p8 -nocrypt" - onchanges: - - x509: etc_elasticfleet_key + - x509: etc_elasticfleet_agent_key efagentperms: file.managed: From 710b800bc22ff64f55c8e82deb16c27499db02fe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 21 Aug 2023 09:00:11 -0400 Subject: [PATCH 161/350] Update config.sls --- salt/suricata/config.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/suricata/config.sls b/salt/suricata/config.sls index c8666ef2b..9da40660e 100644 --- a/salt/suricata/config.sls +++ b/salt/suricata/config.sls @@ -68,6 +68,14 @@ surilogdir: - user: 940 - group: 939 +surinsmdir: + file.directory: + - name: /nsm/suricata + - user: 940 + - group: 939 + - mode: 755 + - makedirs: True + suridatadir: file.directory: - name: /nsm/suricata/extracted From fa31bd4bf7cdf847984ceb65994dbb4946e4ab56 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 21 Aug 2023 09:20:49 -0400 Subject: [PATCH 162/350] Exclude console log --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index cb282aade..d86a441cd 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -13,6 +13,7 @@ elasticfleet: - broker - capture_loss - cluster + - console - ecat_arp_info - known_hosts - known_services From b8d374b2af89e4448c72d595f5e6ae50d6fd6ce2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 21 Aug 2023 09:45:23 -0400 Subject: [PATCH 163/350] add missing containers to soc_docker.yaml. force port bindings to []string --- salt/docker/soc_docker.yaml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/salt/docker/soc_docker.yaml b/salt/docker/soc_docker.yaml index 82f59e4dc..d227a3e85 100644 --- a/salt/docker/soc_docker.yaml +++ b/salt/docker/soc_docker.yaml @@ -8,7 +8,7 @@ docker: helpLink: docker.html advanced: True containers: - so-curator: &dockerOptions + so-dockerregistry: &dockerOptions final_octet: description: Last octet of the container IP address. helpLink: docker.html @@ -20,6 +20,7 @@ docker: helpLink: docker.html advanced: True multiline: True + forcedType: "[]string" custom_bind_mounts: description: List of custom local volume bindings. advanced: True @@ -38,12 +39,8 @@ docker: helpLink: docker.html multiline: True forcedType: "[]string" - so-dockerregistry: *dockerOptions - so-elastalert: *dockerOptions - so-elastic-fleet-package-registry: *dockerOptions so-elastic-fleet: *dockerOptions so-elasticsearch: *dockerOptions - so-idh: *dockerOptions so-idstools: *dockerOptions so-influxdb: *dockerOptions so-kibana: *dockerOptions @@ -53,11 +50,21 @@ docker: so-nginx: *dockerOptions so-playbook: *dockerOptions so-redis: *dockerOptions + so-sensoroni: *dockerOptions so-soc: *dockerOptions so-soctopus: *dockerOptions so-strelka-backend: *dockerOptions - so-strelka-coordinator: *dockerOptions so-strelka-filestream: *dockerOptions so-strelka-frontend: *dockerOptions - so-strelka-gatekeeper: *dockerOptions so-strelka-manager: *dockerOptions + so-strelka-gatekeeper: *dockerOptions + so-strelka-coordinator: *dockerOptions + so-elastalert: *dockerOptions + so-curator: *dockerOptions + so-elastic-fleet-package-registry: *dockerOptions + so-idh: *dockerOptions + so-elastic-agent: *dockerOptions + so-telegraf: *dockerOptions + so-steno: *dockerOptions + so-suricata: *dockerOptions + so-zeek: *dockerOptions From 9e18fe64cf4c69ad4af078c8bbad7bfbab1bc412 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 21 Aug 2023 11:20:47 -0400 Subject: [PATCH 164/350] Remove OSSEC configuration --- salt/soc/defaults.yaml | 50 ------------------------------------------ 1 file changed, 50 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 49be076c0..8ac49ea2e 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -474,19 +474,6 @@ soc: - event.dataset - process.executable - user.name - ':ossec:': - - soc_timestamp - - source.ip - - source.port - - destination.ip - - destination.port - - rule.name - - rule.level - - rule.category - - process.name - - user.name - - user.escalated - - location ':strelka:file': - soc_timestamp - file.name @@ -523,28 +510,6 @@ soc: - message - kibana.log.meta.req.headers.x-real-ip - event.dataset - '::rootcheck': - - soc_timestamp - - host.name - - metadata.ip_address - - log.full - - event.dataset - - event.module - '::ossec': - - soc_timestamp - - host.name - - metadata.ip_address - - log.full - - event.dataset - - event.module - '::syscollector': - - soc_timestamp - - host.name - - metadata.ip_address - - wazuh.data.type - - log.full - - event.dataset - - event.module ':syslog:syslog': - soc_timestamp - host.name @@ -1621,21 +1586,6 @@ soc: - rule.uuid - rule.category - rule.rev - ':ossec:': - - soc_timestamp - - rule.name - - event.severity_label - - source.ip - - source.port - - destination.ip - - destination.port - - rule.level - - rule.category - - process.name - - user.name - - user.escalated - - location - - process.name queryBaseFilter: tags:alert queryToggleFilters: - name: acknowledged From 563a495725d5aca4b8b0b70a7963b5173ce2a53a Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 21 Aug 2023 11:24:07 -0400 Subject: [PATCH 165/350] Add Playbook --- salt/soc/defaults.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 8ac49ea2e..ff8b240ec 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1586,6 +1586,15 @@ soc: - rule.uuid - rule.category - rule.rev + ':playbook:': + - soc_timestamp + - rule.name + - event.severity_label + - event_data.event.module + - event_data.event.category + - event_data.process.executable + - event_data.process.pid + - event_data.winlog.computer_name queryBaseFilter: tags:alert queryToggleFilters: - name: acknowledged From 84d5d52ec850d157ab58461ac5ef807899318cae Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 21 Aug 2023 15:36:57 -0400 Subject: [PATCH 166/350] 2.4.10 Hotfix --- DOWNLOAD_AND_VERIFY_ISO.md | 22 ++++++++++----------- sigs/securityonion-2.4.10-20230821.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.4.10-20230821.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 816c4f827..1e6299a8e 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.4.10-20230815 ISO image released on 2023/08/15 +### 2.4.10-20230821 ISO image released on 2023/08/21 ### Download and Verify -2.4.10-20230815 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso +2.4.10-20230821 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230821.iso -MD5: 97AEC929FB1FC22F106C0C93E3476FAB -SHA1: 78AF37FD19FDC34BA324C1A661632D19D1F2284A -SHA256: D04BA45D1664FC3CF7EA2188CB7E570642F6390C3959B4AFBB8222A853859394 +MD5: 353EB36F807DC947F08F79B3DCFA420E +SHA1: B25E3BEDB81BBEF319DC710267E6D78422F39C56 +SHA256: 3D369E92FEB65D14E1A981E99FA223DA52C92057A037C243AD6332B6B9A6D9BC Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230821.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230815.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230821.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230815.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230821.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.10-20230815.iso.sig securityonion-2.4.10-20230815.iso +gpg --verify securityonion-2.4.10-20230821.iso.sig securityonion-2.4.10-20230821.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Sun 13 Aug 2023 05:30:29 PM EDT using RSA key ID FE507013 +gpg: Signature made Mon 21 Aug 2023 09:47:50 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.10-20230821.iso.sig b/sigs/securityonion-2.4.10-20230821.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..251032166d6230346dbf71728f011eebc5da5576 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%YTB1^@~P5PT3| zxBgIY6Sa5`{10%j$#G-=&7)Yq?msqB@!u8T^uU>^@;#Crl{W6zggRK{<&)>3PB<|W z3QeU+TCg*fDq}`2`@kpe+l91Igs!)ZOmc&uoQQuB^tUYOd*WEMp4^%fbozh-CKmR; zkbRG*xPHN#ORrwOPdyA*I3(_Ms3_!*l@+3*YJiWet3vUg(gexEZ- z3>e?Faf*TjCy9~w$OfJTsO~Dpa{8^kv9Wy%KQ^CnYLTdgozr!IqoXsOvoJ~Trb-m) z7ia>3pr}A2=k4KGs;ed1eNIp380u=}J4hw+vNkq53Cs@BM0-a=2l({(&!+3rEBX{Y z#oJnGgJ5dRbxaSLZ3m(-twNHPUv1eTjpR-!537!>D01}zIBR53z5YCEeSZ%K6LBKz zg?a5yptkv##1Gjf1DRzBkRgZ~OmojzHOkR$i>+2{*LMqgQerR2H(qZiQP<`atWbCw z`nKWfmeeAOFyBG~BmiKF6-IFe|7M;vJ+k(AE?QBn$4klcY49);j2Qi3zGHsNUsZX- z^tsTqc@y5=wjs(=8sgj6llp7&OS*KK_A{)k$TDgWqnT-ZCVD zh_$&2)-T`}yCIFrqvU1>;qnuX5StsZmIbF_-3zS~fFyiK#wIr9abDudJoMmpK@FQu EZ&ou9XaE2J literal 0 HcmV?d00001 From f2c665e4faa9e6264c48ed689c34f93595f35430 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 21 Aug 2023 16:30:02 -0400 Subject: [PATCH 167/350] Update HOTFIX --- HOTFIX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOTFIX b/HOTFIX index 2fd1e16d4..d3f5a12fa 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +1 @@ -20230821 + From 8a751e097d231004580735173910512cb5205fbf Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 23 Aug 2023 14:32:05 -0400 Subject: [PATCH 168/350] cert path refactor --- salt/elasticfleet/enabled.sls | 12 +----------- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 2 +- .../tools/sbin_jinja/so-elastic-fleet-setup | 4 ---- salt/elasticsearch/enabled.sls | 2 +- salt/logstash/enabled.sls | 2 +- salt/redis/enabled.sls | 2 +- salt/ssl/init.sls | 2 +- salt/ssl/remove.sls | 2 +- salt/telegraf/enabled.sls | 2 +- 9 files changed, 8 insertions(+), 22 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index 82c7735db..320b6d6b6 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -68,11 +68,6 @@ so-elastic-fleet: - /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro - {% if GLOBALS.os_family == 'Debian' %} - - /etc/ssl/elasticfleet-server.crt:/etc/ssl/elasticfleet-server.crt:ro - - /etc/ssl/elasticfleet-server.key:/etc/ssl/elasticfleet-server.key:ro - - /etc/ssl/tls/certs/intca.crt:/etc/ssl/tls/certs/intca.crt:ro - {% endif %} - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs {% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %} @@ -87,13 +82,8 @@ so-elastic-fleet: - FLEET_SERVER_POLICY_ID=FleetServer_{{ GLOBALS.hostname }} - FLEET_SERVER_CERT=/etc/pki/elasticfleet-server.crt - FLEET_SERVER_CERT_KEY=/etc/pki/elasticfleet-server.key - {% if GLOBALS.os_family == 'Debian' %} - - FLEET_CA=/etc/ssl/certs/intca.crt - - FLEET_SERVER_ELASTICSEARCH_CA=/etc/ssl/certs/intca.crt - {% else %} - - FLEET_CA=/etc/pki/tls/certs/intca.crt + - FLEET_CA=/etc/pki/tls/certs/intca.crt - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt - {% endif %} - LOGS_PATH=logs {% if DOCKER.containers['so-elastic-fleet'].extra_env %} {% for XTRAENV in DOCKER.containers['so-elastic-fleet'].extra_env %} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index d7d6458c9..c935521fd 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -65,7 +65,7 @@ do if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi printf "\n\n### Generating $GOOS/$GOARCH Installer...\n" docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \ - --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \ + --mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \ --mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \ --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH} diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index ac0ce4db9..83a155ae6 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -6,11 +6,7 @@ # this file except in compliance with the Elastic License 2.0. {% from 'vars/globals.map.jinja' import GLOBALS %} -{% if GLOBALS.os_family == 'Debian' %} -INTCA=/etc/ssl/certs/intca.crt -{% else %} INTCA=/etc/pki/tls/certs/intca.crt -{% endif %} . /usr/sbin/so-elastic-fleet-common diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index e28ca5fdf..8baff4901 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -59,7 +59,7 @@ so-elasticsearch: {% if GLOBALS.is_manager %} - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro {% else %} - - /etc/ssl/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro + - /etc/pki/tls/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro {% endif %} - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index 731ad4ca3..c76f81d21 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -73,7 +73,7 @@ so-logstash: {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %} - /etc/pki/ca.crt:/usr/share/filebeat/ca.crt:ro {% else %} - - /etc/ssl/certs/intca.crt:/usr/share/filebeat/ca.crt:ro + - /etc/pki/tls/certs/intca.crt:/usr/share/filebeat/ca.crt:ro {% endif %} {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-searchnode'] %} - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro diff --git a/salt/redis/enabled.sls b/salt/redis/enabled.sls index 2a4f5a179..4c452bec0 100644 --- a/salt/redis/enabled.sls +++ b/salt/redis/enabled.sls @@ -33,7 +33,7 @@ so-redis: {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %} - /etc/pki/ca.crt:/certs/ca.crt:ro {% else %} - - /etc/ssl/certs/intca.crt:/certs/ca.crt:ro + - /etc/pki/certs/intca.crt:/certs/ca.crt:ro {% endif %} {% if DOCKER.containers['so-redis'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-redis'].custom_bind_mounts %} diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 4e48688f3..9ff3a3a6d 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -41,7 +41,7 @@ include: # Trust the CA trusttheca: x509.pem_managed: - - name: /etc/ssl/certs/intca.crt + - name: /etc/pki/tls/certs/intca.crt - text: {{ trusttheca_text }} # Install packages needed for the sensor diff --git a/salt/ssl/remove.sls b/salt/ssl/remove.sls index 4eb0eb442..43a245288 100644 --- a/salt/ssl/remove.sls +++ b/salt/ssl/remove.sls @@ -1,6 +1,6 @@ trusttheca: file.absent: - - name: /etc/ssl/certs/intca.crt + - name: /etc/pki/tls/certs/intca.crt influxdb_key: file.absent: diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls index 598587e17..d55e536d6 100644 --- a/salt/telegraf/enabled.sls +++ b/salt/telegraf/enabled.sls @@ -46,7 +46,7 @@ so-telegraf: {% if GLOBALS.role in ['so-manager', 'so-eval', 'so-managersearch' ] %} - /etc/pki/ca.crt:/etc/telegraf/ca.crt:ro {% else %} - - /etc/ssl/certs/intca.crt:/etc/telegraf/ca.crt:ro + - /etc/pki/tls/certs/intca.crt:/etc/telegraf/ca.crt:ro {% endif %} - /etc/pki/influxdb.crt:/etc/telegraf/telegraf.crt:ro - /etc/pki/influxdb.key:/etc/telegraf/telegraf.key:ro From 0f24c8e8bb855306eca56bd433ba594ae63e5723 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 23 Aug 2023 19:02:32 +0000 Subject: [PATCH 169/350] Add packages --- salt/elasticfleet/defaults.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index d86a441cd..77fa9dd31 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -26,20 +26,51 @@ elasticfleet: - stderr - stdout packages: + - auditd - aws - azure + - barracuda + - cisco_asa - cloudflare + - crowdstrike + - darktrace - elasticsearch - endpoint + - f5_bigip - fleet_server - fim + - fortinet + - gcp - github - google_workspace + - http_endpoint + - httpjson + - juniper + - juniper_srx + - kafka_log + - lastpass - log + - m365_defender + - microsoft_defender_endpoint + - microsoft_dhcp + - netflow + - o365 + - okta - osquery_manager + - panw + - pfsense - redis + - sentinel_one + - sonicwall_firewall + - symantec_endpoint - system - tcp + - ti_abusech + - ti_misp + - ti_otx + - ti_recordedfuture - udp - windows + - zscaler_zia + - zscaler_zpa - 1password From 3f2793088a28354fdae2a7eb3422b6e4923abdaf Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 23 Aug 2023 19:02:50 +0000 Subject: [PATCH 170/350] Add templates --- salt/elasticsearch/defaults.yaml | 1134 ++++++++++++++++++++++++++++++ 1 file changed, 1134 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 5cb027fd2..1c1d3ec58 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -286,6 +286,24 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so-logs-auditd_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-auditd.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-auditd.log@package" + - "logs-auditd.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false so-logs-aws_x_cloudtrail: index_sorting: False index_template: @@ -646,6 +664,42 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so-logs-barracuda_x_waf: + index_sorting: False + index_template: + index_patterns: + - "logs-barracuda.waf-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-barracuda.waf@package" + - "logs-barracuda.waf@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-cisco_asa_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-cisco_asa.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-cisco_asa.log@package" + - "logs-cisco_asa.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false so-logs-cloudflare_x_audit: index_sorting: False index_template: @@ -682,6 +736,114 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so-logs-crowdstrike_x_falcon: + index_sorting: False + index_template: + index_patterns: + - "logs-crowdstrike.falcon-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-crowdstrike.falcon@package" + - "logs-crowdstrike.falcon@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-crowdstrike_x_fdr: + index_sorting: False + index_template: + index_patterns: + - "logs-crowdstrike.fdr-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-crowdstrike.fdr@package" + - "logs-crowdstrike.fdr@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-darktrace_x_ai_analyst_alert: + index_sorting: False + index_template: + index_patterns: + - "logs-darktrace.ai_analyst_alert-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-darktrace.ai_analyst_alert@package" + - "logs-darktrace.ai_analyst_alert@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-darktrace_x_model_breach_alert: + index_sorting: False + index_template: + index_patterns: + - "logs-darktrace.model_breach_alert-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-darktrace.model_breach_alert@package" + - "logs-darktrace.model_breach_alert@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-darktrace_x_system_status_alert: + index_sorting: False + index_template: + index_patterns: + - "logs-darktrace.system_status_alert-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-darktrace.system_status_alert@package" + - "logs-darktrace.system_status_alert@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-f5_bigip_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-f5_bigip.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-f5_bigip.log@package" + - "logs-f5_bigip.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false so-logs-fim_x_event: index_sorting: False index_template: @@ -700,6 +862,186 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so-logs-fortinet_x_clientendpoint: + index_sorting: False + index_template: + index_patterns: + - "logs-fortinet.clientendpoint-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-fortinet.clientendpoint@package" + - "logs-fortinet.clientendpoint@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-fortinet_x_firewall: + index_sorting: False + index_template: + index_patterns: + - "logs-fortinet.firewall-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-fortinet.firewall@package" + - "logs-fortinet.firewall@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-fortinet_x_fortimail: + index_sorting: False + index_template: + index_patterns: + - "logs-fortinet.fortimail-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-fortinet.fortimail@package" + - "logs-fortinet.fortimail@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-fortinet_x_fortimanager: + index_sorting: False + index_template: + index_patterns: + - "logs-fortinet.fortimanager-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-fortinet.fortimanager@package" + - "logs-fortinet.fortimanager@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-fortinet_x_fortigate: + index_sorting: False + index_template: + index_patterns: + - "logs-fortinet.fortigate-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-fortinet.fortigate@package" + - "logs-fortinet.fortigate@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-gcp_x_audit: + index_sorting: False + index_template: + index_patterns: + - "logs-gcp.audit-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-gcp.audit@package" + - "logs-gcp.audit@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-gcp_x_dns: + index_sorting: False + index_template: + index_patterns: + - "logs-gcp.dns-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-gcp.dns@package" + - "logs-gcp.dns@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-gcp_x_firewall: + index_sorting: False + index_template: + index_patterns: + - "logs-gcp.firewall-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-gcp.firewall@package" + - "logs-gcp.firewall@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-gcp_x_loadbalancing_logs: + index_sorting: False + index_template: + index_patterns: + - "logs-gcp.loadbalancing_logs-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-gcp.loadbalancing_logs@package" + - "logs-gcp.loadbalancing_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-gcp_x_vpcflow: + index_sorting: False + index_template: + index_patterns: + - "logs-gcp.vpcflow-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-gcp.vpcflow@package" + - "logs-gcp.vpcflow@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false so-logs-github_x_audit: index_sorting: False index_template: @@ -1042,6 +1384,798 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so-logs-http_endpoint_x_generic: + index_sorting: False + index_template: + index_patterns: + - "logs-http_endpoint.generic-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-http_endpoint.generic@package" + - "logs-http_endpoint.generic@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-httpjson_x_generic: + index_sorting: False + index_template: + index_patterns: + - "logs-httpjson.generic-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-httpjson.generic@package" + - "logs-httpjson.generic@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-juniper_x_junos: + index_sorting: False + index_template: + index_patterns: + - "logs-juniper.junos-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-juniper.junos@package" + - "logs-juniper.junos@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-juniper_x_netscreen: + index_sorting: False + index_template: + index_patterns: + - "logs-juniper.netscreen-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-juniper.netscreen@package" + - "logs-juniper.netscreen@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-juniper_x_srx: + index_sorting: False + index_template: + index_patterns: + - "logs-juniper.srx-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-juniper.srx@package" + - "logs-juniper.srx@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-juniper_srx_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-juniper_srx.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-juniper_srx.log@package" + - "logs-juniper_srx.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-kafka_log_x_generic: + index_sorting: False + index_template: + index_patterns: + - "logs-kafka_log.generic-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-kafka_log.generic@package" + - "logs-kafka_log.generic@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-lastpass_x_detailed_shared_folder: + index_sorting: False + index_template: + index_patterns: + - "logs-lastpass.detailed_shared_folder-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-lastpass.detailed_shared_folder@package" + - "logs-lastpass.detailed_shared_folder@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-lastpass_x_event_report: + index_sorting: False + index_template: + index_patterns: + - "logs-lastpass.event_report-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-lastpass.event_report@package" + - "logs-lastpass.event_report@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-lastpass_x_user: + index_sorting: False + index_template: + index_patterns: + - "logs-lastpass.user-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-lastpass.user@package" + - "logs-lastpass.user@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-m365_defender_x_event: + index_sorting: False + index_template: + index_patterns: + - "logs-m365_defender.event-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-m365_defender.event@package" + - "logs-m365_defender.event@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-m365_defender_x_incident: + index_sorting: False + index_template: + index_patterns: + - "logs-m365_defender.incident-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-m365_defender.incident@package" + - "logs-m365_defender.incident@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-m365_defender_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-m365_defender.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-m365_defender.log@package" + - "logs-m365_defender.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-microsoft_defender_endpoint_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-microsoft_defender_endpoint.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-microsoft_defender_endpoint.log@package" + - "logs-microsoft_defender_endpoint.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-microsoft_dhcp_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-microsoft_dhcp.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-microsoft_dhcp.log@package" + - "logs-microsoft_dhcp.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-netflow_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-netflow.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-netflow.log@package" + - "logs-netflow.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-panw_x_panos: + index_sorting: False + index_template: + index_patterns: + - "logs-panw.panos-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-panw.panos@package" + - "logs-panw.panos@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-pfsense_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-pfsense.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-pfsense.log@package" + - "logs-pfsense.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-sentinel_one_x_activity: + index_sorting: False + index_template: + index_patterns: + - "logs-sentinel_one.activity-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-sentinel_one.activity@package" + - "logs-sentinel_one.activity@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-sentinel_one_x_agent: + index_sorting: False + index_template: + index_patterns: + - "logs-sentinel_one.agent-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-sentinel_one.agent@package" + - "logs-sentinel_one.agent@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-sentinel_one_x_alert: + index_sorting: False + index_template: + index_patterns: + - "logs-sentinel_one.alert-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-sentinel_one.alert@package" + - "logs-sentinel_one.alert@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-sentinel_one_x_group: + index_sorting: False + index_template: + index_patterns: + - "logs-sentinel_one.group-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-sentinel_one.group@package" + - "logs-sentinel_one.group@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-sentinel_one_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-sentinel_one.threat-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-sentinel_one.threat@package" + - "logs-sentinel_one.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-sonicwall_firewall_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-sonicwall_firewall.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-sonicwall_firewall.log@package" + - "logs-sonicwall_firewall.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-symantec_endpoint_x_log: + index_sorting: False + index_template: + index_patterns: + - "logs-symantec_endpoint.log-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-symantec_endpoint.log@package" + - "logs-symantec_endpoint.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-ti_abusech_x_malware: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_abusech.malware-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-ti_abusech.malware@package" + - "logs-ti_abusech.malware@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-ti_abusech_x_malwarebazaar: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_abusech.malwarebazaar-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-ti_abusech.malwarebazaar@package" + - "logs-ti_abusech.malwarebazaar@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-ti_abusech_x_threatfox: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_abusech.threatfox-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-ti_abusech.threatfox@package" + - "logs-ti_abusech.threatfox@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-ti_abusech_x_url: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_abusech.url-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-ti_abusech.url@package" + - "logs-ti_abusech.url@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-ti_misp_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_misp.threat-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-ti_misp.threat@package" + - "logs-ti_misp.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-ti_misp_x_threat_attributes: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_misp.threat_attributes-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-ti_misp.threat_attributes@package" + - "logs-ti_misp.threat_attributes@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-ti_otx_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_otx.threat-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-ti_otx.threat@package" + - "logs-ti_otx.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-ti_recordedfuture_x_latest_ioc-template: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_recordedfuture.latest_ioc-template-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-ti_recordedfuture.latest_ioc-template@package" + - "logs-ti_recordedfuture.latest_ioc-template@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-ti_recordedfuture_x_threat: + index_sorting: False + index_template: + index_patterns: + - "logs-ti_recordedfuture.threat-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-ti_recordedfuture.threat@package" + - "logs-ti_recordedfuture.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zia_x_alerts: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zia.alerts-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zia.alerts@package" + - "logs-zscaler_zia.alerts@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zia_x_dns: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zia.dns-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zia.dns@package" + - "logs-zscaler_zia.dns@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zia_x_firewall: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zia.firewall-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zia.firewall@package" + - "logs-zscaler_zia.firewall@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zia_x_tunnel: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zia.tunnel-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zia.tunnel@package" + - "logs-zscaler_zia.tunnel@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zia_x_web: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zia.web-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zia.web@package" + - "logs-zscaler_zia.web@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zpa_x_app_connector_status: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zpa.app_connector_status-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zpa.app_connector_status@package" + - "logs-zscaler_zpa.app_connector_status@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zpa_x_audit: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zpa.audit-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zpa.audit@package" + - "logs-zscaler_zpa.audit@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zpa_x_browser_access: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zpa.browser_access-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zpa.browser_access@package" + - "logs-zscaler_zpa.browser_access@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zpa_x_user_activity: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zpa.user_activity-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zpa.user_activity@package" + - "logs-zscaler_zpa.user_activity@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-zscaler_zpa_x_user_status: + index_sorting: False + index_template: + index_patterns: + - "logs-zscaler_zpa.user_status-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-zscaler_zpa.user_status@package" + - "logs-zscaler_zpa.user_status@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false so-logs-1password_x_item_usages: index_sorting: False index_template: From a885baf9603061784bb7641749e3c7039376cc9b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 23 Aug 2023 15:24:32 -0400 Subject: [PATCH 171/350] add desktop to grid --- salt/manager/tools/sbin/so-minion | 11 ++++------- setup/so-setup | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index edc0b1404..de55c3a5b 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -187,15 +187,9 @@ function add_logstash_to_minion() { # Security Onion Desktop function add_desktop_to_minion() { printf '%s\n'\ - "host:"\ - " mainint: '$MNIC'"\ "desktop:"\ " gui:"\ - " enabled: true"\ - "sensoroni:"\ - " enabled: True"\ - " config:"\ - " node_description: '${NODE_DESCRIPTION//\'/''}'" >> $PILLARFILE + " enabled: true"\ >> $PILLARFILE } # Add basic host info to the minion file @@ -556,6 +550,9 @@ function createRECEIVER() { add_telegraf_to_minion } +function createDESKTOP() { + add_desktop_to_minion +} function testConnection() { retry 15 3 "salt '$MINION_ID' test.ping" True diff --git a/setup/so-setup b/setup/so-setup index c3172280f..8e8b7af43 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -333,7 +333,7 @@ if [[ $is_desktop ]]; then exit 1 fi -# if ! whiptail_desktop_install; then + if ! whiptail_desktop_install; then if [[ $is_desktop_iso ]]; then if whiptail_desktop_nongrid_iso; then # Remove setup from auto launching @@ -365,7 +365,7 @@ if [[ $is_desktop ]]; then exit 0 fi fi -# fi + fi # If you got this far then you want to join the grid is_minion=true @@ -574,6 +574,17 @@ if ! [[ -f $install_opt_file ]]; then check_manager_connection set_minion_info whiptail_end_settings + + elif [[ $is_desktop ]]; then + info "Setting up as node type desktop" + #check_requirements "desktop" + networking_needful + collect_mngr_hostname + add_mngr_ip_to_hosts + check_manager_connection + set_minion_info + whiptail_end_settings + fi if [[ $waitforstate ]]; then From 2f51349ff817a5cf6325a6f648fff058a3b4f80c Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 23 Aug 2023 20:07:42 +0000 Subject: [PATCH 172/350] Add SOC configuration --- salt/elasticsearch/soc_elasticsearch.yaml | 63 +++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index f269ec014..01de1ec30 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -201,6 +201,7 @@ elasticsearch: so-logs-windows_x_powershell: *indexSettings so-logs-windows_x_powershell_operational: *indexSettings so-logs-windows_x_sysmon_operational: *indexSettings + so-logs-auditd_x_log: *indexSettings so-logs-aws_x_cloudtrail: *indexSettings so-logs-aws_x_cloudwatch_logs: *indexSettings so-logs-aws_x_ec2_logs: *indexSettings @@ -221,9 +222,27 @@ elasticsearch: so-logs-azure_x_provisioning: *indexSettings so-logs-azure_x_signinlogs: *indexSettings so-logs-azure_x_springcloudlogs: *indexSettings + so-logs-barracuda_x_waf: *indexSettings + so-logs-cisco_asa_x_log: *indexSettings so-logs-cloudflare_x_audit: *indexSettings so-logs-cloudflare_x_logpull: *indexSettings + so-logs-crowdstrike_x_falcon: *indexSettings + so-logs-crowdstrike_x_fdr: *indexSettings + so-logs-darktrace_x_ai_analyst_alert: *indexSettings + so-logs-darktrace_x_model_breach_alert: *indexSettings + so-logs-darktrace_x_system_status_alert: *indexSettings + so-logs-f5_bigip_x_log: *indexSettings so-logs-fim_x_event: *indexSettings + so-logs-fortinet_x_clientendpoint: *indexSettings + so-logs-fortinet_x_firewall: *indexSettings + so-logs-fortinet_x_fortimail: *indexSettings + so-logs-fortinet_x_fortimanager: *indexSettings + so-logs-fortinet_x_fortigate: *indexSettings + so-logs-gcp_x_audit: *indexSettings + so-logs-gcp_x_dns: *indexSettings + so-logs-gcp_x_firewall: *indexSettings + so-logs-gcp_x_loadbalancing_logs: *indexSettings + so-logs-gcp_x_vpcflow: *indexSettings so-logs-github_x_audit: *indexSettings so-logs-github_x_code_scanning: *indexSettings so-logs-github_x_dependabot: *indexSettings @@ -243,6 +262,50 @@ elasticsearch: so-logs-google_workspace_x_saml: *indexSettings so-logs-google_workspace_x_token: *indexSettings so-logs-google_workspace_x_user_accounts: *indexSettings + so-logs-http_endpoint_x_generic: *indexSettings + so-logs-httpjson_x_generic: *indexSettings + so-logs-juniper_x_junos: *indexSettings + so-logs-juniper_x_netscreen: *indexSettings + so-logs-juniper_x_srx: *indexSettings + so-logs-juniper_srx_x_log: *indexSettings + so-logs-kafka_log_x_generic: *indexSettings + so-logs-lastpass_x_detailed_shared_folder: *indexSettings + so-logs-lastpass_x_event_report: *indexSettings + so-logs-lastpass_x_user: *indexSettings + so-logs-m365_defender_x_event: *indexSettings + so-logs-m365_defender_x_incident: *indexSettings + so-logs-m365_defender_x_log: *indexSettings + so-logs-microsoft_defender_endpoint_x_log: *indexSettings + so-logs-microsoft_dhcp_x_log: *indexSettings + so-logs-netflow_x_log: *indexSettings + so-logs-panw_x_panos: *indexSettings + so-logs-pfsense_x_log: *indexSettings + so-logs-sentinel_one_x_activity: *indexSettings + so-logs-sentinel_one_x_agent: *indexSettings + so-logs-sentinel_one_x_alert: *indexSettings + so-logs-sentinel_one_x_group: *indexSettings + so-logs-sentinel_one_x_threat: *indexSettings + so-logs-sonicwall_firewall_x_log: *indexSettings + so-logs-symantec_endpoint_x_log: *indexSettings + so-logs-ti_abusech_x_malware: *indexSettings + so-logs-ti_abusech_x_malwarebazaar: *indexSettings + so-logs-ti_abusech_x_threatfox: *indexSettings + so-logs-ti_abusech_x_url: *indexSettings + so-logs-ti_misp_x_threat: *indexSettings + so-logs-ti_misp_x_threat_attributes: *indexSettings + so-logs-ti_otx_x_threat: *indexSettings + so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings + so-logs-ti_recordedfuture_x_threat: *indexSettings + so-logs-zscaler_zia_x_alerts: *indexSettings + so-logs-zscaler_zia_x_dns: *indexSettings + so-logs-zscaler_zia_x_firewall: *indexSettings + so-logs-zscaler_zia_x_tunnel: *indexSettings + so-logs-zscaler_zia_x_web: *indexSettings + so-logs-zscaler_zpa_x_app_connector_status: *indexSettings + so-logs-zscaler_zpa_x_audit: *indexSettings + so-logs-zscaler_zpa_x_browser_access: *indexSettings + so-logs-zscaler_zpa_x_user_activity: *indexSettings + so-logs-zscaler_zpa_x_user_status: *indexSettings so-logs-1password_x_item_usages: *indexSettings so-logs-1password_x_signin_attempts: *indexSettings so-logs-osquery-manager-actions: *indexSettings From 31a49268cb960d26c04d7e8ea28cc5f9c4bf4260 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 23 Aug 2023 20:20:06 +0000 Subject: [PATCH 173/350] Add o365 and okta --- salt/elasticsearch/defaults.yaml | 36 +++++++++++++++++++++++ salt/elasticsearch/soc_elasticsearch.yaml | 2 ++ 2 files changed, 38 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 1c1d3ec58..3ea24c3fd 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1672,6 +1672,42 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so-logs-o365_x_audit: + index_sorting: False + index_template: + index_patterns: + - "logs-o365.audit-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-o365.audit@package" + - "logs-o365.audit@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-okta_x_system: + index_sorting: False + index_template: + index_patterns: + - "logs-okta.system-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-okta.system@package" + - "logs-okta.system@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false so-logs-panw_x_panos: index_sorting: False index_template: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 01de1ec30..e8ecccd2c 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -278,6 +278,8 @@ elasticsearch: so-logs-microsoft_defender_endpoint_x_log: *indexSettings so-logs-microsoft_dhcp_x_log: *indexSettings so-logs-netflow_x_log: *indexSettings + so-logs-okta_x_system: *indexSettings + so-logs-o365_x_audit: *indexSettings so-logs-panw_x_panos: *indexSettings so-logs-pfsense_x_log: *indexSettings so-logs-sentinel_one_x_activity: *indexSettings From d2d0d53eefb476c109b47e82ef8d1880f065535a Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 23 Aug 2023 20:20:44 +0000 Subject: [PATCH 174/350] Change order --- salt/elasticsearch/soc_elasticsearch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index e8ecccd2c..a960facd1 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -278,8 +278,8 @@ elasticsearch: so-logs-microsoft_defender_endpoint_x_log: *indexSettings so-logs-microsoft_dhcp_x_log: *indexSettings so-logs-netflow_x_log: *indexSettings - so-logs-okta_x_system: *indexSettings so-logs-o365_x_audit: *indexSettings + so-logs-okta_x_system: *indexSettings so-logs-panw_x_panos: *indexSettings so-logs-pfsense_x_log: *indexSettings so-logs-sentinel_one_x_activity: *indexSettings From b8dc9ea5600e31fd08b569b45bc2d999f2aee9b2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 23 Aug 2023 17:50:08 -0400 Subject: [PATCH 175/350] cert work --- salt/ssl/init.sls | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 9ff3a3a6d..80164c622 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -36,7 +36,10 @@ include: {% set ca_server = global_ca_server[0] %} {% endif %} - +cacertdir: + file.directory: + - name: /etc/pki/tls/certs + - makedirs: True # Trust the CA trusttheca: @@ -44,6 +47,13 @@ trusttheca: - name: /etc/pki/tls/certs/intca.crt - text: {{ trusttheca_text }} +{% if GLOBALS.os_family == 'Debian' %} +symlinkca: + file.symlink: + - source: /etc/pki/tls/certs/intca.crt + - name: /etc/ssl/certs/intca.crt +{% end %} + # Install packages needed for the sensor m2cryptopkgs: pkg.installed: From 4484e2d031d9b6ffc0d761e4a109a7ff0238bcda Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 23 Aug 2023 18:16:49 -0400 Subject: [PATCH 176/350] cert work --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 80164c622..1131eec12 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -52,7 +52,7 @@ symlinkca: file.symlink: - source: /etc/pki/tls/certs/intca.crt - name: /etc/ssl/certs/intca.crt -{% end %} +{% endif %} # Install packages needed for the sensor m2cryptopkgs: From f4be5641daca889155f965787303d2c2172dbdd6 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 23 Aug 2023 20:49:37 -0400 Subject: [PATCH 177/350] cert work --- salt/ssl/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 1131eec12..ef93a9072 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -50,7 +50,7 @@ trusttheca: {% if GLOBALS.os_family == 'Debian' %} symlinkca: file.symlink: - - source: /etc/pki/tls/certs/intca.crt + - target: /etc/pki/tls/certs/intca.crt - name: /etc/ssl/certs/intca.crt {% endif %} From 82529242031eb02bafeb704ecd89969fd5d950ab Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 24 Aug 2023 12:16:25 -0400 Subject: [PATCH 178/350] allow testing runs to proceed with unsupported os --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index c3172280f..22a9e9238 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -69,7 +69,7 @@ detect_os # Ubuntu/Debian whiptail pallete to make it look the same as CentOS and Rocky. set_palette >> $setup_log 2>&1 -if [[ $not_supported ]]; then +if [[ $not_supported ]] && [ -z "$TESTING" ]; then if [[ "$OSVER" == "focal" ]]; then if (whiptail_focal_warning); then true From 43e4cf632ad70d66f4ee1994ae8254c323bf6a54 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 24 Aug 2023 12:57:35 -0400 Subject: [PATCH 179/350] use the correct var --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 22a9e9238..14d6b2304 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -69,7 +69,7 @@ detect_os # Ubuntu/Debian whiptail pallete to make it look the same as CentOS and Rocky. set_palette >> $setup_log 2>&1 -if [[ $not_supported ]] && [ -z "$TESTING" ]; then +if [[ $not_supported ]] && [ -z "$test_profile" ]; then if [[ "$OSVER" == "focal" ]]; then if (whiptail_focal_warning); then true From e57cc0308424cf288aad574fc2b07e69437c68e3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 24 Aug 2023 14:41:04 -0400 Subject: [PATCH 180/350] fix centos install --- salt/common/packages.sls | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 5f4a348e7..5f013e40b 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -63,11 +63,7 @@ commonpkgs: - httpd-tools - jq - lvm2 - {% if GLOBALS.os == 'CentOS Stream' %} - - MariaDB-devel - {% else %} - mariadb-devel - {% endif %} - net-tools - nmap-ncat - openssl From 4a489afb893077be5575076359fe9a2be42b7df5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 25 Aug 2023 08:55:00 -0400 Subject: [PATCH 181/350] remove old and install new watchdog package --- salt/common/packages.sls | 2 -- salt/strelka/filestream/config.sls | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 5f4a348e7..fe36a1fa1 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -21,7 +21,6 @@ commonpkgs: - python3-dateutil - python3-docker - python3-packaging - - python3-watchdog - python3-lxml - git - rsync @@ -78,7 +77,6 @@ commonpkgs: - python3-packaging - python3-pyyaml - python3-rich - - python3-watchdog - rsync - sqlite - tcpdump diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index 993a59650..a254e9253 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -47,6 +47,14 @@ filestream_config: FILESTREAMCONFIG: {{ STRELKAMERGED.filestream.config }} # Filecheck Section +remove_old_watchdog: + pkg.removed: + - name: python3-watchdog + +install_watchdog: + pkg.installed: + - name: securityonion-python39-watchdog + filecheck_logdir: file.directory: - name: /opt/so/log/strelka From ab1d97c985130bb3504ec3eee4ea330953cdb595 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 25 Aug 2023 09:39:16 -0400 Subject: [PATCH 182/350] restart filecheck if watchdog pkg changes --- salt/strelka/filestream/config.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index a254e9253..a84ab5ba1 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -135,6 +135,7 @@ filecheck_restart: - onchanges: - file: filecheck_script - file: filecheck_conf + - pkg: install_watchdog filcheck_history_clean: cron.present: From 0a88c812e867b51d19eb643d47dab1f9f7c24df3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 25 Aug 2023 13:03:33 -0400 Subject: [PATCH 183/350] differnet watchdog package names for debian vs redhat fams --- salt/strelka/filestream/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index a84ab5ba1..833a08505 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -47,6 +47,12 @@ filestream_config: FILESTREAMCONFIG: {{ STRELKAMERGED.filestream.config }} # Filecheck Section +{% if GLOBALS.os_family == 'Debian' %} +install_watchdog: + pkg.installed: + - name: python3-watchdog + +{% elif GLOBALS.os_family == 'RedHat' %} remove_old_watchdog: pkg.removed: - name: python3-watchdog @@ -54,6 +60,7 @@ remove_old_watchdog: install_watchdog: pkg.installed: - name: securityonion-python39-watchdog +{% endif %} filecheck_logdir: file.directory: From c22f9687fb1f23f5232c1a21e4dfa59555def7ec Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 25 Aug 2023 13:40:34 -0400 Subject: [PATCH 184/350] sync local repo in soup --- salt/manager/tools/sbin/soup | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 934cef2ee..21933c1a8 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -303,6 +303,7 @@ check_log_size_limit() { check_os_updates() { # Check to see if there are OS updates + echo "Checking for OS updates." NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated." OSUPDATES=$(dnf -q list updates | grep -v docker | grep -v containerd | grep -v salt | grep -v Available | wc -l) if [[ "$OSUPDATES" -gt 0 ]]; then @@ -437,6 +438,11 @@ post_to_2.4.20() { POSTVERSION=2.4.20 } +repo_sync() { + echo "Sync the local repo." + su socore -c '/usr/sbin/so-repo-sync' +} + stop_salt_master() { # kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts set +e @@ -762,9 +768,7 @@ main() { fi echo "Verifying we have the latest soup script." verify_latest_update_script - echo "Checking for OS updates." - check_os_updates - + echo "Let's see if we need to update Security Onion." upgrade_check upgrade_space @@ -776,6 +780,10 @@ main() { if [[ $is_airgap -eq 0 ]]; then yum clean all check_os_updates + elif [[ $OS == 'oracle' || $OS == 'redhat'|| $OS == 'centos' ]]; then + # sync remote repo down to local if not airgap + repo_sync + check_os_updates fi if [ "$is_hotfix" == "true" ]; then From 388c90f64113af0f750fec4aa091bda4064571b0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 25 Aug 2023 14:56:42 -0400 Subject: [PATCH 185/350] add oel to set_os --- salt/common/tools/sbin/so-common | 4 ++++ salt/manager/tools/sbin/soup | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index a76aab1f1..03b19d756 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -446,6 +446,10 @@ set_os() { OS=centos OSVER=9 is_centos=true + elif grep -q "Oracle Linux Server release 9" /etc/system-release; then + OS=oel + OSVER=9 + is_oracle=true fi cron_service_name="crond" else diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 21933c1a8..5cb59d6ac 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -780,7 +780,7 @@ main() { if [[ $is_airgap -eq 0 ]]; then yum clean all check_os_updates - elif [[ $OS == 'oracle' || $OS == 'redhat'|| $OS == 'centos' ]]; then + elif [[ $OS == 'oel' || $OS == 'rocky'|| $OS == 'centos' ]]; then # sync remote repo down to local if not airgap repo_sync check_os_updates From 022ee36bca46ae016b0e14868dfcf1cf726c68dd Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 25 Aug 2023 16:44:03 -0400 Subject: [PATCH 186/350] ingest pfsense sample data --- salt/common/tools/sbin/so-test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/common/tools/sbin/so-test b/salt/common/tools/sbin/so-test index 8d6bcf4e1..90309766b 100755 --- a/salt/common/tools/sbin/so-test +++ b/salt/common/tools/sbin/so-test @@ -5,4 +5,10 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +set -e + +# Playback live sample data onto monitor interface so-tcpreplay /opt/samples/* 2> /dev/null + +# Ingest sample pfsense log entry +echo "<134>$(date '+%b %d %H:%M:%S') filterlog[31624]: 84,,,1567509287,igb0.244,match,pass,in,4,0x0,,64,0,0,DF,6,tcp,64,192.168.1.1,10.10.10.10,56320,443,0,S,3333585167,,65535,,mss;nop;wscale;nop;nop;TS;sackOK;eol" | nc -uv -w1 localhost 514 From 5879eeabfa12a370feed8b7a462ed36ee379230e Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 25 Aug 2023 16:45:31 -0400 Subject: [PATCH 187/350] ingest pfsense sample data --- salt/common/tools/sbin/so-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-test b/salt/common/tools/sbin/so-test index 90309766b..7286a35a8 100755 --- a/salt/common/tools/sbin/so-test +++ b/salt/common/tools/sbin/so-test @@ -11,4 +11,4 @@ set -e so-tcpreplay /opt/samples/* 2> /dev/null # Ingest sample pfsense log entry -echo "<134>$(date '+%b %d %H:%M:%S') filterlog[31624]: 84,,,1567509287,igb0.244,match,pass,in,4,0x0,,64,0,0,DF,6,tcp,64,192.168.1.1,10.10.10.10,56320,443,0,S,3333585167,,65535,,mss;nop;wscale;nop;nop;TS;sackOK;eol" | nc -uv -w1 localhost 514 +echo "<134>$(date '+%b %d %H:%M:%S') filterlog[31624]: 84,,,1567509287,igb0.244,match,pass,in,4,0x0,,64,0,0,DF,6,tcp,64,192.168.1.1,10.10.10.10,56320,443,0,S,3333585167,,65535,,mss;nop;wscale;nop;nop;TS;sackOK;eol" | nc -uv -w1 localhost 514 > /dev/null 2>&1 From 1ef4d2cde11d581dd5b3f871460306f554f90a0c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Aug 2023 09:37:45 -0400 Subject: [PATCH 188/350] dont need to repo_sync rocky or centos --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 5cb59d6ac..37c9b3ba5 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -780,7 +780,7 @@ main() { if [[ $is_airgap -eq 0 ]]; then yum clean all check_os_updates - elif [[ $OS == 'oel' || $OS == 'rocky'|| $OS == 'centos' ]]; then + elif [[ $OS == 'oel' ]]; then # sync remote repo down to local if not airgap repo_sync check_os_updates From a8ec3717c44d1fd76343b321babaa7e44ab64bea Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Aug 2023 10:20:53 -0400 Subject: [PATCH 189/350] fail soup if so-repo-sync fails --- salt/manager/tools/sbin/so-repo-sync | 4 +++- salt/manager/tools/sbin/soup | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-repo-sync b/salt/manager/tools/sbin/so-repo-sync index 3e129cd0d..84384fcdf 100644 --- a/salt/manager/tools/sbin/so-repo-sync +++ b/salt/manager/tools/sbin/so-repo-sync @@ -11,6 +11,8 @@ set_version set_os salt_minion_count +set -e + curl --retry 5 --retry-delay 60 -A "reposync/$VERSION/$OS/$(uname -r)/$MINIONCOUNT" https://sigs.securityonion.net/checkup --output /tmp/checkup dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/ -createrepo /nsm/repo \ No newline at end of file +createrepo /nsm/repo diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 37c9b3ba5..45e3df530 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -440,7 +440,7 @@ post_to_2.4.20() { repo_sync() { echo "Sync the local repo." - su socore -c '/usr/sbin/so-repo-sync' + su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." } stop_salt_master() { From c10e686ec6f91d55bc53c8bd3b73c7f431b77bb9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 28 Aug 2023 11:07:28 -0400 Subject: [PATCH 190/350] fix path to intermediate ca cert on heavy nodes --- salt/redis/enabled.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/redis/enabled.sls b/salt/redis/enabled.sls index 4c452bec0..27177d217 100644 --- a/salt/redis/enabled.sls +++ b/salt/redis/enabled.sls @@ -33,7 +33,7 @@ so-redis: {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %} - /etc/pki/ca.crt:/certs/ca.crt:ro {% else %} - - /etc/pki/certs/intca.crt:/certs/ca.crt:ro + - /etc/pki/tls/certs/intca.crt:/certs/ca.crt:ro {% endif %} {% if DOCKER.containers['so-redis'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-redis'].custom_bind_mounts %} From 6b0fbe4634609603fdbddcc86d7eeea96e406a3b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Aug 2023 11:53:45 -0400 Subject: [PATCH 191/350] include so-repo-sync in soup_manager_scripts state --- salt/common/soup_scripts.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/soup_scripts.sls b/salt/common/soup_scripts.sls index 8dff85ddb..041649200 100644 --- a/salt/common/soup_scripts.sls +++ b/salt/common/soup_scripts.sls @@ -19,4 +19,5 @@ soup_manager_scripts: - source: salt://manager/tools/sbin - include_pat: - so-firewall - - soup \ No newline at end of file + - so-repo-sync + - soup From bd61ee22be5ea6e0568505f4dc1381322efd70fe Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 28 Aug 2023 14:41:06 -0400 Subject: [PATCH 192/350] Update defaults.map.jinja --- salt/soc/defaults.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 7720e7027..2587051c5 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -16,7 +16,7 @@ {# add nodes from the logstash:nodes pillar to soc.server.modules.elastic.remoteHostUrls #} {% for node_type, minions in salt['pillar.get']('logstash:nodes', {}).items() %} {% for m in minions.keys() %} -{% do SOCDEFAULTS.soc.config.server.modules.elastic.remoteHostUrls.append(m) %} +{% do SOCDEFAULTS.soc.config.server.modules.elastic.remoteHostUrls.append('https://' ~ m ~ ':9200') %} {% endfor %} {% endfor %} From 1c3d3d703ce4baf6e445721afd8facd53905c232 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Aug 2023 08:56:01 -0400 Subject: [PATCH 193/350] add desktop.map.jinja for global vars --- salt/vars/desktop.map.jinja | 1 + 1 file changed, 1 insertion(+) create mode 100644 salt/vars/desktop.map.jinja diff --git a/salt/vars/desktop.map.jinja b/salt/vars/desktop.map.jinja new file mode 100644 index 000000000..964f69663 --- /dev/null +++ b/salt/vars/desktop.map.jinja @@ -0,0 +1 @@ +{% set ROLE_GLOBALS = {} %} From a1b1294247d2464b66a56a22a400bb7341daff1c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Aug 2023 09:05:01 -0400 Subject: [PATCH 194/350] desktop doesnt need docker state --- salt/logrotate/init.sls | 1 + salt/top.sls | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/logrotate/init.sls b/salt/logrotate/init.sls index 1b096f9db..bdfc3b86c 100644 --- a/salt/logrotate/init.sls +++ b/salt/logrotate/init.sls @@ -3,6 +3,7 @@ logrotateconfdir: file.directory: - name: /opt/so/conf/logrotate + - makedirs: True commonlogrotatescript: file.managed: diff --git a/salt/top.sls b/salt/top.sls index 2323731a1..4a605b13c 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -28,12 +28,12 @@ base: - motd - salt.minion-check - salt.lasthighstate - - docker 'not *_desktop and G@saltversion:{{saltversion}}': - match: compound - common - + - docker + '*_sensor and G@saltversion:{{saltversion}}': - match: compound - sensor From 67ea7d31e110da56301dda1f49b84df7f1888df8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Aug 2023 09:32:10 -0400 Subject: [PATCH 195/350] dont exec so-setup desktop --- setup/so-functions | 4 +--- setup/so-whiptail | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index fc0876248..9f7e61fa1 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1664,9 +1664,7 @@ process_installtype() { elif [ "$install_type" = 'RECEIVER' ]; then is_receiver=true elif [ "$install_type" = 'DESKTOP' ]; then - if [ "$setup_type" != 'desktop' ]; then - exec bash so-setup desktop - fi + is_desktop=true fi } diff --git a/setup/so-whiptail b/setup/so-whiptail index c55e2db8f..01c0ffde9 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -678,9 +678,7 @@ whiptail_install_type_dist_existing() { elif [ "$install_type" = 'RECEIVER' ]; then is_receiver=true elif [ "$install_type" = 'DESKTOP' ]; then - if [ "$setup_type" != 'desktop' ]; then - exec bash so-setup desktop - fi + is_desktop=true fi local exitstatus=$? From 532b2c222a1180663dbc5420fe7cbd157c7c49d9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Aug 2023 10:16:51 -0400 Subject: [PATCH 196/350] edit other/desktop install whiptail --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 01c0ffde9..702949813 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -692,7 +692,7 @@ whiptail_install_type_other() { install_type=$(whiptail --title "$whiptail_title" --menu \ "Choose node type:" 10 65 2 \ - "DESKTOP" "Setup will run 'so-setup desktop' " 3>&1 1>&2 2>&3) + "DESKTOP" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 0455063a39500fad28ba6fb8c0521244211dd01c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Aug 2023 10:26:29 -0400 Subject: [PATCH 197/350] edit other/desktop install whiptail --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 702949813..6188406cb 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -692,7 +692,7 @@ whiptail_install_type_other() { install_type=$(whiptail --title "$whiptail_title" --menu \ "Choose node type:" 10 65 2 \ - "DESKTOP" 3>&1 1>&2 2>&3) + "DESKTOP" "Install Security Onion Desktop " 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From d40bbf6b090fce2fc922e6cf7d5e1f078195da46 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 29 Aug 2023 10:59:40 -0400 Subject: [PATCH 198/350] Add Apache templates --- salt/elasticsearch/defaults.yaml | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 3ea24c3fd..8ae75f984 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -286,6 +286,42 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false + so-logs-apache_x_access: + index_sorting: False + index_template: + index_patterns: + - "logs-apache.access-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-apache.access@package" + - "logs-apache.access@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + so-logs-apache_x_error: + index_sorting: False + index_template: + index_patterns: + - "logs-apache.error-*" + template: + settings: + index: + number_of_replicas: 0 + composed_of: + - "logs-apache.error@package" + - "logs-apache.error@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false so-logs-auditd_x_log: index_sorting: False index_template: From f118e25e8c8e424cd80110f04eeac77e4470aa95 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 29 Aug 2023 11:00:31 -0400 Subject: [PATCH 199/350] Add Apache references --- salt/elasticsearch/soc_elasticsearch.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index a960facd1..1823337b5 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -201,7 +201,8 @@ elasticsearch: so-logs-windows_x_powershell: *indexSettings so-logs-windows_x_powershell_operational: *indexSettings so-logs-windows_x_sysmon_operational: *indexSettings - so-logs-auditd_x_log: *indexSettings + so-logs-apache_x_access: *indexSettings + so-logs-apache_x_error: *indexSettings so-logs-aws_x_cloudtrail: *indexSettings so-logs-aws_x_cloudwatch_logs: *indexSettings so-logs-aws_x_ec2_logs: *indexSettings From c01a9006a6609a0d928bea0c640fe997d42c415e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 29 Aug 2023 11:01:22 -0400 Subject: [PATCH 200/350] Add Apache package --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 77fa9dd31..55e70113f 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -26,6 +26,7 @@ elasticfleet: - stderr - stdout packages: + - apache - auditd - aws - azure From d2063c7e119d77f31139c6c9c94a5e5de1f18b3e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 29 Aug 2023 11:14:49 -0400 Subject: [PATCH 201/350] Add auditd reference back --- salt/elasticsearch/soc_elasticsearch.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 1823337b5..e4de29e00 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -203,6 +203,7 @@ elasticsearch: so-logs-windows_x_sysmon_operational: *indexSettings so-logs-apache_x_access: *indexSettings so-logs-apache_x_error: *indexSettings + so-logs-auditd_x_log: *indexSettings so-logs-aws_x_cloudtrail: *indexSettings so-logs-aws_x_cloudwatch_logs: *indexSettings so-logs-aws_x_ec2_logs: *indexSettings From a4dc48237215eb3f87e377a28d5077f08be915ba Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Aug 2023 13:10:06 -0400 Subject: [PATCH 202/350] add is_desktop_grid var --- setup/so-functions | 2 +- setup/so-whiptail | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 9f7e61fa1..4e105dcd6 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1883,7 +1883,7 @@ securityonion_repo() { if [ -n "$(ls -A /etc/yum.repos.d/ 2>/dev/null)" ]; then logCmd "mv -v /etc/yum.repos.d/* /root/oldrepos/" fi - if [[ $is_desktop_iso ]]; then + if [[ ! $is_desktop_grid ]]; then gpg_rpm_import if [[ ! $is_airgap ]]; then echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /etc/yum/mirror.txt diff --git a/setup/so-whiptail b/setup/so-whiptail index 6188406cb..62f60a84a 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -58,6 +58,8 @@ whiptail_desktop_install() { whiptail --title "$whiptail_title" \ --yesno "$message" 11 75 --defaultno + is_desktop_grid=$? + } whiptail_desktop_nongrid_iso() { From 706a6e2d56ca045f97e2393d270e01a406334928 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 08:34:04 -0400 Subject: [PATCH 203/350] Make sure a data stream is created for syslog --- salt/elasticsearch/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8ae75f984..33362825f 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -4187,6 +4187,7 @@ elasticsearch: so-syslog: index_sorting: False index_template: + data_stream: {} index_patterns: - logs-syslog-so* template: From ce05f29dc4e436060a05cf02adfe3aa9578e3ee6 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 13:03:28 +0000 Subject: [PATCH 204/350] Add port_bindings for port 514 --- salt/docker/defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index e39feaf06..a5d6c5d6d 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -178,6 +178,9 @@ docker: extra_env: [] 'so-elastic-agent': final_octet: 46 + port_bindings: + - 0.0.0.0:514:514/tcp + - 0.0.0.0:514:514/udp custom_bind_mounts: [] extra_hosts: [] extra_env: [] From 655eea2b007124d9abe0674b5281435817cf290d Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 13:03:56 +0000 Subject: [PATCH 205/350] Add port_bindings --- salt/elasticagent/enabled.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls index 963b8549b..7d0f401e9 100644 --- a/salt/elasticagent/enabled.sls +++ b/salt/elasticagent/enabled.sls @@ -31,6 +31,10 @@ so-elastic-agent: - {{ XTRAHOST }} {% endfor %} {% endif %} + - port_bindings: + {% for BINDING in DOCKER.containers['so-elastic-agent'].port_bindings %} + - {{ BINDING }} + {% endfor %} - binds: - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro - /opt/so/log/elasticagent:/usr/share/elastic-agent/logs From 0e22acc255cc62af53810156eabaf9471d8bbcae Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 13:04:32 +0000 Subject: [PATCH 206/350] Add tcp and udp integration --- .../files/elastic-agent.yml.jinja | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/salt/elasticagent/files/elastic-agent.yml.jinja b/salt/elasticagent/files/elastic-agent.yml.jinja index 92aacfa44..7d0b93344 100644 --- a/salt/elasticagent/files/elastic-agent.yml.jinja +++ b/salt/elasticagent/files/elastic-agent.yml.jinja @@ -430,3 +430,54 @@ inputs: exclude_files: - >- broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$ + - id: udp-udp-35051de0-46a5-11ee-8d5d-9f98c8182f60 + name: syslog-udp-514 + revision: 3 + type: udp + use_output: default + meta: + package: + name: udp + version: 1.10.0 + data_stream: + namespace: so + package_policy_id: 35051de0-46a5-11ee-8d5d-9f98c8182f60 + streams: + - id: udp-udp.generic-35051de0-46a5-11ee-8d5d-9f98c8182f60 + data_stream: + dataset: syslog + pipeline: syslog + host: '0.0.0.0:514' + max_message_size: 10KiB + processors: + - add_fields: + fields: + module: syslog + target: event + tags: + - syslog + - id: tcp-tcp-33d37bb0-46a5-11ee-8d5d-9f98c8182f60 + name: syslog-tcp-514 + revision: 3 + type: tcp + use_output: default + meta: + package: + name: tcp + version: 1.10.0 + data_stream: + namespace: so + package_policy_id: 33d37bb0-46a5-11ee-8d5d-9f98c8182f60 + streams: + - id: tcp-tcp.generic-33d37bb0-46a5-11ee-8d5d-9f98c8182f60 + data_stream: + dataset: syslog + pipeline: syslog + host: '0.0.0.0:514' + processors: + - add_fields: + fields: + module: syslog + target: event + tags: + - syslog From 60b0af5ab793fa1b6a592743f162e6905b797798 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 13:05:30 +0000 Subject: [PATCH 207/350] Allow external syslog --- salt/firewall/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index ff127c419..ecb4bad6b 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -1141,6 +1141,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: From ae01da780e242ddc084e3fb7907a952d0599cb88 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Aug 2023 09:10:59 -0400 Subject: [PATCH 208/350] desktop network install nongrid --- setup/so-functions | 2 +- setup/so-setup | 61 +++++++++++++++++++++++----------------------- setup/so-whiptail | 6 ++++- 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 4e105dcd6..1c0cad2a7 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1883,7 +1883,7 @@ securityonion_repo() { if [ -n "$(ls -A /etc/yum.repos.d/ 2>/dev/null)" ]; then logCmd "mv -v /etc/yum.repos.d/* /root/oldrepos/" fi - if [[ ! $is_desktop_grid ]]; then + if ! $is_desktop_grid; then gpg_rpm_import if [[ ! $is_airgap ]]; then echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /etc/yum/mirror.txt diff --git a/setup/so-setup b/setup/so-setup index b946c06c8..61c0d88e3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -333,38 +333,39 @@ if [[ $is_desktop ]]; then exit 1 fi - if ! whiptail_desktop_install; then - if [[ $is_desktop_iso ]]; then - if whiptail_desktop_nongrid_iso; then - # Remove setup from auto launching - parse_install_username - sed -i '$ d' /home/$INSTALLUSERNAME/.bash_profile >> "$setup_log" 2>&1 - securityonion_repo - info "Enabling graphical interface and setting it to load at boot" - systemctl set-default graphical.target - info "Setting desktop background" - set_desktop_background - echo "Desktop Install Complete!" - echo "" - echo "Please reboot to start graphical interface." - exit 0 + whiptail_desktop_install + if ! is_desktop_grid; then + if [[ $is_desktop_iso ]]; then + if whiptail_desktop_nongrid_iso; then + # Remove setup from auto launching + parse_install_username + sed -i '$ d' /home/$INSTALLUSERNAME/.bash_profile >> "$setup_log" 2>&1 + securityonion_repo + info "Enabling graphical interface and setting it to load at boot" + systemctl set-default graphical.target + info "Setting desktop background" + set_desktop_background + echo "Desktop Install Complete!" + echo "" + echo "Please reboot to start graphical interface." + exit 0 + else + # Abort! + exit 0 + fi else - # Abort! - exit 0 + if whiptail_desktop_nongrid_network; then + info "" + info "" + info "Kicking off the automated setup of the Security Onion Desktop. This can take a while depending on your network connection." + info "" + info "" + desktop_salt_local + else + # Abort! + exit 0 + fi fi - else - if whiptail_desktop_nongrid_network; then - info "" - info "" - info "Kicking off the automated setup of the Security Onion Desktop. This can take a while depending on your network connection." - info "" - info "" - desktop_salt_local - else - # Abort! - exit 0 - fi - fi fi # If you got this far then you want to join the grid diff --git a/setup/so-whiptail b/setup/so-whiptail index 62f60a84a..8fd3b5fdd 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -58,7 +58,11 @@ whiptail_desktop_install() { whiptail --title "$whiptail_title" \ --yesno "$message" 11 75 --defaultno - is_desktop_grid=$? + if [ $? -eq 0 ]; then + is_desktop_grid=true + else + is_desktop_grid=false + fi } From 97587064f8dd55b427d362c76748e73c581ce936 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Aug 2023 09:48:52 -0400 Subject: [PATCH 209/350] remove packages from nongrid desktop install --- setup/so-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 1c0cad2a7..eab98b849 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -91,7 +91,7 @@ desktop_salt_local() { securityonion_repo gpg_rpm_import # Install salt - logCmd "yum -y install salt-minion-$SALTVERSION httpd-tools python3 python36-docker python36-dateutil python36-m2crypto python36-mysql python36-packaging python36-lxml yum-utils device-mapper-persistent-data lvm2 openssl jq" + logCmd "yum -y install salt-minion-$SALTVERSION httpd-tools python3 python3-dateutil yum-utils device-mapper-persistent-data lvm2 openssl jq" logCmd "yum -y update --exclude=salt*" logCmd "salt-call state.apply desktop --local --file-root=../salt/ -l info" From a3eeba4761991448fbaf3eb0ae48383e5681a3dc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Aug 2023 09:51:09 -0400 Subject: [PATCH 210/350] do networking_needful for nongrid desktop network install --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index 61c0d88e3..de117331d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -355,6 +355,7 @@ if [[ $is_desktop ]]; then fi else if whiptail_desktop_nongrid_network; then + networking_needful info "" info "" info "Kicking off the automated setup of the Security Onion Desktop. This can take a while depending on your network connection." From 8381fa1d4220d0943eb5efc703c8040bf505d923 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Aug 2023 10:26:24 -0400 Subject: [PATCH 211/350] cant import globals because of nongrid desktop install~ --- salt/desktop/xwindows.sls | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/desktop/xwindows.sls b/salt/desktop/xwindows.sls index b18109d45..66e4c9a05 100644 --- a/salt/desktop/xwindows.sls +++ b/salt/desktop/xwindows.sls @@ -1,7 +1,5 @@ -{% from 'vars/globals.map.jinja' import GLOBALS %} - {# we only want this state to run it is CentOS #} -{% if GLOBALS.os == 'OEL' %} +{% if grains.os == 'OEL' %} include: - desktop.packages From b14614ae53f055cb75be9fab8744dfcf7a811639 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Aug 2023 10:32:13 -0400 Subject: [PATCH 212/350] need $ for vars --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index de117331d..f6e5c8c4e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -334,7 +334,7 @@ if [[ $is_desktop ]]; then fi whiptail_desktop_install - if ! is_desktop_grid; then + if ! $is_desktop_grid; then if [[ $is_desktop_iso ]]; then if whiptail_desktop_nongrid_iso; then # Remove setup from auto launching From b45e114ef25389be5fb97d1a8dc896e8b872d887 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Aug 2023 10:41:34 -0400 Subject: [PATCH 213/350] cant use GLOBALS var due to desktop nongrid install --- salt/desktop/packages.sls | 4 +--- salt/desktop/remove_gui.sls | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/salt/desktop/packages.sls b/salt/desktop/packages.sls index 524c2c266..3817f2e80 100644 --- a/salt/desktop/packages.sls +++ b/salt/desktop/packages.sls @@ -1,7 +1,5 @@ -{% from 'vars/globals.map.jinja' import GLOBALS %} - {# we only want this state to run it is CentOS #} -{% if GLOBALS.os == 'OEL' %} +{% if grains.os == 'OEL' %} desktop_packages: pkg.installed: diff --git a/salt/desktop/remove_gui.sls b/salt/desktop/remove_gui.sls index 53d927cbe..d8de07a9a 100644 --- a/salt/desktop/remove_gui.sls +++ b/salt/desktop/remove_gui.sls @@ -1,7 +1,5 @@ -{% from 'vars/globals.map.jinja' import GLOBALS %} - {# we only want this state to run it is CentOS #} -{% if GLOBALS.os == 'OEL' %} +{% if grains.os == 'OEL' %} remove_graphical_target: file.symlink: From fe690922de38a2d6ef9dd9da8afad312e39ad97e Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 19:16:05 +0000 Subject: [PATCH 214/350] Add analyzer configuration to the defaults file --- salt/sensoroni/defaults.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/salt/sensoroni/defaults.yaml b/salt/sensoroni/defaults.yaml index 4ccc11ce9..f53646ac2 100644 --- a/salt/sensoroni/defaults.yaml +++ b/salt/sensoroni/defaults.yaml @@ -8,3 +8,31 @@ sensoroni: node_checkin_interval_ms: 10000 sensoronikey: soc_host: + analyzers: + emailrep: + base_url: https://emailrep.io/ + api_key: + greynoise: + base_url: https://api.greynoise.io/ + api_key: + api_version: community + localfile: + file_path: [] + otx: + base_url: https://otx.alienvault.com/api/v1/ + api_key: + pulsedive: + base_url: https://pulsedive.com/api/ + api_key: + spamhaus: + lookup_host: zen.spamhaus.org + nameservers: [] + urlscan: + base_url: https://urlscan.io/api/v1/ + api_key: + enabled: False + visibility: public + timeout: 180 + virustotal: + base_url: https://www.virustotal.com/api/v3/search?query= + api_key: From 8cc19b0748c6804abe01abb8f7dd3df9dc23784f Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 30 Aug 2023 19:16:38 +0000 Subject: [PATCH 215/350] Add analyzer configuration description --- salt/sensoroni/soc_sensoroni.yaml | 142 ++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index 8a35272ea..6a728ef9c 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -37,3 +37,145 @@ sensoroni: helpLink: sensoroni.html global: True advanced: True + analyzers: + emailrep: + api_key: + description: API key for the EmailRep analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the EmailRep analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + greynoise: + api_key: + description: API key for the GreyNoise analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + api_version: + description: API key for the GreyNoise analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + base_url: + description: Base URL for the GreyNoise analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + localfile: + file_path: + description: File path for the LocalFile analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: "[]string" + otx: + api_key: + description: API key for the OTX analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the OTX analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + pulsedive: + api_key: + description: API key for the Pulsedive analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the Pulsedive analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + spamhaus: + lookup_host: + description: Host to use for lookups. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + nameservers: + description: Nameservers used for queries. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedTypes: "[]string" + urlscan: + api_key: + description: API key for the Urlscan analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the Urlscan analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + enabled: + description: Analyzer enabled + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: bool + timeout: + description: Timeout for the Urlscan analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: int + visibility: + description: Type of visibility. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string + virustotal: + api_key: + description: API key for the VirusTotal analyzer. + helpLink: sensoroni.html + global: True + sensitive: True + advanced: True + forcedType: string + base_url: + description: Base URL for the VirusTotal analyzer. + helpLink: sensoroni.html + global: True + sensitive: False + advanced: True + forcedType: string From 78915f900b8aad6ebb9e4038ec1f4b0ad916add6 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 15:37:30 -0400 Subject: [PATCH 216/350] Add fortigate package --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 55e70113f..979e795f7 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -41,6 +41,7 @@ elasticfleet: - fleet_server - fim - fortinet + - fortinet_fortigate - gcp - github - google_workspace From d090852895fb899fb9d029c57ae2c54e879a9722 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 15:40:40 -0400 Subject: [PATCH 217/350] Correct fortigate template name --- salt/elasticsearch/defaults.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 33362825f..cc2f5e1cd 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -970,18 +970,18 @@ elasticsearch: data_stream: hidden: false allow_custom_routing: false - so-logs-fortinet_x_fortigate: + so-logs-fortinet_fortigate_x_log: index_sorting: False index_template: index_patterns: - - "logs-fortinet.fortigate-*" + - "logs-fortinet_fortigate.log-*" template: settings: index: number_of_replicas: 0 composed_of: - - "logs-fortinet.fortigate@package" - - "logs-fortinet.fortigate@custom" + - "logs-fortinet_fortigate.log@package" + - "logs-fortinet_fortigate.log@custom" - "so-fleet_globals-1" - "so-fleet_agent_id_verification-1" priority: 501 From 21e91a753701b2672fedc9a5025982b05138fb6b Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 16:10:38 -0400 Subject: [PATCH 218/350] Fix api_version --- salt/sensoroni/soc_sensoroni.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index 6a728ef9c..2d1536191 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -62,7 +62,7 @@ sensoroni: advanced: True forcedType: string api_version: - description: API key for the GreyNoise analyzer. + description: API version for the GreyNoise analyzer. helpLink: sensoroni.html global: True sensitive: False From 41300af944c1c537ef9bf99bb2411d80bafdbb4e Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 30 Aug 2023 16:30:32 -0400 Subject: [PATCH 219/350] Set global to false --- salt/sensoroni/soc_sensoroni.yaml | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index 2d1536191..eb63dbe25 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -42,14 +42,14 @@ sensoroni: api_key: description: API key for the EmailRep analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the EmailRep analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -57,21 +57,21 @@ sensoroni: api_key: description: API key for the GreyNoise analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string api_version: description: API version for the GreyNoise analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string base_url: description: Base URL for the GreyNoise analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -79,7 +79,7 @@ sensoroni: file_path: description: File path for the LocalFile analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: "[]string" @@ -87,14 +87,14 @@ sensoroni: api_key: description: API key for the OTX analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the OTX analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -102,14 +102,14 @@ sensoroni: api_key: description: API key for the Pulsedive analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the Pulsedive analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -117,14 +117,14 @@ sensoroni: lookup_host: description: Host to use for lookups. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string nameservers: description: Nameservers used for queries. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedTypes: "[]string" @@ -132,35 +132,35 @@ sensoroni: api_key: description: API key for the Urlscan analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the Urlscan analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string enabled: description: Analyzer enabled helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: bool timeout: description: Timeout for the Urlscan analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: int visibility: description: Type of visibility. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string @@ -168,14 +168,14 @@ sensoroni: api_key: description: API key for the VirusTotal analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the VirusTotal analyzer. helpLink: sensoroni.html - global: True + global: False sensitive: False advanced: True forcedType: string From 14a62805310e59d4e16f3b6809fa0d74860edb81 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Aug 2023 16:49:17 -0400 Subject: [PATCH 220/350] iso desktop join grid - set install_type and minion_type --- setup/so-functions | 2 +- setup/so-setup | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index eab98b849..4f973d147 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1269,7 +1269,7 @@ get_redirect() { get_minion_type() { local minion_type case "$install_type" in - 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'SEARCHNODE' | 'FLEET' | 'IDH' | 'STANDALONE' | 'IMPORT' | 'RECEIVER') + 'EVAL' | 'MANAGERSEARCH' | 'MANAGER' | 'SENSOR' | 'HEAVYNODE' | 'SEARCHNODE' | 'FLEET' | 'IDH' | 'STANDALONE' | 'IMPORT' | 'RECEIVER' | 'DESKTOP') minion_type=$(echo "$install_type" | tr '[:upper:]' '[:lower:]') ;; esac diff --git a/setup/so-setup b/setup/so-setup index f6e5c8c4e..cdc7e67d6 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -104,6 +104,7 @@ if [ "$setup_type" = 'desktop' ]; then # Check to see if this is an ISO. Usually this dir on exists on ISO installs. if [ -d /root/SecurityOnion ]; then is_desktop_iso=true + install_type='DESKTOP' fi fi From a615fc8e47f444ea9dc87390c897626b5b226216 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Wed, 30 Aug 2023 15:33:01 -0600 Subject: [PATCH 221/350] New Config Default: longRelayTimeoutMs Salt is getting a second timeout for operations known to take a long time such as sending and importing files. There's also an entry in soc_soc.yaml so the value can be changed in SOC's config page. --- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ff8b240ec..05543cd19 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1012,6 +1012,7 @@ soc: verifyCert: false salt: queueDir: /opt/sensoroni/queue + longRelayTimeoutMs: 120000 sostatus: refreshIntervalMs: 30000 offlineThresholdMs: 900000 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index b2ed893f6..e94144069 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -111,6 +111,11 @@ soc: description: Duration (in milliseconds) that must elapse after a grid node fails to check-in before the node will be marked offline (fault). global: True advanced: True + salt: + longRelayTimeoutMs: + description: Duration (in milliseconds) to wait for a response from the Salt API when executing tasks known for being long running before giving up and showing an error on the SOC UI. + global: True + advanced: True client: enableReverseLookup: description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI. From c812c3991ef952eb3b9e472ea4b1353221e72695 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 31 Aug 2023 08:54:13 -0400 Subject: [PATCH 222/350] we dont need to run convert-gnome-classic script --- salt/desktop/scripts/convert-gnome-classic.sh | 4 ---- salt/desktop/xwindows.sls | 5 +---- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 salt/desktop/scripts/convert-gnome-classic.sh diff --git a/salt/desktop/scripts/convert-gnome-classic.sh b/salt/desktop/scripts/convert-gnome-classic.sh deleted file mode 100644 index e69a43b2d..000000000 --- a/salt/desktop/scripts/convert-gnome-classic.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -echo "Setting default session to gnome-classic" -cp /usr/share/accountsservice/user-templates/standard /etc/accountsservice/user-templates/ -sed -i 's|Session=gnome|Session=gnome-classic|g' /etc/accountsservice/user-templates/standard diff --git a/salt/desktop/xwindows.sls b/salt/desktop/xwindows.sls index 66e4c9a05..85da0590c 100644 --- a/salt/desktop/xwindows.sls +++ b/salt/desktop/xwindows.sls @@ -12,10 +12,7 @@ graphical_target: - require: - desktop_packages -convert_gnome_classic: - cmd.script: - - name: salt://desktop/scripts/convert-gnome-classic.sh - +{# set users to use gnome-classic #} {% for username in salt['file.find'](path='/home/',mindepth=1,maxdepth=1,type='d') %} {% set username = username.split('/')[2] %} {% if username != 'zeek' %} From da56a421e5b0a89c2fa7750249066762777191cd Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 31 Aug 2023 09:17:33 -0400 Subject: [PATCH 223/350] Update motd.md --- salt/soc/files/soc/motd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index cf22e863d..d6b0d3d27 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -1,6 +1,6 @@ ## Getting Started -New to Security Onion 2? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheatsheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion 2 Essentials online course, available on our [Training](https://securityonionsolutions.com/training) website. +New to Security Onion 2? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheat Sheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion 2 Essentials online course, available on our [Training](https://securityonionsolutions.com/training) website. If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. Then go to the [Dashboards](/#/dashboards) interface for a general overview of all logs collected or go to the [Hunt](/#/hunt) interface for more focused threat hunting. Once you've found something of interest, escalate it to [Cases](/#/cases) to then collect evidence and analyze observables as you work towards closing the case. From a60c34d5488bc88fbbe9a6f817dffd3327eecf8e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 31 Aug 2023 09:40:54 -0400 Subject: [PATCH 224/350] exclude unnecessary pillars from desktop nodes --- pillar/top.sls | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 4893c44f9..bf28b6474 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -2,21 +2,23 @@ base: '*': - global.soc_global - global.adv_global - - docker.soc_docker - - docker.adv_docker - - firewall.soc_firewall - - firewall.adv_firewall - - influxdb.token - logrotate.soc_logrotate - logrotate.adv_logrotate - - nginx.soc_nginx - - nginx.adv_nginx - - node_data.ips - ntp.soc_ntp - ntp.adv_ntp - patch.needs_restarting - patch.soc_patch - patch.adv_patch + + '* and not *_desktop': + - docker.soc_docker + - docker.adv_docker + - firewall.soc_firewall + - firewall.adv_firewall + - influxdb.token + - nginx.soc_nginx + - nginx.adv_nginx + - node_data.ips - sensoroni.soc_sensoroni - sensoroni.adv_sensoroni - telegraf.soc_telegraf From ee848b8a8c7c8b547d70885e5f03ea99a0f1e25e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 31 Aug 2023 09:51:55 -0400 Subject: [PATCH 225/350] comments for desktop install --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index cdc7e67d6..a9c7776c3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -578,9 +578,9 @@ if ! [[ -f $install_opt_file ]]; then set_minion_info whiptail_end_settings + # desktop install will only get this far if joining the grid elif [[ $is_desktop ]]; then info "Setting up as node type desktop" - #check_requirements "desktop" networking_needful collect_mngr_hostname add_mngr_ip_to_hosts From 1a3b3b21fbd9dfbc0395659fb7183bea9f1c9d4d Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 31 Aug 2023 15:09:19 +0000 Subject: [PATCH 226/350] Change entropy value syntax --- salt/elasticsearch/files/ingest/strelka.file | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/strelka.file b/salt/elasticsearch/files/ingest/strelka.file index 741e20aa1..a74a7c622 100644 --- a/salt/elasticsearch/files/ingest/strelka.file +++ b/salt/elasticsearch/files/ingest/strelka.file @@ -63,8 +63,8 @@ { "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 50 && ctx.rule?.score <=69", "field": "event.severity", "value": 2, "override": true } }, { "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 70 && ctx.rule?.score <=89", "field": "event.severity", "value": 3, "override": true } }, { "set": { "if": "ctx.rule?.score != null && ctx.rule?.score >= 90", "field": "event.severity", "value": 4, "override": true } }, - { "set": { "if": "ctx.scan?.entropy?.entropy == 0", "field": "scan.entropy.entropy", "value": "0.0", "override": true } }, - { "set": { "if": "ctx.scan?.pe?.image_version == 0", "field": "scan.pe.image_version", "value": "0.0", "override": true } }, + { "set": { "if": "ctx.scan?.entropy?.entropy == '0'", "field": "scan.entropy.entropy", "value": "0.0", "override": true } }, + { "set": { "if": "ctx.scan?.pe?.image_version == '0'", "field": "scan.pe.image_version", "value": "0.0", "override": true } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" }}, { "convert" : { "field" : "scan.exiftool","type": "string", "ignore_missing":true }}, { "remove": { "field": ["host", "path", "message", "exiftool", "scan.yara.meta"], "ignore_missing": true } }, From 0fed757b11c56af7106badc3ca5cb38786108b32 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 31 Aug 2023 15:10:27 +0000 Subject: [PATCH 227/350] Add entropy mapping --- .../templates/component/so/so-scan-mappings.json | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/templates/component/so/so-scan-mappings.json b/salt/elasticsearch/templates/component/so/so-scan-mappings.json index 60dc5b928..8ddbe6077 100644 --- a/salt/elasticsearch/templates/component/so/so-scan-mappings.json +++ b/salt/elasticsearch/templates/component/so/so-scan-mappings.json @@ -33,10 +33,17 @@ } } } - } + }, + "entropy": { + "properties": { + "entropy": { + "type": "float" + } + } + } } } } } } -} \ No newline at end of file +} From b010919099acda92e7473ab605a9e2ae60f2049c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 31 Aug 2023 13:21:32 -0400 Subject: [PATCH 228/350] add sensoroni, telegraf, common states to desktop. allow docker_registry connection to managers for desktop --- pillar/top.sls | 10 +++++----- salt/firewall/defaults.yaml | 9 +++++++++ salt/top.sls | 26 +++----------------------- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index bf28b6474..9f21a2c99 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -2,6 +2,7 @@ base: '*': - global.soc_global - global.adv_global + - influxdb.token - logrotate.soc_logrotate - logrotate.adv_logrotate - ntp.soc_ntp @@ -9,20 +10,19 @@ base: - patch.needs_restarting - patch.soc_patch - patch.adv_patch + - sensoroni.soc_sensoroni + - sensoroni.adv_sensoroni + - telegraf.soc_telegraf + - telegraf.adv_telegraf '* and not *_desktop': - docker.soc_docker - docker.adv_docker - firewall.soc_firewall - firewall.adv_firewall - - influxdb.token - nginx.soc_nginx - nginx.adv_nginx - node_data.ips - - sensoroni.soc_sensoroni - - sensoroni.adv_sensoroni - - telegraf.soc_telegraf - - telegraf.adv_telegraf '*_manager or *_managersearch': - match: compound diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index ecb4bad6b..578a242f9 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -463,6 +463,9 @@ firewall: - endgame desktop: portgroups: + - docker_registry + - influxdb + - sensoroni - yum customhostgroup0: portgroups: [] @@ -651,6 +654,9 @@ firewall: - endgame desktop: portgroups: + - docker_registry + - influxdb + - sensoroni - yum customhostgroup0: portgroups: [] @@ -847,6 +853,9 @@ firewall: - strelka_frontend desktop: portgroups: + - docker_registry + - influxdb + - sensoroni - yum customhostgroup0: portgroups: [] diff --git a/salt/top.sls b/salt/top.sls index 4a605b13c..ccad30307 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -28,18 +28,18 @@ base: - motd - salt.minion-check - salt.lasthighstate + - common + - sensoroni + - telegraf 'not *_desktop and G@saltversion:{{saltversion}}': - match: compound - - common - docker '*_sensor and G@saltversion:{{saltversion}}': - match: compound - sensor - ssl - - sensoroni - - telegraf - firewall - nginx - pcap @@ -57,11 +57,9 @@ base: - ca - ssl - registry - - sensoroni - manager - backup.config_backup - nginx - - telegraf - influxdb - soc - kratos @@ -92,9 +90,7 @@ base: - ca - ssl - registry - - sensoroni - nginx - - telegraf - influxdb - soc - kratos @@ -124,11 +120,9 @@ base: - ca - ssl - registry - - sensoroni - manager - backup.config_backup - nginx - - telegraf - influxdb - soc - kratos @@ -157,9 +151,7 @@ base: '*_searchnode and G@saltversion:{{saltversion}}': - match: compound - ssl - - sensoroni - nginx - - telegraf - firewall - elasticsearch - logstash @@ -172,9 +164,7 @@ base: - ca - ssl - registry - - sensoroni - nginx - - telegraf - influxdb - soc - kratos @@ -201,9 +191,7 @@ base: - match: compound - sensor - ssl - - sensoroni - nginx - - telegraf - firewall - elasticsearch - logstash @@ -224,10 +212,8 @@ base: - ca - ssl - registry - - sensoroni - manager - nginx - - telegraf - influxdb - soc - kratos @@ -247,8 +233,6 @@ base: '*_receiver and G@saltversion:{{saltversion}}': - match: compound - ssl - - sensoroni - - telegraf - firewall - logstash - redis @@ -258,8 +242,6 @@ base: '*_idh and G@saltversion:{{saltversion}}': - match: compound - ssl - - sensoroni - - telegraf - firewall - elasticfleet.install_agent_grid - docker_clean @@ -268,8 +250,6 @@ base: '*_fleet and G@saltversion:{{saltversion}}': - match: compound - ssl - - sensoroni - - telegraf - firewall - logstash - elasticfleet From 1871d48f7f1fe38686024804f0cde9c93ff75f27 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 31 Aug 2023 20:42:00 -0400 Subject: [PATCH 229/350] remove unnecesary OTHER submenu --- setup/so-whiptail | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 8fd3b5fdd..9622ad44a 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -563,7 +563,7 @@ whiptail_install_type() { "EVAL" "Evaluation mode (not for production) " \ "STANDALONE" "Standalone production install " \ "DISTRIBUTED" "Distributed install submenu " \ - "OTHER" "Other install types" \ + "DESKTOP" "Install Security Onion Desktop" \ 3>&1 1>&2 2>&3 ) elif [[ "$OSVER" == "focal" ]]; then @@ -584,8 +584,6 @@ whiptail_install_type() { else whiptail_install_type_dist_existing fi - elif [[ $install_type == "OTHER" ]]; then - whiptail_install_type_other fi export install_type @@ -691,21 +689,6 @@ whiptail_install_type_dist_existing() { whiptail_check_exitstatus $exitstatus } - -whiptail_install_type_other() { - - [ -n "$TESTING" ] && return - - install_type=$(whiptail --title "$whiptail_title" --menu \ - "Choose node type:" 10 65 2 \ - "DESKTOP" "Install Security Onion Desktop " 3>&1 1>&2 2>&3) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - - export install_type -} - whiptail_invalid_input() { # TODO: This should accept a list of arguments to specify what general pattern the input should follow [ -n "$TESTING" ] && return From b64fa512688239d06ba8b09bc3261c595f943ee8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 1 Sep 2023 09:16:24 -0400 Subject: [PATCH 230/350] give desktop docker state and pillars --- pillar/top.sls | 4 ++-- salt/top.sls | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 9f21a2c99..53ec8a330 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -2,6 +2,8 @@ base: '*': - global.soc_global - global.adv_global + - docker.soc_docker + - docker.adv_docker - influxdb.token - logrotate.soc_logrotate - logrotate.adv_logrotate @@ -16,8 +18,6 @@ base: - telegraf.adv_telegraf '* and not *_desktop': - - docker.soc_docker - - docker.adv_docker - firewall.soc_firewall - firewall.adv_firewall - nginx.soc_nginx diff --git a/salt/top.sls b/salt/top.sls index ccad30307..e278635b5 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -29,13 +29,10 @@ base: - salt.minion-check - salt.lasthighstate - common + - docker - sensoroni - telegraf - 'not *_desktop and G@saltversion:{{saltversion}}': - - match: compound - - docker - '*_sensor and G@saltversion:{{saltversion}}': - match: compound - sensor From 0fb00d569e4417f382a211b8b89d3685c5ea46a1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 1 Sep 2023 09:39:39 -0400 Subject: [PATCH 231/350] allow states for desktop. give all nodes docker_clean, order it last --- salt/allowed_states.map.jinja | 2 ++ salt/docker_clean/init.sls | 1 + salt/top.sls | 12 +----------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index a3c5c75ab..6932e8c84 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -188,6 +188,8 @@ 'docker_clean' ], 'so-desktop': [ + 'docker_clean', + 'telegraf' ], }, grain='role') %} diff --git a/salt/docker_clean/init.sls b/salt/docker_clean/init.sls index c11af4f56..ee60f5591 100644 --- a/salt/docker_clean/init.sls +++ b/salt/docker_clean/init.sls @@ -9,6 +9,7 @@ prune_images: cmd.run: - name: so-docker-prune + - order: last {% else %} diff --git a/salt/top.sls b/salt/top.sls index e278635b5..2409aec82 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -32,6 +32,7 @@ base: - docker - sensoroni - telegraf + - docker_clean '*_sensor and G@saltversion:{{saltversion}}': - match: compound @@ -44,7 +45,6 @@ base: - healthcheck - zeek - strelka - - docker_clean - elasticfleet.install_agent_grid '*_eval and G@saltversion:{{saltversion}}': @@ -79,7 +79,6 @@ base: - playbook - redis - elasticfleet - - docker_clean '*_manager and G@saltversion:{{saltversion}}': - match: compound @@ -108,7 +107,6 @@ base: - soctopus - playbook - elasticfleet - - docker_clean '*_standalone and G@saltversion:{{saltversion}}': - match: compound @@ -143,7 +141,6 @@ base: - soctopus - playbook - elasticfleet - - docker_clean '*_searchnode and G@saltversion:{{saltversion}}': - match: compound @@ -153,7 +150,6 @@ base: - elasticsearch - logstash - elasticfleet.install_agent_grid - - docker_clean '*_managersearch and G@saltversion:{{saltversion}}': - match: compound @@ -182,7 +178,6 @@ base: - soctopus - playbook - elasticfleet - - docker_clean '*_heavynode and G@saltversion:{{saltversion}}': - match: compound @@ -200,7 +195,6 @@ base: - zeek - elasticfleet.install_agent_grid - elasticagent - - docker_clean '*_import and G@saltversion:{{saltversion}}': - match: compound @@ -225,7 +219,6 @@ base: - suricata - zeek - elasticfleet - - docker_clean '*_receiver and G@saltversion:{{saltversion}}': - match: compound @@ -234,14 +227,12 @@ base: - logstash - redis - elasticfleet.install_agent_grid - - docker_clean '*_idh and G@saltversion:{{saltversion}}': - match: compound - ssl - firewall - elasticfleet.install_agent_grid - - docker_clean - idh '*_fleet and G@saltversion:{{saltversion}}': @@ -252,7 +243,6 @@ base: - elasticfleet - elasticfleet.install_agent_grid - schedule - - docker_clean 'J@desktop:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:OEL )': - match: compound From b64d4e36584bca8ea60c7f7f4d9a99407cb52251 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 1 Sep 2023 09:53:26 -0400 Subject: [PATCH 232/350] add telegraf pillar to desktop --- salt/manager/tools/sbin/so-minion | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index de55c3a5b..075632985 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -552,6 +552,7 @@ function createRECEIVER() { function createDESKTOP() { add_desktop_to_minion + add_telegraf_to_minion } function testConnection() { From 546c562ef0ad9779c41adacc03378e515d826c50 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 1 Sep 2023 10:31:02 -0400 Subject: [PATCH 233/350] expose standard relay timeout in config UI; up default to 45s to accommodate sluggish pillar.get calls --- salt/soc/defaults.yaml | 1 + salt/soc/soc_soc.yaml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 05543cd19..6d8ed5bfd 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1012,6 +1012,7 @@ soc: verifyCert: false salt: queueDir: /opt/sensoroni/queue + timeoutMs: 45000 longRelayTimeoutMs: 120000 sostatus: refreshIntervalMs: 30000 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index e94144069..291f564ed 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -116,6 +116,10 @@ soc: description: Duration (in milliseconds) to wait for a response from the Salt API when executing tasks known for being long running before giving up and showing an error on the SOC UI. global: True advanced: True + relayTimeoutMs: + description: Duration (in milliseconds) to wait for a response from the Salt API when executing common grid management tasks before giving up and showing an error on the SOC UI. + global: True + advanced: True client: enableReverseLookup: description: Set to true to enable reverse DNS lookups for IP addresses in the SOC UI. From 765a22e6f0c435e4d9b36d957623b5bcf8b2cd30 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 1 Sep 2023 11:31:23 -0400 Subject: [PATCH 234/350] Add so-elastic-agent --- salt/firewall/containers.map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/firewall/containers.map.jinja b/salt/firewall/containers.map.jinja index 617b4a216..02e8a4644 100644 --- a/salt/firewall/containers.map.jinja +++ b/salt/firewall/containers.map.jinja @@ -58,6 +58,7 @@ {% set NODE_CONTAINERS = [ 'so-curator', 'so-elasticsearch', + 'so-elastic-agent', 'so-logstash', 'so-nginx', 'so-redis', From 3434d0f200ea1a79d4ad5c58c33fa3b04cb7093b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 1 Sep 2023 12:02:30 -0400 Subject: [PATCH 235/350] add sensoroni and telegraf back to individual nodes. add seperate block for desktop --- salt/top.sls | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index 2409aec82..b8ca0f14e 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -30,14 +30,14 @@ base: - salt.lasthighstate - common - docker - - sensoroni - - telegraf - docker_clean '*_sensor and G@saltversion:{{saltversion}}': - match: compound - sensor - ssl + - sensoroni + - telegraf - firewall - nginx - pcap @@ -60,6 +60,8 @@ base: - influxdb - soc - kratos + - sensoroni + - telegraf - firewall - idstools - suricata.manager @@ -92,6 +94,8 @@ base: - kratos - firewall - manager + - sensoroni + - telegraf - backup.config_backup - idstools - suricata.manager @@ -122,6 +126,8 @@ base: - soc - kratos - firewall + - sensoroni + - telegraf - idstools - suricata.manager - healthcheck @@ -145,6 +151,8 @@ base: '*_searchnode and G@saltversion:{{saltversion}}': - match: compound - ssl + - sensoroni + - telegraf - nginx - firewall - elasticsearch @@ -163,6 +171,8 @@ base: - kratos - firewall - manager + - sensoroni + - telegraf - backup.config_backup - idstools - suricata.manager @@ -183,6 +193,8 @@ base: - match: compound - sensor - ssl + - sensoroni + - telegraf - nginx - firewall - elasticsearch @@ -208,6 +220,8 @@ base: - influxdb - soc - kratos + - sensoroni + - telegraf - firewall - idstools - suricata.manager @@ -223,6 +237,8 @@ base: '*_receiver and G@saltversion:{{saltversion}}': - match: compound - ssl + - sensoroni + - telegraf - firewall - logstash - redis @@ -231,6 +247,8 @@ base: '*_idh and G@saltversion:{{saltversion}}': - match: compound - ssl + - sensoroni + - telegraf - firewall - elasticfleet.install_agent_grid - idh @@ -238,12 +256,19 @@ base: '*_fleet and G@saltversion:{{saltversion}}': - match: compound - ssl + - sensoroni + - telegraf - firewall - logstash - elasticfleet - elasticfleet.install_agent_grid - schedule + '*_desktop and G@saltversion:{{saltversion}}': + - ssl + - sensoroni + - telegraf + 'J@desktop:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:OEL )': - match: compound - desktop From 490669d3782fbc037a4a5474c2eafcd34bf89c8d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 1 Sep 2023 12:03:01 -0400 Subject: [PATCH 236/350] add ssl to desktop for allowed_states --- salt/allowed_states.map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 6932e8c84..4e3e57f9c 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -188,6 +188,7 @@ 'docker_clean' ], 'so-desktop': [ + 'ssl', 'docker_clean', 'telegraf' ], From aebfb19ab77c409210935bfbdcdecce31a0b7c37 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 1 Sep 2023 12:05:28 -0400 Subject: [PATCH 237/350] add sostatus.sh to desktop for telegraf scripts --- salt/telegraf/defaults.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index 36ef679f0..a87fa952b 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -87,4 +87,5 @@ telegraf: - sostatus.sh fleet: - sostatus.sh - desktop: [] + desktop: + - sostatus.sh From 585fba4bc69a244aafe3717c87eb80b498e05347 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 1 Sep 2023 12:40:01 -0400 Subject: [PATCH 238/350] add functions salt_install_module_deps and salt_patch_x509_v2 --- setup/so-functions | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 4f973d147..0300e8d21 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -94,6 +94,9 @@ desktop_salt_local() { logCmd "yum -y install salt-minion-$SALTVERSION httpd-tools python3 python3-dateutil yum-utils device-mapper-persistent-data lvm2 openssl jq" logCmd "yum -y update --exclude=salt*" + salt_install_module_deps + salt_patch_x509_v2 + logCmd "salt-call state.apply desktop --local --file-root=../salt/ -l info" read -r -d '' message <<- EOM Finished Security Onion Desktop installation. @@ -2073,21 +2076,27 @@ saltify() { fi logCmd "mkdir -p /etc/salt/minion.d" + salt_install_module_deps + salt_patch_x509_v2 + +} + +# Run a salt command to generate the minion key +salt_firstcheckin() { + salt-call state.show_top >> /dev/null 2>&1 # send output to /dev/null because we don't actually care about the ouput +} + +salt_install_module_deps() { logCmd "salt-pip install docker --no-index --only-binary=:all: --find-links files/salt_module_deps/docker/" logCmd "salt-pip install pymysql --no-index --only-binary=:all: --find-links files/salt_module_deps/pymysql/" +} +salt_patch_x509_v2() { # this can be removed when https://github.com/saltstack/salt/issues/64195 is resolved if [ $SALTVERSION == "3006.1" ]; then info "Salt version 3006.1 found. Patching /opt/saltstack/salt/lib/python3.10/site-packages/salt/states/x509_v2.py" \cp -v ./files/patch/states/x509_v2.py /opt/saltstack/salt/lib/python3.10/site-packages/salt/states/x509_v2.py fi - -} - - -# Run a salt command to generate the minion key -salt_firstcheckin() { - salt-call state.show_top >> /dev/null 2>&1 # send output to /dev/null because we don't actually care about the ouput } # Create an secrets pillar so that passwords survive re-install From 8093e5ce7c44612225a27515aef4fed0dfbac468 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 1 Sep 2023 13:01:17 -0400 Subject: [PATCH 239/350] use IP to avoid host issues --- salt/common/tools/sbin/so-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-test b/salt/common/tools/sbin/so-test index 7286a35a8..1758a44bb 100755 --- a/salt/common/tools/sbin/so-test +++ b/salt/common/tools/sbin/so-test @@ -11,4 +11,4 @@ set -e so-tcpreplay /opt/samples/* 2> /dev/null # Ingest sample pfsense log entry -echo "<134>$(date '+%b %d %H:%M:%S') filterlog[31624]: 84,,,1567509287,igb0.244,match,pass,in,4,0x0,,64,0,0,DF,6,tcp,64,192.168.1.1,10.10.10.10,56320,443,0,S,3333585167,,65535,,mss;nop;wscale;nop;nop;TS;sackOK;eol" | nc -uv -w1 localhost 514 > /dev/null 2>&1 +echo "<134>$(date '+%b %d %H:%M:%S') filterlog[31624]: 84,,,1567509287,igb0.244,match,pass,in,4,0x0,,64,0,0,DF,6,tcp,64,192.168.1.1,10.10.10.10,56320,443,0,S,3333585167,,65535,,mss;nop;wscale;nop;nop;TS;sackOK;eol" | nc -uv -w1 127.0.0.1 514 > /dev/null 2>&1 From 07ed93de19a011ddd18ccae0aa3f93ffe1fd5bf0 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 1 Sep 2023 14:33:32 -0400 Subject: [PATCH 240/350] add elastic agent to desktop --- salt/firewall/defaults.yaml | 17 ++++++++++++++--- salt/top.sls | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 578a242f9..75df49b25 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -289,6 +289,11 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update + desktop: + portgroups: + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update customhostgroup0: portgroups: [] customhostgroup1: @@ -467,6 +472,9 @@ firewall: - influxdb - sensoroni - yum + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update customhostgroup0: portgroups: [] customhostgroup1: @@ -658,6 +666,9 @@ firewall: - influxdb - sensoroni - yum + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update customhostgroup0: portgroups: [] customhostgroup1: @@ -857,6 +868,9 @@ firewall: - influxdb - sensoroni - yum + - elastic_agent_control + - elastic_agent_data + - elastic_agent_update customhostgroup0: portgroups: [] customhostgroup1: @@ -1214,9 +1228,6 @@ firewall: analyst: portgroups: - nginx - desktop: - portgroups: - - yum customhostgroup0: portgroups: [] customhostgroup1: diff --git a/salt/top.sls b/salt/top.sls index b8ca0f14e..6db19b361 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -268,6 +268,7 @@ base: - ssl - sensoroni - telegraf + - elasticfleet.install_agent_grid 'J@desktop:gui:enabled:^[Tt][Rr][Uu][Ee]$ and ( G@saltversion:{{saltversion}} and G@os:OEL )': - match: compound From 335aaa55944ae951153c07b59d3ea2e53aa6c6be Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 1 Sep 2023 15:30:53 -0400 Subject: [PATCH 241/350] add additional test modes --- setup/so-setup | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index a9c7776c3..99a7c672e 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -183,9 +183,26 @@ if [ -n "$test_profile" ]; then install_type=SEARCHNODE HOSTNAME=search MSRVIP_OFFSET=-1 - else + elif [[ "$test_profile" =~ "-managersearch" ]]; then + install_type=MANAGERSEARCH + elif [[ "$test_profile" =~ "-heavynode" ]]; then + install_type=HEAVYNODE + HOSTNAME=sensor + MSRVIP_OFFSET=-1 + elif [[ "$test_profile" =~ "-desktop" ]]; then + install_type=DESKTOP + MSRVIP_OFFSET=-3 + is_desktop_grid=true + fi + + if [[ -z "$HOSTNAME" ]]; then HOSTNAME=manager fi + + if [[ "$install_type" =~ "DESKTOP" ]]; then + is_desktop=true + hostname=desktop + fi info "Activating test profile; profile=$test_profile; install_type=$install_type" From 863db14b61b4d691ce38f7d28c2a47dc4fcf9b7b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 1 Sep 2023 16:27:02 -0400 Subject: [PATCH 242/350] add additional test modes --- setup/so-setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 99a7c672e..ce13af06b 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -201,7 +201,7 @@ if [ -n "$test_profile" ]; then if [[ "$install_type" =~ "DESKTOP" ]]; then is_desktop=true - hostname=desktop + HOSTNAME=desktop fi info "Activating test profile; profile=$test_profile; install_type=$install_type" From a11259c6832ce3331f55e62e704caaf6a6624cdb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 1 Sep 2023 17:08:27 -0400 Subject: [PATCH 243/350] add additional test modes --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index ce13af06b..c6ff27198 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -174,6 +174,7 @@ if [ -n "$test_profile" ]; then # The below settings are hardcoded purely for automated testing purposes. TESTING=true + is_desktop_grid=false if [[ "$test_profile" =~ "-sensor" ]]; then install_type=SENSOR From 6efdf1b9d0444ddb9f39127589c7cbba66d4af83 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 1 Sep 2023 17:24:12 -0400 Subject: [PATCH 244/350] add additional test modes --- setup/so-functions | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 4f973d147..efa6c800f 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -101,8 +101,10 @@ desktop_salt_local() { Press the Enter key to reboot. EOM - whiptail --title "$whiptail_title" --msgbox "$message" 12 75 - reboot + if [[ -z "$TESTING" ]]; then + whiptail --title "$whiptail_title" --msgbox "$message" 12 75 + reboot + fi exit 0 } From 0aae107155a18b89f49a8f5048ade3af5a49e4d4 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 1 Sep 2023 20:30:53 -0400 Subject: [PATCH 245/350] ensure hostname is set --- setup/so-setup | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index c6ff27198..8537aa7c3 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -194,9 +194,7 @@ if [ -n "$test_profile" ]; then install_type=DESKTOP MSRVIP_OFFSET=-3 is_desktop_grid=true - fi - - if [[ -z "$HOSTNAME" ]]; then + else HOSTNAME=manager fi From 0cf913a7c145c8213fa39176409a77291150f66b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 2 Sep 2023 06:05:37 -0400 Subject: [PATCH 246/350] ensure hostname is set --- setup/so-setup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup/so-setup b/setup/so-setup index 8537aa7c3..7c419fae2 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -174,7 +174,6 @@ if [ -n "$test_profile" ]; then # The below settings are hardcoded purely for automated testing purposes. TESTING=true - is_desktop_grid=false if [[ "$test_profile" =~ "-sensor" ]]; then install_type=SENSOR @@ -201,6 +200,9 @@ if [ -n "$test_profile" ]; then if [[ "$install_type" =~ "DESKTOP" ]]; then is_desktop=true HOSTNAME=desktop + if [[ -z "$is_desktop_grid" ]]; then + is_desktop_grid=false + fi fi info "Activating test profile; profile=$test_profile; install_type=$install_type" From 8e2bed7f91c93542777114325dfecdad7546b3ac Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 3 Sep 2023 19:56:40 -0400 Subject: [PATCH 247/350] MS testing --- setup/so-setup | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-setup b/setup/so-setup index 7c419fae2..c1d92ec62 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -185,6 +185,7 @@ if [ -n "$test_profile" ]; then MSRVIP_OFFSET=-1 elif [[ "$test_profile" =~ "-managersearch" ]]; then install_type=MANAGERSEARCH + HOSTNAME=manager elif [[ "$test_profile" =~ "-heavynode" ]]; then install_type=HEAVYNODE HOSTNAME=sensor From cf19c8f8c2fa88adf5deb26ee1c466e332622f0e Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 5 Sep 2023 13:43:41 +0000 Subject: [PATCH 248/350] Remove templates --- .../logs-elastic_agent.apm_server@custom.json | 12 - ...logs-elastic_agent.apm_server@package.json | 329 --- .../logs-elastic_agent.auditbeat@custom.json | 12 - .../logs-elastic_agent.auditbeat@package.json | 329 --- .../logs-elastic_agent.cloudbeat@custom.json | 12 - .../logs-elastic_agent.cloudbeat@package.json | 339 --- ...lastic_agent.endpoint_security@custom.json | 12 - ...astic_agent.endpoint_security@package.json | 329 --- .../logs-elastic_agent.filebeat@custom.json | 12 - .../logs-elastic_agent.filebeat@package.json | 329 --- ...ogs-elastic_agent.fleet_server@custom.json | 12 - ...gs-elastic_agent.fleet_server@package.json | 329 --- .../logs-elastic_agent.heartbeat@custom.json | 12 - .../logs-elastic_agent.heartbeat@package.json | 329 --- .../logs-elastic_agent.metricbeat@custom.json | 12 - ...logs-elastic_agent.metricbeat@package.json | 329 --- ...logs-elastic_agent.osquerybeat@custom.json | 12 - ...ogs-elastic_agent.osquerybeat@package.json | 329 --- .../logs-elastic_agent.packetbeat@custom.json | 12 - ...logs-elastic_agent.packetbeat@package.json | 322 --- .../logs-system.application@custom.json | 12 - .../logs-system.application@package.json | 952 ------ .../logs-system.auth@custom.json | 12 - .../logs-system.auth@package.json | 530 ---- .../logs-system.security@custom.json | 12 - .../logs-system.security@package.json | 1840 ------------ .../logs-system.syslog@custom.json | 12 - .../logs-system.syslog@package.json | 327 --- .../logs-system.system@custom.json | 12 - .../logs-system.system@package.json | 986 ------- .../logs-windows.forwarded@custom.json | 12 - .../logs-windows.forwarded@package.json | 2544 ----------------- .../logs-windows.powershell@custom.json | 12 - .../logs-windows.powershell@package.json | 1335 --------- ...windows.powershell_operational@custom.json | 12 - ...indows.powershell_operational@package.json | 1334 --------- ...ogs-windows.sysmon_operational@custom.json | 12 - ...gs-windows.sysmon_operational@package.json | 1752 ------------ 38 files changed, 15121 deletions(-) delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.apm_server@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.apm_server@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.auditbeat@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.auditbeat@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.cloudbeat@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.cloudbeat@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.endpoint_security@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.endpoint_security@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.filebeat@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.filebeat@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.fleet_server@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.fleet_server@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.heartbeat@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.heartbeat@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.metricbeat@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.metricbeat@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.osquerybeat@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.osquerybeat@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.packetbeat@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.packetbeat@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.application@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.application@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.auth@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.auth@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.security@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.security@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.system@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-system.system@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-windows.forwarded@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-windows.forwarded@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell_operational@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell_operational@package.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-windows.sysmon_operational@custom.json delete mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-windows.sysmon_operational@package.json diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.apm_server@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.apm_server@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.apm_server@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.apm_server@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.apm_server@package.json deleted file mode 100644 index 919763caa..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.apm_server@package.json +++ /dev/null @@ -1,329 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.apm_server-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } - } - }, - "message": { - "type": "text" - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.auditbeat@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.auditbeat@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.auditbeat@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.auditbeat@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.auditbeat@package.json deleted file mode 100644 index 175ad4431..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.auditbeat@package.json +++ /dev/null @@ -1,329 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.auditbeat-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } - } - }, - "message": { - "type": "text" - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.cloudbeat@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.cloudbeat@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.cloudbeat@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.cloudbeat@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.cloudbeat@package.json deleted file mode 100644 index a96480471..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.cloudbeat@package.json +++ /dev/null @@ -1,339 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.cloudbeat-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "decision_id", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "message": { - "type": "match_only_text" - }, - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "result": { - "type": "object" - }, - "input": { - "type": "object" - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "decision_id": { - "type": "text" - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } - } - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.endpoint_security@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.endpoint_security@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.endpoint_security@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.endpoint_security@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.endpoint_security@package.json deleted file mode 100644 index 5f16d18de..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.endpoint_security@package.json +++ /dev/null @@ -1,329 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.endpoint_security-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } - } - }, - "message": { - "type": "text" - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.filebeat@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.filebeat@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.filebeat@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.filebeat@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.filebeat@package.json deleted file mode 100644 index f5b1ab12a..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.filebeat@package.json +++ /dev/null @@ -1,329 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.filebeat-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } - } - }, - "message": { - "type": "text" - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.fleet_server@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.fleet_server@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.fleet_server@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.fleet_server@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.fleet_server@package.json deleted file mode 100644 index a61d9f7a9..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.fleet_server@package.json +++ /dev/null @@ -1,329 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.fleet_server-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } - } - }, - "message": { - "type": "text" - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.heartbeat@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.heartbeat@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.heartbeat@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.heartbeat@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.heartbeat@package.json deleted file mode 100644 index d7e244dc2..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.heartbeat@package.json +++ /dev/null @@ -1,329 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.heartbeat-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "message": { - "type": "text" - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } - } - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.metricbeat@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.metricbeat@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.metricbeat@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.metricbeat@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.metricbeat@package.json deleted file mode 100644 index 7b0c81283..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.metricbeat@package.json +++ /dev/null @@ -1,329 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.metricbeat-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } - } - }, - "message": { - "type": "text" - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.osquerybeat@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.osquerybeat@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.osquerybeat@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.osquerybeat@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.osquerybeat@package.json deleted file mode 100644 index 2a6780e69..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.osquerybeat@package.json +++ /dev/null @@ -1,329 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.osquerybeat-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "event": { - "properties": { - "dataset": { - "type": "constant_keyword" - } - } - }, - "message": { - "type": "text" - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.packetbeat@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.packetbeat@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.packetbeat@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.packetbeat@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.packetbeat@package.json deleted file mode 100644 index 973427be1..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent.packetbeat@package.json +++ /dev/null @@ -1,322 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-elastic_agent.packetbeat-1.7.0", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "ecs.version", - "agent.build.original", - "agent.ephemeral_id", - "agent.id", - "agent.name", - "agent.type", - "agent.version", - "log.level", - "message", - "elastic_agent.id", - "elastic_agent.process", - "elastic_agent.version" - ] - } - } - }, - "mappings": { - "dynamic": false, - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "agent": { - "properties": { - "build": { - "properties": { - "original": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ephemeral_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "elastic_agent": { - "properties": { - "process": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "snapshot": { - "type": "boolean" - } - } - }, - "message": { - "type": "text" - } - } - } - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.application@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.application@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.application@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.application@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.application@package.json deleted file mode 100644 index 05741a4f0..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.application@package.json +++ /dev/null @@ -1,952 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-system.application-1.6.4", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "event.code", - "event.original", - "error.message", - "message", - "winlog.api", - "winlog.activity_id", - "winlog.computer_name", - "winlog.event_data.AuthenticationPackageName", - "winlog.event_data.Binary", - "winlog.event_data.BitlockerUserInputTime", - "winlog.event_data.BootMode", - "winlog.event_data.BootType", - "winlog.event_data.BuildVersion", - "winlog.event_data.Company", - "winlog.event_data.CorruptionActionState", - "winlog.event_data.CreationUtcTime", - "winlog.event_data.Description", - "winlog.event_data.Detail", - "winlog.event_data.DeviceName", - "winlog.event_data.DeviceNameLength", - "winlog.event_data.DeviceTime", - "winlog.event_data.DeviceVersionMajor", - "winlog.event_data.DeviceVersionMinor", - "winlog.event_data.DriveName", - "winlog.event_data.DriverName", - "winlog.event_data.DriverNameLength", - "winlog.event_data.DwordVal", - "winlog.event_data.EntryCount", - "winlog.event_data.ExtraInfo", - "winlog.event_data.FailureName", - "winlog.event_data.FailureNameLength", - "winlog.event_data.FileVersion", - "winlog.event_data.FinalStatus", - "winlog.event_data.Group", - "winlog.event_data.IdleImplementation", - "winlog.event_data.IdleStateCount", - "winlog.event_data.ImpersonationLevel", - "winlog.event_data.IntegrityLevel", - "winlog.event_data.IpAddress", - "winlog.event_data.IpPort", - "winlog.event_data.KeyLength", - "winlog.event_data.LastBootGood", - "winlog.event_data.LastShutdownGood", - "winlog.event_data.LmPackageName", - "winlog.event_data.LogonGuid", - "winlog.event_data.LogonId", - "winlog.event_data.LogonProcessName", - "winlog.event_data.LogonType", - "winlog.event_data.MajorVersion", - "winlog.event_data.MaximumPerformancePercent", - "winlog.event_data.MemberName", - "winlog.event_data.MemberSid", - "winlog.event_data.MinimumPerformancePercent", - "winlog.event_data.MinimumThrottlePercent", - "winlog.event_data.MinorVersion", - "winlog.event_data.NewProcessId", - "winlog.event_data.NewProcessName", - "winlog.event_data.NewSchemeGuid", - "winlog.event_data.NewTime", - "winlog.event_data.NominalFrequency", - "winlog.event_data.Number", - "winlog.event_data.OldSchemeGuid", - "winlog.event_data.OldTime", - "winlog.event_data.OriginalFileName", - "winlog.event_data.Path", - "winlog.event_data.PerformanceImplementation", - "winlog.event_data.PreviousCreationUtcTime", - "winlog.event_data.PreviousTime", - "winlog.event_data.PrivilegeList", - "winlog.event_data.ProcessId", - "winlog.event_data.ProcessName", - "winlog.event_data.ProcessPath", - "winlog.event_data.ProcessPid", - "winlog.event_data.Product", - "winlog.event_data.PuaCount", - "winlog.event_data.PuaPolicyId", - "winlog.event_data.QfeVersion", - "winlog.event_data.Reason", - "winlog.event_data.SchemaVersion", - "winlog.event_data.ScriptBlockText", - "winlog.event_data.ServiceName", - "winlog.event_data.ServiceVersion", - "winlog.event_data.ShutdownActionType", - "winlog.event_data.ShutdownEventCode", - "winlog.event_data.ShutdownReason", - "winlog.event_data.Signature", - "winlog.event_data.SignatureStatus", - "winlog.event_data.Signed", - "winlog.event_data.StartTime", - "winlog.event_data.State", - "winlog.event_data.Status", - "winlog.event_data.StopTime", - "winlog.event_data.SubjectDomainName", - "winlog.event_data.SubjectLogonId", - "winlog.event_data.SubjectUserName", - "winlog.event_data.SubjectUserSid", - "winlog.event_data.TSId", - "winlog.event_data.TargetDomainName", - "winlog.event_data.TargetInfo", - "winlog.event_data.TargetLogonGuid", - "winlog.event_data.TargetLogonId", - "winlog.event_data.TargetServerName", - "winlog.event_data.TargetUserName", - "winlog.event_data.TargetUserSid", - "winlog.event_data.TerminalSessionId", - "winlog.event_data.TokenElevationType", - "winlog.event_data.TransmittedServices", - "winlog.event_data.UserSid", - "winlog.event_data.Version", - "winlog.event_data.Workstation", - "winlog.event_data.param1", - "winlog.event_data.param2", - "winlog.event_data.param3", - "winlog.event_data.param4", - "winlog.event_data.param5", - "winlog.event_data.param6", - "winlog.event_data.param7", - "winlog.event_data.param8", - "winlog.event_id", - "winlog.keywords", - "winlog.channel", - "winlog.record_id", - "winlog.related_activity_id", - "winlog.opcode", - "winlog.provider_guid", - "winlog.provider_name", - "winlog.task", - "winlog.user.identifier", - "winlog.user.name", - "winlog.user.domain", - "winlog.user.type" - ] - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - }, - { - "winlog.user_data": { - "path_match": "winlog.user_data.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "winlog": { - "properties": { - "related_activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "computer_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "process": { - "properties": { - "pid": { - "type": "long" - }, - "thread": { - "properties": { - "id": { - "type": "long" - } - } - } - } - }, - "keywords": { - "ignore_above": 1024, - "type": "keyword" - }, - "channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_data": { - "properties": { - "SignatureStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "OriginalFileName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootMode": { - "ignore_above": 1024, - "type": "keyword" - }, - "Product": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "FileVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "StopTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Status": { - "ignore_above": 1024, - "type": "keyword" - }, - "CorruptionActionState": { - "ignore_above": 1024, - "type": "keyword" - }, - "KeyLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousCreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "PerformanceImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Group": { - "ignore_above": 1024, - "type": "keyword" - }, - "Description": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownActionType": { - "ignore_above": 1024, - "type": "keyword" - }, - "DwordVal": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMajor": { - "ignore_above": 1024, - "type": "keyword" - }, - "ScriptBlockText": { - "ignore_above": 1024, - "type": "keyword" - }, - "TransmittedServices": { - "ignore_above": 1024, - "type": "keyword" - }, - "MaximumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "FinalStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleStateCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "MajorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "Path": { - "ignore_above": 1024, - "type": "keyword" - }, - "SchemaVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "TokenElevationType": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "QfeVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMinor": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Company": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaPolicyId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IntegrityLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastShutdownGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpPort": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "LmPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastBootGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Version": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signed": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownEventCode": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "State": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootType": { - "ignore_above": 1024, - "type": "keyword" - }, - "Binary": { - "ignore_above": 1024, - "type": "keyword" - }, - "ImpersonationLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Detail": { - "ignore_above": 1024, - "type": "keyword" - }, - "TerminalSessionId": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "CreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownReason": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetServerName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Number": { - "ignore_above": 1024, - "type": "keyword" - }, - "BuildVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TSId": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PrivilegeList": { - "ignore_above": 1024, - "type": "keyword" - }, - "param7": { - "ignore_above": 1024, - "type": "keyword" - }, - "param8": { - "ignore_above": 1024, - "type": "keyword" - }, - "param5": { - "ignore_above": 1024, - "type": "keyword" - }, - "param6": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriveName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ExtraInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "param3": { - "ignore_above": 1024, - "type": "keyword" - }, - "param4": { - "ignore_above": 1024, - "type": "keyword" - }, - "param1": { - "ignore_above": 1024, - "type": "keyword" - }, - "param2": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "Workstation": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signature": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumThrottlePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "EntryCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "BitlockerUserInputTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuthenticationPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NominalFrequency": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "opcode": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "type": "long" - }, - "record_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "task": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_guid": { - "ignore_above": 1024, - "type": "keyword" - }, - "activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "api": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "properties": { - "identifier": { - "ignore_above": 1024, - "type": "keyword" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "ingested": { - "type": "date" - }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "module": { - "type": "constant_keyword", - "value": "system" - }, - "dataset": { - "type": "constant_keyword", - "value": "system.application" - } - } - }, - "error": { - "properties": { - "message": { - "type": "match_only_text" - } - } - }, - "message": { - "type": "match_only_text" - } - } - } - }, - "_meta": { - "package": { - "name": "system" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.auth@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.auth@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.auth@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.auth@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.auth@package.json deleted file mode 100644 index 51e707850..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.auth@package.json +++ /dev/null @@ -1,530 +0,0 @@ -{ - "template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-system.auth-1.6.4", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.os.full", - "host.type", - "event.action", - "event.category", - "event.code", - "event.kind", - "event.outcome", - "event.provider", - "event.type", - "ecs.version", - "error.message", - "group.id", - "group.name", - "message", - "process.name", - "related.hosts", - "related.user", - "source.as.organization.name", - "source.geo.city_name", - "source.geo.continent_name", - "source.geo.country_iso_code", - "source.geo.country_name", - "source.geo.region_iso_code", - "source.geo.region_name", - "user.effective.name", - "user.id", - "user.name", - "system.auth.ssh.method", - "system.auth.ssh.signature", - "system.auth.ssh.event", - "system.auth.sudo.error", - "system.auth.sudo.tty", - "system.auth.sudo.pwd", - "system.auth.sudo.user", - "system.auth.sudo.command", - "system.auth.useradd.home", - "system.auth.useradd.shell", - "version" - ] - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - } - } - }, - "source": { - "properties": { - "geo": { - "properties": { - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "as": { - "properties": { - "number": { - "type": "long" - }, - "organization": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "port": { - "type": "long" - }, - "ip": { - "type": "ip" - } - } - }, - "error": { - "properties": { - "message": { - "type": "match_only_text" - } - } - }, - "message": { - "type": "match_only_text" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "@timestamp": { - "type": "date" - }, - "system": { - "properties": { - "auth": { - "properties": { - "ssh": { - "properties": { - "method": { - "ignore_above": 1024, - "type": "keyword" - }, - "dropped_ip": { - "type": "ip" - }, - "signature": { - "ignore_above": 1024, - "type": "keyword" - }, - "event": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "sudo": { - "properties": { - "tty": { - "ignore_above": 1024, - "type": "keyword" - }, - "error": { - "ignore_above": 1024, - "type": "keyword" - }, - "pwd": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "command": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "useradd": { - "properties": { - "shell": { - "ignore_above": 1024, - "type": "keyword" - }, - "home": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "related": { - "properties": { - "hosts": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword", - "value": "logs" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "sequence": { - "type": "long" - }, - "ingested": { - "type": "date" - }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "type": "constant_keyword", - "value": "system" - }, - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "type": "constant_keyword", - "value": "system.auth" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user": { - "properties": { - "effective": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "group": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - }, - "_meta": { - "package": { - "name": "system" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.security@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.security@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.security@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.security@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.security@package.json deleted file mode 100644 index a74cd4a70..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.security@package.json +++ /dev/null @@ -1,1840 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-system.security-1.6.4", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "event.action", - "event.category", - "event.code", - "event.kind", - "event.outcome", - "event.provider", - "event.type", - "tags", - "input.type", - "ecs.version", - "group.domain", - "group.id", - "group.name", - "log.file.path", - "log.level", - "message", - "process.args", - "process.command_line", - "process.entity_id", - "process.executable", - "process.name", - "process.parent.executable", - "process.parent.name", - "process.title", - "related.hash", - "related.hosts", - "related.user", - "service.name", - "service.type", - "source.domain", - "user.domain", - "user.id", - "user.name", - "user.effective.domain", - "user.effective.id", - "user.effective.name", - "user.target.group.domain", - "user.target.group.id", - "user.target.group.name", - "user.target.name", - "user.target.domain", - "user.target.id", - "user.changes.name", - "winlog.logon.type", - "winlog.logon.id", - "winlog.logon.failure.reason", - "winlog.logon.failure.status", - "winlog.logon.failure.sub_status", - "winlog.api", - "winlog.activity_id", - "winlog.channel", - "winlog.computer_name", - "winlog.computerObject.domain", - "winlog.computerObject.id", - "winlog.computerObject.name", - "winlog.event_data.AccessGranted", - "winlog.event_data.AccessList", - "winlog.event_data.AccessListDescription", - "winlog.event_data.AccessMask", - "winlog.event_data.AccessMaskDescription", - "winlog.event_data.AccessRemoved", - "winlog.event_data.AccountDomain", - "winlog.event_data.AccountExpires", - "winlog.event_data.AccountName", - "winlog.event_data.AllowedToDelegateTo", - "winlog.event_data.AuditPolicyChanges", - "winlog.event_data.AuditPolicyChangesDescription", - "winlog.event_data.AuditSourceName", - "winlog.event_data.AuthenticationPackageName", - "winlog.event_data.Binary", - "winlog.event_data.BitlockerUserInputTime", - "winlog.event_data.BootMode", - "winlog.event_data.BootType", - "winlog.event_data.BuildVersion", - "winlog.event_data.CallerProcessId", - "winlog.event_data.CallerProcessName", - "winlog.event_data.Category", - "winlog.event_data.CategoryId", - "winlog.event_data.ClientAddress", - "winlog.event_data.ClientName", - "winlog.event_data.CommandLine", - "winlog.event_data.Company", - "winlog.event_data.CorruptionActionState", - "winlog.event_data.CrashOnAuditFailValue", - "winlog.event_data.CreationUtcTime", - "winlog.event_data.Description", - "winlog.event_data.Detail", - "winlog.event_data.DeviceName", - "winlog.event_data.DeviceNameLength", - "winlog.event_data.DeviceTime", - "winlog.event_data.DeviceVersionMajor", - "winlog.event_data.DeviceVersionMinor", - "winlog.event_data.DisplayName", - "winlog.event_data.DomainBehaviorVersion", - "winlog.event_data.DomainName", - "winlog.event_data.DomainPolicyChanged", - "winlog.event_data.DomainSid", - "winlog.event_data.DriveName", - "winlog.event_data.DriverName", - "winlog.event_data.DriverNameLength", - "winlog.event_data.Dummy", - "winlog.event_data.DwordVal", - "winlog.event_data.EntryCount", - "winlog.event_data.EventSourceId", - "winlog.event_data.ExtraInfo", - "winlog.event_data.FailureName", - "winlog.event_data.FailureNameLength", - "winlog.event_data.FailureReason", - "winlog.event_data.FileVersion", - "winlog.event_data.FinalStatus", - "winlog.event_data.Group", - "winlog.event_data.GroupTypeChange", - "winlog.event_data.HandleId", - "winlog.event_data.HomeDirectory", - "winlog.event_data.HomePath", - "winlog.event_data.IdleImplementation", - "winlog.event_data.IdleStateCount", - "winlog.event_data.ImpersonationLevel", - "winlog.event_data.IntegrityLevel", - "winlog.event_data.IpAddress", - "winlog.event_data.IpPort", - "winlog.event_data.KerberosPolicyChange", - "winlog.event_data.KeyLength", - "winlog.event_data.LastBootGood", - "winlog.event_data.LastShutdownGood", - "winlog.event_data.LmPackageName", - "winlog.event_data.LogonGuid", - "winlog.event_data.LogonHours", - "winlog.event_data.LogonId", - "winlog.event_data.LogonID", - "winlog.event_data.LogonProcessName", - "winlog.event_data.LogonType", - "winlog.event_data.MachineAccountQuota", - "winlog.event_data.MajorVersion", - "winlog.event_data.MandatoryLabel", - "winlog.event_data.MaximumPerformancePercent", - "winlog.event_data.MemberName", - "winlog.event_data.MemberSid", - "winlog.event_data.MinimumPerformancePercent", - "winlog.event_data.MinimumThrottlePercent", - "winlog.event_data.MinorVersion", - "winlog.event_data.MixedDomainMode", - "winlog.event_data.NewProcessId", - "winlog.event_data.NewProcessName", - "winlog.event_data.NewSchemeGuid", - "winlog.event_data.NewSd", - "winlog.event_data.NewSdDacl0", - "winlog.event_data.NewSdDacl1", - "winlog.event_data.NewSdDacl2", - "winlog.event_data.NewSdSacl0", - "winlog.event_data.NewSdSacl1", - "winlog.event_data.NewSdSacl2", - "winlog.event_data.NewTargetUserName", - "winlog.event_data.NewTime", - "winlog.event_data.NewUACList", - "winlog.event_data.NewUacValue", - "winlog.event_data.NominalFrequency", - "winlog.event_data.Number", - "winlog.event_data.ObjectName", - "winlog.event_data.ObjectServer", - "winlog.event_data.ObjectType", - "winlog.event_data.OemInformation", - "winlog.event_data.OldSchemeGuid", - "winlog.event_data.OldSd", - "winlog.event_data.OldSdDacl0", - "winlog.event_data.OldSdDacl1", - "winlog.event_data.OldSdDacl2", - "winlog.event_data.OldSdSacl0", - "winlog.event_data.OldSdSacl1", - "winlog.event_data.OldSdSacl2", - "winlog.event_data.OldTargetUserName", - "winlog.event_data.OldTime", - "winlog.event_data.OldUacValue", - "winlog.event_data.OriginalFileName", - "winlog.event_data.PackageName", - "winlog.event_data.PasswordLastSet", - "winlog.event_data.PasswordHistoryLength", - "winlog.event_data.Path", - "winlog.event_data.ParentProcessName", - "winlog.event_data.PerformanceImplementation", - "winlog.event_data.PreviousCreationUtcTime", - "winlog.event_data.PreAuthType", - "winlog.event_data.PreviousTime", - "winlog.event_data.PrimaryGroupId", - "winlog.event_data.PrivilegeList", - "winlog.event_data.ProcessId", - "winlog.event_data.ProcessName", - "winlog.event_data.ProcessPath", - "winlog.event_data.ProcessPid", - "winlog.event_data.Product", - "winlog.event_data.ProfilePath", - "winlog.event_data.PuaCount", - "winlog.event_data.PuaPolicyId", - "winlog.event_data.QfeVersion", - "winlog.event_data.Reason", - "winlog.event_data.ResourceAttributes", - "winlog.event_data.SamAccountName", - "winlog.event_data.SchemaVersion", - "winlog.event_data.ScriptPath", - "winlog.event_data.SidHistory", - "winlog.event_data.ScriptBlockText", - "winlog.event_data.Service", - "winlog.event_data.ServiceAccount", - "winlog.event_data.ServiceFileName", - "winlog.event_data.ServiceName", - "winlog.event_data.ServiceSid", - "winlog.event_data.ServiceStartType", - "winlog.event_data.ServiceType", - "winlog.event_data.ServiceVersion", - "winlog.event_data.SessionName", - "winlog.event_data.ShutdownActionType", - "winlog.event_data.ShutdownEventCode", - "winlog.event_data.ShutdownReason", - "winlog.event_data.SidFilteringEnabled", - "winlog.event_data.Signature", - "winlog.event_data.SignatureStatus", - "winlog.event_data.Signed", - "winlog.event_data.StartTime", - "winlog.event_data.State", - "winlog.event_data.Status", - "winlog.event_data.StatusDescription", - "winlog.event_data.StopTime", - "winlog.event_data.SubCategory", - "winlog.event_data.SubCategoryGuid", - "winlog.event_data.SubcategoryGuid", - "winlog.event_data.SubCategoryId", - "winlog.event_data.SubcategoryId", - "winlog.event_data.SubjectDomainName", - "winlog.event_data.SubjectLogonId", - "winlog.event_data.SubjectUserName", - "winlog.event_data.SubjectUserSid", - "winlog.event_data.SubStatus", - "winlog.event_data.TSId", - "winlog.event_data.TargetDomainName", - "winlog.event_data.TargetInfo", - "winlog.event_data.TargetLogonGuid", - "winlog.event_data.TargetLogonId", - "winlog.event_data.TargetServerName", - "winlog.event_data.TargetSid", - "winlog.event_data.TargetUserName", - "winlog.event_data.TargetUserSid", - "winlog.event_data.TdoAttributes", - "winlog.event_data.TdoDirection", - "winlog.event_data.TdoType", - "winlog.event_data.TerminalSessionId", - "winlog.event_data.TicketEncryptionType", - "winlog.event_data.TicketEncryptionTypeDescription", - "winlog.event_data.TicketOptions", - "winlog.event_data.TicketOptionsDescription", - "winlog.event_data.TokenElevationType", - "winlog.event_data.TransmittedServices", - "winlog.event_data.UserAccountControl", - "winlog.event_data.UserParameters", - "winlog.event_data.UserPrincipalName", - "winlog.event_data.UserSid", - "winlog.event_data.UserWorkstations", - "winlog.event_data.Version", - "winlog.event_data.Workstation", - "winlog.event_data.WorkstationName", - "winlog.event_data.param1", - "winlog.event_data.param2", - "winlog.event_data.param3", - "winlog.event_data.param4", - "winlog.event_data.param5", - "winlog.event_data.param6", - "winlog.event_data.param7", - "winlog.event_data.param8", - "winlog.event_id", - "winlog.keywords", - "winlog.level", - "winlog.outcome", - "winlog.record_id", - "winlog.related_activity_id", - "winlog.opcode", - "winlog.provider_guid", - "winlog.provider_name", - "winlog.task", - "winlog.time_created", - "winlog.trustAttribute", - "winlog.trustDirection", - "winlog.trustType", - "winlog.user_data.BackupPath", - "winlog.user_data.Channel", - "winlog.user_data.SubjectDomainName", - "winlog.user_data.SubjectLogonId", - "winlog.user_data.SubjectUserName", - "winlog.user_data.SubjectUserSid", - "winlog.user_data.xml_name", - "winlog.user.identifier", - "winlog.user.name", - "winlog.user.domain", - "winlog.user.type" - ] - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" - }, - "parent": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "executable": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - }, - "args_count": { - "type": "long" - }, - "entity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "title": { - "ignore_above": 1024, - "type": "keyword" - }, - "command_line": { - "ignore_above": 1024, - "type": "wildcard" - }, - "executable": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "winlog": { - "properties": { - "related_activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "keywords": { - "ignore_above": 1024, - "type": "keyword" - }, - "logon": { - "properties": { - "failure": { - "properties": { - "reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "sub_status": { - "ignore_above": 1024, - "type": "keyword" - }, - "status": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_data": { - "properties": { - "ProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "OriginalFileName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootMode": { - "ignore_above": 1024, - "type": "keyword" - }, - "Product": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonHours": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "FileVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "TicketOptions": { - "ignore_above": 1024, - "type": "keyword" - }, - "AllowedToDelegateTo": { - "ignore_above": 1024, - "type": "keyword" - }, - "TdoAttributes": { - "ignore_above": 1024, - "type": "keyword" - }, - "StopTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Status": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessMask": { - "ignore_above": 1024, - "type": "keyword" - }, - "KeyLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "ResourceAttributes": { - "ignore_above": 1024, - "type": "keyword" - }, - "SessionName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PasswordHistoryLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSd": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Group": { - "ignore_above": 1024, - "type": "keyword" - }, - "PackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownActionType": { - "ignore_above": 1024, - "type": "keyword" - }, - "DwordVal": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMajor": { - "ignore_above": 1024, - "type": "keyword" - }, - "SidHistory": { - "ignore_above": 1024, - "type": "keyword" - }, - "TransmittedServices": { - "ignore_above": 1024, - "type": "keyword" - }, - "WorkstationName": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleStateCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Path": { - "ignore_above": 1024, - "type": "keyword" - }, - "SchemaVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "CrashOnAuditFailValue": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMinor": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "HandleId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastShutdownGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpPort": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "LmPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastBootGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessListDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Version": { - "ignore_above": 1024, - "type": "keyword" - }, - "MachineAccountQuota": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldUacValue": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserParameters": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signed": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubCategoryId": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldTargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewUacValue": { - "ignore_above": 1024, - "type": "keyword" - }, - "CallerProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProfilePath": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "State": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureReason": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootType": { - "ignore_above": 1024, - "type": "keyword" - }, - "Binary": { - "ignore_above": 1024, - "type": "keyword" - }, - "ImpersonationLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DomainPolicyChanged": { - "ignore_above": 1024, - "type": "keyword" - }, - "CategoryId": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreAuthType": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccountDomain": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewUACList": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubcategoryGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownReason": { - "ignore_above": 1024, - "type": "keyword" - }, - "SidFilteringEnabled": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetServerName": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuditPolicyChanges": { - "ignore_above": 1024, - "type": "keyword" - }, - "Number": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "EventSourceId": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriveName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ExtraInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "PrimaryGroupId": { - "ignore_above": 1024, - "type": "keyword" - }, - "ObjectName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "Workstation": { - "ignore_above": 1024, - "type": "keyword" - }, - "PasswordLastSet": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumThrottlePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "GroupTypeChange": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessList": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuthenticationPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NominalFrequency": { - "ignore_above": 1024, - "type": "keyword" - }, - "SignatureStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "DomainSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ScriptPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "TicketEncryptionType": { - "ignore_above": 1024, - "type": "keyword" - }, - "TicketOptionsDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ObjectServer": { - "ignore_above": 1024, - "type": "keyword" - }, - "HomePath": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserWorkstations": { - "ignore_above": 1024, - "type": "keyword" - }, - "SamAccountName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "CorruptionActionState": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuditSourceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubCategoryGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousCreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuditPolicyChangesDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessMaskDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccountName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PerformanceImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "TicketEncryptionTypeDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceAccount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Description": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ScriptBlockText": { - "ignore_above": 1024, - "type": "keyword" - }, - "ObjectType": { - "ignore_above": 1024, - "type": "keyword" - }, - "MaximumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "KerberosPolicyChange": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "FinalStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "MajorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "MandatoryLabel": { - "ignore_above": 1024, - "type": "keyword" - }, - "HomeDirectory": { - "ignore_above": 1024, - "type": "keyword" - }, - "TokenElevationType": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "QfeVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccountExpires": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceStartType": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserPrincipalName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdSacl1": { - "ignore_above": 1024, - "type": "keyword" - }, - "Dummy": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdSacl0": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdSacl2": { - "ignore_above": 1024, - "type": "keyword" - }, - "Company": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaPolicyId": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdSacl2": { - "ignore_above": 1024, - "type": "keyword" - }, - "IntegrityLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdSacl1": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdSacl0": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSd": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewTargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "ClientName": { - "ignore_above": 1024, - "type": "keyword" - }, - "StatusDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdDacl0": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdDacl2": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdDacl1": { - "ignore_above": 1024, - "type": "keyword" - }, - "DomainBehaviorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessGranted": { - "ignore_above": 1024, - "type": "keyword" - }, - "ParentProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubcategoryId": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessRemoved": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownEventCode": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "MixedDomainMode": { - "ignore_above": 1024, - "type": "keyword" - }, - "Detail": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdDacl1": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdDacl0": { - "ignore_above": 1024, - "type": "keyword" - }, - "Category": { - "ignore_above": 1024, - "type": "keyword" - }, - "TerminalSessionId": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdDacl2": { - "ignore_above": 1024, - "type": "keyword" - }, - "ClientAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "CreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "CallerProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TdoType": { - "ignore_above": 1024, - "type": "keyword" - }, - "Reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceFileName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DisplayName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BuildVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TSId": { - "ignore_above": 1024, - "type": "keyword" - }, - "PrivilegeList": { - "ignore_above": 1024, - "type": "keyword" - }, - "param7": { - "ignore_above": 1024, - "type": "keyword" - }, - "param8": { - "ignore_above": 1024, - "type": "keyword" - }, - "param5": { - "ignore_above": 1024, - "type": "keyword" - }, - "param6": { - "ignore_above": 1024, - "type": "keyword" - }, - "Service": { - "ignore_above": 1024, - "type": "keyword" - }, - "TdoDirection": { - "ignore_above": 1024, - "type": "keyword" - }, - "param3": { - "ignore_above": 1024, - "type": "keyword" - }, - "param4": { - "ignore_above": 1024, - "type": "keyword" - }, - "param1": { - "ignore_above": 1024, - "type": "keyword" - }, - "param2": { - "ignore_above": 1024, - "type": "keyword" - }, - "CommandLine": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserAccountControl": { - "ignore_above": 1024, - "type": "keyword" - }, - "OemInformation": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signature": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubCategory": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "EntryCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonID": { - "ignore_above": 1024, - "type": "keyword" - }, - "BitlockerUserInputTime": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "opcode": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_guid": { - "ignore_above": 1024, - "type": "keyword" - }, - "activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "time_created": { - "ignore_above": 1024, - "type": "keyword" - }, - "trustDirection": { - "ignore_above": 1024, - "type": "keyword" - }, - "api": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - }, - "computer_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "process": { - "properties": { - "pid": { - "type": "long" - }, - "thread": { - "properties": { - "id": { - "type": "long" - } - } - } - } - }, - "trustAttribute": { - "ignore_above": 1024, - "type": "keyword" - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "computerObject": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_data": { - "properties": { - "SubjectUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BackupPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "Channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "xml_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "version": { - "type": "long" - }, - "record_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "task": { - "ignore_above": 1024, - "type": "keyword" - }, - "trustType": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "properties": { - "identifier": { - "ignore_above": 1024, - "type": "keyword" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "log": { - "properties": { - "file": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "source": { - "properties": { - "port": { - "type": "long" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - } - } - }, - "message": { - "type": "match_only_text" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "input": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "related": { - "properties": { - "hosts": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword", - "value": "logs" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "service": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "sequence": { - "type": "long" - }, - "ingested": { - "type": "date" - }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "type": "constant_keyword", - "value": "system" - }, - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "type": "constant_keyword", - "value": "system.security" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user": { - "properties": { - "effective": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "changes": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "target": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "group": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } - }, - "group": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - }, - "_meta": { - "package": { - "name": "system" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@package.json deleted file mode 100644 index 30576a635..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@package.json +++ /dev/null @@ -1,327 +0,0 @@ -{ - "template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-system.syslog-1.6.4", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.os.full", - "host.type", - "event.action", - "event.category", - "event.code", - "event.kind", - "event.outcome", - "event.provider", - "event.type", - "ecs.version", - "message", - "process.name" - ] - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "pid": { - "type": "long" - } - } - }, - "@timestamp": { - "type": "date" - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword", - "value": "logs" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "sequence": { - "type": "long" - }, - "ingested": { - "type": "date" - }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "type": "constant_keyword", - "value": "system" - }, - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "type": "constant_keyword", - "value": "system.syslog" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "message": { - "type": "match_only_text" - } - } - } - }, - "_meta": { - "package": { - "name": "system" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.system@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.system@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.system@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-system.system@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-system.system@package.json deleted file mode 100644 index 068e6846b..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-system.system@package.json +++ /dev/null @@ -1,986 +0,0 @@ -{ - "template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-system.system-1.6.4", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "event.action", - "event.category", - "event.code", - "event.kind", - "event.original", - "event.outcome", - "event.provider", - "event.type", - "error.message", - "message", - "winlog.api", - "winlog.activity_id", - "winlog.computer_name", - "winlog.event_data.AuthenticationPackageName", - "winlog.event_data.Binary", - "winlog.event_data.BitlockerUserInputTime", - "winlog.event_data.BootMode", - "winlog.event_data.BootType", - "winlog.event_data.BuildVersion", - "winlog.event_data.Company", - "winlog.event_data.CorruptionActionState", - "winlog.event_data.CreationUtcTime", - "winlog.event_data.Description", - "winlog.event_data.Detail", - "winlog.event_data.DeviceName", - "winlog.event_data.DeviceNameLength", - "winlog.event_data.DeviceTime", - "winlog.event_data.DeviceVersionMajor", - "winlog.event_data.DeviceVersionMinor", - "winlog.event_data.DriveName", - "winlog.event_data.DriverName", - "winlog.event_data.DriverNameLength", - "winlog.event_data.DwordVal", - "winlog.event_data.EntryCount", - "winlog.event_data.ExtraInfo", - "winlog.event_data.FailureName", - "winlog.event_data.FailureNameLength", - "winlog.event_data.FileVersion", - "winlog.event_data.FinalStatus", - "winlog.event_data.Group", - "winlog.event_data.IdleImplementation", - "winlog.event_data.IdleStateCount", - "winlog.event_data.ImpersonationLevel", - "winlog.event_data.IntegrityLevel", - "winlog.event_data.IpAddress", - "winlog.event_data.IpPort", - "winlog.event_data.KeyLength", - "winlog.event_data.LastBootGood", - "winlog.event_data.LastShutdownGood", - "winlog.event_data.LmPackageName", - "winlog.event_data.LogonGuid", - "winlog.event_data.LogonId", - "winlog.event_data.LogonProcessName", - "winlog.event_data.LogonType", - "winlog.event_data.MajorVersion", - "winlog.event_data.MaximumPerformancePercent", - "winlog.event_data.MemberName", - "winlog.event_data.MemberSid", - "winlog.event_data.MinimumPerformancePercent", - "winlog.event_data.MinimumThrottlePercent", - "winlog.event_data.MinorVersion", - "winlog.event_data.NewProcessId", - "winlog.event_data.NewProcessName", - "winlog.event_data.NewSchemeGuid", - "winlog.event_data.NewTime", - "winlog.event_data.NominalFrequency", - "winlog.event_data.Number", - "winlog.event_data.OldSchemeGuid", - "winlog.event_data.OldTime", - "winlog.event_data.OriginalFileName", - "winlog.event_data.Path", - "winlog.event_data.PerformanceImplementation", - "winlog.event_data.PreviousCreationUtcTime", - "winlog.event_data.PreviousTime", - "winlog.event_data.PrivilegeList", - "winlog.event_data.ProcessId", - "winlog.event_data.ProcessName", - "winlog.event_data.ProcessPath", - "winlog.event_data.ProcessPid", - "winlog.event_data.Product", - "winlog.event_data.PuaCount", - "winlog.event_data.PuaPolicyId", - "winlog.event_data.QfeVersion", - "winlog.event_data.Reason", - "winlog.event_data.SchemaVersion", - "winlog.event_data.ScriptBlockText", - "winlog.event_data.ServiceName", - "winlog.event_data.ServiceVersion", - "winlog.event_data.ShutdownActionType", - "winlog.event_data.ShutdownEventCode", - "winlog.event_data.ShutdownReason", - "winlog.event_data.Signature", - "winlog.event_data.SignatureStatus", - "winlog.event_data.Signed", - "winlog.event_data.StartTime", - "winlog.event_data.State", - "winlog.event_data.Status", - "winlog.event_data.StopTime", - "winlog.event_data.SubjectDomainName", - "winlog.event_data.SubjectLogonId", - "winlog.event_data.SubjectUserName", - "winlog.event_data.SubjectUserSid", - "winlog.event_data.TSId", - "winlog.event_data.TargetDomainName", - "winlog.event_data.TargetInfo", - "winlog.event_data.TargetLogonGuid", - "winlog.event_data.TargetLogonId", - "winlog.event_data.TargetServerName", - "winlog.event_data.TargetUserName", - "winlog.event_data.TargetUserSid", - "winlog.event_data.TerminalSessionId", - "winlog.event_data.TokenElevationType", - "winlog.event_data.TransmittedServices", - "winlog.event_data.UserSid", - "winlog.event_data.Version", - "winlog.event_data.Workstation", - "winlog.event_data.param1", - "winlog.event_data.param2", - "winlog.event_data.param3", - "winlog.event_data.param4", - "winlog.event_data.param5", - "winlog.event_data.param6", - "winlog.event_data.param7", - "winlog.event_data.param8", - "winlog.event_id", - "winlog.keywords", - "winlog.channel", - "winlog.record_id", - "winlog.related_activity_id", - "winlog.opcode", - "winlog.provider_guid", - "winlog.provider_name", - "winlog.task", - "winlog.user.identifier", - "winlog.user.name", - "winlog.user.domain", - "winlog.user.type" - ] - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - }, - { - "winlog.user_data": { - "path_match": "winlog.user_data.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "winlog": { - "properties": { - "related_activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "computer_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "process": { - "properties": { - "pid": { - "type": "long" - }, - "thread": { - "properties": { - "id": { - "type": "long" - } - } - } - } - }, - "keywords": { - "ignore_above": 1024, - "type": "keyword" - }, - "channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_data": { - "properties": { - "SignatureStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "OriginalFileName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootMode": { - "ignore_above": 1024, - "type": "keyword" - }, - "Product": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "FileVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "StopTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Status": { - "ignore_above": 1024, - "type": "keyword" - }, - "CorruptionActionState": { - "ignore_above": 1024, - "type": "keyword" - }, - "KeyLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousCreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "PerformanceImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Group": { - "ignore_above": 1024, - "type": "keyword" - }, - "Description": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownActionType": { - "ignore_above": 1024, - "type": "keyword" - }, - "DwordVal": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMajor": { - "ignore_above": 1024, - "type": "keyword" - }, - "ScriptBlockText": { - "ignore_above": 1024, - "type": "keyword" - }, - "TransmittedServices": { - "ignore_above": 1024, - "type": "keyword" - }, - "MaximumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "FinalStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleStateCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "MajorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "Path": { - "ignore_above": 1024, - "type": "keyword" - }, - "SchemaVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "TokenElevationType": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "QfeVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMinor": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Company": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaPolicyId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IntegrityLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastShutdownGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpPort": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "LmPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastBootGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Version": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signed": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownEventCode": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "State": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootType": { - "ignore_above": 1024, - "type": "keyword" - }, - "Binary": { - "ignore_above": 1024, - "type": "keyword" - }, - "ImpersonationLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Detail": { - "ignore_above": 1024, - "type": "keyword" - }, - "TerminalSessionId": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "CreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownReason": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetServerName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Number": { - "ignore_above": 1024, - "type": "keyword" - }, - "BuildVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TSId": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PrivilegeList": { - "ignore_above": 1024, - "type": "keyword" - }, - "param7": { - "ignore_above": 1024, - "type": "keyword" - }, - "param8": { - "ignore_above": 1024, - "type": "keyword" - }, - "param5": { - "ignore_above": 1024, - "type": "keyword" - }, - "param6": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriveName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ExtraInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "param3": { - "ignore_above": 1024, - "type": "keyword" - }, - "param4": { - "ignore_above": 1024, - "type": "keyword" - }, - "param1": { - "ignore_above": 1024, - "type": "keyword" - }, - "param2": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "Workstation": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signature": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumThrottlePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "EntryCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "BitlockerUserInputTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuthenticationPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NominalFrequency": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "opcode": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "type": "long" - }, - "record_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "task": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_guid": { - "ignore_above": 1024, - "type": "keyword" - }, - "activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "api": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "properties": { - "identifier": { - "ignore_above": 1024, - "type": "keyword" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "type": "constant_keyword", - "value": "system" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "sequence": { - "type": "long" - }, - "ingested": { - "type": "date" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "type": "constant_keyword", - "value": "system.system" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "error": { - "properties": { - "message": { - "type": "match_only_text" - } - } - }, - "message": { - "type": "match_only_text" - } - } - } - }, - "_meta": { - "package": { - "name": "system" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.forwarded@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-windows.forwarded@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.forwarded@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.forwarded@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-windows.forwarded@package.json deleted file mode 100644 index 967641107..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.forwarded@package.json +++ /dev/null @@ -1,2544 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-windows.forwarded-1.20.1", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "analysis": { - "analyzer": { - "powershell_script_analyzer": { - "pattern": "[\\W&&[^-]]+", - "type": "pattern" - } - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "event.action", - "event.category", - "event.code", - "event.kind", - "event.outcome", - "event.provider", - "event.type", - "tags", - "input.type", - "destination.domain", - "destination.user.domain", - "destination.user.id", - "destination.user.name", - "dns.answers.class", - "dns.answers.data", - "dns.answers.name", - "dns.answers.type", - "dns.header_flags", - "dns.id", - "dns.op_code", - "dns.question.class", - "dns.question.name", - "dns.question.registered_domain", - "dns.question.subdomain", - "dns.question.top_level_domain", - "dns.question.type", - "dns.response_code", - "dns.type", - "ecs.version", - "file.code_signature.status", - "file.code_signature.subject_name", - "file.directory", - "file.extension", - "file.hash.md5", - "file.hash.sha1", - "file.hash.sha256", - "file.hash.sha512", - "file.name", - "file.path", - "file.pe.architecture", - "file.pe.company", - "file.pe.description", - "file.pe.file_version", - "file.pe.imphash", - "file.pe.original_file_name", - "file.pe.product", - "group.domain", - "group.id", - "group.name", - "log.file.path", - "log.level", - "message", - "network.community_id", - "network.direction", - "network.protocol", - "network.transport", - "network.type", - "process.args", - "process.command_line", - "process.entity_id", - "process.executable", - "process.hash.md5", - "process.hash.sha1", - "process.hash.sha256", - "process.hash.sha512", - "process.name", - "process.parent.args", - "process.parent.command_line", - "process.parent.entity_id", - "process.parent.executable", - "process.parent.hash.md5", - "process.parent.hash.sha1", - "process.parent.hash.sha256", - "process.parent.hash.sha512", - "process.parent.name", - "process.parent.pe.architecture", - "process.parent.pe.company", - "process.parent.pe.description", - "process.parent.pe.file_version", - "process.parent.pe.imphash", - "process.parent.pe.original_file_name", - "process.parent.pe.product", - "process.parent.title", - "process.pe.architecture", - "process.pe.company", - "process.pe.description", - "process.pe.file_version", - "process.pe.imphash", - "process.pe.original_file_name", - "process.pe.product", - "process.title", - "process.working_directory", - "registry.data.strings", - "registry.data.type", - "registry.hive", - "registry.key", - "registry.path", - "registry.value", - "related.hash", - "related.hosts", - "related.user", - "rule.name", - "service.name", - "service.type", - "source.domain", - "source.user.domain", - "source.user.id", - "source.user.name", - "user.domain", - "user.id", - "user.name", - "user.target.group.domain", - "user.target.group.id", - "user.target.group.name", - "user.target.name", - "sysmon.dns.status", - "winlog.logon.type", - "winlog.logon.id", - "winlog.logon.failure.reason", - "winlog.logon.failure.status", - "winlog.logon.failure.sub_status", - "winlog.api", - "winlog.activity_id", - "winlog.computer_name", - "winlog.level", - "winlog.outcome", - "winlog.trustAttribute", - "winlog.trustDirection", - "winlog.trustType", - "winlog.computerObject.domain", - "winlog.computerObject.id", - "winlog.computerObject.name", - "winlog.event_data.AccessGranted", - "winlog.event_data.AccessMask", - "winlog.event_data.AccessMaskDescription", - "winlog.event_data.AccessRemoved", - "winlog.event_data.AccountDomain", - "winlog.event_data.AccountExpires", - "winlog.event_data.AccountName", - "winlog.event_data.AllowedToDelegateTo", - "winlog.event_data.AuditPolicyChanges", - "winlog.event_data.AuditPolicyChangesDescription", - "winlog.event_data.AuditSourceName", - "winlog.event_data.AuthenticationPackageName", - "winlog.event_data.Binary", - "winlog.event_data.BitlockerUserInputTime", - "winlog.event_data.BootMode", - "winlog.event_data.BootType", - "winlog.event_data.BuildVersion", - "winlog.event_data.CallerProcessId", - "winlog.event_data.CallerProcessName", - "winlog.event_data.Category", - "winlog.event_data.CategoryId", - "winlog.event_data.ClientAddress", - "winlog.event_data.ClientInfo", - "winlog.event_data.ClientName", - "winlog.event_data.CommandLine", - "winlog.event_data.Company", - "winlog.event_data.ComputerAccountChange", - "winlog.event_data.Configuration", - "winlog.event_data.CorruptionActionState", - "winlog.event_data.CrashOnAuditFailValue", - "winlog.event_data.CreationUtcTime", - "winlog.event_data.Description", - "winlog.event_data.Detail", - "winlog.event_data.DeviceName", - "winlog.event_data.DeviceNameLength", - "winlog.event_data.DeviceTime", - "winlog.event_data.DeviceVersionMajor", - "winlog.event_data.DeviceVersionMinor", - "winlog.event_data.DisplayName", - "winlog.event_data.DnsHostName", - "winlog.event_data.DomainBehaviorVersion", - "winlog.event_data.DomainName", - "winlog.event_data.DomainPolicyChanged", - "winlog.event_data.DomainSid", - "winlog.event_data.DriveName", - "winlog.event_data.DriverName", - "winlog.event_data.DriverNameLength", - "winlog.event_data.Dummy", - "winlog.event_data.DwordVal", - "winlog.event_data.EntryCount", - "winlog.event_data.EventSourceId", - "winlog.event_data.EventType", - "winlog.event_data.ExtraInfo", - "winlog.event_data.FailureName", - "winlog.event_data.FailureNameLength", - "winlog.event_data.FailureReason", - "winlog.event_data.FileVersion", - "winlog.event_data.FinalStatus", - "winlog.event_data.Group", - "winlog.event_data.GroupTypeChange", - "winlog.event_data.HandleId", - "winlog.event_data.HomeDirectory", - "winlog.event_data.HomePath", - "winlog.event_data.IdleImplementation", - "winlog.event_data.IdleStateCount", - "winlog.event_data.ImpersonationLevel", - "winlog.event_data.IntegrityLevel", - "winlog.event_data.IpAddress", - "winlog.event_data.IpPort", - "winlog.event_data.KerberosPolicyChange", - "winlog.event_data.KeyLength", - "winlog.event_data.LastBootGood", - "winlog.event_data.LastShutdownGood", - "winlog.event_data.LmPackageName", - "winlog.event_data.LogonGuid", - "winlog.event_data.LogonHours", - "winlog.event_data.LogonId", - "winlog.event_data.LogonID", - "winlog.event_data.LogonProcessName", - "winlog.event_data.LogonType", - "winlog.event_data.MachineAccountQuota", - "winlog.event_data.MajorVersion", - "winlog.event_data.MandatoryLabel", - "winlog.event_data.MaximumPerformancePercent", - "winlog.event_data.MemberName", - "winlog.event_data.MemberSid", - "winlog.event_data.MinimumPerformancePercent", - "winlog.event_data.MinimumThrottlePercent", - "winlog.event_data.MinorVersion", - "winlog.event_data.MixedDomainMode", - "winlog.event_data.NewProcessId", - "winlog.event_data.NewProcessName", - "winlog.event_data.NewSchemeGuid", - "winlog.event_data.NewSd", - "winlog.event_data.NewSdDacl0", - "winlog.event_data.NewSdDacl1", - "winlog.event_data.NewSdDacl2", - "winlog.event_data.NewSdSacl0", - "winlog.event_data.NewSdSacl1", - "winlog.event_data.NewSdSacl2", - "winlog.event_data.NewTargetUserName", - "winlog.event_data.NewTime", - "winlog.event_data.NewUACList", - "winlog.event_data.NewUacValue", - "winlog.event_data.NominalFrequency", - "winlog.event_data.Number", - "winlog.event_data.ObjectName", - "winlog.event_data.ObjectServer", - "winlog.event_data.ObjectType", - "winlog.event_data.OemInformation", - "winlog.event_data.OldSchemeGuid", - "winlog.event_data.OldSd", - "winlog.event_data.OldSdDacl0", - "winlog.event_data.OldSdDacl1", - "winlog.event_data.OldSdDacl2", - "winlog.event_data.OldSdSacl0", - "winlog.event_data.OldSdSacl1", - "winlog.event_data.OldSdSacl2", - "winlog.event_data.OldTargetUserName", - "winlog.event_data.OldTime", - "winlog.event_data.OldUacValue", - "winlog.event_data.OriginalFileName", - "winlog.event_data.PackageName", - "winlog.event_data.PasswordLastSet", - "winlog.event_data.PasswordHistoryLength", - "winlog.event_data.Path", - "winlog.event_data.ParentProcessName", - "winlog.event_data.PerformanceImplementation", - "winlog.event_data.PreviousCreationUtcTime", - "winlog.event_data.PreAuthType", - "winlog.event_data.PreviousTime", - "winlog.event_data.PrimaryGroupId", - "winlog.event_data.PrivilegeList", - "winlog.event_data.ProcessId", - "winlog.event_data.ProcessName", - "winlog.event_data.ProcessPath", - "winlog.event_data.ProcessPid", - "winlog.event_data.Product", - "winlog.event_data.ProfilePath", - "winlog.event_data.PuaCount", - "winlog.event_data.PuaPolicyId", - "winlog.event_data.QfeVersion", - "winlog.event_data.Reason", - "winlog.event_data.SamAccountName", - "winlog.event_data.SchemaVersion", - "winlog.event_data.ScriptPath", - "winlog.event_data.Session", - "winlog.event_data.SidHistory", - "winlog.event_data.ScriptBlockText", - "winlog.event_data.Service", - "winlog.event_data.ServiceAccount", - "winlog.event_data.ServiceFileName", - "winlog.event_data.ServiceName", - "winlog.event_data.ServicePrincipalNames", - "winlog.event_data.ServiceSid", - "winlog.event_data.ServiceStartType", - "winlog.event_data.ServiceType", - "winlog.event_data.ServiceVersion", - "winlog.event_data.SessionName", - "winlog.event_data.ShutdownActionType", - "winlog.event_data.ShutdownEventCode", - "winlog.event_data.ShutdownReason", - "winlog.event_data.SidFilteringEnabled", - "winlog.event_data.Signature", - "winlog.event_data.SignatureStatus", - "winlog.event_data.Signed", - "winlog.event_data.StartTime", - "winlog.event_data.State", - "winlog.event_data.Status", - "winlog.event_data.StatusDescription", - "winlog.event_data.StopTime", - "winlog.event_data.SubCategory", - "winlog.event_data.SubCategoryGuid", - "winlog.event_data.SubcategoryGuid", - "winlog.event_data.SubCategoryId", - "winlog.event_data.SubcategoryId", - "winlog.event_data.SubjectDomainName", - "winlog.event_data.SubjectLogonId", - "winlog.event_data.SubjectUserName", - "winlog.event_data.SubjectUserSid", - "winlog.event_data.SubStatus", - "winlog.event_data.TSId", - "winlog.event_data.TargetDomainName", - "winlog.event_data.TargetInfo", - "winlog.event_data.TargetLogonGuid", - "winlog.event_data.TargetLogonId", - "winlog.event_data.TargetServerName", - "winlog.event_data.TargetSid", - "winlog.event_data.TargetUserName", - "winlog.event_data.TargetUserSid", - "winlog.event_data.TdoAttributes", - "winlog.event_data.TdoDirection", - "winlog.event_data.TdoType", - "winlog.event_data.TerminalSessionId", - "winlog.event_data.TicketEncryptionType", - "winlog.event_data.TicketEncryptionTypeDescription", - "winlog.event_data.TicketOptions", - "winlog.event_data.TicketOptionsDescription", - "winlog.event_data.TokenElevationType", - "winlog.event_data.TransmittedServices", - "winlog.event_data.UserAccountControl", - "winlog.event_data.UserParameters", - "winlog.event_data.UserPrincipalName", - "winlog.event_data.UserSid", - "winlog.event_data.UserWorkstations", - "winlog.event_data.Version", - "winlog.event_data.Workstation", - "winlog.event_data.WorkstationName", - "winlog.event_data.param1", - "winlog.event_data.param2", - "winlog.event_data.param3", - "winlog.event_data.param4", - "winlog.event_data.param5", - "winlog.event_data.param6", - "winlog.event_data.param7", - "winlog.event_data.param8", - "winlog.event_id", - "winlog.keywords", - "winlog.channel", - "winlog.record_id", - "winlog.related_activity_id", - "winlog.opcode", - "winlog.provider_guid", - "winlog.provider_name", - "winlog.task", - "winlog.user_data.BackupPath", - "winlog.user_data.Channel", - "winlog.user_data.SubjectDomainName", - "winlog.user_data.SubjectLogonId", - "winlog.user_data.SubjectUserName", - "winlog.user_data.SubjectUserSid", - "winlog.user_data.xml_name", - "winlog.user.identifier", - "winlog.user.name", - "winlog.user.domain", - "winlog.user.type", - "powershell.id", - "powershell.pipeline_id", - "powershell.runspace_id", - "powershell.command.path", - "powershell.command.name", - "powershell.command.type", - "powershell.command.value", - "powershell.connected_user.domain", - "powershell.connected_user.name", - "powershell.engine.version", - "powershell.engine.previous_state", - "powershell.engine.new_state", - "powershell.file.script_block_id", - "powershell.file.script_block_text", - "powershell.process.executable_version", - "powershell.provider.new_state", - "powershell.provider.name" - ] - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "sysmon": { - "properties": { - "file": { - "properties": { - "archived": { - "type": "boolean" - }, - "is_executable": { - "type": "boolean" - } - } - }, - "dns": { - "properties": { - "status": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "log": { - "properties": { - "file": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "destination": { - "properties": { - "port": { - "type": "long" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "rule": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "source": { - "properties": { - "port": { - "type": "long" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "network": { - "properties": { - "community_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "protocol": { - "ignore_above": 1024, - "type": "keyword" - }, - "transport": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "direction": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "file": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "extension": { - "ignore_above": 1024, - "type": "keyword" - }, - "code_signature": { - "properties": { - "valid": { - "type": "boolean" - }, - "trusted": { - "type": "boolean" - }, - "subject_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "exists": { - "type": "boolean" - }, - "status": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pe": { - "properties": { - "file_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "product": { - "ignore_above": 1024, - "type": "keyword" - }, - "imphash": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "ignore_above": 1024, - "type": "keyword" - }, - "company": { - "ignore_above": 1024, - "type": "keyword" - }, - "original_file_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "directory": { - "ignore_above": 1024, - "type": "keyword" - }, - "hash": { - "properties": { - "sha1": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha256": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha512": { - "ignore_above": 1024, - "type": "keyword" - }, - "md5": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "related": { - "properties": { - "hosts": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "powershell": { - "properties": { - "sequence": { - "type": "long" - }, - "total": { - "type": "long" - }, - "connected_user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "executable_version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "file": { - "properties": { - "script_block_text": { - "search_analyzer": "powershell_script_analyzer", - "analyzer": "powershell_script_analyzer", - "type": "text" - }, - "script_block_id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "engine": { - "properties": { - "previous_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_state": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "runspace_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "pipeline_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "command": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "value": { - "type": "text" - } - } - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "sequence": { - "type": "long" - }, - "ingested": { - "type": "date" - }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "type": "constant_keyword", - "value": "windows" - }, - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "type": "constant_keyword", - "value": "windows.forwarded" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "group": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "registry": { - "properties": { - "hive": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "data": { - "properties": { - "strings": { - "ignore_above": 1024, - "type": "wildcard" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "value": { - "ignore_above": 1024, - "type": "keyword" - }, - "key": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" - }, - "parent": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" - }, - "pe": { - "properties": { - "file_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "product": { - "ignore_above": 1024, - "type": "keyword" - }, - "imphash": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "ignore_above": 1024, - "type": "keyword" - }, - "company": { - "ignore_above": 1024, - "type": "keyword" - }, - "original_file_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "start": { - "type": "date" - }, - "pid": { - "type": "long" - }, - "args_count": { - "type": "long" - }, - "entity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "title": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "command_line": { - "ignore_above": 1024, - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "executable": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "hash": { - "properties": { - "sha1": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha256": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha512": { - "ignore_above": 1024, - "type": "keyword" - }, - "md5": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "pe": { - "properties": { - "file_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "product": { - "ignore_above": 1024, - "type": "keyword" - }, - "imphash": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "ignore_above": 1024, - "type": "keyword" - }, - "company": { - "ignore_above": 1024, - "type": "keyword" - }, - "original_file_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "pid": { - "type": "long" - }, - "working_directory": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "args_count": { - "type": "long" - }, - "entity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "title": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "command_line": { - "ignore_above": 1024, - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "executable": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "hash": { - "properties": { - "sha1": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha256": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha512": { - "ignore_above": 1024, - "type": "keyword" - }, - "md5": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "winlog": { - "properties": { - "related_activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "keywords": { - "ignore_above": 1024, - "type": "keyword" - }, - "logon": { - "properties": { - "failure": { - "properties": { - "reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "sub_status": { - "ignore_above": 1024, - "type": "keyword" - }, - "status": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_data": { - "properties": { - "ProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Configuration": { - "ignore_above": 1024, - "type": "keyword" - }, - "OriginalFileName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootMode": { - "ignore_above": 1024, - "type": "keyword" - }, - "Product": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonHours": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "FileVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "TicketOptions": { - "ignore_above": 1024, - "type": "keyword" - }, - "AllowedToDelegateTo": { - "ignore_above": 1024, - "type": "keyword" - }, - "TdoAttributes": { - "ignore_above": 1024, - "type": "keyword" - }, - "StopTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Status": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessMask": { - "ignore_above": 1024, - "type": "keyword" - }, - "KeyLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "SessionName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PasswordHistoryLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSd": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Group": { - "ignore_above": 1024, - "type": "keyword" - }, - "PackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownActionType": { - "ignore_above": 1024, - "type": "keyword" - }, - "DwordVal": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMajor": { - "ignore_above": 1024, - "type": "keyword" - }, - "SidHistory": { - "ignore_above": 1024, - "type": "keyword" - }, - "TransmittedServices": { - "ignore_above": 1024, - "type": "keyword" - }, - "WorkstationName": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleStateCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Path": { - "ignore_above": 1024, - "type": "keyword" - }, - "SchemaVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "CrashOnAuditFailValue": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMinor": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "HandleId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "DnsHostName": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastShutdownGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpPort": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "LmPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastBootGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Version": { - "ignore_above": 1024, - "type": "keyword" - }, - "MachineAccountQuota": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldUacValue": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserParameters": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signed": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubCategoryId": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldTargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewUacValue": { - "ignore_above": 1024, - "type": "keyword" - }, - "CallerProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProfilePath": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "State": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureReason": { - "ignore_above": 1024, - "type": "keyword" - }, - "ComputerAccountChange": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootType": { - "ignore_above": 1024, - "type": "keyword" - }, - "Binary": { - "ignore_above": 1024, - "type": "keyword" - }, - "ImpersonationLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DomainPolicyChanged": { - "ignore_above": 1024, - "type": "keyword" - }, - "CategoryId": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreAuthType": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccountDomain": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewUACList": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubcategoryGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownReason": { - "ignore_above": 1024, - "type": "keyword" - }, - "SidFilteringEnabled": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetServerName": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuditPolicyChanges": { - "ignore_above": 1024, - "type": "keyword" - }, - "Number": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "EventSourceId": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriveName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ExtraInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "PrimaryGroupId": { - "ignore_above": 1024, - "type": "keyword" - }, - "ObjectName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "Workstation": { - "ignore_above": 1024, - "type": "keyword" - }, - "PasswordLastSet": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumThrottlePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "GroupTypeChange": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuthenticationPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NominalFrequency": { - "ignore_above": 1024, - "type": "keyword" - }, - "SignatureStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "DomainSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ScriptPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "TicketEncryptionType": { - "ignore_above": 1024, - "type": "keyword" - }, - "TicketOptionsDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ObjectServer": { - "ignore_above": 1024, - "type": "keyword" - }, - "HomePath": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserWorkstations": { - "ignore_above": 1024, - "type": "keyword" - }, - "SamAccountName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "CorruptionActionState": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuditSourceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubCategoryGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousCreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuditPolicyChangesDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessMaskDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccountName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PerformanceImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "TicketEncryptionTypeDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceAccount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Description": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ScriptBlockText": { - "ignore_above": 1024, - "type": "keyword" - }, - "ObjectType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServicePrincipalNames": { - "ignore_above": 1024, - "type": "keyword" - }, - "MaximumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "KerberosPolicyChange": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "FinalStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "MajorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "MandatoryLabel": { - "ignore_above": 1024, - "type": "keyword" - }, - "HomeDirectory": { - "ignore_above": 1024, - "type": "keyword" - }, - "TokenElevationType": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "QfeVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccountExpires": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceStartType": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserPrincipalName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdSacl1": { - "ignore_above": 1024, - "type": "keyword" - }, - "Dummy": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdSacl0": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdSacl2": { - "ignore_above": 1024, - "type": "keyword" - }, - "Company": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaPolicyId": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdSacl2": { - "ignore_above": 1024, - "type": "keyword" - }, - "EventType": { - "ignore_above": 1024, - "type": "keyword" - }, - "IntegrityLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdSacl1": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdSacl0": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSd": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewTargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "ClientName": { - "ignore_above": 1024, - "type": "keyword" - }, - "StatusDescription": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdDacl0": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdDacl2": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSdDacl1": { - "ignore_above": 1024, - "type": "keyword" - }, - "DomainBehaviorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessGranted": { - "ignore_above": 1024, - "type": "keyword" - }, - "ParentProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubcategoryId": { - "ignore_above": 1024, - "type": "keyword" - }, - "AccessRemoved": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownEventCode": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "MixedDomainMode": { - "ignore_above": 1024, - "type": "keyword" - }, - "ClientInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "Detail": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdDacl1": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdDacl0": { - "ignore_above": 1024, - "type": "keyword" - }, - "Category": { - "ignore_above": 1024, - "type": "keyword" - }, - "TerminalSessionId": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSdDacl2": { - "ignore_above": 1024, - "type": "keyword" - }, - "ClientAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "CreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "CallerProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TdoType": { - "ignore_above": 1024, - "type": "keyword" - }, - "Reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceFileName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DisplayName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BuildVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TSId": { - "ignore_above": 1024, - "type": "keyword" - }, - "PrivilegeList": { - "ignore_above": 1024, - "type": "keyword" - }, - "param7": { - "ignore_above": 1024, - "type": "keyword" - }, - "param8": { - "ignore_above": 1024, - "type": "keyword" - }, - "param5": { - "ignore_above": 1024, - "type": "keyword" - }, - "param6": { - "ignore_above": 1024, - "type": "keyword" - }, - "Service": { - "ignore_above": 1024, - "type": "keyword" - }, - "TdoDirection": { - "ignore_above": 1024, - "type": "keyword" - }, - "param3": { - "ignore_above": 1024, - "type": "keyword" - }, - "param4": { - "ignore_above": 1024, - "type": "keyword" - }, - "param1": { - "ignore_above": 1024, - "type": "keyword" - }, - "param2": { - "ignore_above": 1024, - "type": "keyword" - }, - "CommandLine": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserAccountControl": { - "ignore_above": 1024, - "type": "keyword" - }, - "OemInformation": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signature": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubCategory": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "EntryCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonID": { - "ignore_above": 1024, - "type": "keyword" - }, - "BitlockerUserInputTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Session": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "opcode": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_guid": { - "ignore_above": 1024, - "type": "keyword" - }, - "activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "time_created": { - "type": "date" - }, - "trustDirection": { - "ignore_above": 1024, - "type": "keyword" - }, - "api": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - }, - "computer_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "trustAttribute": { - "ignore_above": 1024, - "type": "keyword" - }, - "process": { - "properties": { - "pid": { - "type": "long" - }, - "thread": { - "properties": { - "id": { - "type": "long" - } - } - } - } - }, - "level": { - "ignore_above": 1024, - "type": "keyword" - }, - "computerObject": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "user_data": { - "properties": { - "SubjectUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BackupPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "Channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "xml_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "version": { - "type": "long" - }, - "record_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "task": { - "ignore_above": 1024, - "type": "keyword" - }, - "trustType": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "properties": { - "identifier": { - "ignore_above": 1024, - "type": "keyword" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "dns": { - "properties": { - "op_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "resolved_ip": { - "type": "ip" - }, - "response_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "question": { - "properties": { - "registered_domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "top_level_domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "subdomain": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "class": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "answers": { - "properties": { - "data": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "class": { - "ignore_above": 1024, - "type": "keyword" - }, - "ttl": { - "type": "long" - } - } - }, - "header_flags": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "message": { - "type": "match_only_text" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "input": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "service": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "dataset": { - "properties": { - "name": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - } - } - }, - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "target": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "group": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } - } - } - } - }, - "_meta": { - "package": { - "name": "windows" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell@package.json deleted file mode 100644 index ad0ff857e..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell@package.json +++ /dev/null @@ -1,1335 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-windows.powershell-1.20.1", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "analysis": { - "analyzer": { - "powershell_script_analyzer": { - "pattern": "[\\W&&[^-]]+", - "type": "pattern" - } - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "event.action", - "event.category", - "event.code", - "event.kind", - "event.outcome", - "event.provider", - "event.type", - "tags", - "input.type", - "destination.user.domain", - "destination.user.id", - "destination.user.name", - "ecs.version", - "file.directory", - "file.extension", - "file.name", - "file.path", - "log.level", - "message", - "process.args", - "process.command_line", - "process.entity_id", - "process.executable", - "process.name", - "process.title", - "related.hash", - "related.hosts", - "related.user", - "source.user.domain", - "source.user.id", - "source.user.name", - "user.domain", - "user.id", - "user.name", - "powershell.id", - "powershell.pipeline_id", - "powershell.runspace_id", - "powershell.command.path", - "powershell.command.name", - "powershell.command.type", - "powershell.command.value", - "powershell.connected_user.domain", - "powershell.connected_user.name", - "powershell.engine.version", - "powershell.engine.previous_state", - "powershell.engine.new_state", - "powershell.file.script_block_id", - "powershell.file.script_block_text", - "powershell.process.executable_version", - "powershell.provider.new_state", - "powershell.provider.name", - "winlog.api", - "winlog.activity_id", - "winlog.computer_name", - "winlog.event_data.AuthenticationPackageName", - "winlog.event_data.Binary", - "winlog.event_data.BitlockerUserInputTime", - "winlog.event_data.BootMode", - "winlog.event_data.BootType", - "winlog.event_data.BuildVersion", - "winlog.event_data.Company", - "winlog.event_data.CorruptionActionState", - "winlog.event_data.CreationUtcTime", - "winlog.event_data.Description", - "winlog.event_data.Detail", - "winlog.event_data.DeviceName", - "winlog.event_data.DeviceNameLength", - "winlog.event_data.DeviceTime", - "winlog.event_data.DeviceVersionMajor", - "winlog.event_data.DeviceVersionMinor", - "winlog.event_data.DriveName", - "winlog.event_data.DriverName", - "winlog.event_data.DriverNameLength", - "winlog.event_data.DwordVal", - "winlog.event_data.EntryCount", - "winlog.event_data.ExtraInfo", - "winlog.event_data.FailureName", - "winlog.event_data.FailureNameLength", - "winlog.event_data.FileVersion", - "winlog.event_data.FinalStatus", - "winlog.event_data.Group", - "winlog.event_data.IdleImplementation", - "winlog.event_data.IdleStateCount", - "winlog.event_data.ImpersonationLevel", - "winlog.event_data.IntegrityLevel", - "winlog.event_data.IpAddress", - "winlog.event_data.IpPort", - "winlog.event_data.KeyLength", - "winlog.event_data.LastBootGood", - "winlog.event_data.LastShutdownGood", - "winlog.event_data.LmPackageName", - "winlog.event_data.LogonGuid", - "winlog.event_data.LogonId", - "winlog.event_data.LogonProcessName", - "winlog.event_data.LogonType", - "winlog.event_data.MajorVersion", - "winlog.event_data.MaximumPerformancePercent", - "winlog.event_data.MemberName", - "winlog.event_data.MemberSid", - "winlog.event_data.MinimumPerformancePercent", - "winlog.event_data.MinimumThrottlePercent", - "winlog.event_data.MinorVersion", - "winlog.event_data.NewProcessId", - "winlog.event_data.NewProcessName", - "winlog.event_data.NewSchemeGuid", - "winlog.event_data.NewTime", - "winlog.event_data.NominalFrequency", - "winlog.event_data.Number", - "winlog.event_data.OldSchemeGuid", - "winlog.event_data.OldTime", - "winlog.event_data.OriginalFileName", - "winlog.event_data.Path", - "winlog.event_data.PerformanceImplementation", - "winlog.event_data.PreviousCreationUtcTime", - "winlog.event_data.PreviousTime", - "winlog.event_data.PrivilegeList", - "winlog.event_data.ProcessId", - "winlog.event_data.ProcessName", - "winlog.event_data.ProcessPath", - "winlog.event_data.ProcessPid", - "winlog.event_data.Product", - "winlog.event_data.PuaCount", - "winlog.event_data.PuaPolicyId", - "winlog.event_data.QfeVersion", - "winlog.event_data.Reason", - "winlog.event_data.SchemaVersion", - "winlog.event_data.ScriptBlockText", - "winlog.event_data.ServiceName", - "winlog.event_data.ServiceVersion", - "winlog.event_data.ShutdownActionType", - "winlog.event_data.ShutdownEventCode", - "winlog.event_data.ShutdownReason", - "winlog.event_data.Signature", - "winlog.event_data.SignatureStatus", - "winlog.event_data.Signed", - "winlog.event_data.StartTime", - "winlog.event_data.State", - "winlog.event_data.Status", - "winlog.event_data.StopTime", - "winlog.event_data.SubjectDomainName", - "winlog.event_data.SubjectLogonId", - "winlog.event_data.SubjectUserName", - "winlog.event_data.SubjectUserSid", - "winlog.event_data.TSId", - "winlog.event_data.TargetDomainName", - "winlog.event_data.TargetInfo", - "winlog.event_data.TargetLogonGuid", - "winlog.event_data.TargetLogonId", - "winlog.event_data.TargetServerName", - "winlog.event_data.TargetUserName", - "winlog.event_data.TargetUserSid", - "winlog.event_data.TerminalSessionId", - "winlog.event_data.TokenElevationType", - "winlog.event_data.TransmittedServices", - "winlog.event_data.UserSid", - "winlog.event_data.Version", - "winlog.event_data.Workstation", - "winlog.event_data.param1", - "winlog.event_data.param2", - "winlog.event_data.param3", - "winlog.event_data.param4", - "winlog.event_data.param5", - "winlog.event_data.param6", - "winlog.event_data.param7", - "winlog.event_data.param8", - "winlog.event_id", - "winlog.keywords", - "winlog.channel", - "winlog.record_id", - "winlog.related_activity_id", - "winlog.opcode", - "winlog.provider_guid", - "winlog.provider_name", - "winlog.task", - "winlog.user.identifier", - "winlog.user.name", - "winlog.user.domain", - "winlog.user.type" - ] - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - }, - { - "winlog.user_data": { - "path_match": "winlog.user_data.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "pid": { - "type": "long" - }, - "args_count": { - "type": "long" - }, - "entity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "title": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "command_line": { - "ignore_above": 1024, - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "executable": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - "winlog": { - "properties": { - "related_activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "computer_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "process": { - "properties": { - "pid": { - "type": "long" - }, - "thread": { - "properties": { - "id": { - "type": "long" - } - } - } - } - }, - "keywords": { - "ignore_above": 1024, - "type": "keyword" - }, - "channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_data": { - "properties": { - "SignatureStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "OriginalFileName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootMode": { - "ignore_above": 1024, - "type": "keyword" - }, - "Product": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "FileVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "StopTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Status": { - "ignore_above": 1024, - "type": "keyword" - }, - "CorruptionActionState": { - "ignore_above": 1024, - "type": "keyword" - }, - "KeyLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousCreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "PerformanceImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Group": { - "ignore_above": 1024, - "type": "keyword" - }, - "Description": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownActionType": { - "ignore_above": 1024, - "type": "keyword" - }, - "DwordVal": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMajor": { - "ignore_above": 1024, - "type": "keyword" - }, - "ScriptBlockText": { - "ignore_above": 1024, - "type": "keyword" - }, - "TransmittedServices": { - "ignore_above": 1024, - "type": "keyword" - }, - "MaximumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "FinalStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleStateCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "MajorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "Path": { - "ignore_above": 1024, - "type": "keyword" - }, - "SchemaVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "TokenElevationType": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "QfeVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMinor": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Company": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaPolicyId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IntegrityLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastShutdownGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpPort": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "LmPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastBootGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Version": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signed": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownEventCode": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "State": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootType": { - "ignore_above": 1024, - "type": "keyword" - }, - "Binary": { - "ignore_above": 1024, - "type": "keyword" - }, - "ImpersonationLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Detail": { - "ignore_above": 1024, - "type": "keyword" - }, - "TerminalSessionId": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "CreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownReason": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetServerName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Number": { - "ignore_above": 1024, - "type": "keyword" - }, - "BuildVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TSId": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PrivilegeList": { - "ignore_above": 1024, - "type": "keyword" - }, - "param7": { - "ignore_above": 1024, - "type": "keyword" - }, - "param8": { - "ignore_above": 1024, - "type": "keyword" - }, - "param5": { - "ignore_above": 1024, - "type": "keyword" - }, - "param6": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriveName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ExtraInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "param3": { - "ignore_above": 1024, - "type": "keyword" - }, - "param4": { - "ignore_above": 1024, - "type": "keyword" - }, - "param1": { - "ignore_above": 1024, - "type": "keyword" - }, - "param2": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "Workstation": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signature": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumThrottlePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "EntryCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "BitlockerUserInputTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuthenticationPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NominalFrequency": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "opcode": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "type": "long" - }, - "record_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "task": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_guid": { - "ignore_above": 1024, - "type": "keyword" - }, - "activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "api": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "properties": { - "identifier": { - "ignore_above": 1024, - "type": "keyword" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "destination": { - "properties": { - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "source": { - "properties": { - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "message": { - "type": "match_only_text" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "input": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "file": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "extension": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "directory": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "related": { - "properties": { - "hosts": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "powershell": { - "properties": { - "sequence": { - "type": "long" - }, - "total": { - "type": "long" - }, - "connected_user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "executable_version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "file": { - "properties": { - "script_block_text": { - "search_analyzer": "powershell_script_analyzer", - "analyzer": "powershell_script_analyzer", - "type": "text" - }, - "script_block_id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "engine": { - "properties": { - "previous_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_state": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "runspace_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "pipeline_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "command": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "value": { - "type": "text" - } - } - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "sequence": { - "type": "long" - }, - "ingested": { - "type": "date" - }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "type": "constant_keyword", - "value": "windows" - }, - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "type": "constant_keyword", - "value": "windows.powershell" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "dataset": { - "properties": { - "name": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - } - } - }, - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - }, - "_meta": { - "package": { - "name": "windows" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell_operational@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell_operational@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell_operational@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell_operational@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell_operational@package.json deleted file mode 100644 index b5cc588c9..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.powershell_operational@package.json +++ /dev/null @@ -1,1334 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-windows.powershell_operational-1.20.1", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "analysis": { - "analyzer": { - "powershell_script_analyzer": { - "pattern": "[\\W&&[^-]]+", - "type": "pattern" - } - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "event.action", - "event.category", - "event.code", - "event.kind", - "event.outcome", - "event.provider", - "event.type", - "tags", - "input.type", - "destination.user.domain", - "destination.user.id", - "destination.user.name", - "ecs.version", - "file.directory", - "file.extension", - "file.name", - "file.path", - "log.level", - "message", - "process.args", - "process.command_line", - "process.entity_id", - "process.executable", - "process.name", - "process.title", - "related.hash", - "related.hosts", - "related.user", - "source.user.domain", - "source.user.id", - "source.user.name", - "user.domain", - "user.id", - "user.name", - "powershell.id", - "powershell.pipeline_id", - "powershell.runspace_id", - "powershell.command.path", - "powershell.command.name", - "powershell.command.type", - "powershell.command.value", - "powershell.connected_user.domain", - "powershell.connected_user.name", - "powershell.engine.version", - "powershell.engine.previous_state", - "powershell.engine.new_state", - "powershell.file.script_block_id", - "powershell.file.script_block_text", - "powershell.process.executable_version", - "powershell.provider.new_state", - "powershell.provider.name", - "winlog.api", - "winlog.activity_id", - "winlog.computer_name", - "winlog.event_data.AuthenticationPackageName", - "winlog.event_data.Binary", - "winlog.event_data.BitlockerUserInputTime", - "winlog.event_data.BootMode", - "winlog.event_data.BootType", - "winlog.event_data.BuildVersion", - "winlog.event_data.Company", - "winlog.event_data.CorruptionActionState", - "winlog.event_data.CreationUtcTime", - "winlog.event_data.Description", - "winlog.event_data.Detail", - "winlog.event_data.DeviceName", - "winlog.event_data.DeviceNameLength", - "winlog.event_data.DeviceTime", - "winlog.event_data.DeviceVersionMajor", - "winlog.event_data.DeviceVersionMinor", - "winlog.event_data.DriveName", - "winlog.event_data.DriverName", - "winlog.event_data.DriverNameLength", - "winlog.event_data.DwordVal", - "winlog.event_data.EntryCount", - "winlog.event_data.ExtraInfo", - "winlog.event_data.FailureName", - "winlog.event_data.FailureNameLength", - "winlog.event_data.FileVersion", - "winlog.event_data.FinalStatus", - "winlog.event_data.Group", - "winlog.event_data.IdleImplementation", - "winlog.event_data.IdleStateCount", - "winlog.event_data.ImpersonationLevel", - "winlog.event_data.IntegrityLevel", - "winlog.event_data.IpAddress", - "winlog.event_data.IpPort", - "winlog.event_data.KeyLength", - "winlog.event_data.LastBootGood", - "winlog.event_data.LastShutdownGood", - "winlog.event_data.LmPackageName", - "winlog.event_data.LogonGuid", - "winlog.event_data.LogonId", - "winlog.event_data.LogonProcessName", - "winlog.event_data.LogonType", - "winlog.event_data.MajorVersion", - "winlog.event_data.MaximumPerformancePercent", - "winlog.event_data.MemberName", - "winlog.event_data.MemberSid", - "winlog.event_data.MinimumPerformancePercent", - "winlog.event_data.MinimumThrottlePercent", - "winlog.event_data.MinorVersion", - "winlog.event_data.NewProcessId", - "winlog.event_data.NewProcessName", - "winlog.event_data.NewSchemeGuid", - "winlog.event_data.NewTime", - "winlog.event_data.NominalFrequency", - "winlog.event_data.Number", - "winlog.event_data.OldSchemeGuid", - "winlog.event_data.OldTime", - "winlog.event_data.OriginalFileName", - "winlog.event_data.Path", - "winlog.event_data.PerformanceImplementation", - "winlog.event_data.PreviousCreationUtcTime", - "winlog.event_data.PreviousTime", - "winlog.event_data.PrivilegeList", - "winlog.event_data.ProcessId", - "winlog.event_data.ProcessName", - "winlog.event_data.ProcessPath", - "winlog.event_data.ProcessPid", - "winlog.event_data.Product", - "winlog.event_data.PuaCount", - "winlog.event_data.PuaPolicyId", - "winlog.event_data.QfeVersion", - "winlog.event_data.Reason", - "winlog.event_data.SchemaVersion", - "winlog.event_data.ScriptBlockText", - "winlog.event_data.ServiceName", - "winlog.event_data.ServiceVersion", - "winlog.event_data.ShutdownActionType", - "winlog.event_data.ShutdownEventCode", - "winlog.event_data.ShutdownReason", - "winlog.event_data.Signature", - "winlog.event_data.SignatureStatus", - "winlog.event_data.Signed", - "winlog.event_data.StartTime", - "winlog.event_data.State", - "winlog.event_data.Status", - "winlog.event_data.StopTime", - "winlog.event_data.SubjectDomainName", - "winlog.event_data.SubjectLogonId", - "winlog.event_data.SubjectUserName", - "winlog.event_data.SubjectUserSid", - "winlog.event_data.TSId", - "winlog.event_data.TargetDomainName", - "winlog.event_data.TargetInfo", - "winlog.event_data.TargetLogonGuid", - "winlog.event_data.TargetLogonId", - "winlog.event_data.TargetServerName", - "winlog.event_data.TargetUserName", - "winlog.event_data.TargetUserSid", - "winlog.event_data.TerminalSessionId", - "winlog.event_data.TokenElevationType", - "winlog.event_data.TransmittedServices", - "winlog.event_data.UserSid", - "winlog.event_data.Version", - "winlog.event_data.Workstation", - "winlog.event_data.param1", - "winlog.event_data.param2", - "winlog.event_data.param3", - "winlog.event_data.param4", - "winlog.event_data.param5", - "winlog.event_data.param6", - "winlog.event_data.param7", - "winlog.event_data.param8", - "winlog.event_id", - "winlog.keywords", - "winlog.channel", - "winlog.record_id", - "winlog.related_activity_id", - "winlog.opcode", - "winlog.provider_guid", - "winlog.provider_name", - "winlog.task", - "winlog.user.identifier", - "winlog.user.name", - "winlog.user.domain", - "winlog.user.type" - ] - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - }, - { - "winlog.user_data": { - "path_match": "winlog.user_data.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "pid": { - "type": "long" - }, - "args_count": { - "type": "long" - }, - "entity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "title": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "command_line": { - "ignore_above": 1024, - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "executable": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - "winlog": { - "properties": { - "related_activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "computer_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "process": { - "properties": { - "pid": { - "type": "long" - }, - "thread": { - "properties": { - "id": { - "type": "long" - } - } - } - } - }, - "keywords": { - "ignore_above": 1024, - "type": "keyword" - }, - "channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_data": { - "properties": { - "SignatureStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "OriginalFileName": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootMode": { - "ignore_above": 1024, - "type": "keyword" - }, - "Product": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "FileVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "StopTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Status": { - "ignore_above": 1024, - "type": "keyword" - }, - "CorruptionActionState": { - "ignore_above": 1024, - "type": "keyword" - }, - "KeyLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousCreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "PerformanceImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Group": { - "ignore_above": 1024, - "type": "keyword" - }, - "Description": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownActionType": { - "ignore_above": 1024, - "type": "keyword" - }, - "DwordVal": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMajor": { - "ignore_above": 1024, - "type": "keyword" - }, - "ScriptBlockText": { - "ignore_above": 1024, - "type": "keyword" - }, - "TransmittedServices": { - "ignore_above": 1024, - "type": "keyword" - }, - "MaximumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "FinalStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleStateCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "MajorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "Path": { - "ignore_above": 1024, - "type": "keyword" - }, - "SchemaVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "TokenElevationType": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "QfeVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMinor": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Company": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaPolicyId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IntegrityLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastShutdownGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpPort": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "LmPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastBootGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Version": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signed": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownEventCode": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "State": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootType": { - "ignore_above": 1024, - "type": "keyword" - }, - "Binary": { - "ignore_above": 1024, - "type": "keyword" - }, - "ImpersonationLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Detail": { - "ignore_above": 1024, - "type": "keyword" - }, - "TerminalSessionId": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "CreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownReason": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetServerName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Number": { - "ignore_above": 1024, - "type": "keyword" - }, - "BuildVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TSId": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PrivilegeList": { - "ignore_above": 1024, - "type": "keyword" - }, - "param7": { - "ignore_above": 1024, - "type": "keyword" - }, - "param8": { - "ignore_above": 1024, - "type": "keyword" - }, - "param5": { - "ignore_above": 1024, - "type": "keyword" - }, - "param6": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriveName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ExtraInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "param3": { - "ignore_above": 1024, - "type": "keyword" - }, - "param4": { - "ignore_above": 1024, - "type": "keyword" - }, - "param1": { - "ignore_above": 1024, - "type": "keyword" - }, - "param2": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "Workstation": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signature": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumThrottlePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "EntryCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "BitlockerUserInputTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuthenticationPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NominalFrequency": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "opcode": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "type": "long" - }, - "record_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "task": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_guid": { - "ignore_above": 1024, - "type": "keyword" - }, - "activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "api": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "properties": { - "identifier": { - "ignore_above": 1024, - "type": "keyword" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "destination": { - "properties": { - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "source": { - "properties": { - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "message": { - "type": "match_only_text" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "input": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "file": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "extension": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "directory": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "related": { - "properties": { - "hosts": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "powershell": { - "properties": { - "sequence": { - "type": "long" - }, - "total": { - "type": "long" - }, - "connected_user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "executable_version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "file": { - "properties": { - "script_block_text": { - "analyzer": "powershell_script_analyzer", - "type": "text" - }, - "script_block_id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "engine": { - "properties": { - "previous_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_state": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "new_state": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "runspace_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "pipeline_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "command": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "value": { - "type": "text" - } - } - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "sequence": { - "type": "long" - }, - "ingested": { - "type": "date" - }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "type": "constant_keyword", - "value": "windows" - }, - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "type": "constant_keyword", - "value": "windows.powershell_operational" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "dataset": { - "properties": { - "name": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - } - } - }, - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - }, - "_meta": { - "package": { - "name": "windows" - }, - "managed_by": "fleet", - "managed": true - } - } diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.sysmon_operational@custom.json b/salt/elasticsearch/templates/component/elastic-agent/logs-windows.sysmon_operational@custom.json deleted file mode 100644 index fe77af1db..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.sysmon_operational@custom.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "template": { - "settings": {} - }, - "_meta": { - "package": { - "name": "elastic_agent" - }, - "managed_by": "fleet", - "managed": true - } -} diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.sysmon_operational@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-windows.sysmon_operational@package.json deleted file mode 100644 index 451eaf7aa..000000000 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-windows.sysmon_operational@package.json +++ /dev/null @@ -1,1752 +0,0 @@ - {"template": { - "settings": { - "index": { - "lifecycle": { - "name": "logs" - }, - "codec": "best_compression", - "default_pipeline": "logs-windows.sysmon_operational-1.20.1", - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "query": { - "default_field": [ - "cloud.account.id", - "cloud.availability_zone", - "cloud.instance.id", - "cloud.instance.name", - "cloud.machine.type", - "cloud.provider", - "cloud.region", - "cloud.project.id", - "cloud.image.id", - "container.id", - "container.image.name", - "container.name", - "host.architecture", - "host.hostname", - "host.id", - "host.mac", - "host.name", - "host.os.family", - "host.os.kernel", - "host.os.name", - "host.os.platform", - "host.os.version", - "host.os.build", - "host.os.codename", - "host.type", - "event.action", - "event.category", - "event.code", - "event.kind", - "event.outcome", - "event.provider", - "event.type", - "tags", - "input.type", - "destination.domain", - "dns.answers.class", - "dns.answers.data", - "dns.answers.name", - "dns.answers.type", - "dns.header_flags", - "dns.id", - "dns.op_code", - "dns.question.class", - "dns.question.name", - "dns.question.registered_domain", - "dns.question.subdomain", - "dns.question.top_level_domain", - "dns.question.type", - "dns.response_code", - "dns.type", - "ecs.version", - "error.code", - "error.message", - "file.code_signature.status", - "file.code_signature.subject_name", - "file.directory", - "file.extension", - "file.hash.md5", - "file.hash.sha1", - "file.hash.sha256", - "file.hash.sha512", - "file.name", - "file.path", - "file.pe.architecture", - "file.pe.company", - "file.pe.description", - "file.pe.file_version", - "file.pe.imphash", - "file.pe.original_file_name", - "file.pe.product", - "group.domain", - "group.id", - "group.name", - "log.level", - "message", - "network.community_id", - "network.direction", - "network.protocol", - "network.transport", - "network.type", - "process.args", - "process.command_line", - "process.entity_id", - "process.executable", - "process.hash.md5", - "process.hash.sha1", - "process.hash.sha256", - "process.hash.sha512", - "process.name", - "process.parent.args", - "process.parent.command_line", - "process.parent.entity_id", - "process.parent.executable", - "process.parent.name", - "process.pe.architecture", - "process.pe.company", - "process.pe.description", - "process.pe.file_version", - "process.pe.imphash", - "process.pe.original_file_name", - "process.pe.product", - "process.title", - "process.working_directory", - "registry.data.strings", - "registry.data.type", - "registry.hive", - "registry.key", - "registry.path", - "registry.value", - "related.hash", - "related.hosts", - "related.user", - "rule.name", - "service.name", - "service.type", - "source.domain", - "user.domain", - "user.id", - "user.name", - "user.target.group.domain", - "user.target.group.id", - "user.target.group.name", - "user.target.name", - "sysmon.dns.status", - "winlog.api", - "winlog.activity_id", - "winlog.computer_name", - "winlog.event_data.AuthenticationPackageName", - "winlog.event_data.Binary", - "winlog.event_data.BitlockerUserInputTime", - "winlog.event_data.BootMode", - "winlog.event_data.BootType", - "winlog.event_data.BuildVersion", - "winlog.event_data.CallTrace", - "winlog.event_data.ClientInfo", - "winlog.event_data.Company", - "winlog.event_data.Configuration", - "winlog.event_data.CorruptionActionState", - "winlog.event_data.CreationUtcTime", - "winlog.event_data.Description", - "winlog.event_data.Detail", - "winlog.event_data.DeviceName", - "winlog.event_data.DeviceNameLength", - "winlog.event_data.DeviceTime", - "winlog.event_data.DeviceVersionMajor", - "winlog.event_data.DeviceVersionMinor", - "winlog.event_data.DriveName", - "winlog.event_data.DriverName", - "winlog.event_data.DriverNameLength", - "winlog.event_data.DwordVal", - "winlog.event_data.EntryCount", - "winlog.event_data.EventType", - "winlog.event_data.EventNamespace", - "winlog.event_data.ExtraInfo", - "winlog.event_data.FailureName", - "winlog.event_data.FailureNameLength", - "winlog.event_data.FileVersion", - "winlog.event_data.FinalStatus", - "winlog.event_data.GrantedAccess", - "winlog.event_data.Group", - "winlog.event_data.IdleImplementation", - "winlog.event_data.IdleStateCount", - "winlog.event_data.ImpersonationLevel", - "winlog.event_data.IntegrityLevel", - "winlog.event_data.IpAddress", - "winlog.event_data.IpPort", - "winlog.event_data.KeyLength", - "winlog.event_data.LastBootGood", - "winlog.event_data.LastShutdownGood", - "winlog.event_data.LmPackageName", - "winlog.event_data.LogonGuid", - "winlog.event_data.LogonId", - "winlog.event_data.LogonProcessName", - "winlog.event_data.LogonType", - "winlog.event_data.MajorVersion", - "winlog.event_data.MaximumPerformancePercent", - "winlog.event_data.MemberName", - "winlog.event_data.MemberSid", - "winlog.event_data.MinimumPerformancePercent", - "winlog.event_data.MinimumThrottlePercent", - "winlog.event_data.MinorVersion", - "winlog.event_data.Name", - "winlog.event_data.NewProcessId", - "winlog.event_data.NewProcessName", - "winlog.event_data.NewSchemeGuid", - "winlog.event_data.NewThreadId", - "winlog.event_data.NewTime", - "winlog.event_data.NominalFrequency", - "winlog.event_data.Number", - "winlog.event_data.OldSchemeGuid", - "winlog.event_data.OldTime", - "winlog.event_data.Operation", - "winlog.event_data.OriginalFileName", - "winlog.event_data.Path", - "winlog.event_data.PerformanceImplementation", - "winlog.event_data.PreviousCreationUtcTime", - "winlog.event_data.PreviousTime", - "winlog.event_data.PrivilegeList", - "winlog.event_data.ProcessId", - "winlog.event_data.ProcessName", - "winlog.event_data.ProcessPath", - "winlog.event_data.ProcessPid", - "winlog.event_data.Product", - "winlog.event_data.PuaCount", - "winlog.event_data.PuaPolicyId", - "winlog.event_data.QfeVersion", - "winlog.event_data.Query", - "winlog.event_data.Reason", - "winlog.event_data.SchemaVersion", - "winlog.event_data.ScriptBlockText", - "winlog.event_data.ServiceName", - "winlog.event_data.ServiceVersion", - "winlog.event_data.Session", - "winlog.event_data.ShutdownActionType", - "winlog.event_data.ShutdownEventCode", - "winlog.event_data.ShutdownReason", - "winlog.event_data.Signature", - "winlog.event_data.SignatureStatus", - "winlog.event_data.Signed", - "winlog.event_data.StartAddress", - "winlog.event_data.StartFunction", - "winlog.event_data.StartModule", - "winlog.event_data.StartTime", - "winlog.event_data.State", - "winlog.event_data.Status", - "winlog.event_data.StopTime", - "winlog.event_data.SubjectDomainName", - "winlog.event_data.SubjectLogonId", - "winlog.event_data.SubjectUserName", - "winlog.event_data.SubjectUserSid", - "winlog.event_data.TSId", - "winlog.event_data.TargetDomainName", - "winlog.event_data.TargetImage", - "winlog.event_data.TargetInfo", - "winlog.event_data.TargetLogonGuid", - "winlog.event_data.TargetLogonId", - "winlog.event_data.TargetProcessGUID", - "winlog.event_data.TargetProcessId", - "winlog.event_data.TargetServerName", - "winlog.event_data.TargetUserName", - "winlog.event_data.TargetUserSid", - "winlog.event_data.TerminalSessionId", - "winlog.event_data.TokenElevationType", - "winlog.event_data.TransmittedServices", - "winlog.event_data.Type", - "winlog.event_data.UserSid", - "winlog.event_data.Version", - "winlog.event_data.Workstation", - "winlog.event_data.param1", - "winlog.event_data.param2", - "winlog.event_data.param3", - "winlog.event_data.param4", - "winlog.event_data.param5", - "winlog.event_data.param6", - "winlog.event_data.param7", - "winlog.event_data.param8", - "winlog.event_id", - "winlog.keywords", - "winlog.channel", - "winlog.record_id", - "winlog.related_activity_id", - "winlog.opcode", - "winlog.provider_guid", - "winlog.provider_name", - "winlog.task", - "winlog.user.identifier", - "winlog.user.name", - "winlog.user.domain", - "winlog.user.type" - ] - } - } - }, - "mappings": { - "dynamic_templates": [ - { - "container.labels": { - "path_match": "container.labels.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - }, - { - "winlog.user_data": { - "path_match": "winlog.user_data.*", - "mapping": { - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "properties": { - "container": { - "properties": { - "image": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "sysmon": { - "properties": { - "file": { - "properties": { - "archived": { - "type": "boolean" - }, - "is_executable": { - "type": "boolean" - } - } - }, - "dns": { - "properties": { - "status": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "log": { - "properties": { - "level": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "destination": { - "properties": { - "port": { - "type": "long" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - } - } - }, - "rule": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "source": { - "properties": { - "port": { - "type": "long" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - } - } - }, - "error": { - "properties": { - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "message": { - "type": "match_only_text" - } - } - }, - "network": { - "properties": { - "community_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "protocol": { - "ignore_above": 1024, - "type": "keyword" - }, - "transport": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "direction": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "cloud": { - "properties": { - "availability_zone": { - "ignore_above": 1024, - "type": "keyword" - }, - "image": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "instance": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "machine": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "project": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "region": { - "ignore_above": 1024, - "type": "keyword" - }, - "account": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "file": { - "properties": { - "path": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "extension": { - "ignore_above": 1024, - "type": "keyword" - }, - "code_signature": { - "properties": { - "valid": { - "type": "boolean" - }, - "trusted": { - "type": "boolean" - }, - "subject_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "exists": { - "type": "boolean" - }, - "status": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "pe": { - "properties": { - "file_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "product": { - "ignore_above": 1024, - "type": "keyword" - }, - "imphash": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "ignore_above": 1024, - "type": "keyword" - }, - "company": { - "ignore_above": 1024, - "type": "keyword" - }, - "original_file_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "directory": { - "ignore_above": 1024, - "type": "keyword" - }, - "hash": { - "properties": { - "sha1": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha256": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha512": { - "ignore_above": 1024, - "type": "keyword" - }, - "md5": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "related": { - "properties": { - "hosts": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "user": { - "ignore_above": 1024, - "type": "keyword" - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "host": { - "properties": { - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "os": { - "properties": { - "build": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "codename": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "text" - } - } - }, - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "containerized": { - "type": "boolean" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "event": { - "properties": { - "sequence": { - "type": "long" - }, - "ingested": { - "type": "date" - }, - "code": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "type": "constant_keyword", - "value": "windows" - }, - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "type": "constant_keyword", - "value": "windows.sysmon_operational" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "group": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "registry": { - "properties": { - "hive": { - "ignore_above": 1024, - "type": "keyword" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "data": { - "properties": { - "strings": { - "ignore_above": 1024, - "type": "wildcard" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "value": { - "ignore_above": 1024, - "type": "keyword" - }, - "key": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "process": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" - }, - "parent": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "pid": { - "type": "long" - }, - "args_count": { - "type": "long" - }, - "entity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "command_line": { - "ignore_above": 1024, - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "executable": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - } - } - }, - "pe": { - "properties": { - "file_version": { - "ignore_above": 1024, - "type": "keyword" - }, - "product": { - "ignore_above": 1024, - "type": "keyword" - }, - "imphash": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "ignore_above": 1024, - "type": "keyword" - }, - "company": { - "ignore_above": 1024, - "type": "keyword" - }, - "original_file_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "architecture": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "pid": { - "type": "long" - }, - "working_directory": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "args_count": { - "type": "long" - }, - "entity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "title": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "command_line": { - "ignore_above": 1024, - "type": "wildcard", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "executable": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "hash": { - "properties": { - "sha1": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha256": { - "ignore_above": 1024, - "type": "keyword" - }, - "sha512": { - "ignore_above": 1024, - "type": "keyword" - }, - "md5": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "winlog": { - "properties": { - "related_activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "computer_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "process": { - "properties": { - "pid": { - "type": "long" - }, - "thread": { - "properties": { - "id": { - "type": "long" - } - } - } - } - }, - "keywords": { - "ignore_above": 1024, - "type": "keyword" - }, - "channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_data": { - "properties": { - "SignatureStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Configuration": { - "ignore_above": 1024, - "type": "keyword" - }, - "OriginalFileName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Query": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootMode": { - "ignore_above": 1024, - "type": "keyword" - }, - "Product": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "FileVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "CallTrace": { - "ignore_above": 1024, - "type": "keyword" - }, - "StopTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Status": { - "ignore_above": 1024, - "type": "keyword" - }, - "GrantedAccess": { - "ignore_above": 1024, - "type": "keyword" - }, - "CorruptionActionState": { - "ignore_above": 1024, - "type": "keyword" - }, - "KeyLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousCreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "PerformanceImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Group": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewThreadId": { - "ignore_above": 1024, - "type": "keyword" - }, - "Description": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownActionType": { - "ignore_above": 1024, - "type": "keyword" - }, - "DwordVal": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMajor": { - "ignore_above": 1024, - "type": "keyword" - }, - "ScriptBlockText": { - "ignore_above": 1024, - "type": "keyword" - }, - "TransmittedServices": { - "ignore_above": 1024, - "type": "keyword" - }, - "MaximumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "FinalStatus": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleStateCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "MajorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "Path": { - "ignore_above": 1024, - "type": "keyword" - }, - "SchemaVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "TokenElevationType": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinorVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "IdleImplementation": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessPath": { - "ignore_above": 1024, - "type": "keyword" - }, - "QfeVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceVersionMinor": { - "ignore_above": 1024, - "type": "keyword" - }, - "Type": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpAddress": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Company": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaPolicyId": { - "ignore_above": 1024, - "type": "keyword" - }, - "EventType": { - "ignore_above": 1024, - "type": "keyword" - }, - "IntegrityLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastShutdownGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "IpPort": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "LmPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Name": { - "ignore_above": 1024, - "type": "keyword" - }, - "UserSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "LastBootGood": { - "ignore_above": 1024, - "type": "keyword" - }, - "PuaCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "Version": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetProcessGUID": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signed": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownEventCode": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "ServiceName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PreviousTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "State": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartFunction": { - "ignore_above": 1024, - "type": "keyword" - }, - "BootType": { - "ignore_above": 1024, - "type": "keyword" - }, - "Binary": { - "ignore_above": 1024, - "type": "keyword" - }, - "ClientInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "ImpersonationLevel": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Detail": { - "ignore_above": 1024, - "type": "keyword" - }, - "TerminalSessionId": { - "ignore_above": 1024, - "type": "keyword" - }, - "MemberSid": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriverName": { - "ignore_above": 1024, - "type": "keyword" - }, - "DeviceNameLength": { - "ignore_above": 1024, - "type": "keyword" - }, - "OldSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Operation": { - "ignore_above": 1024, - "type": "keyword" - }, - "CreationUtcTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "Reason": { - "ignore_above": 1024, - "type": "keyword" - }, - "ShutdownReason": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetServerName": { - "ignore_above": 1024, - "type": "keyword" - }, - "Number": { - "ignore_above": 1024, - "type": "keyword" - }, - "BuildVersion": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetImage": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumPerformancePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonProcessName": { - "ignore_above": 1024, - "type": "keyword" - }, - "TSId": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetDomainName": { - "ignore_above": 1024, - "type": "keyword" - }, - "PrivilegeList": { - "ignore_above": 1024, - "type": "keyword" - }, - "param7": { - "ignore_above": 1024, - "type": "keyword" - }, - "param8": { - "ignore_above": 1024, - "type": "keyword" - }, - "param5": { - "ignore_above": 1024, - "type": "keyword" - }, - "param6": { - "ignore_above": 1024, - "type": "keyword" - }, - "DriveName": { - "ignore_above": 1024, - "type": "keyword" - }, - "EventNamespace": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "LogonType": { - "ignore_above": 1024, - "type": "keyword" - }, - "ExtraInfo": { - "ignore_above": 1024, - "type": "keyword" - }, - "StartModule": { - "ignore_above": 1024, - "type": "keyword" - }, - "param3": { - "ignore_above": 1024, - "type": "keyword" - }, - "param4": { - "ignore_above": 1024, - "type": "keyword" - }, - "param1": { - "ignore_above": 1024, - "type": "keyword" - }, - "param2": { - "ignore_above": 1024, - "type": "keyword" - }, - "TargetLogonId": { - "ignore_above": 1024, - "type": "keyword" - }, - "Workstation": { - "ignore_above": 1024, - "type": "keyword" - }, - "SubjectUserName": { - "ignore_above": 1024, - "type": "keyword" - }, - "FailureName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NewSchemeGuid": { - "ignore_above": 1024, - "type": "keyword" - }, - "Signature": { - "ignore_above": 1024, - "type": "keyword" - }, - "MinimumThrottlePercent": { - "ignore_above": 1024, - "type": "keyword" - }, - "ProcessId": { - "ignore_above": 1024, - "type": "keyword" - }, - "EntryCount": { - "ignore_above": 1024, - "type": "keyword" - }, - "BitlockerUserInputTime": { - "ignore_above": 1024, - "type": "keyword" - }, - "AuthenticationPackageName": { - "ignore_above": 1024, - "type": "keyword" - }, - "NominalFrequency": { - "ignore_above": 1024, - "type": "keyword" - }, - "Session": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "opcode": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "type": "long" - }, - "record_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "task": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_guid": { - "ignore_above": 1024, - "type": "keyword" - }, - "activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "api": { - "ignore_above": 1024, - "type": "keyword" - }, - "provider_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "properties": { - "identifier": { - "ignore_above": 1024, - "type": "keyword" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "dns": { - "properties": { - "op_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "resolved_ip": { - "type": "ip" - }, - "response_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "question": { - "properties": { - "registered_domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "top_level_domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "subdomain": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "class": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "answers": { - "properties": { - "data": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "class": { - "ignore_above": 1024, - "type": "keyword" - }, - "ttl": { - "type": "long" - } - } - }, - "header_flags": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "message": { - "type": "match_only_text" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - }, - "input": { - "properties": { - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "@timestamp": { - "type": "date" - }, - "data_stream": { - "properties": { - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - }, - "dataset": { - "type": "constant_keyword" - } - } - }, - "service": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "dataset": { - "properties": { - "name": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - }, - "type": { - "type": "constant_keyword" - } - } - }, - "user": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "target": { - "properties": { - "name": { - "ignore_above": 1024, - "type": "keyword", - "fields": { - "text": { - "type": "match_only_text" - } - } - }, - "group": { - "properties": { - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } - } - } - } - }, - "_meta": { - "package": { - "name": "windows" - }, - "managed_by": "fleet", - "managed": true - } - } From b66be9c22640b3809a86134d01cbd42edcf4917a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 5 Sep 2023 12:46:49 -0400 Subject: [PATCH 249/350] only ingest pfsense on sensor nodes --- salt/common/tools/sbin/so-test | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-test b/salt/common/tools/sbin/so-test index 1758a44bb..01b4da637 100755 --- a/salt/common/tools/sbin/so-test +++ b/salt/common/tools/sbin/so-test @@ -5,10 +5,14 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. +. /usr/sbin/so-common + set -e # Playback live sample data onto monitor interface so-tcpreplay /opt/samples/* 2> /dev/null # Ingest sample pfsense log entry -echo "<134>$(date '+%b %d %H:%M:%S') filterlog[31624]: 84,,,1567509287,igb0.244,match,pass,in,4,0x0,,64,0,0,DF,6,tcp,64,192.168.1.1,10.10.10.10,56320,443,0,S,3333585167,,65535,,mss;nop;wscale;nop;nop;TS;sackOK;eol" | nc -uv -w1 127.0.0.1 514 > /dev/null 2>&1 +if is_sensor_node; then + echo "<134>$(date '+%b %d %H:%M:%S') filterlog[31624]: 84,,,1567509287,igb0.244,match,pass,in,4,0x0,,64,0,0,DF,6,tcp,64,192.168.1.1,10.10.10.10,56320,443,0,S,3333585167,,65535,,mss;nop;wscale;nop;nop;TS;sackOK;eol" | nc -uv -w1 127.0.0.1 514 > /dev/null 2>&1 +fi From ffaab4a1b47d7949ab6ff061d97d0b59f95ad049 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 6 Sep 2023 14:19:53 -0400 Subject: [PATCH 250/350] only add endgame to action if it is populated --- salt/soc/merged.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index dc2f889bb..c17c23b25 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -35,7 +35,7 @@ {% endif %} {% set standard_actions = SOCMERGED.config.pop('actions') %} -{% if pillar.global.endgamehost is defined %} +{% if pillar.global.endgamehost != '' %} {% set endgame_dict = { "name": "Endgame", "description": "Endgame Endpoint Investigation and Response", From 60f1947eb4b4f5a6d2b5c43507164246a92e63cd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 7 Sep 2023 14:01:19 -0400 Subject: [PATCH 251/350] prevent endgame_dict from being added to standard_actions if it is already present --- salt/soc/merged.map.jinja | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index c17c23b25..052ff9941 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -35,7 +35,17 @@ {% endif %} {% set standard_actions = SOCMERGED.config.pop('actions') %} + {% if pillar.global.endgamehost != '' %} +{# this is added to prevent endgame_dict from being added to standard_actions for each time this file is rendered #} +{% set endgame = namespace(add=true) %} +{% for d in standard_actions %} +{% if d.name is defined %} +{% if d.name == 'Endgame' %} +{% set endgame.add = false %} +{% endif %} +{% endif %} +{% endfor %} {% set endgame_dict = { "name": "Endgame", "description": "Endgame Endpoint Investigation and Response", @@ -44,7 +54,9 @@ "links": ["https://" ~ pillar.global.endgamehost ~ "/endpoints/{:agent.id}"] } %} -{% do standard_actions.append(endgame_dict) %} +{% if endgame.add %} +{% do standard_actions.append(endgame_dict) %} +{% endif %} {% endif %} {% do SOCMERGED.config.server.client.hunt.update({'actions': standard_actions}) %} From 35157f2e8b27c313235a4cbd95fa4e0bb77ea12f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 7 Sep 2023 15:46:04 -0400 Subject: [PATCH 252/350] add comment --- salt/soc/merged.map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 052ff9941..33c0070ad 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -38,6 +38,7 @@ {% if pillar.global.endgamehost != '' %} {# this is added to prevent endgame_dict from being added to standard_actions for each time this file is rendered #} +{# since this map file is rendered 3 times, it causes endgame_dict to appened 3 times if custom actions are defined in the pillar #} {% set endgame = namespace(add=true) %} {% for d in standard_actions %} {% if d.name is defined %} From f8ae3f12e65aeb6a5efa851b6a55f81adeab94df Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 7 Sep 2023 17:22:10 -0400 Subject: [PATCH 253/350] addl node types --- setup/so-setup | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup/so-setup b/setup/so-setup index c1d92ec62..030afdf47 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -194,6 +194,18 @@ if [ -n "$test_profile" ]; then install_type=DESKTOP MSRVIP_OFFSET=-3 is_desktop_grid=true + elif [[ "$test_profile" =~ "-idh" ]]; then + install_type=IDH + HOSTNAME=idh + MSRVIP_OFFSET=-4 + elif [[ "$test_profile" =~ "-receiver" ]]; then + install_type=RECEIVER + HOSTNAME=receiver + MSRVIP_OFFSET=-5 + elif [[ "$test_profile" =~ "-fleet" ]]; then + install_type=FLEET + HOSTNAME=fleet + MSRVIP_OFFSET=-6 else HOSTNAME=manager fi From 598515e5b447770bb9cee1dae8d8e7974ff08112 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 8 Sep 2023 09:21:13 -0400 Subject: [PATCH 254/350] give priority to presets --- setup/so-functions | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 5015b4bff..5d6ada340 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -398,20 +398,22 @@ collect_mngr_hostname() { sed -i "/$MSRV/d" /etc/hosts fi - if ! getent hosts "$MSRV"; then - whiptail_manager_ip + if [[ -z "$MSRVIP" ]]; then + if ! getent hosts "$MSRV"; then + whiptail_manager_ip - while ! valid_ip4 "$MSRVIP" || [[ $MSRVIP == "$MAINIP" || $MSRVIP == "127.0.0.1" ]]; do - whiptail_invalid_input + while ! valid_ip4 "$MSRVIP" || [[ $MSRVIP == "$MAINIP" || $MSRVIP == "127.0.0.1" ]]; do + whiptail_invalid_input + whiptail_manager_ip "$MSRVIP" + done + else + MSRVIP=$(getent hosts "$MSRV" | awk 'NR==1{print $1}') whiptail_manager_ip "$MSRVIP" - done - else - MSRVIP=$(getent hosts "$MSRV" | awk 'NR==1{print $1}') - whiptail_manager_ip "$MSRVIP" - while ! valid_ip4 "$MSRVIP" || [[ $MSRVIP == "$MAINIP" || $MSRVIP == "127.0.0.1" ]]; do - whiptail_invalid_input - whiptail_manager_ip "$MSRVIP" - done + while ! valid_ip4 "$MSRVIP" || [[ $MSRVIP == "$MAINIP" || $MSRVIP == "127.0.0.1" ]]; do + whiptail_invalid_input + whiptail_manager_ip "$MSRVIP" + done + fi fi } From e814a3409f4dc2ac56fcdc5c32d79c6231dff1c7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 8 Sep 2023 15:28:24 -0400 Subject: [PATCH 255/350] fix rule location for rulecat.conf. run so-rule-update if rules change in /opt/so/rules/nids --- salt/idstools/enabled.sls | 1 + salt/idstools/etc/rulecat.conf | 4 ++-- salt/idstools/sorules/extraction.rules | 26 -------------------------- salt/idstools/sorules/filters.rules | 11 ----------- 4 files changed, 3 insertions(+), 39 deletions(-) delete mode 100644 salt/idstools/sorules/extraction.rules delete mode 100644 salt/idstools/sorules/filters.rules diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index 3f5acda19..31afc5113 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -77,6 +77,7 @@ run_so-rule-update: - docker_container: so-idstools - onchanges: - file: idstoolsetcsync + - file: synclocalnidsrules - order: last {% else %} diff --git a/salt/idstools/etc/rulecat.conf b/salt/idstools/etc/rulecat.conf index 8be3aa1ce..d6f3d93d8 100644 --- a/salt/idstools/etc/rulecat.conf +++ b/salt/idstools/etc/rulecat.conf @@ -3,8 +3,8 @@ --merged=/opt/so/rules/nids/all.rules --local=/opt/so/rules/nids/local.rules {%- if GLOBALS.md_engine == "SURICATA" %} ---local=/opt/so/rules/nids/sorules/extraction.rules ---local=/opt/so/rules/nids/sorules/filters.rules +--local=/opt/so/rules/nids/extraction.rules +--local=/opt/so/rules/nids/filters.rules {%- endif %} --url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules --disable=/opt/so/idstools/etc/disable.conf diff --git a/salt/idstools/sorules/extraction.rules b/salt/idstools/sorules/extraction.rules deleted file mode 100644 index bccfc69d6..000000000 --- a/salt/idstools/sorules/extraction.rules +++ /dev/null @@ -1,26 +0,0 @@ -# Extract all PDF mime type -alert http any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100000; rev:1;) -alert smtp any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100001; rev:1;) -alert nfs any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100002; rev:1;) -alert smb any any -> any any (msg:"FILE pdf detected"; filemagic:"PDF document"; filestore; sid:1100003; rev:1;) -# Extract EXE/DLL file types -alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100004; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100005; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100006; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"PE32 executable"; filestore; sid:1100007; rev:1;) -alert http any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100008; rev:1;) -alert smtp any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100009; rev:1;) -alert nfs any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100010; rev:1;) -alert smb any any -> any any (msg:"FILE EXE detected"; filemagic:"MS-DOS executable"; filestore; sid:1100011; rev:1;) - -# Extract all Zip files -alert http any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100012; rev:1;) -alert smtp any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100013; rev:1;) -alert nfs any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100014; rev:1;) -alert smb any any -> any any (msg:"FILE ZIP detected"; filemagic:"Zip"; filestore; sid:1100015; rev:1;) - -# Extract Word Docs -alert http any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100016; rev:1;) -alert smtp any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100017; rev:1;) -alert nfs any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100018; rev:1;) -alert smb any any -> any any (msg:"FILE WORDDOC detected"; filemagic:"Composite Document File V2 Document"; filestore; sid:1100019; rev:1;) \ No newline at end of file diff --git a/salt/idstools/sorules/filters.rules b/salt/idstools/sorules/filters.rules deleted file mode 100644 index 051d1913f..000000000 --- a/salt/idstools/sorules/filters.rules +++ /dev/null @@ -1,11 +0,0 @@ -# Start the filters at sid 1200000 -# Example of filtering out *google.com from being in the dns log. -#config dns any any -> any any (dns.query; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200000;) -# Example of filtering out *google.com from being in the http log. -#config http any any -> any any (http.host; content:"google.com"; config: logging disable, type tx, scope tx; sid:1200001;) -# Example of filtering out someuseragent from being in the http log. -#config http any any -> any any (http.user_agent; content:"someuseragent"; config: logging disable, type tx, scope tx; sid:1200002;) -# Example of filtering out Google's certificate from being in the ssl log. -#config tls any any -> any any (tls.fingerprint; content:"4f:a4:5e:58:7e:d9:db:20:09:d7:b6:c7:ff:58:c4:7b:dc:3f:55:b4"; config: logging disable, type tx, scope tx; sid:1200003;) -# Example of filtering out a md5 of a file from being in the files log. -#config fileinfo any any -> any any (fileinfo.filemd5; content:"7a125dc69c82d5caf94d3913eecde4b5"; config: logging disable, type tx, scope tx; sid:1200004;) From f1d0db81714941b6652337638291cac55124e8e8 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 11 Sep 2023 13:30:11 +0000 Subject: [PATCH 256/350] /app to /kibana/app --- salt/nginx/etc/nginx.conf | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 05da0b5d8..b2616e946 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -230,7 +230,19 @@ http { proxy_cookie_path /api/ /influxdb/api/; } - location /kibana/ { + location /app/ { + auth_request /auth/sessions/whoami; + rewrite /app/(.*) /app/$1 break; + proxy_pass http://{{ GLOBALS.manager }}:5601/app/; + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + + location /kibana/ { auth_request /auth/sessions/whoami; rewrite /kibana/(.*) /$1 break; proxy_pass http://{{ GLOBALS.manager }}:5601/; From 35ebbc974c370ac0ac4a4f0d8c8403d677656c85 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 11 Sep 2023 13:52:16 +0000 Subject: [PATCH 257/350] Change description to indicate that opencanary modules only apply to IDH nodes --- salt/idh/soc_idh.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/idh/soc_idh.yaml b/salt/idh/soc_idh.yaml index f792812e4..1d6918405 100644 --- a/salt/idh/soc_idh.yaml +++ b/salt/idh/soc_idh.yaml @@ -23,7 +23,7 @@ idh: class: *loggingOptions filename: *loggingOptions portscan_x_enabled: &serviceOptions - description: To enable this opencanary module, set this value to true. To disable set to false. + description: To enable this opencanary module, set this value to true. To disable set to false. This option only applies to IDH nodes within your grid. helpLink: idh.html portscan_x_logfile: *loggingOptions portscan_x_synrate: From 30c3255cb28f5d62dc9fcae4186a7a2b4554faf7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 12 Sep 2023 08:39:42 -0400 Subject: [PATCH 258/350] dont manage sorules --- salt/idstools/sync_files.sls | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/salt/idstools/sync_files.sls b/salt/idstools/sync_files.sls index e8d5edda6..64479e937 100644 --- a/salt/idstools/sync_files.sls +++ b/salt/idstools/sync_files.sls @@ -26,13 +26,6 @@ rulesdir: - group: 939 - makedirs: True -SOrulesdir: - file.directory: - - name: /opt/so/rules/nids/sorules - - user: 939 - - group: 939 - - makedirs: True - # Don't show changes because all.rules can be large synclocalnidsrules: file.recurse: @@ -42,13 +35,3 @@ synclocalnidsrules: - group: 939 - show_changes: False - include_pat: 'E@.rules' - -# Don't show changes because all.rules can be large -syncnidsSOrules: - file.recurse: - - name: /opt/so/rules/nids/sorules - - source: salt://idstools/sorules/ - - user: 939 - - group: 939 - - show_changes: False - - include_pat: 'E@.rules' From 11b8e1341885118bd22ccee21d5c6e1776f474f8 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 13 Sep 2023 07:37:54 -0400 Subject: [PATCH 259/350] FIX: SOC Config pcap doc links should point to steno docs #11302 --- salt/pcap/soc_pcap.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/pcap/soc_pcap.yaml b/salt/pcap/soc_pcap.yaml index 0f4b7e1e4..32204a23a 100644 --- a/salt/pcap/soc_pcap.yaml +++ b/salt/pcap/soc_pcap.yaml @@ -1,35 +1,35 @@ pcap: enabled: description: You can enable or disable Stenographer on all sensors or a single sensor. - helpLink: pcap.html + helpLink: stenographer.html config: maxdirectoryfiles: description: The maximum number of packet/index files to create before deleting old files. - helpLink: pcap.html + helpLink: stenographer.html diskfreepercentage: description: The disk space percent to always keep free for PCAP - helpLink: pcap.html + helpLink: stenographer.html blocks: description: The number of 1MB packet blocks used by AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. advanced: True - helpLink: pcap.html + helpLink: stenographer.html preallocate_file_mb: description: File size to pre-allocate for individual PCAP files. You shouldn't need to change this. advanced: True - helpLink: pcap.html + helpLink: stenographer.html aiops: description: The max number of async writes to allow at once. advanced: True - helpLink: pcap.html + helpLink: stenographer.html pin_to_cpu: description: Enable CPU pinning for PCAP. advanced: True - helpLink: pcap.html + helpLink: stenographer.html cpus_to_pin_to: description: CPU to pin PCAP to. Currently only a single CPU is supported. advanced: True - helpLink: pcap.html + helpLink: stenographer.html disks: description: List of disks to use for PCAP. This is currently not used. advanced: True - helpLink: pcap.html + helpLink: stenographer.html From 22c0323bdae337f5cb4431ff6a1f54a0ddcc9f81 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Sep 2023 10:57:45 -0400 Subject: [PATCH 260/350] Update so-minion --- salt/manager/tools/sbin/so-minion | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 075632985..01a58585f 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -239,6 +239,10 @@ function add_sensor_to_minion() { echo " threads: '$CORECOUNT'" >> $PILLARFILE echo "pcap:" >> $PILLARFILE echo " enabled: True" >> $PILLARFILE + if [[ $is_pcaplimit ]]; then + echo " config:" >> $PILLARFILE + echo " diskfreepercentage: 40" >> $PILLARFILE + fi echo " " >> $PILLARFILE } @@ -409,6 +413,7 @@ function apply_ES_state() { salt-call state.apply elasticsearch concurrent=True } function createEVAL() { + is_pcaplimit=true add_elasticsearch_to_minion add_sensor_to_minion add_strelka_to_minion @@ -429,6 +434,7 @@ function createEVAL() { } function createSTANDALONE() { + is_pcaplimit=true add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion @@ -520,8 +526,9 @@ function createIDH() { } function createHEAVYNODE() { + is_pcaplimit=true add_elasticsearch_to_minion - add_elastic_agent_to_minion + add_elastic_agent_to_minion add_logstash_to_minion add_sensor_to_minion add_strelka_to_minion From 33d68478b6678e6707061b4f8e2755f2b705b0a6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Sep 2023 11:48:16 -0400 Subject: [PATCH 261/350] Update so-minion --- salt/manager/tools/sbin/so-minion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 01a58585f..64084dbd0 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -241,7 +241,7 @@ function add_sensor_to_minion() { echo " enabled: True" >> $PILLARFILE if [[ $is_pcaplimit ]]; then echo " config:" >> $PILLARFILE - echo " diskfreepercentage: 40" >> $PILLARFILE + echo " diskfreepercentage: 60" >> $PILLARFILE fi echo " " >> $PILLARFILE } From e067b7134e3ed344af40eabfe6a281567705e6e7 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 14 Sep 2023 07:38:07 -0400 Subject: [PATCH 262/350] exclude docker pull unauth errors from failing setup since they'll be retried --- setup/so-verify | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-verify b/setup/so-verify index 07d24d114..e907e8bdc 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -52,6 +52,7 @@ log_has_errors() { grep -vE "/nsm/rules/yara*" | \ grep -vE "Failed to restart snapd" | \ grep -vE "Login Failed Details" | \ + grep -vE "response from daemon: unauthorized" | \ grep -vE "Running scope as unit" &> "$error_log" if [[ $? -eq 0 ]]; then From 59d077f3ffb94df96c183b817786b738ff5d0432 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Thu, 14 Sep 2023 08:32:17 -0400 Subject: [PATCH 263/350] Fix regex --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index c935521fd..275bc6a11 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -46,7 +46,7 @@ do done printf "\n### Stripping out unused components" -find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -regex '.*fleet.*\|.*packet.*\|.*apm*.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete +find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -maxdepth 1 -regex '.*fleet.*\|.*packet.*\|.*apm.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete printf "\n### Tarring everything up again" for OS in "${OSARCH[@]}" From 0c11a9b7337c523cf24396cdd264cdcf69c6d979 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Thu, 14 Sep 2023 09:33:17 -0400 Subject: [PATCH 264/350] Add transform role --- salt/elasticsearch/config.map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index ed4a5033f..37447cabb 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -21,7 +21,7 @@ {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} {% endfor %} {% if grains.id.split('_') | last == 'manager' %} - {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master','data','remote_cluster_client']}) %} + {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master','data','remote_cluster_client','transform']}) %} {% else %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'roles': ['master', 'data_hot', 'remote_cluster_client']}) %} {% endif %} From c65c9777bdb8590faf056039f7602e2254249137 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 14 Sep 2023 17:42:25 -0400 Subject: [PATCH 265/350] improvents for checking system requirements --- setup/so-functions | 58 ++++++++++++++++++++++++++++++++++------------ setup/so-setup | 22 +++++++++--------- setup/so-variables | 8 +++---- setup/so-whiptail | 17 +------------- 4 files changed, 59 insertions(+), 46 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 5d6ada340..3707e3141 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -707,8 +707,6 @@ checkin_at_boot() { } check_requirements() { - local standalone_or_dist=$1 - local node_type=$2 # optional local req_mem local req_cores local req_storage @@ -716,27 +714,57 @@ check_requirements() { readarray -t nic_list <<< "$(ip link| awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2}' | grep -vwe "bond0" | sed 's/ //g' | sed -r 's/(.*)(\.[0-9]+)@\1/\1\2/g')" local num_nics=${#nic_list[@]} - if [[ "$standalone_or_dist" == 'standalone' ]]; then + if [[ $is_eval ]]; then req_mem=12 req_cores=4 req_nics=2 - elif [[ "$standalone_or_dist" == 'dist' ]]; then - req_mem=8 + elif [[ $is_standalone ]]; then + req_mem=24 req_cores=4 - if [[ "$node_type" == 'sensor' ]]; then req_nics=2; else req_nics=1; fi - if [[ "$node_type" == 'fleet' ]]; then req_mem=4; fi - if [[ "$node_type" == 'idh' ]]; then req_mem=1 req_cores=2; fi - elif [[ "$standalone_or_dist" == 'import' ]]; then + req_nics=2 + elif [[ $is_manager ]]; then + req_mem=16 + req_cores=4 + req_nics=1 + elif [[ $is_managersearch ]]; then + req_mem=16 + req_cores=8 + req_nics=1 + elif [[ $is_sensor ]]; then + req_mem=12 + req_cores=4 + req_nics=2 + elif [[ $is_fleet ]]; then req_mem=4 + req_cores=4 + req_nics=1 + elif [[ $is_searchnode ]]; then + req_mem=16 + req_cores=4 + req_nics=1 + elif [[ $is_heavynode ]]; then + req_mem=24 + req_cores=4 + req_nics=2 + elif [[ $is_idh ]]; then + req_mem=1 + req_cores=2 + req_nics=1 + elif [[ $is_import ]]; then + req_mem=4 + req_cores=2 + req_nics=1 + elif [[ $is_receiver ]]; then + req_mem=8 req_cores=2 req_nics=1 fi if [[ $setup_type == 'network' ]] ; then - if [[ -n $nsm_mount ]]; then - if [[ "$standalone_or_dist" == 'import' ]]; then + if [[ -n $nsm_mount ]]; then # does a /nsm mount exist + if [[ $is_import ]]; then req_storage=50 - elif [[ "$node_type" == 'idh' ]]; then + elif [[ $is_idh ]]; then req_storage=12 else req_storage=100 @@ -748,10 +776,10 @@ check_requirements() { whiptail_storage_requirements "/nsm" "${free_space_nsm} GB" "${req_storage} GB" fi else - if [[ "$standalone_or_dist" == 'import' ]]; then + if [[ $is_import ]]; then req_storage=50 - elif [[ "$node_type" == 'idh' ]]; then - req_storage=12 + elif [[ $is_idh ]]; then + req_storage=12 else req_storage=200 fi diff --git a/setup/so-setup b/setup/so-setup index 030afdf47..e35dde579 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -422,7 +422,7 @@ if ! [[ -f $install_opt_file ]]; then # If it is an install from ISO is this airgap? [[ $is_iso ]] && whiptail_airgap # Make sure minimum requirements are met - check_requirements "manager" + check_requirements # Do networking things networking_needful # Do we need a proxy? @@ -453,7 +453,7 @@ if ! [[ -f $install_opt_file ]]; then monints=true check_elastic_license [[ $is_iso ]] && whiptail_airgap - check_requirements "manager" + check_requirements networking_needful [[ ! $is_airgap ]] && collect_net_method collect_dockernet @@ -474,7 +474,7 @@ if ! [[ -f $install_opt_file ]]; then check_elastic_license waitforstate=true [[ $is_iso ]] && whiptail_airgap - check_requirements "manager" + check_requirements networking_needful [[ ! $is_airgap ]] && collect_net_method collect_dockernet @@ -494,7 +494,7 @@ if ! [[ -f $install_opt_file ]]; then check_elastic_license waitforstate=true [[ $is_iso ]] && whiptail_airgap - check_requirements "manager" + check_requirements networking_needful [[ ! $is_airgap ]] && collect_net_method collect_dockernet @@ -512,7 +512,7 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_sensor ]]; then info "Setting up as node type sensor" monints=true - check_requirements "sensor" + check_requirements calculate_useable_cores networking_needful check_network_manager_conf @@ -527,7 +527,7 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_fleet ]]; then info "Setting up as node type fleet" - check_requirements "fleet" + check_requirements networking_needful check_network_manager_conf set_network_dev_status_list @@ -540,7 +540,7 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_searchnode ]]; then info "Setting up as node type searchnode" - check_requirements "elasticsearch" + check_requirements networking_needful check_network_manager_conf set_network_dev_status_list @@ -554,7 +554,7 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_heavynode ]]; then info "Setting up as node type heavynode" monints=true - check_requirements "heavynode" + check_requirements calculate_useable_cores networking_needful check_network_manager_conf @@ -569,7 +569,7 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_idh ]]; then info "Setting up as node type idh" - check_requirements "idh" + check_requirements networking_needful collect_mngr_hostname add_mngr_ip_to_hosts @@ -583,7 +583,7 @@ if ! [[ -f $install_opt_file ]]; then waitforstate=true [[ $is_iso ]] && whiptail_airgap check_elastic_license - check_requirements "import" + check_requirements networking_needful [[ ! $is_airgap ]] && detect_cloud collect_dockernet @@ -601,7 +601,7 @@ if ! [[ -f $install_opt_file ]]; then elif [[ $is_receiver ]]; then info "Setting up as node type receiver" - check_requirements "receiver" + check_requirements networking_needful collect_mngr_hostname add_mngr_ip_to_hosts diff --git a/setup/so-variables b/setup/so-variables index 7c5e51c6c..7f6522487 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -5,7 +5,7 @@ mkdir -p /nsm total_mem=$(grep MemTotal /proc/meminfo | awk '{print $2}' | sed -r 's/.{3}$//') export total_mem -total_mem_hr=$(grep MemTotal /proc/meminfo | awk '{ printf("%.0f", $2/1024/1024); }') +total_mem_hr=$(grep MemTotal /proc/meminfo | awk '{ printf("%.0f", $2/1000/1000); }') export total_mem_hr num_cpu_cores=$(nproc) @@ -32,10 +32,10 @@ export filesystem_root filesystem_nsm=$(df /nsm | awk '$3 ~ /[0-9]+/ { print $2 * 1000 }') export filesystem_nsm -free_space_nsm=$(df -Pk /nsm | sed 1d | grep -v used | awk '{ print $4 / 1048576 }' | awk '{ printf("%.0f", $1) }') +free_space_nsm=$(df -Pk /nsm | sed 1d | grep -v used | awk '{ print $4 / 1042803 }' | awk '{ printf("%.0f", $1) }') export free_space_nsm -free_space_root=$(df -Pk / | sed 1d | grep -v used | awk '{ print $4 / 1048576 }' | awk '{ printf("%.0f", $1) }') +free_space_root=$(df -Pk / | sed 1d | grep -v used | awk '{ print $4 / 1042803 }' | awk '{ printf("%.0f", $1) }') export free_space_root readarray -t mountpoints <<< "$(lsblk -nlo MOUNTPOINT)" @@ -218,4 +218,4 @@ patch_pillar_file="$local_salt_dir/pillar/patch/soc_patch.sls" export patch_pillar_file adv_patch_pillar_file="$local_salt_dir/pillar/patch/adv_patch.sls" -export adv_patch_pillar_file \ No newline at end of file +export adv_patch_pillar_file diff --git a/setup/so-whiptail b/setup/so-whiptail index 9622ad44a..ede138d26 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -232,7 +232,7 @@ whiptail_requirements_error() { [ -n "$TESTING" ] && return - if [[ $(echo "$requirement_needed" | tr '[:upper:]' '[:lower:]') == 'nics' ]]; then + if [[ $(echo "$requirement_needed" | tr '[:upper:]' '[:lower:]') =~ 'nic' ]]; then whiptail --title "$whiptail_title" \ --msgbox "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select OK to exit setup and reconfigure the machine." 10 75 @@ -1184,21 +1184,6 @@ whiptail_reinstall() { whiptail_check_exitstatus $exitstatus } -whiptail_requirements_error() { - - local requirement_needed=$1 - local current_val=$2 - local needed_val=$3 - - [ -n "$TESTING" ] && return - - whiptail --title "$whiptail_title" \ - --yesno "This machine currently has $current_val $requirement_needed, but needs $needed_val to meet minimum requirements. Select YES to continue anyway, or select NO to cancel." 10 75 - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus -} - whiptail_sensor_config() { [ -n "$TESTING" ] && return From 98499c3963e564af5694d11f736ca93877e930c6 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 15 Sep 2023 13:51:46 +0000 Subject: [PATCH 266/350] Clean component template directory --- salt/elasticsearch/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 8baff4901..fa0f824b4 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -108,6 +108,7 @@ escomponenttemplates: - source: salt://elasticsearch/templates/component - user: 930 - group: 939 + - clean: True - onchanges_in: - cmd: so-elasticsearch-templates From f9cbde10a6787d5eefa11696d85cdc35b1d5e3d3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 18 Sep 2023 11:19:21 -0400 Subject: [PATCH 267/350] avoid volume sprawl --- salt/influxdb/config.sls | 8 ++++++++ salt/influxdb/enabled.sls | 1 + salt/playbook/config.sls | 8 ++++++++ salt/playbook/enabled.sls | 1 + salt/redis/config.sls | 7 +++++++ salt/redis/enabled.sls | 1 + 6 files changed, 26 insertions(+) diff --git a/salt/influxdb/config.sls b/salt/influxdb/config.sls index 54e20b713..3520e46b3 100644 --- a/salt/influxdb/config.sls +++ b/salt/influxdb/config.sls @@ -25,6 +25,14 @@ influxlogdir: - group: 939 - makedirs: True +influxlogdir: + file.directory: + - name: /opt/so/conf/influxdb/etc + - dir_mode: 750 + - user: 939 + - group: 939 + - makedirs: True + influxdbdir: file.directory: - name: /nsm/influxdb diff --git a/salt/influxdb/enabled.sls b/salt/influxdb/enabled.sls index 70f4c404f..c0733c12c 100644 --- a/salt/influxdb/enabled.sls +++ b/salt/influxdb/enabled.sls @@ -38,6 +38,7 @@ so-influxdb: - binds: - /opt/so/log/influxdb/:/log:rw - /opt/so/conf/influxdb/config.yaml:/conf/config.yaml:ro + - /opt/so/conf/influxdb/etc:/etc/influxdb2:rw - /nsm/influxdb:/var/lib/influxdb2:rw - /etc/pki/influxdb.crt:/conf/influxdb.crt:ro - /etc/pki/influxdb.key:/conf/influxdb.key:ro diff --git a/salt/playbook/config.sls b/salt/playbook/config.sls index 7d37f8873..f4c2cf137 100644 --- a/salt/playbook/config.sls +++ b/salt/playbook/config.sls @@ -91,6 +91,14 @@ playbooklogdir: - group: 939 - makedirs: True +playbookfilesdir: + file.directory: + - name: /opt/so/conf/playbook/redmine-files + - dir_mode: 775 + - user: 939 + - group: 939 + - makedirs: True + {% if 'idh' in salt['cmd.shell']("ls /opt/so/saltstack/local/pillar/minions/|awk -F'_' {'print $2'}|awk -F'.' {'print $1'}").split() %} idh-plays: file.recurse: diff --git a/salt/playbook/enabled.sls b/salt/playbook/enabled.sls index 434cb18e4..e70fec693 100644 --- a/salt/playbook/enabled.sls +++ b/salt/playbook/enabled.sls @@ -33,6 +33,7 @@ so-playbook: - sobridge: - ipv4_address: {{ DOCKER.containers['so-playbook'].ip }} - binds: + - /opt/so/conf/playbook/redmine-files:/usr/src/redmine/files:rw - /opt/so/log/playbook:/playbook/log:rw {% if DOCKER.containers['so-playbook'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-playbook'].custom_bind_mounts %} diff --git a/salt/redis/config.sls b/salt/redis/config.sls index d698040f8..053d46707 100644 --- a/salt/redis/config.sls +++ b/salt/redis/config.sls @@ -25,6 +25,13 @@ redisworkdir: - group: 939 - makedirs: True +redisdatadir: + file.directory: + - name: /nsm/redis/data + - user: 939 + - group: 939 + - makedirs: True + redislogdir: file.directory: - name: /opt/so/log/redis diff --git a/salt/redis/enabled.sls b/salt/redis/enabled.sls index 27177d217..fc206e3cb 100644 --- a/salt/redis/enabled.sls +++ b/salt/redis/enabled.sls @@ -28,6 +28,7 @@ so-redis: - /opt/so/log/redis:/var/log/redis:rw - /opt/so/conf/redis/etc/redis.conf:/usr/local/etc/redis/redis.conf:ro - /opt/so/conf/redis/working:/redis:rw + - /nsm/redis/data:/data:rw - /etc/pki/redis.crt:/certs/redis.crt:ro - /etc/pki/redis.key:/certs/redis.key:ro {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import'] %} From bbef96ac25fafe91f29b432133a4ba0773b7a367 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 18 Sep 2023 12:12:57 -0400 Subject: [PATCH 268/350] use unique name --- salt/influxdb/config.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/config.sls b/salt/influxdb/config.sls index 3520e46b3..66c681a0d 100644 --- a/salt/influxdb/config.sls +++ b/salt/influxdb/config.sls @@ -25,7 +25,7 @@ influxlogdir: - group: 939 - makedirs: True -influxlogdir: +influxetcdir: file.directory: - name: /opt/so/conf/influxdb/etc - dir_mode: 750 From 66bb1272aef598de001d9f134847ceecdd36a4fe Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 18 Sep 2023 13:39:56 -0400 Subject: [PATCH 269/350] avoid volume sprawl --- salt/strelka/config.sls | 14 ++++++++++++++ salt/strelka/coordinator/enabled.sls | 1 + salt/strelka/gatekeeper/enabled.sls | 1 + 3 files changed, 16 insertions(+) diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index bf3ac3dca..1d0f75adf 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -43,6 +43,20 @@ strelka_sbin: - group: 939 - file_mode: 755 +strelkagkredisdatadir: + file.directory: + - name: /nsm/strelka/gk-redis-data + - user: 939 + - group: 939 + - makedirs: True + +strelkacoordredisdatadir: + file.directory: + - name: /nsm/strelka/coord-redis-data + - user: 939 + - group: 939 + - makedirs: True + {% else %} {{sls}}_state_not_allowed: diff --git a/salt/strelka/coordinator/enabled.sls b/salt/strelka/coordinator/enabled.sls index 7a156bc9a..1222378f7 100644 --- a/salt/strelka/coordinator/enabled.sls +++ b/salt/strelka/coordinator/enabled.sls @@ -39,6 +39,7 @@ strelka_coordinator: {% endif %} {% if DOCKER.containers['so-strelka-coordinator'].custom_bind_mounts %} - binds: + - /nsm/strelka/coord-redis-data:/data:rw {% for BIND in DOCKER.containers['so-strelka-coordinator'].custom_bind_mounts %} - {{ BIND }} {% endfor %} diff --git a/salt/strelka/gatekeeper/enabled.sls b/salt/strelka/gatekeeper/enabled.sls index b309403f4..185910f83 100644 --- a/salt/strelka/gatekeeper/enabled.sls +++ b/salt/strelka/gatekeeper/enabled.sls @@ -33,6 +33,7 @@ strelka_gatekeeper: {% endfor %} {% if DOCKER.containers['so-strelka-gatekeeper'].custom_bind_mounts %} - binds: + - /nsm/strelka/gk-redis-data:/data:rw {% for BIND in DOCKER.containers['so-strelka-gatekeeper'].custom_bind_mounts %} - {{ BIND }} {% endfor %} From bb3632d1b262a8cdaaead837be15edd0f33019a9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 18 Sep 2023 14:38:15 -0400 Subject: [PATCH 270/350] fix bind if statement --- salt/strelka/coordinator/enabled.sls | 4 ++-- salt/strelka/gatekeeper/enabled.sls | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/strelka/coordinator/enabled.sls b/salt/strelka/coordinator/enabled.sls index 1222378f7..3440cd5a4 100644 --- a/salt/strelka/coordinator/enabled.sls +++ b/salt/strelka/coordinator/enabled.sls @@ -37,13 +37,13 @@ strelka_coordinator: - {{ XTRAENV }} {% endfor %} {% endif %} - {% if DOCKER.containers['so-strelka-coordinator'].custom_bind_mounts %} - binds: - /nsm/strelka/coord-redis-data:/data:rw + {% if DOCKER.containers['so-strelka-coordinator'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-strelka-coordinator'].custom_bind_mounts %} - {{ BIND }} {% endfor %} - {% endif %} + {% endif %} delete_so-strelka-coordinator_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/strelka/gatekeeper/enabled.sls b/salt/strelka/gatekeeper/enabled.sls index 185910f83..8d06ddf6a 100644 --- a/salt/strelka/gatekeeper/enabled.sls +++ b/salt/strelka/gatekeeper/enabled.sls @@ -31,13 +31,13 @@ strelka_gatekeeper: {% for BINDING in DOCKER.containers['so-strelka-gatekeeper'].port_bindings %} - {{ BINDING }} {% endfor %} - {% if DOCKER.containers['so-strelka-gatekeeper'].custom_bind_mounts %} - binds: - /nsm/strelka/gk-redis-data:/data:rw - {% for BIND in DOCKER.containers['so-strelka-gatekeeper'].custom_bind_mounts %} + {% if DOCKER.containers['so-strelka-gatekeeper'].custom_bind_mounts %} + {% for BIND in DOCKER.containers['so-strelka-gatekeeper'].custom_bind_mounts %} - {{ BIND }} - {% endfor %} - {% endif %} + {% endfor %} + {% endif %} {% if DOCKER.containers['so-strelka-gatekeeper'].extra_env %} - environment: {% for XTRAENV in DOCKER.containers['so-strelka-gatekeeper'].extra_env %} From a914a022732a3691ab52dff7bf10f37d4121cffc Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 18 Sep 2023 14:43:02 -0400 Subject: [PATCH 271/350] prune unused volumes during upgrade --- salt/manager/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 45e3df530..1251f9a57 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -434,7 +434,8 @@ post_to_2.4.10() { } post_to_2.4.20() { - echo "Nothing to apply" + echo "Pruning unused volumes" + docker volume prune -f POSTVERSION=2.4.20 } From 151e8bfc4e4c19e8daaffbd902cfd0169ced9721 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 18 Sep 2023 15:21:45 -0400 Subject: [PATCH 272/350] fix idstool extra_env for container --- salt/idstools/enabled.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/idstools/enabled.sls b/salt/idstools/enabled.sls index 31afc5113..decc5a5b2 100644 --- a/salt/idstools/enabled.sls +++ b/salt/idstools/enabled.sls @@ -26,8 +26,8 @@ so-idstools: - http_proxy={{ proxy }} - https_proxy={{ proxy }} - no_proxy={{ salt['pillar.get']('manager:no_proxy') }} - {% if DOCKER.containers['so-elastalert'].extra_env %} - {% for XTRAENV in DOCKER.containers['so-elastalert'].extra_env %} + {% if DOCKER.containers['so-idstools'].extra_env %} + {% for XTRAENV in DOCKER.containers['so-idstools'].extra_env %} - {{ XTRAENV }} {% endfor %} {% endif %} From 5bac1e4d15f65ba01e3337acf7be0921e6a6fa99 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 18 Sep 2023 21:31:15 +0000 Subject: [PATCH 273/350] Show correct dates and Kibana URL for already processed EVTX files --- salt/common/tools/sbin_jinja/so-import-evtx | 51 +++++++++++---------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index 59a13612c..f48f935bc 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -80,8 +80,8 @@ function evtx2es() { -e "SHIFTTS=$SHIFTDATE" \ -v "$EVTX:/tmp/data.evtx" \ -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \ - -v "/nsm/import/evtx-end_newest:/tmp/newest" \ - -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \ + -v "/nsm/import/$HASH/evtx-end_newest:/tmp/newest" \ + -v "/nsm/import/$HASH/evtx-start_oldest:/tmp/oldest" \ --entrypoint "/evtx_calc_timestamps.sh" \ {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} >> $LOG_FILE 2>&1 } @@ -111,12 +111,6 @@ INVALID_EVTXS_COUNT=0 VALID_EVTXS_COUNT=0 SKIPPED_EVTXS_COUNT=0 -touch /nsm/import/evtx-start_oldest -touch /nsm/import/evtx-end_newest - -echo $START_OLDEST > /nsm/import/evtx-start_oldest -echo $END_NEWEST > /nsm/import/evtx-end_newest - # paths must be quoted in case they include spaces for EVTX in $INPUT_FILES; do EVTX=$(/usr/bin/realpath "$EVTX") @@ -141,8 +135,15 @@ for EVTX in $INPUT_FILES; do status "- this EVTX has already been imported; skipping" SKIPPED_EVTXS_COUNT=$((SKIPPED_EVTXS_COUNT + 1)) else + # create EVTX directory EVTX_DIR=$HASH_DIR/evtx mkdir -p $EVTX_DIR + # create import timestamp files + for i in evtx-start_oldest evtx-end_newest; do + if ! [ -f "$i" ]; then + touch /nsm/import/$HASH/$i + fi + done # import evtx and write them to import ingest pipeline status "- importing logs to Elasticsearch..." @@ -154,28 +155,28 @@ for EVTX in $INPUT_FILES; do VALID_EVTXS_COUNT=$((VALID_EVTXS_COUNT + 1)) fi - # compare $START to $START_OLDEST - START=$(cat /nsm/import/evtx-start_oldest) - START_COMPARE=$(date -d $START +%s) - START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s) - if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then - START_OLDEST=$START - fi - - # compare $ENDNEXT to $END_NEWEST - END=$(cat /nsm/import/evtx-end_newest) - ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"` - ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s) - END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s) - if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then - END_NEWEST=$ENDNEXT - fi - cp -f "${EVTX}" "${EVTX_DIR}"/data.evtx chmod 644 "${EVTX_DIR}"/data.evtx fi # end of valid evtx + # compare $START to $START_OLDEST + START=$(cat /nsm/import/$HASH/evtx-start_oldest) + START_COMPARE=$(date -d $START +%s) + START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s) + if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then + START_OLDEST=$START + fi + + # compare $ENDNEXT to $END_NEWEST + END=$(cat /nsm/import/$HASH/evtx-end_newest) + ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"` + ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s) + END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s) + if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then + END_NEWEST=$ENDNEXT + fi + status done # end of for-loop processing evtx files From 47e611682a5849c08ad93e2170567c658fd64b7c Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 19 Sep 2023 09:24:12 -0400 Subject: [PATCH 274/350] ignore debian apt update output --- setup/so-verify | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-verify b/setup/so-verify index e907e8bdc..e9a8a375c 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -53,6 +53,7 @@ log_has_errors() { grep -vE "Failed to restart snapd" | \ grep -vE "Login Failed Details" | \ grep -vE "response from daemon: unauthorized" | \ + grep -vE "Reading first line of patchfile" | \ grep -vE "Running scope as unit" &> "$error_log" if [[ $? -eq 0 ]]; then From a1e963f834918a909245c74551f98bb37933b7a8 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 19 Sep 2023 13:28:20 +0000 Subject: [PATCH 275/350] Reverse timestamps where necessary --- salt/common/tools/sbin_jinja/so-import-evtx | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-import-evtx b/salt/common/tools/sbin_jinja/so-import-evtx index f48f935bc..d12f34593 100755 --- a/salt/common/tools/sbin_jinja/so-import-evtx +++ b/salt/common/tools/sbin_jinja/so-import-evtx @@ -160,8 +160,18 @@ for EVTX in $INPUT_FILES; do fi # end of valid evtx - # compare $START to $START_OLDEST + # determine start and end and make sure they aren't reversed START=$(cat /nsm/import/$HASH/evtx-start_oldest) + END=$(cat /nsm/import/$HASH/evtx-end_newest) + START_EPOCH=`date -d "$START" +"%s"` + END_EPOCH=`date -d "$END" +"%s"` + if [ "$START_EPOCH" -gt "$END_EPOCH" ]; then + TEMP=$START + START=$END + END=$TEMP + fi + + # compare $START to $START_OLDEST START_COMPARE=$(date -d $START +%s) START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s) if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then @@ -169,7 +179,6 @@ for EVTX in $INPUT_FILES; do fi # compare $ENDNEXT to $END_NEWEST - END=$(cat /nsm/import/$HASH/evtx-end_newest) ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"` ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s) END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s) From 508260bd468bbeafaa86f0b05b879df75a32ec70 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 19 Sep 2023 13:32:03 +0000 Subject: [PATCH 276/350] Use event.created for timestamp --- salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 index 688000fb7..52b6bae7a 100644 --- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 +++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1 @@ -80,6 +80,7 @@ { "set": { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } }, { "set": { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } }, + { "date": { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } }, { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true } }, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } ], From 2e0ea3f37412b766773a7725ddd19e4d15df0590 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 19 Sep 2023 13:33:12 +0000 Subject: [PATCH 277/350] Set final pipeline --- salt/elasticsearch/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index cc2f5e1cd..91e5191f6 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3689,6 +3689,7 @@ elasticsearch: refresh_interval: 30s number_of_shards: 1 number_of_replicas: 0 + final_pipeline: ".fleet_final_pipeline-1" composed_of: - agent-mappings - dtc-agent-mappings From 3fa3f83007e216f29579778595a66654d24518d4 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Wed, 20 Sep 2023 08:22:52 -0400 Subject: [PATCH 278/350] Update soc_sensoroni.yaml --- salt/sensoroni/soc_sensoroni.yaml | 54 +++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/salt/sensoroni/soc_sensoroni.yaml b/salt/sensoroni/soc_sensoroni.yaml index eb63dbe25..db51da358 100644 --- a/salt/sensoroni/soc_sensoroni.yaml +++ b/salt/sensoroni/soc_sensoroni.yaml @@ -2,53 +2,53 @@ sensoroni: enabled: description: Enable or disable Sensoroni. advanced: True - helpLink: sensoroni.html + helpLink: grid.html config: analyze: enabled: description: Enable or disable the analyzer. advanced: True - helpLink: sensoroni.html + helpLink: cases.html timeout_ms: description: Timeout period for the analyzer. advanced: True - helpLink: sensoroni.html + helpLink: cases.html parallel_limit: description: Parallel limit for the analyzer. advanced: True - helpLink: sensoroni.html + helpLink: cases.html node_checkin_interval_ms: description: Interval in ms to checkin to the soc_host. advanced: True - helpLink: sensoroni.html + helpLink: grid.html node_description: description: Description of the specific node. - helpLink: sensoroni.html + helpLink: grid.html node: True forcedType: string sensoronikey: description: Shared key for sensoroni authentication. - helpLink: sensoroni.html + helpLink: grid.html global: True sensitive: True advanced: True soc_host: description: Host for sensoroni agents to connect to. - helpLink: sensoroni.html + helpLink: grid.html global: True advanced: True analyzers: emailrep: api_key: description: API key for the EmailRep analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the EmailRep analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -56,21 +56,21 @@ sensoroni: greynoise: api_key: description: API key for the GreyNoise analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string api_version: description: API version for the GreyNoise analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: string base_url: description: Base URL for the GreyNoise analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -78,7 +78,7 @@ sensoroni: localfile: file_path: description: File path for the LocalFile analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -86,14 +86,14 @@ sensoroni: otx: api_key: description: API key for the OTX analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the OTX analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -101,14 +101,14 @@ sensoroni: pulsedive: api_key: description: API key for the Pulsedive analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the Pulsedive analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -116,14 +116,14 @@ sensoroni: spamhaus: lookup_host: description: Host to use for lookups. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: string nameservers: description: Nameservers used for queries. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -131,35 +131,35 @@ sensoroni: urlscan: api_key: description: API key for the Urlscan analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the Urlscan analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: string enabled: description: Analyzer enabled - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: bool timeout: description: Timeout for the Urlscan analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True forcedType: int visibility: description: Type of visibility. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True @@ -167,14 +167,14 @@ sensoroni: virustotal: api_key: description: API key for the VirusTotal analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: True advanced: True forcedType: string base_url: description: Base URL for the VirusTotal analyzer. - helpLink: sensoroni.html + helpLink: cases.html global: False sensitive: False advanced: True From fa3a79a7875e03dc115cd8d93fb98c93764d532f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 21 Sep 2023 09:41:44 -0400 Subject: [PATCH 279/350] Update soup to prune in background --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 1251f9a57..c3f9f29d4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -434,8 +434,8 @@ post_to_2.4.10() { } post_to_2.4.20() { - echo "Pruning unused volumes" - docker volume prune -f + echo "Pruning unused docker volumes on all nodes - This process will run in the background." + salt --async \* cmd.run "docker volume prune -f" POSTVERSION=2.4.20 } From eeeae08ec885ae8be7d56b96f1d52ca3530688e5 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 21 Sep 2023 18:39:06 +0000 Subject: [PATCH 280/350] /app/ to /app/dashboards/ --- salt/nginx/etc/nginx.conf | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index b2616e946..795663384 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -230,18 +230,19 @@ http { proxy_cookie_path /api/ /influxdb/api/; } - location /app/ { - auth_request /auth/sessions/whoami; - rewrite /app/(.*) /app/$1 break; - proxy_pass http://{{ GLOBALS.manager }}:5601/app/; - proxy_read_timeout 300; - proxy_connect_timeout 300; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Proxy ""; - proxy_set_header X-Forwarded-Proto $scheme; - + location /app/dashboards/ { + auth_request /auth/sessions/whoami; + rewrite /app/dashboards/(.*) /app/dashboards/$1 break; + proxy_pass http://{{ GLOBALS.manager }}:5601/app/; + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + proxy_set_header X-Forwarded-Proto $scheme; + } + location /kibana/ { auth_request /auth/sessions/whoami; rewrite /kibana/(.*) /$1 break; From c95af6b9922d7b8cbf13d9b2b951243594ca7c2d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 25 Sep 2023 14:39:33 -0400 Subject: [PATCH 281/350] Add a note about testing analyzers outside of the Sensoroni Docker container --- salt/sensoroni/files/analyzers/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index 8b1f44f29..a75799558 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -141,7 +141,6 @@ Additionally, to support airgapped users, the dependency packages themselves, an pip download -r /requirements.txt -d /source-packages ``` - ### Analyzer Architecture The Sensoroni Docker container is responsible for executing analyzers. Only the manager's Sensoroni container will process analyzer jobs. Other nodes in the grid, such as sensors and search nodes, will not be assigned analyzer jobs. @@ -154,6 +153,12 @@ The analyzer itself will only run when a user in SOC enqueues an analyzer job, s python -m urlhaus '{"artifactType":"url","value":"https://bigbadbotnet.invalid",...}' ``` +To manually test an analyzer outside of the Sensoroni Docker container, use a command similar to the following: + +```bash +PYTHONPATH=. python urlhaus/urlhaus.py '{"artifactType":"url","value":"https://bigbadbotnet.invalid",...}' +``` + It is up to each analyzer to determine whether the provided input is compatible with that analyzer. This is assisted by the analyzer metadata, as described earlier in this document, with the use of the `supportedTypes` list. Once the analyzer completes its functionality, it must terminate promptly. See the following sections for more details on expected internal behavior of the analyzer. From 7cb9b5f2577b92cbcd8d908050eaffb40812807d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 25 Sep 2023 14:41:20 -0400 Subject: [PATCH 282/350] Add the blank line that was removed from the previous commit --- salt/sensoroni/files/analyzers/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sensoroni/files/analyzers/README.md b/salt/sensoroni/files/analyzers/README.md index a75799558..19335a545 100644 --- a/salt/sensoroni/files/analyzers/README.md +++ b/salt/sensoroni/files/analyzers/README.md @@ -141,6 +141,7 @@ Additionally, to support airgapped users, the dependency packages themselves, an pip download -r /requirements.txt -d /source-packages ``` + ### Analyzer Architecture The Sensoroni Docker container is responsible for executing analyzers. Only the manager's Sensoroni container will process analyzer jobs. Other nodes in the grid, such as sensors and search nodes, will not be assigned analyzer jobs. From e25d1c0ff34a37a895c2725dc1e247141b5d6e59 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 26 Sep 2023 10:01:21 -0400 Subject: [PATCH 283/350] so-salt-minion-check is jinja template --- salt/common/tools/{sbin => sbin_jinja}/so-salt-minion-check | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/common/tools/{sbin => sbin_jinja}/so-salt-minion-check (100%) diff --git a/salt/common/tools/sbin/so-salt-minion-check b/salt/common/tools/sbin_jinja/so-salt-minion-check similarity index 100% rename from salt/common/tools/sbin/so-salt-minion-check rename to salt/common/tools/sbin_jinja/so-salt-minion-check From 0bba68769bf6602dfeb8a1484adfa246c17d1fd0 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 26 Sep 2023 14:05:12 +0000 Subject: [PATCH 284/350] Make scan.pe.image_version type of 'float' --- .../templates/component/so/so-scan-mappings.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/so/so-scan-mappings.json b/salt/elasticsearch/templates/component/so/so-scan-mappings.json index 8ddbe6077..008a6ab10 100644 --- a/salt/elasticsearch/templates/component/so/so-scan-mappings.json +++ b/salt/elasticsearch/templates/component/so/so-scan-mappings.json @@ -20,7 +20,10 @@ "type": "float" } } - } + }, + "image_version": { + "type": "float" + } } }, "elf": { From 2abf434ebefb502b5bf2abfcf8c3cec3b173cde4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 26 Sep 2023 10:56:20 -0400 Subject: [PATCH 285/350] create snapshots of default, local salt and pillars during soup. rsync soup with --delete --- salt/common/tools/sbin/so-common | 6 ++---- salt/manager/tools/sbin/soup | 12 ++++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 03b19d756..0dfb19bbe 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -154,13 +154,11 @@ check_salt_minion_status() { return $status } - - copy_new_files() { # Copy new files over to the salt dir cd $UPDATE_DIR - rsync -a salt $DEFAULT_SALT_DIR/ - rsync -a pillar $DEFAULT_SALT_DIR/ + rsync -a salt $DEFAULT_SALT_DIR/ --delete + rsync -a pillar $DEFAULT_SALT_DIR/ --delete chown -R socore:socore $DEFAULT_SALT_DIR/ chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh cd /tmp diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 1251f9a57..8ec9f9bad 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -171,6 +171,13 @@ airgap_update_dockers() { fi } +backup_old_states_pillars() { + + tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_default_states_pillars.tar.gz /opt/so/saltstack/default/ + tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_local_states_pillars.tar.gz /opt/so/saltstack/local/ + +} + update_registry() { docker stop so-dockerregistry docker rm so-dockerregistry @@ -789,6 +796,7 @@ main() { if [ "$is_hotfix" == "true" ]; then echo "Applying $HOTFIXVERSION hotfix" + backup_old_states_pillars copy_new_files apply_hotfix echo "Hotfix applied" @@ -845,6 +853,10 @@ main() { update_centos_repo fi + echo "" + echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/" + backup_old_states_pillars + echo "" echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR." copy_new_files From 48801da44e9df1c589a165fa42f7778bfed26b93 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 26 Sep 2023 18:12:20 -0400 Subject: [PATCH 286/350] log check tool initial --- salt/common/tools/sbin/so-log-check | 174 ++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100755 salt/common/tools/sbin/so-log-check diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check new file mode 100755 index 000000000..6a3ca9876 --- /dev/null +++ b/salt/common/tools/sbin/so-log-check @@ -0,0 +1,174 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +RECENT_LOG_LINES=200 +EXCLUDE_STARTUP_ERRORS=N +EXCLUDE_FALSE_POSITIVE_ERRORS=N +EXCLUDE_KNOWN_ERRORS=N + +while [[ $# -gt 0 ]]; do + case $1 in + --exclude-connection-errors) + EXCLUDE_STARTUP_ERRORS=Y + ;; + --exclude-false-positives) + EXCLUDE_FALSE_POSITIVE_ERRORS=Y + ;; + --exclude-known-errors) + EXCLUDE_KNOWN_ERRORS=Y + ;; + --unknown) + EXCLUDE_STARTUP_ERRORS=Y + EXCLUDE_FALSE_POSITIVE_ERRORS=Y + EXCLUDE_KNOWN_ERRORS=Y + ;; + --recent-log-lines) + shift + RECENT_LOG_LINES=$1 + ;; + *) + echo "Usage: $0 [options]" + echo "" + echo "where options are:" + echo " --recent-log-lines N looks at the most recent N log lines per file or container; defaults to 200" + echo " --exclude-connection-errors exclude errors caused by a recent server or container restart" + echo " --exclude-false-positives exclude logs that are not actual errors but contain the error string" + echo " --exclude-known-errors exclude errors that are known and non-critical issues" + echo " --unknown exclude everthing mentioned above; only show unknown errors" + echo "" + echo "A non-zero return value indicates errors were found" + exit 1 + ;; + esac + shift +done + +echo "Security Onion Log Check - $(date)" +echo "-------------------------------------------" +echo "" +echo "- RECENT_LOG_LINES: $RECENT_LOG_LINES" +echo "- EXCLUDE_STARTUP_ERRORS: $EXCLUDE_STARTUP_ERRORS" +echo "- EXCLUDE_FALSE_POSITIVE_ERRORS: $EXCLUDE_FALSE_POSITIVE_ERRORS" +echo "- EXCLUDE_KNOWN_ERRORS: $EXCLUDE_KNOWN_ERRORS" +echo "" + +function status() { + header "$1" +} + +function exclude_container() { + name=$1 + + exclude_id=$(docker ps | grep "$name" | awk '{print $1}') + if [[ -n "$exclude_id" ]]; then + CONTAINER_IDS=$(echo $CONTAINER_IDS | sed -e "s/$exclude_id//g") + return $? + fi + return $? +} + +function exclude_log() { + name=$1 + + LOG_FILES=$(echo "$LOG_FILES" | sed -e "s/$name//g") +} + +function check_for_errors() { + if cat /tmp/log_check | grep -i error | grep -vEi "$EXCLUDED_ERRORS"; then + RESULT=1 + fi +} + +EXCLUDED_ERRORS="__LOG_CHECK_PLACEHOLDER_EXCLUSION__" + +if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|database is locked" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|econnreset" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unreachable" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request.py" # server not yet ready (python stack output) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|httperror" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|servfail" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connection refused" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards" # server not yet ready +fi + +if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_status_error" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_error.json" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules" # false positive (error in rulename) +fi + +if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|bookkeeper" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noindices" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to start transient scope" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # request successful, contained error string in content + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so-user.lock exists" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|systemd-run" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|retcode: 1" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|telemetry-task" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|redisqueue" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fleet_detail_query" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|num errors=0" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/alerting" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/notifiers" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisoning/plugins" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|active-responses.log" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|scanentropy" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integration policy" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|blob unknown" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|token required" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|zeekcaptureloss" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unable to create detection" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error installing new prebuilt rules" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parent.error" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip" # known issue + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail" # zeek + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example" # example test data + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term" # setup in progress, influxdb not yet setup + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded" +fi + +RESULT=0 + +# Check Security Onion container stdout/stderr logs +CONTAINER_IDS=$(docker ps -q) +exclude_container so-kibana +exclude_container so-idstools + +for container_id in $CONTAINER_IDS; do + status "Checking container $container_id" + docker logs -n $RECENT_LOG_LINES $container_id > /tmp/log_check 2>&1 + check_for_errors +done + +# Check Security Onion related log files +LOG_FILES=$(find /opt/so/log/ /nsm -name \*.log) +exclude_log "\s?.*kibana.log" +LOG_FILES="$LOG_FILES /var/log/cron" + +for log_file in $LOG_FILES; do + status "Checking log file $log_file" + tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check + check_for_errors +done + +exit $RESULT \ No newline at end of file From 2c8d413f168fe2dedcf9e7eb91dfc806377ee3b5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 26 Sep 2023 18:14:37 -0400 Subject: [PATCH 287/350] log check tool initial --- salt/common/tools/sbin/so-log-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 6a3ca9876..752a6d51e 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -40,7 +40,7 @@ while [[ $# -gt 0 ]]; do echo " --exclude-connection-errors exclude errors caused by a recent server or container restart" echo " --exclude-false-positives exclude logs that are not actual errors but contain the error string" echo " --exclude-known-errors exclude errors that are known and non-critical issues" - echo " --unknown exclude everthing mentioned above; only show unknown errors" + echo " --unknown exclude everything mentioned above; only show unknown errors" echo "" echo "A non-zero return value indicates errors were found" exit 1 From 9c854a13ccf44b56163ed90d9ae8e26d163a0ff2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 26 Sep 2023 21:41:44 -0400 Subject: [PATCH 288/350] skip zeek spool logs due to test data false positives --- salt/common/tools/sbin/so-log-check | 32 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 752a6d51e..6169e9720 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -38,7 +38,7 @@ while [[ $# -gt 0 ]]; do echo "where options are:" echo " --recent-log-lines N looks at the most recent N log lines per file or container; defaults to 200" echo " --exclude-connection-errors exclude errors caused by a recent server or container restart" - echo " --exclude-false-positives exclude logs that are not actual errors but contain the error string" + echo " --exclude-false-positives exclude logs that are known false positives" echo " --exclude-known-errors exclude errors that are known and non-critical issues" echo " --unknown exclude everything mentioned above; only show unknown errors" echo "" @@ -76,7 +76,8 @@ function exclude_container() { function exclude_log() { name=$1 - LOG_FILES=$(echo "$LOG_FILES" | sed -e "s/$name//g") + cat /tmp/log_check_files | grep -v $name > /tmp/log_check_files.new + mv /tmp/log_check_files.new /tmp/log_check_files } function check_for_errors() { @@ -97,8 +98,10 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request.py" # server not yet ready (python stack output) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|httperror" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|servfail" # server not yet ready - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connection refused" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connect" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes" # server not yet ready (telegraf) fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then @@ -107,11 +110,15 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated" # false positive (playbook) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|could cause errors" # false positive (playbook) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h" # false positive (zeek test data) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules" # false positive (error in rulename) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden" # playbook EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" @@ -161,14 +168,25 @@ for container_id in $CONTAINER_IDS; do done # Check Security Onion related log files -LOG_FILES=$(find /opt/so/log/ /nsm -name \*.log) -exclude_log "\s?.*kibana.log" -LOG_FILES="$LOG_FILES /var/log/cron" +find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files +echo "/var/log/cron" >> /tmp/log_check_files +exclude_log "kibana.log" +exclude_log "spool" -for log_file in $LOG_FILES; do +for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check check_for_errors done +# Cleanup temp files +rm -f /tmp/log_check_files +rm -f /tmp/log_check + +if [[ $RESULT -eq 0 ]]; then + echo -e "\nResult: No errors found" +else + echo -e "\nResult: One or more errors found" +fi + exit $RESULT \ No newline at end of file From b47d915cb6318bfa8af3a29763fe38015764ec2f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 27 Sep 2023 09:30:19 -0400 Subject: [PATCH 289/350] don't inspect imported zeek output --- salt/common/tools/sbin/so-log-check | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 6169e9720..621f0027a 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -101,7 +101,12 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connect" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics" # server not yet ready - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes" # server not yet ready (telegraf) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes" # server not yet ready (telegraf waiting on influx) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|expected field at" # server not yet ready (telegraf waiting on health data) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cached the public key" # server not yet ready (salt minion waiting on key acceptance) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no ingest nodes" # server not yet ready (logstash waiting on elastic) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to poll" # server not yet ready (sensoroni waiting on soc) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|minions returned with non" # server not yet ready (salt waiting on minions) fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then @@ -110,14 +115,18 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error-template" # false positive (elastic templates) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated" # false positive (playbook) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|could cause errors" # false positive (playbook) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h" # false positive (zeek test data) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules" # false positive (error in rulename) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input" # false positive (Invalid user input in hunt query) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror" # idstools connection timeout + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror" # idstools connection timeout EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden" # playbook EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" @@ -146,7 +155,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unable to create detection" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error installing new prebuilt rules" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parent.error" - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip" # known issue + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip" # known issue in GH EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail" # zeek EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example" # example test data EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term" # setup in progress, influxdb not yet setup @@ -172,6 +181,7 @@ find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files echo "/var/log/cron" >> /tmp/log_check_files exclude_log "kibana.log" exclude_log "spool" +exclude_log "import" for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" From 05e7c32cf9f6d9246e458d33d0b13aa43b337d06 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 27 Sep 2023 10:08:08 -0400 Subject: [PATCH 290/350] remove duplicate filecheck_run cron --- salt/strelka/filestream/config.sls | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/strelka/filestream/config.sls b/salt/strelka/filestream/config.sls index 833a08505..0f9f38914 100644 --- a/salt/strelka/filestream/config.sls +++ b/salt/strelka/filestream/config.sls @@ -108,6 +108,11 @@ filecheck_stdout.log: {% if GLOBALS.md_engine == 'ZEEK' %} +remove_filecheck_run: + cron.absent: + - identifier: filecheck_run + - user: socore + filecheck_run_socore: cron.present: - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' @@ -121,6 +126,11 @@ remove_filecheck_run_suricata: {% elif GLOBALS.md_engine == 'SURICATA'%} +remove_filecheck_run: + cron.absent: + - identifier: filecheck_run + - user: suricata + filecheck_run_suricata: cron.present: - name: 'ps -ef | grep filecheck | grep -v grep > /dev/null 2>&1 || python3 /opt/so/conf/strelka/filecheck >> /opt/so/log/strelka/filecheck_stdout.log 2>&1 &' From c4fea9cb9da5a5044ad1c7be9aeca417b4e6ab96 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 27 Sep 2023 11:03:58 -0400 Subject: [PATCH 291/350] Update nginx.conf --- salt/nginx/etc/nginx.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/nginx/etc/nginx.conf b/salt/nginx/etc/nginx.conf index 795663384..3ef0c5c1f 100644 --- a/salt/nginx/etc/nginx.conf +++ b/salt/nginx/etc/nginx.conf @@ -8,6 +8,7 @@ worker_processes auto; error_log /var/log/nginx/error.log; pid /run/nginx.pid; +user nobody; include /usr/share/nginx/modules/*.conf; From 87cc389088eb0f1886c73610f70e3701eb625d20 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 27 Sep 2023 15:36:13 -0400 Subject: [PATCH 292/350] deb OS doesn't use /var/log/cron, skip --- salt/common/tools/sbin/so-log-check | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 621f0027a..d377d0236 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -178,7 +178,9 @@ done # Check Security Onion related log files find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files -echo "/var/log/cron" >> /tmp/log_check_files +if [[ -f /var/log/cron ]]; then + echo "/var/log/cron" >> /tmp/log_check_files +fi exclude_log "kibana.log" exclude_log "spool" exclude_log "import" From f094b1162d7c23cd5bb2d98d7e34fb2f7b6afecf Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 27 Sep 2023 15:48:05 -0400 Subject: [PATCH 293/350] Update defaults.yaml --- salt/zeek/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index 8e6814b2e..783c38820 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -8,9 +8,9 @@ zeek: buffer: 128*1024*1024 zeekctl: MailTo: root@localhost - MailConnectionSummary: 1 + MailConnectionSummary: 0 MinDiskSpace: 5 - MailHostUpDown: 1 + MailHostUpDown: 0 LogRotationInterval: 3600 LogExpireInterval: 0 StatsLogEnable: 1 From 4666916077db773dfb10d94e0decb85620d0c453 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 27 Sep 2023 15:48:52 -0400 Subject: [PATCH 294/350] ignore generic python stack trace log lines of code, rely on actual error messages --- salt/common/tools/sbin/so-log-check | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index d377d0236..9deeba1cd 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -125,6 +125,8 @@ fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof" + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise error" # redis/python generic stack line, rely on other lines for actual error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail(error)" # redis/python generic stack line, rely on other lines for actual error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror" # idstools connection timeout EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror" # idstools connection timeout EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden" # playbook From 2427344dca2fc963b9b9d608f7f665141610d9aa Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 27 Sep 2023 15:58:58 -0400 Subject: [PATCH 295/350] Update defaults.yaml --- salt/zeek/defaults.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index 783c38820..4435670a2 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -28,7 +28,6 @@ zeek: - misc/loaded-scripts - tuning/defaults - misc/capture-loss - - misc/stats - frameworks/software/vulnerable - frameworks/software/version-changes - protocols/ftp/software From 2fb73cd51621a8dea6a6f8d31597273e05ff12fb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 27 Sep 2023 16:07:38 -0400 Subject: [PATCH 296/350] Update defaults.yaml --- salt/telegraf/defaults.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index a87fa952b..ab8679e57 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -11,7 +11,6 @@ telegraf: quiet: 'false' scripts: eval: - - beatseps.sh - checkfiles.sh - influxdbsize.sh - oldpcap.sh @@ -23,7 +22,6 @@ telegraf: - zeekcaptureloss.sh - zeekloss.sh standalone: - - beatseps.sh - checkfiles.sh - eps.sh - influxdbsize.sh @@ -36,13 +34,11 @@ telegraf: - zeekcaptureloss.sh - zeekloss.sh manager: - - beatseps.sh - influxdbsize.sh - raid.sh - redis.sh - sostatus.sh managersearch: - - beatseps.sh - eps.sh - influxdbsize.sh - raid.sh @@ -51,7 +47,6 @@ telegraf: import: - sostatus.sh sensor: - - beatseps.sh - checkfiles.sh - oldpcap.sh - raid.sh @@ -61,7 +56,6 @@ telegraf: - zeekcaptureloss.sh - zeekloss.sh heavynode: - - beatseps.sh - checkfiles.sh - eps.sh - oldpcap.sh @@ -75,12 +69,10 @@ telegraf: idh: - sostatus.sh searchnode: - - beatseps.sh - eps.sh - raid.sh - sostatus.sh receiver: - - beatseps.sh - eps.sh - raid.sh - redis.sh From 039d5ae9aa4a65e8e8bce9d5e45bfd51682c0e84 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 27 Sep 2023 16:09:27 -0400 Subject: [PATCH 297/350] Delete salt/telegraf/scripts/beatseps.sh --- salt/telegraf/scripts/beatseps.sh | 38 ------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 salt/telegraf/scripts/beatseps.sh diff --git a/salt/telegraf/scripts/beatseps.sh b/salt/telegraf/scripts/beatseps.sh deleted file mode 100644 index 5f3db53f8..000000000 --- a/salt/telegraf/scripts/beatseps.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - - - -# if this script isn't already running -if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then - - PREVCOUNTFILE='/tmp/beatseps.txt' - EVENTCOUNTCURRENT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.acked')" - FAILEDEVENTCOUNT="$(curl -s localhost:5066/stats | jq '.libbeat.output.events.failed')" - - if [ ! -z "$EVENTCOUNTCURRENT" ]; then - - if [ -f "$PREVCOUNTFILE" ]; then - EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE` - else - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - exit 0 - fi - - echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE - # the division by 30 is because the agent interval is 30 seconds - EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30)) - if [ "$EVENTS" -lt 0 ]; then - EVENTS=0 - fi - - echo "fbstats eps=${EVENTS%%.*},failed=$FAILEDEVENTCOUNT" - fi - -fi - -exit 0 From 24def3a196efc12506f11de8c32e5d52cc6bd24b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 27 Sep 2023 16:50:01 -0400 Subject: [PATCH 298/350] ignore generic python stack trace log lines of code, rely on actual error messages --- salt/common/tools/sbin/so-log-check | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 9deeba1cd..f89995065 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -107,6 +107,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no ingest nodes" # server not yet ready (logstash waiting on elastic) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to poll" # server not yet ready (sensoroni waiting on soc) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|minions returned with non" # server not yet ready (salt waiting on minions) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term" # server not yet ready (influxdb not yet setup) fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then @@ -121,23 +122,25 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h" # false positive (zeek test data) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules" # false positive (error in rulename) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input" # false positive (Invalid user input in hunt query) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example" # false positive (example test data) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # false positive (request successful, contained error string in content) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof" - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise error" # redis/python generic stack line, rely on other lines for actual error - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail(error)" # redis/python generic stack line, rely on other lines for actual error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise error" # redis/python generic stack line, rely on other lines for actual error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail\\(error\\)" # redis/python generic stack line, rely on other lines for actual error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror" # idstools connection timeout EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror" # idstools connection timeout EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden" # playbook EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context canceled" # elastic agent during shutdown EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|bookkeeper" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noindices" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to start transient scope" - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200" # request successful, contained error string in content EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so-user.lock exists" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|systemd-run" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|retcode: 1" @@ -159,8 +162,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parent.error" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip" # known issue in GH EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail" # zeek - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example" # example test data - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term" # setup in progress, influxdb not yet setup EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded" fi From 76c0b881ff3e6a4011e71c2f6ca9941f290349bb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 27 Sep 2023 18:20:50 -0400 Subject: [PATCH 299/350] exclude import from snapshotting previous version pillars and states --- salt/manager/tools/sbin/soup | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8c6c2b237..8259bf6ab 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -518,7 +518,7 @@ up_to_2.4.10() { } up_to_2.4.20() { - echo "Nothing to do for 2.4.20" + echo "Preupgrade soup changes for 2.4.20" INSTALLEDVERSION=2.4.20 } @@ -796,7 +796,10 @@ main() { if [ "$is_hotfix" == "true" ]; then echo "Applying $HOTFIXVERSION hotfix" - backup_old_states_pillars + # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars + if [[ ! "$MINIONID" =~ "_import" ]]; then + backup_old_states_pillars + fi copy_new_files apply_hotfix echo "Hotfix applied" @@ -853,9 +856,12 @@ main() { update_centos_repo fi - echo "" - echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/" - backup_old_states_pillars + # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars + if [[ ! "$MINIONID" =~ "_import" ]]; then + echo "" + echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/" + backup_old_states_pillars + fi echo "" echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR." From d72e4ae97d7514cf7fe8b3ade06279079f2504aa Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 27 Sep 2023 18:39:23 -0400 Subject: [PATCH 300/350] ignore soctopus errors --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index f89995065..f9393ce8a 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -135,6 +135,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden" # playbook EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context canceled" # elastic agent during shutdown + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exited with code 128" # soctopus errors during forced restart by highstate EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" From 419acab48aea966593207cb396372b6daecb5833 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 27 Sep 2023 19:17:13 -0400 Subject: [PATCH 301/350] revert up_to_2.4.20 --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8259bf6ab..960c50f31 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -518,7 +518,7 @@ up_to_2.4.10() { } up_to_2.4.20() { - echo "Preupgrade soup changes for 2.4.20" + echo "Nothing to do for 2.4.20" INSTALLEDVERSION=2.4.20 } From 49115cde55027eba0fe73cf95acd46f1714ec5a3 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 27 Sep 2023 19:55:46 -0400 Subject: [PATCH 302/350] logcheck improvements --- salt/common/tools/sbin/so-log-check | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index f9393ce8a..e75c9cd60 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -136,6 +136,9 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context canceled" # elastic agent during shutdown EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exited with code 128" # soctopus errors during forced restart by highstate + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|geoip database update" # airgap can't update GeoIP DB + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|filenotfounderror" # bug in 2.4.10 filecheck salt state caused duplicate cronjobs + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check" # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" From 9ee64f93ca25384a3e18fb9308981aa45f7dcdfc Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 27 Sep 2023 20:17:59 -0400 Subject: [PATCH 303/350] logcheck improvements --- salt/common/tools/sbin/so-log-check | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index e75c9cd60..b4b40a90b 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -119,6 +119,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error-template" # false positive (elastic templates) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated" # false positive (playbook) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|could cause errors" # false positive (playbook) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_error.yml" # false positive (playbook) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h" # false positive (zeek test data) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules" # false positive (error in rulename) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input" # false positive (Invalid user input in hunt query) @@ -139,6 +140,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|geoip database update" # airgap can't update GeoIP DB EXCLUDED_ERRORS="$EXCLUDED_ERRORS|filenotfounderror" # bug in 2.4.10 filecheck salt state caused duplicate cronjobs EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check" # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|generating elastalert config" # playbook expected error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" From 621da9e7e319df596b343847480be8ca4fce3d36 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 27 Sep 2023 22:20:54 -0400 Subject: [PATCH 304/350] more exclusions --- salt/common/tools/sbin/so-log-check | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index b4b40a90b..c6a966385 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -101,6 +101,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connect" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|broken pipe" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes" # server not yet ready (telegraf waiting on influx) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|expected field at" # server not yet ready (telegraf waiting on health data) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cached the public key" # server not yet ready (salt minion waiting on key acceptance) @@ -118,6 +119,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error-template" # false positive (elastic templates) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated" # false positive (playbook) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|windows" # false positive (playbook) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|could cause errors" # false positive (playbook) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_error.yml" # false positive (playbook) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h" # false positive (zeek test data) @@ -129,7 +131,7 @@ fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof" - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise error" # redis/python generic stack line, rely on other lines for actual error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise" # redis/python generic stack line, rely on other lines for actual error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail\\(error\\)" # redis/python generic stack line, rely on other lines for actual error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror" # idstools connection timeout EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror" # idstools connection timeout From 89a9c30cc89371979ed8ea50b12a2e00ad978158 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 28 Sep 2023 08:27:31 -0400 Subject: [PATCH 305/350] exclude known issues --- salt/common/tools/sbin/so-log-check | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index c6a966385..865846fac 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -102,6 +102,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|broken pipe" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout exceeded" # server not yet ready (telegraf waiting on elasticsearch) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes" # server not yet ready (telegraf waiting on influx) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|expected field at" # server not yet ready (telegraf waiting on health data) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cached the public key" # server not yet ready (salt minion waiting on key acceptance) @@ -117,6 +118,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fs_errors" # false positive (suricata stats) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error-template" # false positive (elastic templates) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated" # false positive (playbook) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|windows" # false positive (playbook) @@ -143,6 +145,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|filenotfounderror" # bug in 2.4.10 filecheck salt state caused duplicate cronjobs EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check" # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|generating elastalert config" # playbook expected error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|activerecord" # playbook expected error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" @@ -192,9 +195,10 @@ find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files if [[ -f /var/log/cron ]]; then echo "/var/log/cron" >> /tmp/log_check_files fi -exclude_log "kibana.log" -exclude_log "spool" -exclude_log "import" +exclude_log "kibana.log" # kibana error logs are too verbose with large varieties of errors most of which are temporary +exclude_log "spool" # disregard zeek analyze logs +exclude_log "import" # disregard imported test data the contains error strings +exclude_log "update.log" # ignore playbook updates due to known issues for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" From 202eb7e8765ff239734bcecd0c7327bb40dec33a Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 28 Sep 2023 09:16:56 -0400 Subject: [PATCH 306/350] Exclude known_certs --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 979e795f7..6737df17d 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -15,6 +15,7 @@ elasticfleet: - cluster - console - ecat_arp_info + - known_certs - known_hosts - known_services - loaded_scripts From ee45fc31a2894137a82a2e90a6e3fb2aff39c2ba Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Sep 2023 11:04:16 -0400 Subject: [PATCH 307/350] Delete salt/strelka/tools/sbin_jinja/so-yara-download --- .../strelka/tools/sbin_jinja/so-yara-download | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 salt/strelka/tools/sbin_jinja/so-yara-download diff --git a/salt/strelka/tools/sbin_jinja/so-yara-download b/salt/strelka/tools/sbin_jinja/so-yara-download deleted file mode 100644 index a8087173c..000000000 --- a/salt/strelka/tools/sbin_jinja/so-yara-download +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -NOROOT=1 -. /usr/sbin/so-common - -{%- set proxy = salt['pillar.get']('manager:proxy') %} - -# Download the rules from the internet -{%- if proxy %} -export http_proxy={{ proxy }} -export https_proxy={{ proxy }} -export no_proxy=salt['pillar.get']('manager:no_proxy') -{%- endif %} - -mkdir -p /tmp/yara -cd /tmp/yara -git clone https://github.com/Security-Onion-Solutions/securityonion-yara.git -mkdir -p /nsm/rules/yara -rsync -shav --progress /tmp/yara/securityonion-yara/yara /nsm/rules/ -cd /tmp -rm -rf /tmp/yara - From a77a53f20b3bbdd6d6b7965e6bb4e65a146ae154 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Sep 2023 11:10:17 -0400 Subject: [PATCH 308/350] Update init.sls --- salt/manager/init.sls | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index b9d2d3ba9..146bca126 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -26,6 +26,15 @@ repo_log_dir: - user - group +yara_log_dir: + file.directory: + - name: /opt/so/log/yarasync + - user: socore + - group: socore + - recurse: + - user + - group + repo_conf_dir: file.directory: - name: /opt/so/conf/reposync From 7a21b7903dfbdf57518ba2a667b5aa14f4c8f640 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Sep 2023 11:46:43 -0400 Subject: [PATCH 309/350] Fix manager cron logic --- salt/manager/init.sls | 92 +++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 52 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 146bca126..55badaf10 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -61,21 +61,23 @@ manager_sbin: - group: 939 - file_mode: 755 -#manager_sbin_jinja: -# file.recurse: -# - name: /usr/sbin -# - source: salt://manager/tools/sbin_jinja -# - user: 939 -# - group: 939 -# - file_mode: 755 -# - template: jinja +yara_update_scripts: + file.recurse: + - name: /usr/sbin/ + - source: salt://manager/tools/sbin_jinja/ + - user: socore + - group: socore + - file_mode: 755 + - template: jinja + - defaults: + EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }} so-repo-sync: - {% if MANAGERMERGED.reposync.enabled %} + {% if MANAGERMERGED.reposync.enabled or ! GLOBALS.airgap %} cron.present: - {% else %} + {% else %} cron.absent: - {% endif %} + {% endif %} - user: socore - name: '/usr/sbin/so-repo-sync >> /opt/so/log/reposync/reposync.log 2>&1' - identifier: so-repo-sync @@ -91,7 +93,15 @@ socore_own_saltstack: - user - group -{% if STRELKAMERGED.rules.enabled %} +rules_dir: + file.directory: + - name: /nsm/rules/yara + - user: socore + - group: socore + - makedirs: True + +{% if STRELKAMERGED.rules.enabled %} + strelkarepos: file.managed: - name: /opt/so/conf/strelka/repos.txt @@ -100,67 +110,45 @@ strelkarepos: - defaults: STRELKAREPOS: {{ STRELKAMERGED.rules.repos }} - makedirs: True -{% endif %} - -yara_update_scripts: - file.recurse: - - name: /usr/sbin/ - - source: salt://manager/tools/sbin_jinja/ - - user: socore - - group: socore - - file_mode: 755 - - template: jinja - - defaults: - EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }} - -rules_dir: - file.directory: - - name: /nsm/rules/yara - - user: socore - - group: socore - - makedirs: True - -{% if GLOBALS.airgap %} -remove_strelka-yara-download: - cron.absent: - - user: socore - - identifier: strelka-yara-download strelka-yara-update: + {% if MANAGERMERGED.reposync.enabled or ! GLOBALS.airgap %} cron.present: + {% else %} + cron.absent: + {% endif %} - user: socore - - name: '/usr/sbin/so-yara-update >> /nsm/strelka/log/yara-update.log 2>&1' + - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1' - identifier: strelka-yara-update - hour: '7' - minute: '1' -update_yara_rules: - cmd.run: - - name: /usr/sbin/so-yara-update - - onchanges: - - file: yara_update_scripts -{% else %} -remove_strelka-yara-update: - cron.absent: - - user: socore - - identifier: strelka-yara-update - strelka-yara-download: + {% if MANAGERMERGED.reposync.enabled or ! GLOBALS.airgap %} cron.present: + {% else %} + cron.absent: + {% endif %} - user: socore - name: '/usr/sbin/so-yara-download >> /nsm/strelka/log/yara-download.log 2>&1' - identifier: strelka-yara-download - hour: '7' - minute: '1' +{% if ! GLOBALS.airgap %} +update_yara_rules: + cmd.run: + - name: /usr/sbin/so-yara-update + - onchanges: + - file: yara_update_scripts + download_yara_rules: cmd.run: - name: /usr/sbin/so-yara-download - onchanges: - file: yara_update_scripts -{% endif %} - - +{% endif %} +{% endif %} {% else %} {{sls}}_state_not_allowed: From 5040df7551474d521fef76a2872913f046b8fdc5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Sep 2023 12:32:40 -0400 Subject: [PATCH 310/350] Fix manager cron logic --- salt/manager/init.sls | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 55badaf10..68d51c2af 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -73,7 +73,7 @@ yara_update_scripts: EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }} so-repo-sync: - {% if MANAGERMERGED.reposync.enabled or ! GLOBALS.airgap %} + {% if MANAGERMERGED.reposync.enabled or not GLOBALS.airgap %} cron.present: {% else %} cron.absent: @@ -112,7 +112,7 @@ strelkarepos: - makedirs: True strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled or ! GLOBALS.airgap %} + {% if MANAGERMERGED.reposync.enabled or not GLOBALS.airgap %} cron.present: {% else %} cron.absent: @@ -124,18 +124,18 @@ strelka-yara-update: - minute: '1' strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled or ! GLOBALS.airgap %} + {% if MANAGERMERGED.reposync.enabled or not GLOBALS.airgap %} cron.present: {% else %} cron.absent: {% endif %} - user: socore - - name: '/usr/sbin/so-yara-download >> /nsm/strelka/log/yara-download.log 2>&1' + - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1' - identifier: strelka-yara-download - hour: '7' - minute: '1' -{% if ! GLOBALS.airgap %} +{% if not GLOBALS.airgap %} update_yara_rules: cmd.run: - name: /usr/sbin/so-yara-update From 018186ccbd1d63ec1e0785e13a9579f27751264c Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 28 Sep 2023 16:43:56 +0000 Subject: [PATCH 311/350] Upgrade packages and load integrations when packages change --- salt/elasticfleet/config.sls | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 902b5eb4c..82b975697 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -59,6 +59,14 @@ eastatedir: - group: 939 - makedirs: True +eapackageupgrade: + file.managed: + - name: /usr/sbin/so-elastic-fleet-package-upgrade + - source: salt://elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade + - user: 947 + - group: 939 + - template: jinja + {% if GLOBALS.role != "so-fleet" %} eaintegrationsdir: file.directory: @@ -88,6 +96,7 @@ ea-integrations-load: - onchanges: - file: eaintegration - file: eadynamicintegration + - file: eapackageupgrade {% endif %} {% else %} From 95d32cb07689a8792e6b2be213c38314797c8eec Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Sep 2023 12:49:46 -0400 Subject: [PATCH 312/350] Fix manager cron logic --- salt/manager/init.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/init.sls b/salt/manager/init.sls index 68d51c2af..e808325ef 100644 --- a/salt/manager/init.sls +++ b/salt/manager/init.sls @@ -73,7 +73,7 @@ yara_update_scripts: EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }} so-repo-sync: - {% if MANAGERMERGED.reposync.enabled or not GLOBALS.airgap %} + {% if MANAGERMERGED.reposync.enabled %} cron.present: {% else %} cron.absent: @@ -112,7 +112,7 @@ strelkarepos: - makedirs: True strelka-yara-update: - {% if MANAGERMERGED.reposync.enabled or not GLOBALS.airgap %} + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} cron.present: {% else %} cron.absent: @@ -124,7 +124,7 @@ strelka-yara-update: - minute: '1' strelka-yara-download: - {% if MANAGERMERGED.reposync.enabled or not GLOBALS.airgap %} + {% if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} cron.present: {% else %} cron.absent: From ff359460508df28a1a5f022b63507cd076f94047 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Sep 2023 13:06:21 -0400 Subject: [PATCH 313/350] Fix manager cron logic --- setup/so-functions | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 3707e3141..679142e2a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1963,12 +1963,10 @@ securityonion_repo() { fi if [[ $is_rpm ]]; then logCmd "dnf repolist all"; fi if [[ $waitforstate ]]; then - if [[ ! $is_airgap ]]; then - if [[ $is_rpm ]]; then + if [[ $is_rpm ]]; then # Build the repo locally so we can use it echo "Syncing Repos" repo_sync_local - fi fi fi } @@ -1978,7 +1976,7 @@ repo_sync_local() { if [[ $is_supported ]]; then # Sync the repo from the the SO repo locally. # Check for reposync - info "Backing up old repos" + info "Adding Repo Download Configuration" mkdir -p /nsm/repo mkdir -p /opt/so/conf/reposync/cache echo "https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9" > /opt/so/conf/reposync/mirror.txt @@ -2002,10 +2000,10 @@ repo_sync_local() { if [[ ! $is_airgap ]]; then curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install logCmd "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" + # After the download is complete run createrepo + create_repo fi - # After the download is complete run createrepo - create_repo else # Add the proper repos for unsupported stuff echo "Adding Repos" From 8c44481ee15a2776d32cff1b1e0e6e68619300f5 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 28 Sep 2023 17:57:31 +0000 Subject: [PATCH 314/350] Load templates after package changes --- .../tools/sbin_jinja/so-elastic-fleet-package-upgrade | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade index 2fb3f7798..a092e3ecb 100644 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade @@ -15,3 +15,4 @@ elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION" echo {%- endfor %} echo +/usr/sbin/so-elasticsearch-templates-load From 670cd190518ba9337d63af1dbff0d8052c674241 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 28 Sep 2023 18:04:07 +0000 Subject: [PATCH 315/350] Exclude package upgrade script --- salt/elasticfleet/config.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/elasticfleet/config.sls b/salt/elasticfleet/config.sls index 82b975697..d2e357c91 100644 --- a/salt/elasticfleet/config.sls +++ b/salt/elasticfleet/config.sls @@ -37,6 +37,8 @@ elasticfleet_sbin_jinja: - group: 939 - file_mode: 755 - template: jinja + - exclude_pat: + - so-elastic-fleet-package-upgrade # exclude this because we need to watch it for changes eaconfdir: file.directory: From b8aad7f5e605b47aab7b704b75c7dc65c4d9f5b0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Sep 2023 19:44:49 -0400 Subject: [PATCH 316/350] Update defaults.yaml --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 6737df17d..a4862623d 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -13,6 +13,7 @@ elasticfleet: - broker - capture_loss - cluster + - conn-summary - console - ecat_arp_info - known_certs From ec3cc7a854137ab4f701c022aa895ecab864851f Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 29 Sep 2023 10:49:36 -0400 Subject: [PATCH 317/350] exclude all playbook logs --- salt/common/tools/sbin/so-log-check | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 865846fac..03b2e5c68 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -118,6 +118,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding component template" # false positive (elastic security) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index template" # false positive (elastic security) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fs_errors" # false positive (suricata stats) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error-template" # false positive (elastic templates) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated" # false positive (playbook) @@ -141,7 +143,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml" # Elastic ML errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context canceled" # elastic agent during shutdown EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exited with code 128" # soctopus errors during forced restart by highstate - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|geoip database update" # airgap can't update GeoIP DB + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|geoip databases update" # airgap can't update GeoIP DB EXCLUDED_ERRORS="$EXCLUDED_ERRORS|filenotfounderror" # bug in 2.4.10 filecheck salt state caused duplicate cronjobs EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check" # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|generating elastalert config" # playbook expected error @@ -181,11 +183,13 @@ RESULT=0 # Check Security Onion container stdout/stderr logs CONTAINER_IDS=$(docker ps -q) -exclude_container so-kibana -exclude_container so-idstools +exclude_container so-kibana # kibana error logs are too verbose with large varieties of errors most of which are temporary +exclude_container so-idstools # ignore due to known issues and noisy logging +exclude_container so-playbook # ignore due to several playbook known issues for container_id in $CONTAINER_IDS; do - status "Checking container $container_id" + container_name=$(docker ps --format json | jq ". | select(.ID==\"$container_id\")|.Names") + status "Checking container $container_name" docker logs -n $RECENT_LOG_LINES $container_id > /tmp/log_check 2>&1 check_for_errors done @@ -195,10 +199,11 @@ find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files if [[ -f /var/log/cron ]]; then echo "/var/log/cron" >> /tmp/log_check_files fi -exclude_log "kibana.log" # kibana error logs are too verbose with large varieties of errors most of which are temporary -exclude_log "spool" # disregard zeek analyze logs -exclude_log "import" # disregard imported test data the contains error strings -exclude_log "update.log" # ignore playbook updates due to known issues +exclude_log "kibana.log" # kibana error logs are too verbose with large varieties of errors most of which are temporary +exclude_log "spool" # disregard zeek analyze logs as this is data specific +exclude_log "import" # disregard imported test data the contains error strings +exclude_log "update.log" # ignore playbook updates due to several known issues +exclude_log "playbook.log" # ignore due to several playbook known issues for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" From 9d3f6059eed0a07395b0e687b6f7c85b9394c842 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 29 Sep 2023 11:10:08 -0400 Subject: [PATCH 318/350] remove redis from eval --- salt/telegraf/defaults.yaml | 1 - salt/top.sls | 1 - 2 files changed, 2 deletions(-) diff --git a/salt/telegraf/defaults.yaml b/salt/telegraf/defaults.yaml index ab8679e57..0b7d532b1 100644 --- a/salt/telegraf/defaults.yaml +++ b/salt/telegraf/defaults.yaml @@ -15,7 +15,6 @@ telegraf: - influxdbsize.sh - oldpcap.sh - raid.sh - - redis.sh - sostatus.sh - stenoloss.sh - suriloss.sh diff --git a/salt/top.sls b/salt/top.sls index 6db19b361..4f84e17ac 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -79,7 +79,6 @@ base: - utility - soctopus - playbook - - redis - elasticfleet '*_manager and G@saltversion:{{saltversion}}': From d546d520690abd4f9ea549e60de483907b0c1eda Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 29 Sep 2023 14:08:44 -0400 Subject: [PATCH 319/350] exclude logstash --- salt/common/tools/sbin/so-log-check | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 03b2e5c68..b19026cad 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -91,7 +91,10 @@ EXCLUDED_ERRORS="__LOG_CHECK_PLACEHOLDER_EXCLUSION__" if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|database is locked" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|econnreset" # server not yet ready - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unreachable" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unreachable" # server not yet ready (logstash waiting on elastic) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|shutdown process" # server not yet ready (logstash waiting on elastic) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|contain valid certificates" # server not yet ready (logstash waiting on elastic) + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failedaction" # server not yet ready (logstash waiting on elastic) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable" # server not yet ready @@ -148,6 +151,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check" # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors EXCLUDED_ERRORS="$EXCLUDED_ERRORS|generating elastalert config" # playbook expected error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|activerecord" # playbook expected error + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics" # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" From ad01be66ea34cc039fd214c1f8f749942e6545aa Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 29 Sep 2023 14:09:04 -0400 Subject: [PATCH 320/350] remove checkmine engine. add x509.get_pem_entries to managers mine_functions. simplify mine update during soup --- salt/manager/tools/sbin/soup | 2 +- salt/salt/etc/minion.d/mine_functions.conf.jinja | 4 ++++ salt/salt/master.sls | 9 +++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 960c50f31..333be836b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -580,7 +580,7 @@ update_centos_repo() { update_salt_mine() { echo "Populating the mine with network.ip_addrs pillar.host.mainint for each host." set +e - salt \* cmd.run cmd='MAININT=$(salt-call pillar.get host:mainint --out=newline_values_only) && salt-call mine.send name=network.ip_addrs interface="$MAININT"' + salt \* mine.update set -e } diff --git a/salt/salt/etc/minion.d/mine_functions.conf.jinja b/salt/salt/etc/minion.d/mine_functions.conf.jinja index 378d2c435..2ae345cdf 100644 --- a/salt/salt/etc/minion.d/mine_functions.conf.jinja +++ b/salt/salt/etc/minion.d/mine_functions.conf.jinja @@ -2,3 +2,7 @@ mine_interval: 35 mine_functions: network.ip_addrs: - interface: {{ GLOBALS.main_interface }} +{% if GLOBALS.is_manager -%} + x509.get_pem_entries: + - glob_path: '/etc/pki/ca.crt' +{% endif -%} diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 8b2b6c7d0..b10a4df0f 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -18,17 +18,14 @@ salt_master_service: - enable: True checkmine_engine: - file.managed: + file.absent: - name: /etc/salt/engines/checkmine.py - - source: salt://salt/engines/checkmine.py - - makedirs: True - watch_in: - service: salt_minion_service engines_config: - file.managed: + file.absent: - name: /etc/salt/minion.d/engines.conf - - source: salt://salt/files/engines.conf - watch_in: - service: salt_minion_service @@ -38,4 +35,4 @@ engines_config: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} \ No newline at end of file +{% endif %} From e8b67da08bdf5a8239a33d7e6e99450d2d4b49fb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 29 Sep 2023 14:20:20 -0400 Subject: [PATCH 321/350] exclude oom error from cmd line --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index b19026cad..63a33c4ee 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -121,6 +121,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|outofmemoryerror" # false positive (elastic command line) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding component template" # false positive (elastic security) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index template" # false positive (elastic security) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fs_errors" # false positive (suricata stats) From 8690304dffce6fc1ef2a923edf8ec2b80d90079d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 29 Sep 2023 16:17:19 -0400 Subject: [PATCH 322/350] change how mine_functions.conf is managed during setup --- salt/salt/etc/minion.d/mine_functions.conf.jinja | 4 ++-- salt/salt/mine_functions.sls | 5 +++++ salt/salt/minion.sls | 9 +-------- setup/so-functions | 3 +-- 4 files changed, 9 insertions(+), 12 deletions(-) create mode 100644 salt/salt/mine_functions.sls diff --git a/salt/salt/etc/minion.d/mine_functions.conf.jinja b/salt/salt/etc/minion.d/mine_functions.conf.jinja index 2ae345cdf..e3c62e75c 100644 --- a/salt/salt/etc/minion.d/mine_functions.conf.jinja +++ b/salt/salt/etc/minion.d/mine_functions.conf.jinja @@ -1,8 +1,8 @@ mine_interval: 35 mine_functions: network.ip_addrs: - - interface: {{ GLOBALS.main_interface }} -{% if GLOBALS.is_manager -%} + - interface: {{ pillar.host.mainint }} +{% if grains.role in ['so-eval','so-import','so-manager','so-managersearch','so-standalone'] -%} x509.get_pem_entries: - glob_path: '/etc/pki/ca.crt' {% endif -%} diff --git a/salt/salt/mine_functions.sls b/salt/salt/mine_functions.sls new file mode 100644 index 000000000..27a905847 --- /dev/null +++ b/salt/salt/mine_functions.sls @@ -0,0 +1,5 @@ +mine_functions: + file.managed: + - name: /etc/salt/minion.d/mine_functions.conf + - source: salt://salt/etc/minion.d/mine_functions.conf.jinja + - template: jinja diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 43f7539f9..865bd367f 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -12,6 +12,7 @@ include: - salt - systemd.reload - repo.client + - salt.mine_functions {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} @@ -78,14 +79,6 @@ salt_minion_service_unit_file: {% endif %} -mine_functions: - file.managed: - - name: /etc/salt/minion.d/mine_functions.conf - - source: salt://salt/etc/minion.d/mine_functions.conf.jinja - - template: jinja - - defaults: - GLOBALS: {{ GLOBALS }} - # this has to be outside the if statement above since there are _in calls to this state salt_minion_service: service.running: diff --git a/setup/so-functions b/setup/so-functions index 679142e2a..eab7a4add 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -649,8 +649,7 @@ configure_minion() { "log_level_logfile: info"\ "log_file: /opt/so/log/salt/minion" >> "$minion_config" - cp -f ../salt/salt/etc/minion.d/mine_functions.conf.jinja /etc/salt/minion.d/mine_functions.conf - sed -i "s/{{ GLOBALS.main_interface }}/$MNIC/" /etc/salt/minion.d/mine_functions.conf + logCmd "salt-call state.apply salt.mine_functions -l info" { logCmd "systemctl enable salt-minion"; From 827ed7b273cf2a9180fa94e7cd398a2e178dbfcb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 29 Sep 2023 17:08:42 -0400 Subject: [PATCH 323/350] run salt.mine_function state locally and provide pillar info to it --- setup/so-functions | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index eab7a4add..b55ae0def 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -649,7 +649,8 @@ configure_minion() { "log_level_logfile: info"\ "log_file: /opt/so/log/salt/minion" >> "$minion_config" - logCmd "salt-call state.apply salt.mine_functions -l info" + info "Running: salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar='{"host": {"mainint": "$MNIC"}}'" + salt-call state.apply salt.mine_functions --local --file-root=../salt/ -l info pillar="{'host': {'mainint': $MNIC}}" { logCmd "systemctl enable salt-minion"; From 39ea1d317df32c17e96f3566fea64ddfbd33b297 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 29 Sep 2023 17:12:14 -0400 Subject: [PATCH 324/350] add comment --- salt/salt/mine_functions.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/salt/mine_functions.sls b/salt/salt/mine_functions.sls index 27a905847..49a47e524 100644 --- a/salt/salt/mine_functions.sls +++ b/salt/salt/mine_functions.sls @@ -1,3 +1,11 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +# this state was seperated from salt.minion state since it is called during setup +# GLOBALS are imported in the salt.minion state and that is not available at that point in setup +# this state is included in the salt.minion state mine_functions: file.managed: - name: /etc/salt/minion.d/mine_functions.conf From ea085c5ff6aafb1e06b5851e8731c934c6fc3ccf Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 29 Sep 2023 21:38:13 -0400 Subject: [PATCH 325/350] more known errors --- salt/common/tools/sbin/so-log-check | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 63a33c4ee..ba5285bf3 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -105,6 +105,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics" # server not yet ready EXCLUDED_ERRORS="$EXCLUDED_ERRORS|broken pipe" # server not yet ready + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status: 502" # server not yet ready (nginx waiting on upstream) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout exceeded" # server not yet ready (telegraf waiting on elasticsearch) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes" # server not yet ready (telegraf waiting on influx) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|expected field at" # server not yet ready (telegraf waiting on health data) @@ -153,6 +154,8 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|generating elastalert config" # playbook expected error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|activerecord" # playbook expected error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics" # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf does not" # known issue with reposync on pre-2.4.20 + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record" # stenographer corrupt index EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" From 8c7933cd60feabd414036da72cf3c2282212b99d Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 30 Sep 2023 18:11:29 -0400 Subject: [PATCH 326/350] fix exclusion --- salt/common/tools/sbin/so-log-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index ba5285bf3..dac1121bc 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -154,7 +154,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|generating elastalert config" # playbook expected error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|activerecord" # playbook expected error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics" # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf does not" # known issue with reposync on pre-2.4.20 + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf" # known issue with reposync on pre-2.4.20 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record" # stenographer corrupt index EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" From cd8a74290b6f1259d21e294282ac83dc9aeddaa5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 10:36:17 -0400 Subject: [PATCH 327/350] hold openssl version --- salt/common/init.sls | 1 - salt/common/packages.sls | 11 ++++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index f50f0c61b..37ea4239d 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -91,7 +91,6 @@ vimconfig: alwaysupdated: pkg.latest: - pkgs: - - openssl - openssh-server - bash - skip_suggestions: True diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 8b54bdbf5..f5707a377 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -46,6 +46,12 @@ python-rich: {% endif %} {% if GLOBALS.os_family == 'RedHat' %} + +holdversion_openssl: + pkg.held: + - name: - openssl + - version: 1:3.0.7-16.0.1.el9_2 + commonpkgs: pkg.installed: - skip_suggestions: True @@ -65,7 +71,7 @@ commonpkgs: - mariadb-devel - net-tools - nmap-ncat - - openssl + - openssl: 1:3.0.7-16.0.1.el9_2 - procps-ng - python3-dnf-plugin-versionlock - python3-docker @@ -79,4 +85,7 @@ commonpkgs: - unzip - wget - yum-utils + + + {% endif %} From 70a36bafa54b92e258f4e5a2942006c04dcd7b1e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 10:38:54 -0400 Subject: [PATCH 328/350] remove - --- salt/common/packages.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index f5707a377..ae723fd94 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -49,7 +49,7 @@ python-rich: holdversion_openssl: pkg.held: - - name: - openssl + - name: openssl - version: 1:3.0.7-16.0.1.el9_2 commonpkgs: From dfe399291f9398435fd0520955bf19826400bb04 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 10:54:41 -0400 Subject: [PATCH 329/350] hold openssl-libs --- salt/common/packages.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index ae723fd94..f7c8fd5dc 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -52,6 +52,11 @@ holdversion_openssl: - name: openssl - version: 1:3.0.7-16.0.1.el9_2 +holdversion_openssl-libs: + pkg.held: + - name: openssl-libs + - version: 1:3.0.7-16.0.1.el9_2 + commonpkgs: pkg.installed: - skip_suggestions: True From c1ab8952eb727c0cf0cea085c6b75aa468109b0e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 10:59:51 -0400 Subject: [PATCH 330/350] hold openssl-devel --- salt/common/packages.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index f7c8fd5dc..a4a32f15f 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -57,6 +57,11 @@ holdversion_openssl-libs: - name: openssl-libs - version: 1:3.0.7-16.0.1.el9_2 +holdversion_openssl-devel: + pkg.held: + - name: openssl-devel + - version: 1:3.0.7-16.0.1.el9_2 + commonpkgs: pkg.installed: - skip_suggestions: True From f85dd910a302bad9515390d99d7929fe8106fe3c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 11:13:08 -0400 Subject: [PATCH 331/350] hold openssl from update during setup --- salt/common/packages.sls | 2 ++ setup/so-functions | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index a4a32f15f..0bf8616be 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -47,6 +47,8 @@ python-rich: {% if GLOBALS.os_family == 'RedHat' %} +# holding these since openssl-devel-1:3.0.7-16.0.1.el9_2 seems to be a requirement for mariadb-devel-3:10.5.16-2.el9_0 +# https://github.com/Security-Onion-Solutions/securityonion/discussions/11443 holdversion_openssl: pkg.held: - name: openssl diff --git a/setup/so-functions b/setup/so-functions index 679142e2a..26e1b2dab 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2443,7 +2443,8 @@ update_sudoers_for_testing() { update_packages() { if [[ $is_oracle ]]; then logCmd "dnf repolist" - logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*" + # holding openssl https://github.com/Security-Onion-Solutions/securityonion/discussions/11443 + logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*,openssl*" RMREPOFILES=("oracle-linux-ol9.repo" "uek-ol9.repo" "virt-ol9.repo") info "Removing repo files added by oracle-repos package update" for FILE in ${RMREPOFILES[@]}; do From 0f08d5d640a2e0e0fa6767ded9a7ec9d934c15ae Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 11:43:03 -0400 Subject: [PATCH 332/350] install openssl version 1:3.0.7-16.0.1.el9_2 --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index 26e1b2dab..243e89c99 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2444,6 +2444,7 @@ update_packages() { if [[ $is_oracle ]]; then logCmd "dnf repolist" # holding openssl https://github.com/Security-Onion-Solutions/securityonion/discussions/11443 + logCmd "dnf -y install openssl-1:3.0.7-16.0.1.el9_2 openssl-libs-1:3.0.7-16.0.1.el9_2 openssl-devel-1:3.0.7-16.0.1.el9_2" logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*,openssl*" RMREPOFILES=("oracle-linux-ol9.repo" "uek-ol9.repo" "virt-ol9.repo") info "Removing repo files added by oracle-repos package update" From 3a5c6ee43aac37d2f385bd93091f89dd3dd84bc1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 12:09:13 -0400 Subject: [PATCH 333/350] install version lock before we try to hold pkgs --- salt/common/packages.sls | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 0bf8616be..827cc6bf0 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -47,6 +47,11 @@ python-rich: {% if GLOBALS.os_family == 'RedHat' %} +# install versionlock first so we can hold packages in the next states +install_versionlock: + pkg.installed: + - name: python3-dnf-plugin-versionlock + # holding these since openssl-devel-1:3.0.7-16.0.1.el9_2 seems to be a requirement for mariadb-devel-3:10.5.16-2.el9_0 # https://github.com/Security-Onion-Solutions/securityonion/discussions/11443 holdversion_openssl: @@ -85,7 +90,6 @@ commonpkgs: - nmap-ncat - openssl: 1:3.0.7-16.0.1.el9_2 - procps-ng - - python3-dnf-plugin-versionlock - python3-docker - python3-m2crypto - python3-packaging From 6547afe6c07cc064587a44c3ca13b723c92d7375 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 13:35:00 -0400 Subject: [PATCH 334/350] dont hold openssl-devel --- salt/common/packages.sls | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 827cc6bf0..185bf536e 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -64,11 +64,6 @@ holdversion_openssl-libs: - name: openssl-libs - version: 1:3.0.7-16.0.1.el9_2 -holdversion_openssl-devel: - pkg.held: - - name: openssl-devel - - version: 1:3.0.7-16.0.1.el9_2 - commonpkgs: pkg.installed: - skip_suggestions: True @@ -85,10 +80,10 @@ commonpkgs: - httpd-tools - jq - lvm2 + - openssl: 1:3.0.7-16.0.1.el9_2 - mariadb-devel - net-tools - nmap-ncat - - openssl: 1:3.0.7-16.0.1.el9_2 - procps-ng - python3-docker - python3-m2crypto @@ -102,6 +97,4 @@ commonpkgs: - wget - yum-utils - - {% endif %} From 6b90961e87221dcb3e16a5702ff618b237274a28 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 14:26:28 -0400 Subject: [PATCH 335/350] openssl-libs --- salt/common/packages.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 185bf536e..adef3828b 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -81,6 +81,7 @@ commonpkgs: - jq - lvm2 - openssl: 1:3.0.7-16.0.1.el9_2 + - openssl-libs: 1:3.0.7-16.0.1.el9_2 - mariadb-devel - net-tools - nmap-ncat From d7a14d9e00ab8b098a32c4487a09b22332980da2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 15:08:22 -0400 Subject: [PATCH 336/350] update holds --- salt/common/packages.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index adef3828b..b002c62e9 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -67,6 +67,7 @@ holdversion_openssl-libs: commonpkgs: pkg.installed: - skip_suggestions: True + - update_holds: True - pkgs: - curl - device-mapper-persistent-data From 57e76232eca7076451d7075ad400d8156daae718 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 15:48:53 -0400 Subject: [PATCH 337/350] openssl pkgs in own state --- salt/common/packages.sls | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index b002c62e9..ca0326839 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -64,10 +64,18 @@ holdversion_openssl-libs: - name: openssl-libs - version: 1:3.0.7-16.0.1.el9_2 -commonpkgs: +openssl_pkgs: pkg.installed: - skip_suggestions: True - update_holds: True + - pkgs: + - openssl: 1:3.0.7-16.0.1.el9_2 + - openssl-libs: 1:3.0.7-16.0.1.el9_2 + - openssl-devel: 1:3.0.7-16.0.1.el9_2 + +commonpkgs: + pkg.installed: + - skip_suggestions: True - pkgs: - curl - device-mapper-persistent-data @@ -81,8 +89,6 @@ commonpkgs: - httpd-tools - jq - lvm2 - - openssl: 1:3.0.7-16.0.1.el9_2 - - openssl-libs: 1:3.0.7-16.0.1.el9_2 - mariadb-devel - net-tools - nmap-ncat From 8995752c2722116e2cf328d067a24371e68bcd33 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 2 Oct 2023 16:17:26 -0400 Subject: [PATCH 338/350] let openssl-devel be installed with mariadb --- salt/common/packages.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index ca0326839..b4e97a81d 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -71,7 +71,6 @@ openssl_pkgs: - pkgs: - openssl: 1:3.0.7-16.0.1.el9_2 - openssl-libs: 1:3.0.7-16.0.1.el9_2 - - openssl-devel: 1:3.0.7-16.0.1.el9_2 commonpkgs: pkg.installed: From c699c2fe2ab9cb2d94e6460d5a5ea69cc60d38fa Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 3 Oct 2023 09:43:29 -0400 Subject: [PATCH 339/350] exclude known issues --- salt/common/tools/sbin/so-log-check | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index dac1121bc..c2d16fd86 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -118,7 +118,7 @@ fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_status_error" # false positive - EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_error.json" # false positive + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_error" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index" # false positive EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror" # false positive @@ -156,6 +156,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics" # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf" # known issue with reposync on pre-2.4.20 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record" # stenographer corrupt index + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field." # known ingest type collisions issue with earlier versions of SO EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets" EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed" From 66be04e78a8c1a6717134024c89773af2b9d1b7f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 3 Oct 2023 09:53:40 -0400 Subject: [PATCH 340/350] remove mariadb --- salt/common/init.sls | 1 + salt/common/packages.sls | 29 ++++------------------------- salt/common/tools/sbin/so-common | 2 +- setup/so-functions | 3 +-- 4 files changed, 7 insertions(+), 28 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 37ea4239d..f50f0c61b 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -91,6 +91,7 @@ vimconfig: alwaysupdated: pkg.latest: - pkgs: + - openssl - openssh-server - bash - skip_suggestions: True diff --git a/salt/common/packages.sls b/salt/common/packages.sls index b4e97a81d..c5d2729fd 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -47,35 +47,15 @@ python-rich: {% if GLOBALS.os_family == 'RedHat' %} -# install versionlock first so we can hold packages in the next states -install_versionlock: - pkg.installed: - - name: python3-dnf-plugin-versionlock - -# holding these since openssl-devel-1:3.0.7-16.0.1.el9_2 seems to be a requirement for mariadb-devel-3:10.5.16-2.el9_0 -# https://github.com/Security-Onion-Solutions/securityonion/discussions/11443 -holdversion_openssl: - pkg.held: - - name: openssl - - version: 1:3.0.7-16.0.1.el9_2 - -holdversion_openssl-libs: - pkg.held: - - name: openssl-libs - - version: 1:3.0.7-16.0.1.el9_2 - -openssl_pkgs: - pkg.installed: - - skip_suggestions: True - - update_holds: True - - pkgs: - - openssl: 1:3.0.7-16.0.1.el9_2 - - openssl-libs: 1:3.0.7-16.0.1.el9_2 +remove_mariadb: + pkg.removed: + - name: mariadb-devel commonpkgs: pkg.installed: - skip_suggestions: True - pkgs: + - python3-dnf-plugin-versionlock - curl - device-mapper-persistent-data - fuse @@ -88,7 +68,6 @@ commonpkgs: - httpd-tools - jq - lvm2 - - mariadb-devel - net-tools - nmap-ncat - procps-ng diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 0dfb19bbe..f754b34ef 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -240,7 +240,7 @@ gpg_rpm_import() { else local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys" fi - RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub' 'MariaDB-Server-GPG-KEY') + RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub') for RPMKEY in "${RPMKEYS[@]}"; do rpm --import $RPMKEYSLOC/$RPMKEY echo "Imported $RPMKEY" diff --git a/setup/so-functions b/setup/so-functions index 243e89c99..84d6d80f9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2444,8 +2444,7 @@ update_packages() { if [[ $is_oracle ]]; then logCmd "dnf repolist" # holding openssl https://github.com/Security-Onion-Solutions/securityonion/discussions/11443 - logCmd "dnf -y install openssl-1:3.0.7-16.0.1.el9_2 openssl-libs-1:3.0.7-16.0.1.el9_2 openssl-devel-1:3.0.7-16.0.1.el9_2" - logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*,openssl*" + logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*" RMREPOFILES=("oracle-linux-ol9.repo" "uek-ol9.repo" "virt-ol9.repo") info "Removing repo files added by oracle-repos package update" for FILE in ${RMREPOFILES[@]}; do From 2434ce14d3fe1ed8773e085a6696b9d01026d1c5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 3 Oct 2023 10:01:07 -0400 Subject: [PATCH 341/350] remove removing mariadb-devel --- salt/common/packages.sls | 4 ---- setup/so-functions | 1 - 2 files changed, 5 deletions(-) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index c5d2729fd..521f2201c 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -47,10 +47,6 @@ python-rich: {% if GLOBALS.os_family == 'RedHat' %} -remove_mariadb: - pkg.removed: - - name: mariadb-devel - commonpkgs: pkg.installed: - skip_suggestions: True diff --git a/setup/so-functions b/setup/so-functions index 84d6d80f9..679142e2a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2443,7 +2443,6 @@ update_sudoers_for_testing() { update_packages() { if [[ $is_oracle ]]; then logCmd "dnf repolist" - # holding openssl https://github.com/Security-Onion-Solutions/securityonion/discussions/11443 logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*" RMREPOFILES=("oracle-linux-ol9.repo" "uek-ol9.repo" "virt-ol9.repo") info "Removing repo files added by oracle-repos package update" From f3ba28062b48e6bfd9adb55c649057eb6987ca14 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 3 Oct 2023 10:05:56 -0400 Subject: [PATCH 342/350] Remove MySQL --- salt/mysql/config.sls | 2 +- setup/so-functions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/mysql/config.sls b/salt/mysql/config.sls index 5f9010011..274f25d76 100644 --- a/salt/mysql/config.sls +++ b/salt/mysql/config.sls @@ -9,7 +9,7 @@ # MySQL Setup mysqlpkgs: - pkg.installed: + pkg.removed: - skip_suggestions: False - pkgs: {% if grains['os_family'] != 'RedHat' %} diff --git a/setup/so-functions b/setup/so-functions index 679142e2a..aad627a8d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2088,7 +2088,7 @@ saltify() { if [[ $waitforstate ]]; then retry 150 20 "apt-get -y install salt-common=$SALTVERSION salt-minion=$SALTVERSION salt-master=$SALTVERSION" || fail_setup retry 150 20 "apt-mark hold salt-minion salt-common salt-master" || fail_setup - retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" || exit 1 + retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-packaging python3-influxdb python3-lxml" || exit 1 else retry 150 20 "apt-get -y install salt-common=$SALTVERSION salt-minion=$SALTVERSION" || fail_setup retry 150 20 "apt-mark hold salt-minion salt-common" || fail_setup From d78b55873d369e0fa759d8c484ad2e51289ee286 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 3 Oct 2023 10:15:28 -0400 Subject: [PATCH 343/350] remove mariadb-devel --- salt/common/packages.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index 521f2201c..c5d2729fd 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -47,6 +47,10 @@ python-rich: {% if GLOBALS.os_family == 'RedHat' %} +remove_mariadb: + pkg.removed: + - name: mariadb-devel + commonpkgs: pkg.installed: - skip_suggestions: True From d79e27774c06e77787e8cb171990444594b37abb Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 5 Oct 2023 11:27:48 -0400 Subject: [PATCH 344/350] 2.4.20 --- DOWNLOAD_AND_VERIFY_ISO.md | 22 ++++++++++----------- sigs/securityonion-2.4.20-20231006.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.4.20-20231006.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 1e6299a8e..f78ed8045 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,18 +1,18 @@ -### 2.4.10-20230821 ISO image released on 2023/08/21 +### 2.4.20-20231006 ISO image released on 2023/08/21 ### Download and Verify -2.4.10-20230821 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230821.iso +2.4.20-20231006 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.20-20231006.iso -MD5: 353EB36F807DC947F08F79B3DCFA420E -SHA1: B25E3BEDB81BBEF319DC710267E6D78422F39C56 -SHA256: 3D369E92FEB65D14E1A981E99FA223DA52C92057A037C243AD6332B6B9A6D9BC +MD5: 269F00308C53976BF0EAE788D1DB29DB +SHA1: 3F7C2324AE1271112F3B752BA4724AF36688FC27 +SHA256: 542B8B3F4F75AD24DC78007F8FE0857E00DC4CC9F4870154DCB8D5D0C4144B65 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230821.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.20-20231006.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.10-20230821.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.20-20231006.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.10-20230821.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.20-20231006.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.10-20230821.iso.sig securityonion-2.4.10-20230821.iso +gpg --verify securityonion-2.4.20-20231006.iso.sig securityonion-2.4.20-20231006.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Mon 21 Aug 2023 09:47:50 AM EDT using RSA key ID FE507013 +gpg: Signature made Tue 03 Oct 2023 11:40:51 AM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.20-20231006.iso.sig b/sigs/securityonion-2.4.20-20231006.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..b253c67343b7b2cf16a7e755c9994273a0ddcbf3 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%aUF0{{vM5PT3| zxBgIY6PPIv|8AKBz*y#69E5wO6vi|oL@`l#Tb0V-g6qpodgIz_Qz?#fV&dsL)bw7; zEQ5dR`7WH2!A2&_oP21_JVTNA-V)o_J5B1nXhuKq`dW;jVYtmpO|bDSl2_jc>+i~z z!YaS~vE>O;Knj%{_=6*d1>;fqP7xtOq7dlE*J3+rJ@LOtG8j8$gDzLp(Yp+n>O*9Y zZ;w4lR4oDEQ5tlI#JsHlxdTPdpKI;7=?!Mjr=4})v59Qq@d3juf)K@ROkoR{Vtq8j zOfw_a81qDfNkCEs`A^plu`Gznwc-l3IdkH{5K}tU%EmV33V?2_i~4tT02)RIxGEU&aRRnk}JYjQpnhK>`SjSi# ze2u@u?YU>5_R;{3*6BAG1}iQSBOdN#r48V3bvTv`XYcuiUoJom;3*m4_}RLXHbgV~ zRnf9Ycgu(Tuxhq02}9f7VB85+A9}Q#K&Y5;+wQ~#<o@5C8xG literal 0 HcmV?d00001 From c25aed9a2b285631764a5ce2dfd93f01022d81c1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 5 Oct 2023 11:37:49 -0400 Subject: [PATCH 345/350] Update DOWNLOAD_AND_VERIFY_ISO.md --- DOWNLOAD_AND_VERIFY_ISO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index f78ed8045..dabfd285c 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,4 +1,4 @@ -### 2.4.20-20231006 ISO image released on 2023/08/21 +### 2.4.20-20231006 ISO image released on 2023/10/06 From 4dc24b22c79042e8f6959f5e59e21b8fb7249410 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Oct 2023 10:51:59 -0400 Subject: [PATCH 346/350] accept icmp on input chain --- salt/firewall/iptables.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/firewall/iptables.jinja b/salt/firewall/iptables.jinja index c15a54e46..074663e15 100644 --- a/salt/firewall/iptables.jinja +++ b/salt/firewall/iptables.jinja @@ -89,7 +89,6 @@ COMMIT -A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -A INPUT -i lo -j ACCEPT -A INPUT -m conntrack --ctstate INVALID -j DROP --A INPUT -j REJECT --reject-with icmp-host-prohibited -A INPUT -p icmp -j ACCEPT -A INPUT -j LOGGING -A FORWARD -j DOCKER-USER @@ -103,6 +102,7 @@ COMMIT -A FORWARD -m conntrack --ctstate INVALID -j DROP -A FORWARD -j REJECT --reject-with icmp-host-prohibited -A OUTPUT -o lo -j ACCEPT +# block icmp timestamp reply -A OUTPUT -p icmp -m icmp --icmp-type 14 -j DROP {%- for rule in D2 %} From 49ebbf3232fe08091f796b1e7b1100fa9aed7d56 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 10 Oct 2023 11:05:39 -0400 Subject: [PATCH 347/350] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a3ab5389f..8ea99f559 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.20 +2.4.30 From a283e7ea0bea9928ff27ba42f022a5d07934520e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Oct 2023 13:00:54 -0400 Subject: [PATCH 348/350] remove checkmine salt engine --- salt/salt/engines/checkmine.py | 28 ---------------------------- salt/salt/files/engines.conf | 6 ------ 2 files changed, 34 deletions(-) delete mode 100644 salt/salt/engines/checkmine.py delete mode 100644 salt/salt/files/engines.conf diff --git a/salt/salt/engines/checkmine.py b/salt/salt/engines/checkmine.py deleted file mode 100644 index 5cc0a5ad3..000000000 --- a/salt/salt/engines/checkmine.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -import logging -from time import sleep -from os import remove - -log = logging.getLogger(__name__) - -def start(interval=30): - log.info("checkmine engine started") - minionid = __grains__['id'] - while True: - try: - ca_crt = __salt__['saltutil.runner']('mine.get', tgt=minionid, fun='x509.get_pem_entries')[minionid]['/etc/pki/ca.crt'] - log.info('Successfully queried Salt mine for the CA.') - except: - log.error('Could not pull CA from the Salt mine.') - log.info('Removing /var/cache/salt/master/minions/%s/mine.p to force Salt mine to be repopulated.' % minionid) - try: - remove('/var/cache/salt/master/minions/%s/mine.p' % minionid) - log.info('Removed /var/cache/salt/master/minions/%s/mine.p' % minionid) - except FileNotFoundError: - log.error('/var/cache/salt/master/minions/%s/mine.p does not exist' % minionid) - - __salt__['mine.send'](name='x509.get_pem_entries', glob_path='/etc/pki/ca.crt') - log.warning('Salt mine repopulated with /etc/pki/ca.crt') - - sleep(interval) \ No newline at end of file diff --git a/salt/salt/files/engines.conf b/salt/salt/files/engines.conf deleted file mode 100644 index c9e20adf3..000000000 --- a/salt/salt/files/engines.conf +++ /dev/null @@ -1,6 +0,0 @@ -engines_dirs: - - /etc/salt/engines - -engines: - - checkmine: - interval: 30 \ No newline at end of file From 89467adf9c3ba493c397836a942b9f75b9eb183e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Oct 2023 13:05:43 -0400 Subject: [PATCH 349/350] batch the salt mine update --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 333be836b..e4b388e22 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -580,7 +580,7 @@ update_centos_repo() { update_salt_mine() { echo "Populating the mine with network.ip_addrs pillar.host.mainint for each host." set +e - salt \* mine.update + salt \* mine.update -b 50 set -e } From 4193130ed05fc6cd6e34e1432161737c23996a74 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 10 Oct 2023 13:07:12 -0400 Subject: [PATCH 350/350] reduce salt mine interval to 25 minutes --- salt/salt/etc/minion.d/mine_functions.conf.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/etc/minion.d/mine_functions.conf.jinja b/salt/salt/etc/minion.d/mine_functions.conf.jinja index e3c62e75c..3851238fd 100644 --- a/salt/salt/etc/minion.d/mine_functions.conf.jinja +++ b/salt/salt/etc/minion.d/mine_functions.conf.jinja @@ -1,4 +1,4 @@ -mine_interval: 35 +mine_interval: 25 mine_functions: network.ip_addrs: - interface: {{ pillar.host.mainint }}