From 95581f505a84074ae1bbfd9fed83bdda477a6049 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 31 Jul 2023 13:18:57 -0400 Subject: [PATCH 01/18] import DOCKER in idh.enabled --- salt/idh/enabled.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/idh/enabled.sls b/salt/idh/enabled.sls index 480e7eedc..7ef0ebb46 100644 --- a/salt/idh/enabled.sls +++ b/salt/idh/enabled.sls @@ -6,6 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKER %} include: - idh.config From 57562ad5e366a00f2d1d0d6f8af03fe8992c7de5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 31 Jul 2023 13:34:08 -0400 Subject: [PATCH 02/18] add managersearch and standlone fw rules for searchnode --- salt/firewall/defaults.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 57446a5c2..20b966e48 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -866,6 +866,14 @@ firewall: portgroups: - elasticsearch_node - elasticsearch_rest + managersearch: + portgroups: + - elasticsearch_node + - elasticsearch_rest + standalone: + portgroups: + - elasticsearch_node + - elasticsearch_rest dockernet: portgroups: - elasticsearch_node From a89508f1ae63cf80d156fbfd136af23dd4cbde2f Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 31 Jul 2023 15:17:24 -0400 Subject: [PATCH 03/18] Heavy Node fixes --- salt/elasticagent/enabled.sls | 3 + .../files/elastic-agent.yml.jinja | 349 +++++++++++++++++- .../grid-nodes_heavy/elasticsearch-logs.json | 106 ------ .../grid-nodes_heavy/kratos-logs.json | 29 -- .../grid-nodes_heavy/osquery-grid-nodes.json | 2 +- .../grid-nodes_heavy/redis-logs.json | 76 ---- .../grid-nodes_heavy/soc-auth-sync-logs.json | 29 -- .../grid-nodes_heavy/soc-salt-relay-logs.json | 29 -- .../grid-nodes_heavy/soc-sensoroni-logs.json | 29 -- .../grid-nodes_heavy/soc-server-logs.json | 29 -- .../grid-nodes_heavy/system-grid-nodes.json | 2 +- 11 files changed, 336 insertions(+), 347 deletions(-) delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json delete mode 100644 salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json diff --git a/salt/elasticagent/enabled.sls b/salt/elasticagent/enabled.sls index 4c00920ac..b133d94ab 100644 --- a/salt/elasticagent/enabled.sls +++ b/salt/elasticagent/enabled.sls @@ -35,6 +35,7 @@ so-elastic-agent: - /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro - /nsm:/nsm:ro + - /opt/so/log:/opt/so/log:ro {% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %} - {{ BIND }} @@ -47,6 +48,8 @@ so-elastic-agent: - {{ XTRAENV }} {% endfor %} {% endif %} + - require: + - file: create-elastic-agent-config - watch: - file: create-elastic-agent-config diff --git a/salt/elasticagent/files/elastic-agent.yml.jinja b/salt/elasticagent/files/elastic-agent.yml.jinja index 2d32a3b17..92aacfa44 100644 --- a/salt/elasticagent/files/elastic-agent.yml.jinja +++ b/salt/elasticagent/files/elastic-agent.yml.jinja @@ -3,7 +3,7 @@ {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} id: aea1ba80-1065-11ee-a369-97538913b6a9 -revision: 2 +revision: 1 outputs: default: type: elasticsearch @@ -22,56 +22,369 @@ agent: metrics: false features: {} inputs: - - id: logfile-logs-80ffa884-2cfc-459a-964a-34df25714d85 - name: suricata-logs - revision: 1 + - id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62 + name: import-evtx-logs + revision: 2 type: logfile use_output: default meta: package: name: log - version: + version: data_stream: namespace: so - package_policy_id: 80ffa884-2cfc-459a-964a-34df25714d85 + package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62 streams: - - id: logfile-log.log-80ffa884-2cfc-459a-964a-34df25714d85 + - id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62 + data_stream: + dataset: import + paths: + - /nsm/import/*/evtx/*.json + processors: + - dissect: + field: log.file.path + tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}' + target_prefix: '' + - decode_json_fields: + fields: + - message + target: '' + - drop_fields: + ignore_missing: true + fields: + - host + - add_fields: + fields: + dataset: system.security + type: logs + namespace: default + target: data_stream + - add_fields: + fields: + dataset: system.security + module: system + imported: true + target: event + - then: + - add_fields: + fields: + dataset: windows.sysmon_operational + target: data_stream + - add_fields: + fields: + dataset: windows.sysmon_operational + module: windows + imported: true + target: event + if: + equals: + winlog.channel: Microsoft-Windows-Sysmon/Operational + - then: + - add_fields: + fields: + dataset: system.application + target: data_stream + - add_fields: + fields: + dataset: system.application + target: event + if: + equals: + winlog.channel: Application + - then: + - add_fields: + fields: + dataset: system.system + target: data_stream + - add_fields: + fields: + dataset: system.system + target: event + if: + equals: + winlog.channel: System + - then: + - add_fields: + fields: + dataset: windows.powershell_operational + target: data_stream + - add_fields: + fields: + dataset: windows.powershell_operational + module: windows + target: event + if: + equals: + winlog.channel: Microsoft-Windows-PowerShell/Operational + tags: + - import + - id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0 + name: redis-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: redis + version: + data_stream: + namespace: default + package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0 + streams: + - id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0 + data_stream: + dataset: redis.log + type: logs + exclude_files: + - .gz$ + paths: + - /opt/so/log/redis/redis.log + tags: + - redis-log + exclude_lines: + - '^\s+[\-`(''.|_]' + - id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8 + name: import-suricata-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8 + streams: + - id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8 + data_stream: + dataset: import + pipeline: suricata.common + paths: + - /nsm/import/*/suricata/eve*.json + processors: + - add_fields: + fields: + module: suricata + imported: true + category: network + target: event + - dissect: + field: log.file.path + tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}' + target_prefix: '' + - id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d + name: soc-server-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d + streams: + - id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d + data_stream: + dataset: soc + pipeline: common + paths: + - /opt/so/log/soc/sensoroni-server.log + processors: + - decode_json_fields: + add_error_key: true + process_array: true + max_depth: 2 + fields: + - message + target: soc + - add_fields: + fields: + module: soc + dataset_temp: server + category: host + target: event + - rename: + ignore_missing: true + fields: + - from: soc.fields.sourceIp + to: source.ip + - from: soc.fields.status + to: http.response.status_code + - from: soc.fields.method + to: http.request.method + - from: soc.fields.path + to: url.path + - from: soc.message + to: event.action + - from: soc.level + to: log.level + tags: + - so-soc + - id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073 + name: soc-sensoroni-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073 + streams: + - id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073 + data_stream: + dataset: soc + pipeline: common + paths: + - /opt/so/log/sensoroni/sensoroni.log + processors: + - decode_json_fields: + add_error_key: true + process_array: true + max_depth: 2 + fields: + - message + target: sensoroni + - add_fields: + fields: + module: soc + dataset_temp: sensoroni + category: host + target: event + - rename: + ignore_missing: true + fields: + - from: sensoroni.fields.sourceIp + to: source.ip + - from: sensoroni.fields.status + to: http.response.status_code + - from: sensoroni.fields.method + to: http.request.method + - from: sensoroni.fields.path + to: url.path + - from: sensoroni.message + to: event.action + - from: sensoroni.level + to: log.level + - id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515 + name: soc-salt-relay-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515 + streams: + - id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515 + data_stream: + dataset: soc + pipeline: common + paths: + - /opt/so/log/soc/salt-relay.log + processors: + - dissect: + field: message + tokenizer: '%{soc.ts} | %{event.action}' + target_prefix: '' + - add_fields: + fields: + module: soc + dataset_temp: salt_relay + category: host + target: event + tags: + - so-soc + - id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0 + name: soc-auth-sync-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0 + streams: + - id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0 + data_stream: + dataset: soc + pipeline: common + paths: + - /opt/so/log/soc/sync.log + processors: + - dissect: + field: message + tokenizer: '%{event.action}' + target_prefix: '' + - add_fields: + fields: + module: soc + dataset_temp: auth_sync + category: host + target: event + tags: + - so-soc + - id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253 + name: suricata-logs + revision: 2 + type: logfile + use_output: default + meta: + package: + name: log + version: + data_stream: + namespace: so + package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253 + streams: + - id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253 data_stream: dataset: suricata + pipeline: suricata.common paths: - /nsm/suricata/eve*.json processors: - add_fields: - target: event fields: - category: network module: suricata - pipeline: suricata.common - - id: logfile-logs-90103ac4-f6bd-4a4a-b596-952c332390fc + category: network + target: event + - id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327 name: strelka-logs - revision: 1 + revision: 2 type: logfile use_output: default meta: package: name: log - version: + version: data_stream: namespace: so - package_policy_id: 90103ac4-f6bd-4a4a-b596-952c332390fc + package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327 streams: - - id: logfile-log.log-90103ac4-f6bd-4a4a-b596-952c332390fc + - id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327 data_stream: dataset: strelka + pipeline: strelka.file paths: - /nsm/strelka/log/strelka.log processors: - add_fields: - target: event fields: - category: file module: strelka - pipeline: strelka.file + category: file + target: event - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d name: zeek-logs revision: 1 diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json deleted file mode 100644 index 711602775..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-logs.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "package": { - "name": "elasticsearch", - "version": "" - }, - "name": "elasticsearch-logs", - "namespace": "default", - "description": "Elasticsearch Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "elasticsearch-logfile": { - "enabled": true, - "streams": { - "elasticsearch.audit": { - "enabled": false, - "vars": { - "paths": [ - "/var/log/elasticsearch/*_audit.json" - ] - } - }, - "elasticsearch.deprecation": { - "enabled": false, - "vars": { - "paths": [ - "/var/log/elasticsearch/*_deprecation.json" - ] - } - }, - "elasticsearch.gc": { - "enabled": false, - "vars": { - "paths": [ - "/var/log/elasticsearch/gc.log.[0-9]*", - "/var/log/elasticsearch/gc.log" - ] - } - }, - "elasticsearch.server": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/elasticsearch/*.log" - ] - } - }, - "elasticsearch.slowlog": { - "enabled": false, - "vars": { - "paths": [ - "/var/log/elasticsearch/*_index_search_slowlog.json", - "/var/log/elasticsearch/*_index_indexing_slowlog.json" - ] - } - } - } - }, - "elasticsearch-elasticsearch/metrics": { - "enabled": false, - "vars": { - "hosts": [ - "http://localhost:9200" - ], - "scope": "node" - }, - "streams": { - "elasticsearch.stack_monitoring.ccr": { - "enabled": false - }, - "elasticsearch.stack_monitoring.cluster_stats": { - "enabled": false - }, - "elasticsearch.stack_monitoring.enrich": { - "enabled": false - }, - "elasticsearch.stack_monitoring.index": { - "enabled": false - }, - "elasticsearch.stack_monitoring.index_recovery": { - "enabled": false, - "vars": { - "active.only": true - } - }, - "elasticsearch.stack_monitoring.index_summary": { - "enabled": false - }, - "elasticsearch.stack_monitoring.ml_job": { - "enabled": false - }, - "elasticsearch.stack_monitoring.node": { - "enabled": false - }, - "elasticsearch.stack_monitoring.node_stats": { - "enabled": false - }, - "elasticsearch.stack_monitoring.pending_tasks": { - "enabled": false - }, - "elasticsearch.stack_monitoring.shard": { - "enabled": false - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json deleted file mode 100644 index c9e4183de..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "kratos-logs", - "namespace": "so", - "description": "Kratos logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/kratos/kratos.log" - ], - "data_stream.dataset": "kratos", - "tags": ["so-kratos"], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: iam\n module: kratos", - "custom": "pipeline: kratos" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json index d0281c111..b1454d4bd 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/osquery-grid-nodes.json @@ -3,7 +3,7 @@ "name": "osquery_manager", "version": "" }, - "name": "osquery-grid-nodes", + "name": "osquery-grid-nodes_heavy", "namespace": "default", "policy_id": "so-grid-nodes_heavy", "inputs": { diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json deleted file mode 100644 index cddcedfd8..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/redis-logs.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "package": { - "name": "redis", - "version": "" - }, - "name": "redis-logs", - "namespace": "default", - "description": "Redis logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "redis-logfile": { - "enabled": true, - "streams": { - "redis.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/redis/redis.log" - ], - "tags": [ - "redis-log" - ], - "preserve_original_event": false - } - } - } - }, - "redis-redis": { - "enabled": false, - "streams": { - "redis.slowlog": { - "enabled": false, - "vars": { - "hosts": [ - "127.0.0.1:6379" - ], - "password": "" - } - } - } - }, - "redis-redis/metrics": { - "enabled": false, - "vars": { - "hosts": [ - "127.0.0.1:6379" - ], - "idle_timeout": "20s", - "maxconn": 10, - "network": "tcp", - "password": "" - }, - "streams": { - "redis.info": { - "enabled": false, - "vars": { - "period": "10s" - } - }, - "redis.key": { - "enabled": false, - "vars": { - "key.patterns": "- limit: 20\n pattern: *\n", - "period": "10s" - } - }, - "redis.keyspace": { - "enabled": false, - "vars": { - "period": "10s" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json deleted file mode 100644 index 2004c8c5d..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "soc-auth-sync-logs", - "namespace": "so", - "description": "Security Onion - Elastic Auth Sync - Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/soc/sync.log" - ], - "data_stream.dataset": "soc", - "tags": ["so-soc"], - "processors": "- dissect:\n tokenizer: \"%{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: auth_sync", - "custom": "pipeline: common" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json deleted file mode 100644 index b1b6098c1..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "soc-salt-relay-logs", - "namespace": "so", - "description": "Security Onion - Salt Relay - Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/soc/salt-relay.log" - ], - "data_stream.dataset": "soc", - "tags": ["so-soc"], - "processors": "- dissect:\n tokenizer: \"%{soc.ts} | %{event.action}\"\n field: \"message\"\n target_prefix: \"\"\n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: salt_relay", - "custom": "pipeline: common" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json deleted file mode 100644 index 5954e5052..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "soc-sensoroni-logs", - "namespace": "so", - "description": "Security Onion - Sensoroni - Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/sensoroni/sensoroni.log" - ], - "data_stream.dataset": "soc", - "tags": [], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"sensoroni\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: sensoroni\n- rename:\n fields:\n - from: \"sensoroni.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"sensoroni.fields.status\"\n to: \"http.response.status_code\"\n - from: \"sensoroni.fields.method\"\n to: \"http.request.method\"\n - from: \"sensoroni.fields.path\"\n to: \"url.path\"\n - from: \"sensoroni.message\"\n to: \"event.action\"\n - from: \"sensoroni.level\"\n to: \"log.level\"\n ignore_missing: true", - "custom": "pipeline: common" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json deleted file mode 100644 index 89e26563a..000000000 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "package": { - "name": "log", - "version": "" - }, - "name": "soc-server-logs", - "namespace": "so", - "description": "Security Onion Console Logs", - "policy_id": "so-grid-nodes_heavy", - "inputs": { - "logs-logfile": { - "enabled": true, - "streams": { - "log.log": { - "enabled": true, - "vars": { - "paths": [ - "/opt/so/log/soc/sensoroni-server.log" - ], - "data_stream.dataset": "soc", - "tags": ["so-soc"], - "processors": "- decode_json_fields:\n fields: [\"message\"]\n target: \"soc\"\n process_array: true\n max_depth: 2\n add_error_key: true \n- add_fields:\n target: event\n fields:\n category: host\n module: soc\n dataset_temp: server\n- rename:\n fields:\n - from: \"soc.fields.sourceIp\"\n to: \"source.ip\"\n - from: \"soc.fields.status\"\n to: \"http.response.status_code\"\n - from: \"soc.fields.method\"\n to: \"http.request.method\"\n - from: \"soc.fields.path\"\n to: \"url.path\"\n - from: \"soc.message\"\n to: \"event.action\"\n - from: \"soc.level\"\n to: \"log.level\"\n ignore_missing: true", - "custom": "pipeline: common" - } - } - } - } - } -} diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json index 31d30d4e0..3df514f0b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/system-grid-nodes.json @@ -4,7 +4,7 @@ "name": "system", "version": "" }, - "name": "system-grid-nodes", + "name": "system-grid-nodes_heavy", "namespace": "default", "inputs": { "system-logfile": { From b6dd347eb8ba085b9452b705aa860fe88f89e8d0 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 31 Jul 2023 15:22:29 -0400 Subject: [PATCH 04/18] Heavy Node add manager --- salt/logstash/enabled.sls | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index a88e97b19..cd9d6dd7e 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -9,6 +9,11 @@ {% from 'docker/docker.map.jinja' import DOCKER %} {% from 'logstash/map.jinja' import LOGSTASH_MERGED %} {% from 'logstash/map.jinja' import REDIS_NODES %} +{# we append the manager here so that it is added to extra_hosts so the heavynode can resolve it #} +{# we cannont append in the logstash/map.jinja because then it would be added to the 0900_input_redis.conf #} +{% if GLOBALS.role == 'so-heavynode' %} +{% do REDIS_NODES.append({GLOBALS.manager:GLOBALS.manager_ip}) %} +{% endif %} {% set lsheap = LOGSTASH_MERGED.settings.lsheap %} include: From 6a55a8e5c08c0cbfd3f7fae2ec2c3fda12eece82 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 22:17:22 -0400 Subject: [PATCH 05/18] Elastic 8.2.2 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 9b69eb781..a2dedd324 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.1","id": "8.7.1","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.8.2","id": "8.8.2","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 44c926ba8d0672a6545fdf31f596a7e5797bf8a2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 22:18:07 -0400 Subject: [PATCH 06/18] Elastic 8.8.2 --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index e65955178..159a69e68 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.8.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 5dd5f9fc1caa8c613226faf801c6f7f83796eedc Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 22:18:43 -0400 Subject: [PATCH 07/18] Elastic 8.8.2 --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 53c8664d2..f9459587d 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -5,7 +5,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -ELASTIC_AGENT_TARBALL_VERSION="8.7.1" +ELASTIC_AGENT_TARBALL_VERSION="8.8.2" DEFAULT_SALT_DIR=/opt/so/saltstack/default DOC_BASE_URL="https://docs.securityonion.net/en/2.4" From 29b64eadd42306852873047bd883900b558ea958 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 1 Aug 2023 02:20:22 +0000 Subject: [PATCH 08/18] Change log.log to log.logs --- .../grid-nodes_general/import-zeek-logs.json | 2 +- .../integrations-dynamic/grid-nodes_general/zeek-logs.json | 2 +- .../files/integrations/grid-nodes_general/idh-logs.json | 2 +- .../files/integrations/grid-nodes_general/import-evtx-logs.json | 2 +- .../integrations/grid-nodes_general/import-suricata-logs.json | 2 +- .../files/integrations/grid-nodes_general/kratos-logs.json | 2 +- .../integrations/grid-nodes_general/soc-auth-sync-logs.json | 2 +- .../integrations/grid-nodes_general/soc-salt-relay-logs.json | 2 +- .../integrations/grid-nodes_general/soc-sensoroni-logs.json | 2 +- .../files/integrations/grid-nodes_general/soc-server-logs.json | 2 +- .../files/integrations/grid-nodes_general/strelka-logs.json | 2 +- .../files/integrations/grid-nodes_general/suricata-logs.json | 2 +- .../files/integrations/grid-nodes_heavy/kratos-logs.json | 2 +- .../files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json | 2 +- .../integrations/grid-nodes_heavy/soc-salt-relay-logs.json | 2 +- .../files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json | 2 +- .../files/integrations/grid-nodes_heavy/soc-server-logs.json | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json index 4c22f0446..0979f98b6 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json @@ -13,7 +13,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json index 2cec88bf2..32bff857b 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json @@ -14,7 +14,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json index 32055112a..29cc1a879 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json index d9f8daeb9..178b6ed53 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json @@ -12,7 +12,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json index f17ee33d1..3b8cffcc1 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json index c342b57bd..b1fb71077 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json index 84e9ae94d..3aa740881 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json index 07bd89b89..840f36f6b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json index bee14ebf5..60ee95f45 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json index 285d79148..b789adc1d 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json index 6f6beca99..089b5d4f8 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json index 7ff43c3a8..a9d857b24 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json index c9e4183de..684cfd59b 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/kratos-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json index 2004c8c5d..e031fe08c 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-auth-sync-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json index b1b6098c1..1c8399bca 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-salt-relay-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json index 5954e5052..a5e4b6217 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-sensoroni-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ diff --git a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json index 89e26563a..f36a00c37 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes_heavy/soc-server-logs.json @@ -11,7 +11,7 @@ "logs-logfile": { "enabled": true, "streams": { - "log.log": { + "log.logs": { "enabled": true, "vars": { "paths": [ From 48d9c14563fe44e2c28a978140a1944cfe73e1cc Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 1 Aug 2023 02:20:43 +0000 Subject: [PATCH 09/18] Enable log package by default --- salt/elasticfleet/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index 46d496955..3d806d63f 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -32,4 +32,5 @@ elasticfleet: - fim - github - google_workspace + - log - 1password From 9d59e4250f39b56023a87cd0c5d39fb5a67a9311 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 22:23:54 -0400 Subject: [PATCH 10/18] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 59aa62c1f..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.5 +2.4.0-foxtrot From f84b0a3219d3f2046f48138b39d310afaef4937a Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 23:16:46 -0400 Subject: [PATCH 11/18] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f..59aa62c1f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.5 From 527a6ba454e26f48bec1af1abd409019e1075f2d Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 31 Jul 2023 23:52:38 -0400 Subject: [PATCH 12/18] Use asterisk when searching 'msg' since it is now a keyword --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 53db2c838..cb7d400a0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1140,7 +1140,7 @@ soc: showSubtitle: true - name: SOC - Auth description: Users authenticated to SOC grouped by IP address and identity - query: 'event.dataset:kratos.audit AND msg:authenticated | groupby http_request.headers.x-real-ip identity_id' + query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby http_request.headers.x-real-ip identity_id' showSubtitle: true - name: SOC - App description: Logs generated by the Security Onion Console (SOC) server and modules @@ -1405,7 +1405,7 @@ soc: query: '* | groupby -sankey event.dataset event.category* | groupby -pie event.category | groupby -bar event.module* | groupby event.dataset | groupby event.module* | groupby event.category | groupby observer.name | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' - name: SOC Auth description: SOC (Security Onion Console) authentication logs - query: 'event.dataset:kratos.audit AND msg:authenticated | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent' + query: 'event.dataset:kratos.audit AND msg:*authenticated* | groupby -sankey http_request.headers.x-real-ip identity_id | groupby http_request.headers.x-real-ip | groupby identity_id | groupby http_request.headers.user-agent' - name: Elastalerts description: Elastalert logs query: '_index: "*:elastalert*" | groupby rule_name | groupby alert_info.type' From 2875a7a2e5163fae947e58e354154c8c64fa5366 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Aug 2023 09:48:44 -0400 Subject: [PATCH 13/18] Sensor NIC offload --- salt/sensor/files/99-so-checksum-offload-disable | 12 ++++++++++++ salt/sensor/init.sls | 11 +++++++++++ salt/top.sls | 5 +++++ 3 files changed, 28 insertions(+) create mode 100755 salt/sensor/files/99-so-checksum-offload-disable create mode 100644 salt/sensor/init.sls diff --git a/salt/sensor/files/99-so-checksum-offload-disable b/salt/sensor/files/99-so-checksum-offload-disable new file mode 100755 index 000000000..fdce54f5e --- /dev/null +++ b/salt/sensor/files/99-so-checksum-offload-disable @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + + + +. /usr/sbin/so-common + +init_monitor $MNIC diff --git a/salt/sensor/init.sls b/salt/sensor/init.sls new file mode 100644 index 000000000..34133e488 --- /dev/null +++ b/salt/sensor/init.sls @@ -0,0 +1,11 @@ +offload_script: + file.managed: + - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable + - source: salt://sensor/files/99-so-checksum-offload-disable + - mode: 755 + +execute_checksum: + cmd.run: + - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable + - onchanges: + - file: offload_script \ No newline at end of file diff --git a/salt/top.sls b/salt/top.sls index e53895324..bc51c2db1 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -36,6 +36,7 @@ base: '*_sensor and G@saltversion:{{saltversion}}': - match: compound + - sensor - ssl - sensoroni - telegraf @@ -52,6 +53,7 @@ base: '*_eval and G@saltversion:{{saltversion}}': - match: compound - salt.master + - sensor - ca - ssl - registry @@ -118,6 +120,7 @@ base: '*_standalone and G@saltversion:{{saltversion}}': - match: compound - salt.master + - sensor - ca - ssl - registry @@ -196,6 +199,7 @@ base: '*_heavynode and G@saltversion:{{saltversion}}': - match: compound + - sensor - ssl - sensoroni - nginx @@ -216,6 +220,7 @@ base: '*_import and G@saltversion:{{saltversion}}': - match: compound - salt.master + - sensor - ca - ssl - registry From 87a5d20ac968f811338556d71d66edcf066eb9dd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Aug 2023 10:03:59 -0400 Subject: [PATCH 14/18] Sensor NIC offload --- salt/sensor/files/99-so-checksum-offload-disable | 4 +++- salt/sensor/init.sls | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/sensor/files/99-so-checksum-offload-disable b/salt/sensor/files/99-so-checksum-offload-disable index fdce54f5e..72f7838db 100755 --- a/salt/sensor/files/99-so-checksum-offload-disable +++ b/salt/sensor/files/99-so-checksum-offload-disable @@ -9,4 +9,6 @@ . /usr/sbin/so-common -init_monitor $MNIC +{% set MNIC = salt['pillar.get']('sensor:interface') %} + +init_monitor {{ MNIC }} diff --git a/salt/sensor/init.sls b/salt/sensor/init.sls index 34133e488..53cd808c6 100644 --- a/salt/sensor/init.sls +++ b/salt/sensor/init.sls @@ -3,6 +3,7 @@ offload_script: - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable - source: salt://sensor/files/99-so-checksum-offload-disable - mode: 755 + - template: jinja execute_checksum: cmd.run: From b6579d7d45474c229316cfa1653bc86565e3e725 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Aug 2023 10:13:44 -0400 Subject: [PATCH 15/18] Sensor NIC offload --- salt/common/tools/sbin/so-common | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index f9459587d..0581c09c6 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -225,12 +225,15 @@ init_monitor() { if [[ $MONITORNIC == "bond0" ]]; then BIFACES=$(lookup_bond_interfaces) + for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload; do + ethtool -K "$MONITORNIC" "$i" off; + done else BIFACES=$MONITORNIC fi for DEVICE_IFACE in $BIFACES; do - for i in rx tx sg tso ufo gso gro lro; do + for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload; do ethtool -K "$DEVICE_IFACE" "$i" off; done ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on From 4adaddf13f2e5b42dc16362d4bc24726277ad5bf Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 1 Aug 2023 10:14:59 -0400 Subject: [PATCH 16/18] Move syslog to the INPUT chain where needed --- salt/firewall/defaults.yaml | 79 +++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/salt/firewall/defaults.yaml b/salt/firewall/defaults.yaml index 20b966e48..3095c052e 100644 --- a/salt/firewall/defaults.yaml +++ b/salt/firewall/defaults.yaml @@ -198,9 +198,6 @@ firewall: portgroups: - redis - elasticsearch_node - self: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -218,9 +215,6 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - syslog: - portgroups: - - syslog analyst: portgroups: - nginx @@ -255,6 +249,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -425,12 +425,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - self: - portgroups: - - syslog - syslog: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -497,6 +491,12 @@ firewall: receiver: portgroups: - salt_manager + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -588,9 +588,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - self: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -608,9 +605,6 @@ firewall: endgame: portgroups: - endgame - syslog: - portgroups: - - syslog analyst: portgroups: - nginx @@ -660,6 +654,12 @@ firewall: receiver: portgroups: - salt_manager + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -760,9 +760,6 @@ firewall: - elastic_agent_control - elastic_agent_data - elastic_agent_update - self: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -783,9 +780,6 @@ firewall: strelka_frontend: portgroups: - strelka_frontend - syslog: - portgroups: - - syslog analyst: portgroups: - nginx @@ -838,6 +832,12 @@ firewall: receiver: portgroups: - salt_manager + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -884,9 +884,6 @@ firewall: searchnode: portgroups: - elasticsearch_node - self: - portgroups: - - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -918,6 +915,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -942,9 +945,6 @@ firewall: chain: DOCKER-USER: hostgroups: - self: - portgroups: - - syslog strelka_frontend: portgroups: - strelka_frontend @@ -979,6 +979,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -1030,6 +1036,9 @@ firewall: strelka_frontend: portgroups: - strelka_frontend + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: @@ -1189,11 +1198,7 @@ firewall: self: portgroups: - redis - - syslog - beats_5644 - syslog: - portgroups: - - syslog beats_endpoint: portgroups: - beats_5044 @@ -1234,6 +1239,12 @@ firewall: localhost: portgroups: - all + self: + portgroups: + - syslog + syslog: + portgroups: + - syslog customhostgroup0: portgroups: [] customhostgroup1: From f35f42c83d58c4aabee7ca7c7a48a8d16b344c97 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 1 Aug 2023 10:23:45 -0400 Subject: [PATCH 17/18] Sensor NIC offload --- salt/common/tools/sbin/so-common | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 0581c09c6..3c79110b3 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -225,7 +225,7 @@ init_monitor() { if [[ $MONITORNIC == "bond0" ]]; then BIFACES=$(lookup_bond_interfaces) - for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload; do + for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do ethtool -K "$MONITORNIC" "$i" off; done else @@ -233,7 +233,7 @@ init_monitor() { fi for DEVICE_IFACE in $BIFACES; do - for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload; do + for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do ethtool -K "$DEVICE_IFACE" "$i" off; done ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on From 3fa0a98830682de91c80c0eaa862bfd0fa5516a1 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 1 Aug 2023 12:45:09 -0400 Subject: [PATCH 18/18] Update verbiage and links in soc_sensor.yaml --- salt/sensor/soc_sensor.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/sensor/soc_sensor.yaml b/salt/sensor/soc_sensor.yaml index 0774e9bcf..9ab0c236e 100644 --- a/salt/sensor/soc_sensor.yaml +++ b/salt/sensor/soc_sensor.yaml @@ -1,9 +1,9 @@ sensor: interface: description: Main sensor monitoring interface. - helpLink: sensor.html + helpLink: network.html readonly: True mtu: - description: Main IP address of the grid host. - helpLink: host.html + description: Maximum Transmission Unit (MTU) of the sensor monitoring interface. + helpLink: network.html readonly: True