From b5f656ae58d2ea2580238211f278262d82e65a3f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 23 May 2024 13:22:22 -0400 Subject: [PATCH 1/8] dont render pillar each time so-tcpreplay runs --- salt/common/tools/{sbin => sbin_jinja}/so-tcpreplay | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename salt/common/tools/{sbin => sbin_jinja}/so-tcpreplay (92%) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin_jinja/so-tcpreplay similarity index 92% rename from salt/common/tools/sbin/so-tcpreplay rename to salt/common/tools/sbin_jinja/so-tcpreplay index 99314c289..6f3f02983 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin_jinja/so-tcpreplay @@ -10,7 +10,7 @@ . /usr/sbin/so-common . /usr/sbin/so-image-common -REPLAYIFACE=${REPLAYIFACE:-$(lookup_pillar interface sensor)} +REPLAYIFACE=${REPLAYIFACE:-"{{pillar.sensor.interface}}"} REPLAYSPEED=${REPLAYSPEED:-10} mkdir -p /opt/so/samples @@ -57,8 +57,8 @@ if ! docker ps | grep -q so-tcpreplay; then fi if is_sensor_node; then - echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface ${REPLAYIFACE}..." - docker exec so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i ${REPLAYIFACE} -M${REPLAYSPEED} $@" + echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface $REPLAYIFACE..." + docker exec so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i $REPLAYIFACE -M${REPLAYSPEED} $@" echo "Replay completed. Warnings shown above are typically expected." elif is_manager_node; then From f396247838bd3c3cf8f3f0c49d20b500a5f4551b Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 31 May 2024 17:46:19 +0000 Subject: [PATCH 2/8] Add index templates and lifecycle policies --- salt/elasticsearch/defaults.yaml | 72 ++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 6ecdc96a1..36d673d70 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -170,6 +170,78 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-items: + index_sorting: false + index_template: + composed_of: + - so-items-mappings + index_patterns: + - .items-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-items-logs + rollover_alias: ".items-default" + routing: + allocation: + include: + _tier_preference: "data_content" + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms + so-lists: + index_sorting: false + index_template: + composed_of: + - so-lists-mappings + index_patterns: + - .lists-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-lists-logs + rollover_alias: ".lists-default" + routing: + allocation: + include: + _tier_preference: "data_content" + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms so-case: index_sorting: false index_template: From a8c231ad8c59a09de4c134b6068d3f38bebe90d5 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 31 May 2024 17:47:01 +0000 Subject: [PATCH 3/8] Add component templates --- .../elastic-agent/so-items-mappings.json | 112 ++++++++++++++++++ .../elastic-agent/so-lists-mappings.json | 55 +++++++++ 2 files changed, 167 insertions(+) create mode 100644 salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json create mode 100644 salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json b/salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json new file mode 100644 index 000000000..85e6c1984 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/so-items-mappings.json @@ -0,0 +1,112 @@ +{ + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "binary": { + "type": "binary" + }, + "boolean": { + "type": "boolean" + }, + "byte": { + "type": "byte" + }, + "created_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, + "date": { + "type": "date" + }, + "date_nanos": { + "type": "date_nanos" + }, + "date_range": { + "type": "date_range" + }, + "deserializer": { + "type": "keyword" + }, + "double": { + "type": "double" + }, + "double_range": { + "type": "double_range" + }, + "float": { + "type": "float" + }, + "float_range": { + "type": "float_range" + }, + "geo_point": { + "type": "geo_point" + }, + "geo_shape": { + "type": "geo_shape" + }, + "half_float": { + "type": "half_float" + }, + "integer": { + "type": "integer" + }, + "integer_range": { + "type": "integer_range" + }, + "ip": { + "type": "ip" + }, + "ip_range": { + "type": "ip_range" + }, + "keyword": { + "type": "keyword" + }, + "list_id": { + "type": "keyword" + }, + "long": { + "type": "long" + }, + "long_range": { + "type": "long_range" + }, + "meta": { + "type": "object", + "enabled": false + }, + "serializer": { + "type": "keyword" + }, + "shape": { + "type": "shape" + }, + "short": { + "type": "short" + }, + "text": { + "type": "text" + }, + "tie_breaker_id": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "type": "keyword" + } + } + }, + "aliases": {} + }, + "version": 2, + "_meta": { + "managed": true, + "description": "default mappings for the .items index template installed by Kibana/Security" + } +} diff --git a/salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json b/salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json new file mode 100644 index 000000000..b2b5fda23 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/so-lists-mappings.json @@ -0,0 +1,55 @@ +{ + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "created_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "deserializer": { + "type": "keyword" + }, + "immutable": { + "type": "boolean" + }, + "meta": { + "type": "object", + "enabled": false + }, + "name": { + "type": "keyword" + }, + "serializer": { + "type": "keyword" + }, + "tie_breaker_id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "aliases": {} + }, + "version": 2, + "_meta": { + "managed": true, + "description": "default mappings for the .lists index template installed by Kibana/Security" + } +} From c88b731793045bd957b28e4d519cab0cebfc2f34 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 3 Jun 2024 15:27:08 -0400 Subject: [PATCH 4/8] revert to 3006.6 --- salt/salt/master.defaults.yaml | 2 +- salt/salt/minion.defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/master.defaults.yaml b/salt/salt/master.defaults.yaml index 24ba29d98..19677f70b 100644 --- a/salt/salt/master.defaults.yaml +++ b/salt/salt/master.defaults.yaml @@ -1,4 +1,4 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: master: - version: 3006.8 + version: 3006.6 diff --git a/salt/salt/minion.defaults.yaml b/salt/salt/minion.defaults.yaml index dddd6683b..2e4ebc93e 100644 --- a/salt/salt/minion.defaults.yaml +++ b/salt/salt/minion.defaults.yaml @@ -1,6 +1,6 @@ # version cannot be used elsewhere in this pillar as soup is grepping for it to determine if Salt needs to be patched salt: minion: - version: 3006.8 + version: 3006.6 check_threshold: 3600 # in seconds, threshold used for so-salt-minion-check. any value less than 600 seconds may cause a lot of salt-minion restarts since the job to touch the file occurs every 5-8 minutes by default service_start_delay: 30 # in seconds. From c0b2cf73883954d8e3feff579d7f19d2446c2120 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 Jun 2024 10:28:21 -0400 Subject: [PATCH 5/8] add the curlys --- salt/common/tools/sbin_jinja/so-tcpreplay | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin_jinja/so-tcpreplay b/salt/common/tools/sbin_jinja/so-tcpreplay index 6f3f02983..a9551c0fa 100755 --- a/salt/common/tools/sbin_jinja/so-tcpreplay +++ b/salt/common/tools/sbin_jinja/so-tcpreplay @@ -57,8 +57,8 @@ if ! docker ps | grep -q so-tcpreplay; then fi if is_sensor_node; then - echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface $REPLAYIFACE..." - docker exec so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i $REPLAYIFACE -M${REPLAYSPEED} $@" + echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface ${REPLAYIFACE}..." + docker exec so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i ${REPLAYIFACE} -M${REPLAYSPEED} $@" echo "Replay completed. Warnings shown above are typically expected." elif is_manager_node; then From a2467d0418532cef20e9c13724b13ee1e4e8618f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 Jun 2024 08:24:57 -0400 Subject: [PATCH 6/8] move so-tcpreplay to sensor state --- salt/sensor/init.sls | 19 ++++++++++++++++++- .../tools/sbin_jinja/so-tcpreplay | 0 2 files changed, 18 insertions(+), 1 deletion(-) rename salt/{common => sensor}/tools/sbin_jinja/so-tcpreplay (100%) diff --git a/salt/sensor/init.sls b/salt/sensor/init.sls index 53cd808c6..ca1cf13c2 100644 --- a/salt/sensor/init.sls +++ b/salt/sensor/init.sls @@ -9,4 +9,21 @@ execute_checksum: cmd.run: - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable - onchanges: - - file: offload_script \ No newline at end of file + - file: offload_script + +sensor_sbin: + file.recurse: + - name: /usr/sbin + - source: salt://sensor/tools/sbin + - user: 939 + - group: 939 + - file_mode: 755 + +sensor_sbin_jinja: + file.recurse: + - name: /usr/sbin + - source: salt://sensor/tools/sbin_jinja + - user: 939 + - group: 939 + - file_mode: 755 + - template: jinja diff --git a/salt/common/tools/sbin_jinja/so-tcpreplay b/salt/sensor/tools/sbin_jinja/so-tcpreplay similarity index 100% rename from salt/common/tools/sbin_jinja/so-tcpreplay rename to salt/sensor/tools/sbin_jinja/so-tcpreplay From ff5773c8379d140cb6a239e89f4dd48672b15f7e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 Jun 2024 08:56:32 -0400 Subject: [PATCH 7/8] move so-tcpreplay back to common. return empty string if no sensor.interface pillar --- .../tools/sbin_jinja/so-tcpreplay | 2 +- salt/sensor/init.sls | 18 +----------------- 2 files changed, 2 insertions(+), 18 deletions(-) rename salt/{sensor => common}/tools/sbin_jinja/so-tcpreplay (96%) diff --git a/salt/sensor/tools/sbin_jinja/so-tcpreplay b/salt/common/tools/sbin_jinja/so-tcpreplay similarity index 96% rename from salt/sensor/tools/sbin_jinja/so-tcpreplay rename to salt/common/tools/sbin_jinja/so-tcpreplay index a9551c0fa..969ca699f 100755 --- a/salt/sensor/tools/sbin_jinja/so-tcpreplay +++ b/salt/common/tools/sbin_jinja/so-tcpreplay @@ -10,7 +10,7 @@ . /usr/sbin/so-common . /usr/sbin/so-image-common -REPLAYIFACE=${REPLAYIFACE:-"{{pillar.sensor.interface}}"} +REPLAYIFACE=${REPLAYIFACE:-"{{salt['pillar.get']('sensor:interface', '')}}"} REPLAYSPEED=${REPLAYSPEED:-10} mkdir -p /opt/so/samples diff --git a/salt/sensor/init.sls b/salt/sensor/init.sls index ca1cf13c2..c9c6a6db5 100644 --- a/salt/sensor/init.sls +++ b/salt/sensor/init.sls @@ -10,20 +10,4 @@ execute_checksum: - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable - onchanges: - file: offload_script - -sensor_sbin: - file.recurse: - - name: /usr/sbin - - source: salt://sensor/tools/sbin - - user: 939 - - group: 939 - - file_mode: 755 - -sensor_sbin_jinja: - file.recurse: - - name: /usr/sbin - - source: salt://sensor/tools/sbin_jinja - - user: 939 - - group: 939 - - file_mode: 755 - - template: jinja + \ No newline at end of file From f6a8a21f94715786f6b645b6342de76a836d498a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 Jun 2024 08:58:46 -0400 Subject: [PATCH 8/8] remove space --- salt/sensor/init.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/sensor/init.sls b/salt/sensor/init.sls index c9c6a6db5..730a7c7ad 100644 --- a/salt/sensor/init.sls +++ b/salt/sensor/init.sls @@ -10,4 +10,3 @@ execute_checksum: - name: /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable - onchanges: - file: offload_script - \ No newline at end of file