Compare commits

..
14 changed files with 308 additions and 122 deletions

No files matched your search

-1
View File
@@ -103,7 +103,6 @@ docker:
- 80:80 - 80:80
- 443:443 - 443:443
- 8443:8443 - 8443:8443
- 7788:7788
- 7789:7789 - 7789:7789
custom_bind_mounts: [] custom_bind_mounts: []
extra_hosts: [] extra_hosts: []
-10
View File
@@ -118,10 +118,6 @@ firewall:
tcp: tcp:
- 5601 - 5601
udp: [] udp: []
localrules:
tcp:
- 7788
udp: []
nginx: nginx:
tcp: tcp:
- 80 - 80
@@ -200,7 +196,6 @@ firewall:
- postgres - postgres
- elasticsearch_rest - elasticsearch_rest
- elasticsearch_node - elasticsearch_node
- localrules
sensor: sensor:
portgroups: portgroups:
- beats_5044 - beats_5044
@@ -391,7 +386,6 @@ firewall:
- elastic_agent_control - elastic_agent_control
- elastic_agent_data - elastic_agent_data
- elastic_agent_update - elastic_agent_update
- localrules
- sensoroni - sensoroni
fleet: fleet:
portgroups: portgroups:
@@ -611,7 +605,6 @@ firewall:
- elastic_agent_control - elastic_agent_control
- elastic_agent_data - elastic_agent_data
- elastic_agent_update - elastic_agent_update
- localrules
- sensoroni - sensoroni
fleet: fleet:
portgroups: portgroups:
@@ -829,7 +822,6 @@ firewall:
- elastic_agent_control - elastic_agent_control
- elastic_agent_data - elastic_agent_data
- elastic_agent_update - elastic_agent_update
- localrules
- sensoroni - sensoroni
fleet: fleet:
portgroups: portgroups:
@@ -1058,7 +1050,6 @@ firewall:
- elastic_agent_update - elastic_agent_update
- endgame - endgame
- strelka_frontend - strelka_frontend
- localrules
fleet: fleet:
portgroups: portgroups:
- elasticsearch_rest - elasticsearch_rest
@@ -1518,7 +1509,6 @@ firewall:
- elasticsearch_rest - elasticsearch_rest
- elasticsearch_node - elasticsearch_node
- elastic_agent_control - elastic_agent_control
- localrules
sensor: sensor:
portgroups: portgroups:
- beats_5044 - beats_5044
-3
View File
@@ -138,9 +138,6 @@ firewall:
kibana: kibana:
tcp: *tcpsettings tcp: *tcpsettings
udp: *udpsettings udp: *udpsettings
localrules:
tcp: *tcpsettings
udp: *udpsettings
nginx: nginx:
tcp: *tcpsettings tcp: *tcpsettings
udp: *udpsettings udp: *udpsettings
+29
View File
@@ -20,6 +20,7 @@ def showUsage(args):
print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr) print(' appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr)
print(' removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr) print(' removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' purgelistitem - Remove every occurrence of a list item from all lists nested under a yaml key, if it exists. Requires KEY and LISTITEM args.', file=sys.stderr)
print(' replacelistobject - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr) print(' replacelistobject - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr) print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
print(' get [-r] - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr) print(' get [-r] - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
@@ -199,6 +200,33 @@ def removelistitem(args):
return 0 return 0
def purgeListItem(content, listItem):
if isinstance(content, dict):
for value in content.values():
purgeListItem(value, listItem)
elif isinstance(content, list):
content[:] = [item for item in content if item != listItem]
for item in content:
purgeListItem(item, listItem)
def purgelistitem(args):
if len(args) != 3:
print('Missing filename, key arg, or list item to purge', file=sys.stderr)
showUsage(None)
return 1
filename = args[0]
key = args[1]
listItem = args[2]
content = loadYaml(filename)
purgeListItem(getKeyValue(content, key), convertType(listItem))
writeYaml(filename, content)
return 0
def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject): def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject):
pieces = key.split(".", 1) pieces = key.split(".", 1)
if len(pieces) > 1: if len(pieces) > 1:
@@ -385,6 +413,7 @@ def main():
"append": append, "append": append,
"appendlistobject": appendlistobject, "appendlistobject": appendlistobject,
"removelistitem": removelistitem, "removelistitem": removelistitem,
"purgelistitem": purgelistitem,
"replacelistobject": replacelistobject, "replacelistobject": replacelistobject,
"get": get, "get": get,
"remove": remove, "remove": remove,
+217
View File
@@ -709,6 +709,223 @@ class TestRemoveListItem(unittest.TestCase):
self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue()) self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
class TestPurgeListItem(unittest.TestCase):
def test_purgelistitem_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
sys.argv = ["cmd", "help"]
soyaml.purgelistitem(["file", "key"])
sysmock.assert_called()
self.assertIn("Missing filename, key arg, or list item to purge", mock_stderr.getvalue())
def test_purgelistitem(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [a,b,c], key2: false}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- c\nkey2: false\n"
self.assertEqual(actual, expected)
def test_purgelistitem_nested_lists(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c], child2: { deep1: [b,d], deep2: 45 } }, key2: [b,e]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - c\n child2:\n deep1:\n - d\n deep2: 45\nkey2:\n- b\n- e\n"
self.assertEqual(actual, expected)
def test_purgelistitem_duplicates(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [b,a,b,c,b]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_list_of_objects(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [{name: item1, values: [a,b]}, b, {name: item2, values: [b,c]}]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- name: item1\n values:\n - a\n- name: item2\n values:\n - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_typed(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [1, '1', 2]}")
file.close()
soyaml.purgelistitem([filename, "key1", "1"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- '1'\n- 2\n"
self.assertEqual(actual, expected)
def test_purgelistitem_item_not_in_list(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c] }}")
file.close()
soyaml.purgelistitem([filename, "key1", "d"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - b\n - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_key_noexist(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [a,b,c]}")
file.close()
result = soyaml.purgelistitem([filename, "key2.child1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- b\n- c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_key_noexist_deep(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c] }}")
file.close()
result = soyaml.purgelistitem([filename, "key1.child2.deep1", "b"])
self.assertEqual(result, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - b\n - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_scalar_and_null_values(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: b, child2: null, child3: [b] }}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1: b\n child2: null\n child3: []\n"
self.assertEqual(actual, expected)
def test_purgelistitem_lists_of_lists(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [[a,b], [b,[b,c]], b]}")
file.close()
soyaml.purgelistitem([filename, "key1", "b"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- - a\n- - - c\n"
self.assertEqual(actual, expected)
def test_purgelistitem_file_object(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: [{name: x}, a, {name: y}, {name: x}]}")
file.close()
itemfile = "/tmp/so-yaml_test-purgelistitem-item.yaml"
file = open(itemfile, "w")
file.write("name: x\n")
file.close()
soyaml.purgelistitem([filename, "key1", "file:" + itemfile])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n- a\n- name: y\n"
self.assertEqual(actual, expected)
def test_purgelistitem_main(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: [a,b,c] }, key2: [b]}")
file.close()
with patch('sys.exit', new=MagicMock()) as sysmock:
sys.argv = ["cmd", "purgelistitem", filename, "key1", "b"]
soyaml.main()
sysmock.assert_called_once_with(0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child1:\n - a\n - c\nkey2:\n- b\n"
self.assertEqual(actual, expected)
def test_purgelistitem_with_remove(self):
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
file = open(filename, "w")
file.write("{key1: { child1: { deep1: [a] }, child2: [b] }, key2: { group1: { items: [x,child1,y] }, group2: { items: [child1] } }}")
file.close()
soyaml.remove([filename, "key1.child1"])
soyaml.purgelistitem([filename, "key2", "child1"])
file = open(filename, "r")
actual = file.read()
file.close()
expected = "key1:\n child2:\n - b\nkey2:\n group1:\n items:\n - x\n - y\n group2:\n items: []\n"
self.assertEqual(actual, expected)
class TestAppendListObject(unittest.TestCase): class TestAppendListObject(unittest.TestCase):
def test_appendlistobject_missing_arg(self): def test_appendlistobject_missing_arg(self):
+20
View File
@@ -527,6 +527,7 @@ preupgrade_changes() {
[[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0 [[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
[[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0 [[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
[[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0 [[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
[[ "$INSTALLEDVERSION" == "3.4.0" ]] && up_to_3.5.0
true true
} }
@@ -1202,6 +1203,25 @@ add_detection_runtime_fields() {
} }
### 3.4.0 End ### ### 3.4.0 End ###
### 3.5.0 Scripts ###
up_to_3.5.0() {
remove_localrules_portgroup
INSTALLEDVERSION=3.5.0
}
remove_localrules_portgroup() {
local pillar_file
for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do
grep -qs localrules "$pillar_file" || continue
echo "Removing the localrules portgroup from $pillar_file."
so-yaml.py remove "$pillar_file" firewall.portgroups.localrules
so-yaml.py purgelistitem "$pillar_file" firewall.role localrules
done
}
### 3.5.0 End ###
repo_sync() { repo_sync() {
echo "Sync the local repo." echo "Sync the local repo."
+2 -3
View File
@@ -54,9 +54,9 @@ so-nginx:
- /opt/so/log/nginx/:/var/log/nginx:rw - /opt/so/log/nginx/:/var/log/nginx:rw
- /opt/so/tmp/nginx/:/var/lib/nginx:rw - /opt/so/tmp/nginx/:/var/lib/nginx:rw
- /opt/so/tmp/nginx/:/run:rw - /opt/so/tmp/nginx/:/run:rw
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages - /nsm/elastic-fleet/artifacts/:/opt/socore/elastic-artifacts/artifacts:ro
- /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts
{% if GLOBALS.is_manager %} {% if GLOBALS.is_manager %}
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
- /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro - /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
- /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro - /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
# ATT&CK Navigator binds # ATT&CK Navigator binds
@@ -64,7 +64,6 @@ so-nginx:
- /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
- /nsm/repo:/opt/socore/html/repo:ro - /nsm/repo:/opt/socore/html/repo:ro
- /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
- /nsm/rules:/nsm/rules:ro
{% if NGINXMERGED.external_suricata %} {% if NGINXMERGED.external_suricata %}
- /opt/so/rules/nids/suri:/surirules:ro - /opt/so/rules/nids/suri:/surirules:ro
{% endif %} {% endif %}
+15 -49
View File
@@ -4,6 +4,8 @@
{%- set role = grains.id.split('_') | last %} {%- set role = grains.id.split('_') | last %}
{%- set influxpass = salt['pillar.get']('secrets:influx_pass') %} {%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
{%- set influxauth = ('so:' + influxpass) | base64_encode %} {%- set influxauth = ('so:' + influxpass) | base64_encode %}
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
{%- set full_server_name = all_names | unique | join(' ') %}
worker_processes auto; worker_processes auto;
error_log /var/log/nginx/error.log; error_log /var/log/nginx/error.log;
@@ -39,52 +41,31 @@ http {
include /etc/nginx/conf.d/*.conf; include /etc/nginx/conf.d/*.conf;
{%- if role in ['fleet'] %}
server {
listen 8443;
server_name {{ GLOBALS.hostname }};
root /opt/socore/html;
location /artifacts/ {
try_files $uri =206;
proxy_read_timeout 90;
proxy_connect_timeout 90;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Proxy "";
proxy_set_header X-Forwarded-Proto $scheme;
}
}
{%- endif %}
{%- if GLOBALS.is_manager %} {%- if GLOBALS.is_manager %}
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
{%- set full_server_name = all_names | unique | join(' ') %}
server { server {
listen 80 default_server; listen 80 default_server;
server_name _; server_name _;
return 307 https://{{ GLOBALS.url_base }}$request_uri; return 307 https://{{ GLOBALS.url_base }}$request_uri;
} }
{%- endif %}
{%- if GLOBALS.is_manager or role == 'fleet' %}
server { server {
listen 8443; listen 8443;
{%- if GLOBALS.is_manager %}
server_name {{ full_server_name }}; server_name {{ full_server_name }};
root /opt/socore/html; {%- else %}
server_name {{ GLOBALS.hostname }};
{%- endif %}
root /opt/socore/elastic-artifacts;
location /artifacts/ { location /artifacts/ {
try_files $uri =206; try_files $uri =404;
proxy_read_timeout 90; }
proxy_connect_timeout 90; location / {
proxy_set_header Host $host; return 404;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Proxy "";
proxy_set_header X-Forwarded-Proto $scheme;
} }
} }
{%- endif %}
{%- if GLOBALS.is_manager %}
server { server {
listen 443 ssl default_server; listen 443 ssl default_server;
http2 on; http2 on;
@@ -111,21 +92,6 @@ http {
{%- endif %} {%- endif %}
{%- if GLOBALS.is_manager %} {%- if GLOBALS.is_manager %}
server {
listen 7788;
server_name {{ full_server_name }};
root /nsm/rules;
location / {
allow all;
sendfile on;
sendfile_max_chunk 1m;
autoindex on;
autoindex_exact_size off;
autoindex_format html;
autoindex_localtime on;
}
}
{%- if NGINXMERGED.external_suricata %} {%- if NGINXMERGED.external_suricata %}
server { server {
listen 7789 ssl; listen 7789 ssl;
+7 -25
View File
@@ -138,10 +138,6 @@
# AGGREGATOR PLUGINS # # AGGREGATOR PLUGINS #
############################################################################### ###############################################################################
[[aggregators.merge]]
drop_original = true
namepass = ["system", "swap", "kafka_partition"]
############################################################################### ###############################################################################
# INPUT PLUGINS # # INPUT PLUGINS #
############################################################################### ###############################################################################
@@ -293,7 +289,7 @@
password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}" password = "{{ salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass') }}"
{%- endif %} {%- endif %}
{% if grains.role in ['so-manager','so-managerhype','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%} {% if grains.role in ['so-manager','so-managersearch','so-standalone','so-receiver'] and GLOBALS.pipeline == "KAFKA" -%}
[[inputs.jolokia2_agent]] [[inputs.jolokia2_agent]]
name_prefix= "kafka_" name_prefix= "kafka_"
urls = ["http://{{ NODEIP }}:8778/jolokia"] urls = ["http://{{ NODEIP }}:8778/jolokia"]
@@ -350,26 +346,6 @@
interval = "60s" interval = "60s"
{%- endif %} {%- endif %}
{#- These report state that changes slowly; collecting them every interval only stores duplicate rows #}
{%- set SLOWSCRIPTS = [] %}
{%- for script in ['agentstatus.sh', 'os.sh', 'features.sh', 'raid.sh', 'zeekloss.sh'] %}
{%- if script in TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] %}
{%- do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove(script) %}
{%- do SLOWSCRIPTS.append(script) %}
{%- endif %}
{%- endfor %}
{%- if SLOWSCRIPTS | length > 0 %}
[[inputs.exec]]
commands = [
{%- for script in SLOWSCRIPTS %}
["/scripts/{{script}}"]{% if not loop.last %},{% endif %}
{%- endfor %}
]
data_format = "influx"
timeout = "15s"
interval = "5m"
{%- endif %}
{%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %} {%- if TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]] | length > 0 %}
[[inputs.exec]] [[inputs.exec]]
commands = [ commands = [
@@ -382,6 +358,12 @@
timeout = "15s" timeout = "15s"
{%- endif %} {%- endif %}
{%- if salt['pillar.get']('healthcheck:enabled', False) %}
[[inputs.file]]
files = ["/host/nsm/zeek/logs/zeek_restart.log"]
data_format = "influx"
{%- endif %}
[[inputs.file]] [[inputs.file]]
files = ["/etc/telegraf/node_config.json"] files = ["/etc/telegraf/node_config.json"]
name_override = "node_config" name_override = "node_config"
+1 -1
View File
@@ -18,7 +18,7 @@
{% if GLOBALS.pipeline != 'REDIS' %} {% if GLOBALS.pipeline != 'REDIS' %}
{# When global pipeline is not REDIS remove redis.sh script. KAFKA metrics are collected via jolokia agent. Config in telegraf.conf #} {# When global pipeline is not REDIS remove redis.sh script. KAFKA metrics are collected via jolokia agent. Config in telegraf.conf #}
{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-heavynode'] %} {% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-receiver', 'so-heavynode'] %}
{% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('redis.sh') %} {% do TELEGRAFMERGED.scripts[GLOBALS.role.split('-')[1]].remove('redis.sh') %}
{% endif %} {% endif %}
{% endif %} {% endif %}
+1 -1
View File
@@ -10,7 +10,7 @@
# if this script isn't already running # if this script isn't already running
if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
FILES=$(find /host/nsm/strelka/unprocessed -maxdepth 1 -type f | wc -l) FILES=$(ls -1x /host/nsm/strelka/unprocessed | wc -l)
echo "faffiles files=$FILES" echo "faffiles files=$FILES"
fi fi
+2 -3
View File
@@ -12,9 +12,8 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
CONTAINERSLOG=/var/log/somon/containers.log CONTAINERSLOG=/var/log/somon/containers.log
# the collector rewrites this every minute; report nothing rather than repeating a stale # the collector rewrites this every minute; report nothing rather than repeating a stale
# file as if it were current, in case a run was skipped or the collector is wedged. # file as if it were current, in case a run was skipped or the collector is wedged
# Kept under two collector cycles so a 60s telegraf interval can't emit the same snapshot twice. MAXAGE=150
MAXAGE=90
if [ -r "$CONTAINERSLOG" ]; then if [ -r "$CONTAINERSLOG" ]; then
AGE=$(( $(date +%s) - $(stat -c %Y "$CONTAINERSLOG") )) AGE=$(( $(date +%s) - $(stat -c %Y "$CONTAINERSLOG") ))
+12 -25
View File
@@ -11,40 +11,27 @@
if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
PREVCOUNTFILE='/tmp/eps.txt' PREVCOUNTFILE='/tmp/eps.txt'
STATS="$(curl -s localhost:9600/_node/stats)" EVENTCOUNTCURRENT="$(curl -s localhost:9600/_node/stats | jq '.events.in')"
if echo "$STATS" | jq -e '.pipelines.search' > /dev/null 2>&1; then if [ ! -z "$EVENTCOUNTCURRENT" ]; then
STAGE=search
COUNT="$(echo "$STATS" | jq '.pipelines.search.events.out')" if [ -f "$PREVCOUNTFILE" ]; then
elif echo "$STATS" | jq -e '.pipelines.receiver' > /dev/null 2>&1; then EVENTCOUNTPREVIOUS=`cat $PREVCOUNTFILE`
STAGE=receiver
COUNT="$(echo "$STATS" | jq '.pipelines.receiver.events.in')"
else else
echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
exit 0 exit 0
fi fi
if ! [[ "$COUNT" =~ ^[0-9]+$ ]]; then echo "${EVENTCOUNTCURRENT}" > $PREVCOUNTFILE
exit 0 # the division by 30 is because the agent interval is 30 seconds
EVENTS=$(((EVENTCOUNTCURRENT - EVENTCOUNTPREVIOUS)/30))
if [ "$EVENTS" -lt 0 ]; then
EVENTS=0
fi fi
NOW=$(date +%s) echo "consumptioneps eps=${EVENTS%%.*}"
read -r PREVEPOCH PREVCOUNT 2>/dev/null < "$PREVCOUNTFILE"
echo "${NOW} ${COUNT}" > "$PREVCOUNTFILE"
if ! [[ "$PREVEPOCH" =~ ^[0-9]+$ && "$PREVCOUNT" =~ ^[0-9]+$ ]]; then
exit 0
fi fi
ELAPSED=$((NOW - PREVEPOCH))
DELTA=$((COUNT - PREVCOUNT))
if [ "$ELAPSED" -le 0 ] || [ "$DELTA" -lt 0 ]; then
EPS=0
else
EPS=$((DELTA / ELAPSED))
fi
echo "consumptioneps,stage=${STAGE} eps=${EPS}"
fi fi
exit 0 exit 0
+2 -1
View File
@@ -10,7 +10,8 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then
FPS_ENABLED=$(cat /var/log/sostatus/fps_enabled) FPS_ENABLED=$(cat /var/log/sostatus/fps_enabled)
LKS_ENABLED=$(cat /var/log/sostatus/lks_enabled) LKS_ENABLED=$(cat /var/log/sostatus/lks_enabled)
echo "features fps=$FPS_ENABLED,lks=$LKS_ENABLED" echo "features fps=$FPS_ENABLED"
echo "features lks=$LKS_ENABLED"
fi fi
exit 0 exit 0