diff --git a/HOTFIX b/HOTFIX index aa8e22a9c..e69de29bb 100644 --- a/HOTFIX +++ b/HOTFIX @@ -1 +0,0 @@ -20220202 20220203 diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index bc11da57b..a7677a754 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -249,6 +249,7 @@ lookup_salt_value() { group=$2 kind=$3 output=${4:-newline_values_only} + local=$5 if [ -z "$kind" ]; then kind=pillar @@ -258,7 +259,13 @@ lookup_salt_value() { group=${group}: fi - salt-call --no-color ${kind}.get ${group}${key} --out=${output} + if [[ "$local" == "--local" ]] || [[ "$local" == "local" ]]; then + local="--local" + else + local="" + fi + + salt-call --no-color ${kind}.get ${group}${key} --out=${output} ${local} } lookup_pillar() { diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 33649ab63..73da9bc24 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -245,7 +245,6 @@ check_sudoers() { } check_log_size_limit() { - local num_minion_pillars num_minion_pillars=$(find /opt/so/saltstack/local/pillar/minions/ -type f | wc -l) @@ -255,7 +254,7 @@ check_log_size_limit() { fi else local minion_id - minion_id=$(lookup_salt_value "id" "" "grains") + minion_id=$(lookup_salt_value "id" "" "grains" "" "local") local minion_arr IFS='_' read -ra minion_arr <<< "$minion_id" @@ -263,7 +262,15 @@ check_log_size_limit() { local node_type="${minion_arr[0]}" local current_limit - current_limit=$(lookup_pillar "log_size_limit" "elasticsearch") + # since it is possible for the salt-master service to be stopped when this is run, we need to check the pillar values locally + # we need to combine default local and default pillars before doing this so we can define --pillar-root in salt-call + local epoch_date=$(date +%s%N) + mkdir -vp /opt/so/saltstack/soup_tmp_${epoch_date}/ + cp -r /opt/so/saltstack/default/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/ + # use \cp here to overwrite any pillar files from default with those in local for the tmp directory + \cp -r /opt/so/saltstack/local/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/ + current_limit=$(salt-call pillar.get elasticsearch:log_size_limit --local --pillar-root=/opt/so/saltstack/soup_tmp_${epoch_date}/pillar --out=newline_values_only) + rm -rf /opt/so/saltstack/soup_tmp_${epoch_date}/ local percent case $node_type in @@ -468,12 +475,15 @@ post_to_2.3.90() { post_to_2.3.100() { echo "Post Processing for 2.3.100" + POSTVERSION=2.3.100 } post_to_2.3.110() { echo "Post Processing for 2.3.110" echo "Updating Kibana dashboards" salt-call state.apply kibana.so_savedobjects_defaults queue=True + so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 & + POSTVERSION=2.3.110 } stop_salt_master() { diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index d3b915a15..b9986d051 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -102,6 +102,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -135,6 +136,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -184,6 +186,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -217,6 +220,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -265,6 +269,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -298,6 +303,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -346,6 +352,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -379,6 +386,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -428,6 +436,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -461,6 +470,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -536,6 +546,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -569,6 +580,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -615,6 +627,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -648,6 +661,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -694,6 +708,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -727,6 +742,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -775,6 +791,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -808,6 +825,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -855,6 +873,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -888,6 +907,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -933,6 +953,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -966,6 +987,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1011,6 +1033,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1045,6 +1068,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1090,6 +1114,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1124,6 +1149,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1170,6 +1196,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1203,6 +1230,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1248,6 +1276,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1281,6 +1310,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1326,6 +1356,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1359,6 +1390,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1404,6 +1436,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1438,6 +1471,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1483,6 +1517,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1517,6 +1552,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1562,6 +1598,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1596,6 +1633,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1642,6 +1680,7 @@ elasticsearch: - client-mappings - container-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1673,6 +1712,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - url-mappings @@ -1716,6 +1756,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1749,6 +1790,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - suricata-mappings - threat-mappings - tls-mappings @@ -1795,6 +1837,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1828,6 +1871,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1873,6 +1917,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1906,6 +1951,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -1952,6 +1998,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -1985,6 +2032,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2031,6 +2079,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2065,6 +2114,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2110,6 +2160,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2144,6 +2195,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2190,6 +2242,7 @@ elasticsearch: - client-mappings - container-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2221,6 +2274,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - url-mappings @@ -2264,6 +2318,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2298,6 +2353,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2343,6 +2399,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2377,6 +2434,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2422,6 +2480,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2456,6 +2515,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2501,6 +2561,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2535,6 +2596,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2580,6 +2642,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2613,6 +2676,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2658,6 +2722,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2692,6 +2757,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2737,6 +2803,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2771,6 +2838,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2816,6 +2884,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2849,6 +2918,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2895,6 +2965,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -2928,6 +2999,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -2974,6 +3046,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3007,6 +3080,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3052,6 +3126,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3085,6 +3160,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3130,6 +3206,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3164,6 +3241,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3209,6 +3287,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3242,6 +3321,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3287,6 +3367,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3321,6 +3402,7 @@ elasticsearch: - dtc-service-mappings - snyk-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3366,6 +3448,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3399,6 +3482,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3444,6 +3528,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3478,6 +3563,7 @@ elasticsearch: - dtc-service-mappings - sophos-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3523,6 +3609,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3556,6 +3643,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3601,6 +3689,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3636,6 +3725,7 @@ elasticsearch: - dtc-service-mappings - so-scan-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3681,6 +3771,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3714,6 +3805,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - syslog-mappings - threat-mappings - tls-mappings @@ -3760,6 +3852,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3793,6 +3886,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3838,6 +3932,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3871,6 +3966,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings @@ -3917,6 +4013,7 @@ elasticsearch: - container-mappings - data_stream-mappings - destination-mappings + - pb-override-destination-mappings - dll-mappings - dns-mappings - dtc-dns-mappings @@ -3950,6 +4047,7 @@ elasticsearch: - service-mappings - dtc-service-mappings - source-mappings + - pb-override-source-mappings - threat-mappings - tls-mappings - tracing-mappings diff --git a/salt/elasticsearch/files/ingest/zeek.dns b/salt/elasticsearch/files/ingest/zeek.dns index aaedaca74..9d7f7e93b 100644 --- a/salt/elasticsearch/files/ingest/zeek.dns +++ b/salt/elasticsearch/files/ingest/zeek.dns @@ -23,7 +23,7 @@ { "rename": { "field": "message2.TTLs", "target_field": "dns.ttls", "ignore_missing": true } }, { "rename": { "field": "message2.rejected", "target_field": "dns.query.rejected", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()", "ignore_failure": true } }, - { "set": { "field": "_index", "value": "so-zeek_dns", "override": true } }, + { "set": { "if": "ctx._index == 'so-zeek'", "field": "_index", "value": "so-zeek_dns", "override": true } }, { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, { "pipeline": { "name": "zeek.common" } } ] diff --git a/salt/elasticsearch/init.sls b/salt/elasticsearch/init.sls index 21b5fa992..b0a6b67b3 100644 --- a/salt/elasticsearch/init.sls +++ b/salt/elasticsearch/init.sls @@ -379,6 +379,15 @@ append_so-elasticsearch_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-elasticsearch +so-elasticsearch-templates: + cmd.run: + - name: /usr/sbin/so-elasticsearch-templates-load + - cwd: /opt/so + - template: jinja + - require: + - docker_container: so-elasticsearch + - file: es_sync_scripts + so-elasticsearch-pipelines: cmd.run: - name: /usr/sbin/so-elasticsearch-pipelines {{ grains.host }} @@ -391,15 +400,6 @@ so-elasticsearch-pipelines: - docker_container: so-elasticsearch - file: so-elasticsearch-pipelines-script -so-elasticsearch-templates: - cmd.run: - - name: /usr/sbin/so-elasticsearch-templates-load - - cwd: /opt/so - - template: jinja - - require: - - docker_container: so-elasticsearch - - file: es_sync_scripts - so-elasticsearch-roles-load: cmd.run: - name: /usr/sbin/so-elasticsearch-roles-load diff --git a/salt/elasticsearch/templates/component/ecs/aws.json b/salt/elasticsearch/templates/component/ecs/aws.json index 10c7dd45b..689b74ac2 100644 --- a/salt/elasticsearch/templates/component/ecs/aws.json +++ b/salt/elasticsearch/templates/component/ecs/aws.json @@ -13,8 +13,7 @@ "additional_eventdata": { "fields": { "text": { - "norms": false, - "type": "text" + "type": "match_only_text" } }, "ignore_above": 1024, @@ -228,8 +227,7 @@ "request_parameters": { "fields": { "text": { - "norms": false, - "type": "text" + "type": "match_only_text" } }, "ignore_above": 1024, @@ -269,8 +267,7 @@ "response_elements": { "fields": { "text": { - "norms": false, - "type": "text" + "type": "match_only_text" } }, "ignore_above": 1024, @@ -279,8 +276,7 @@ "service_event_details": { "fields": { "text": { - "norms": false, - "type": "text" + "type": "match_only_text" } }, "ignore_above": 1024, @@ -406,7 +402,12 @@ "properties": { "message": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } } } }, diff --git a/salt/elasticsearch/templates/component/ecs/base.json b/salt/elasticsearch/templates/component/ecs/base.json index 77594f68d..7bba4285c 100644 --- a/salt/elasticsearch/templates/component/ecs/base.json +++ b/salt/elasticsearch/templates/component/ecs/base.json @@ -13,7 +13,12 @@ "type": "object" }, "message": { - "type": "match_only_text" + "type": "match_only_text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "tags": { "ignore_above": 1024, diff --git a/salt/elasticsearch/templates/component/ecs/cyberark.json b/salt/elasticsearch/templates/component/ecs/cyberark.json index 4ed88aa6f..b0277fa0b 100644 --- a/salt/elasticsearch/templates/component/ecs/cyberark.json +++ b/salt/elasticsearch/templates/component/ecs/cyberark.json @@ -534,7 +534,12 @@ }, "reason": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "rfc5424": { "type": "boolean" diff --git a/salt/elasticsearch/templates/component/ecs/logstash.json b/salt/elasticsearch/templates/component/ecs/logstash.json index 2120a0902..0db82492e 100644 --- a/salt/elasticsearch/templates/component/ecs/logstash.json +++ b/salt/elasticsearch/templates/component/ecs/logstash.json @@ -45,8 +45,7 @@ "thread": { "fields": { "text": { - "norms": false, - "type": "text" + "type": "match_only_text" } }, "ignore_above": 1024, @@ -59,8 +58,7 @@ "event": { "fields": { "text": { - "norms": false, - "type": "text" + "type": "match_only_text" } }, "ignore_above": 1024, @@ -87,8 +85,7 @@ "plugin_params": { "fields": { "text": { - "norms": false, - "type": "text" + "type": "match_only_text" } }, "ignore_above": 1024, @@ -109,8 +106,7 @@ "thread": { "fields": { "text": { - "norms": false, - "type": "text" + "type": "match_only_text" } }, "ignore_above": 1024, diff --git a/salt/elasticsearch/templates/component/ecs/misp.json b/salt/elasticsearch/templates/component/ecs/misp.json index d0c7aa519..1d186db3a 100644 --- a/salt/elasticsearch/templates/component/ecs/misp.json +++ b/salt/elasticsearch/templates/component/ecs/misp.json @@ -12,7 +12,12 @@ "properties": { "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -47,11 +52,21 @@ "properties": { "aliases": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "first_seen": { "type": "date" @@ -92,7 +107,12 @@ "properties": { "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -118,11 +138,21 @@ "properties": { "contact_information": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -175,18 +205,33 @@ "properties": { "aliases": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "first_seen": { "type": "date" }, "goals": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -211,15 +256,30 @@ }, "primary_motivation": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "resource_level": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "secondary_motivations": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } } } }, @@ -227,7 +287,12 @@ "properties": { "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -280,7 +345,12 @@ }, "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -346,7 +416,12 @@ "properties": { "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -377,7 +452,12 @@ }, "object_refs": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "published": { "type": "date" @@ -388,15 +468,30 @@ "properties": { "aliases": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "goals": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -427,27 +522,57 @@ }, "personal_motivations": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "primary_motivation": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "resource_level": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "roles": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "secondary_motivations": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "sophistication": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } } } }, @@ -491,11 +616,21 @@ }, "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "feed": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -602,7 +737,12 @@ "properties": { "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, @@ -615,7 +755,12 @@ }, "kill_chain_phases": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "labels": { "ignore_above": 1024, @@ -650,7 +795,12 @@ "properties": { "description": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "id": { "ignore_above": 1024, diff --git a/salt/elasticsearch/templates/component/ecs/o365.json b/salt/elasticsearch/templates/component/ecs/o365.json index a7df16b97..6c093534d 100644 --- a/salt/elasticsearch/templates/component/ecs/o365.json +++ b/salt/elasticsearch/templates/component/ecs/o365.json @@ -165,7 +165,12 @@ }, "Comments": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "CommunicationType": { "ignore_above": 1024, diff --git a/salt/elasticsearch/templates/component/ecs/zeek.json b/salt/elasticsearch/templates/component/ecs/zeek.json index d9dd7aa32..c79a9efdf 100644 --- a/salt/elasticsearch/templates/component/ecs/zeek.json +++ b/salt/elasticsearch/templates/component/ecs/zeek.json @@ -1333,7 +1333,12 @@ }, "email_body_sections": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "email_delay_tokens": { "ignore_above": 1024, @@ -1453,7 +1458,12 @@ }, "peer_descr": { "norms": false, - "type": "text" + "type": "text", + "fields": { + "text": { + "type": "match_only_text" + } + } }, "peer_name": { "ignore_above": 1024, diff --git a/salt/elasticsearch/templates/component/so/pb-override-destination-mappings.json b/salt/elasticsearch/templates/component/so/pb-override-destination-mappings.json new file mode 100644 index 000000000..8e3ab45f3 --- /dev/null +++ b/salt/elasticsearch/templates/component/so/pb-override-destination-mappings.json @@ -0,0 +1,32 @@ +{ + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-destination.html", + "ecs_version": "1.12.2" + }, + "template": { + "mappings": { + "properties": { + "destination": { + "properties": { + "ip": { + "type": "ip", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "port": { + "type": "long", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + } + } + } + } +} diff --git a/salt/elasticsearch/templates/component/so/pb-override-source-mappings.json b/salt/elasticsearch/templates/component/so/pb-override-source-mappings.json new file mode 100644 index 000000000..55f105b8c --- /dev/null +++ b/salt/elasticsearch/templates/component/so/pb-override-source-mappings.json @@ -0,0 +1,33 @@ +{ + "_meta": { + "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-source.html", + "ecs_version": "1.12.2" + }, + "template": { + "mappings": { + "properties": { + "source": { + "properties": { + "ip": { + "type": "ip", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "port": { + "type": "long", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + } + } + } + } +} + diff --git a/salt/kibana/bin/so-kibana-config-load b/salt/kibana/bin/so-kibana-config-load index 0b887d5c7..d98b0e85f 100644 --- a/salt/kibana/bin/so-kibana-config-load +++ b/salt/kibana/bin/so-kibana-config-load @@ -41,7 +41,7 @@ update() { wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}" IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - {{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.16.3" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i " + {{ ELASTICCURL }} -X PUT "localhost:5601/api/saved_objects/config/7.17.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i " done } diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 98db07fa5..e2bd5fe2f 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "7.17.0","id": "7.17.0","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "7.17.1","id": "7.17.1","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}