mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-07-28 03:33:28 +02:00
Compare commits
31
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
52791204e4 | ||
|
|
74b50f6009 | ||
|
|
e89c820b65 | ||
|
|
9ac05a6ad1 | ||
|
|
24ee3318bc | ||
|
|
ce566ba174 | ||
|
|
2635a60a8c | ||
|
|
244a73b7a2 | ||
|
|
1189621ec5 | ||
|
|
d2524a593f | ||
|
|
f2ab2354fd | ||
|
|
64731c73ba | ||
|
|
024fece607 | ||
|
|
249b126312 | ||
|
|
8e38bff0c3 | ||
|
|
b9f2d56932 | ||
|
|
03fa01a705 | ||
|
|
450eacca41 | ||
|
|
b7a13899f7 | ||
|
|
6f273d7d97 | ||
|
|
b328820c01 | ||
|
|
638aca97c8 | ||
|
|
74a5c895e8 | ||
|
|
d56bf01823 | ||
|
|
d29267d9c2 | ||
|
|
72327285b2 | ||
|
|
cc7a237457 | ||
|
|
b068ad2b35 | ||
|
|
4a2177c827 | ||
|
|
affede7f0a | ||
|
|
97366c0496 |
@@ -165,6 +165,8 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
|
|||||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template" # false positive (elasticsearch index or template names contain 'error')
|
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template" # false positive (elasticsearch index or template names contain 'error')
|
||||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
|
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
|
||||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
|
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
|
||||||
|
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No such container" # false positive (telegraf trying to run stats on an old container)
|
||||||
|
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|passwords do not match" # false positive (automated hydra test)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
|
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
|
||||||
|
|||||||
@@ -3958,10 +3958,13 @@ elasticsearch:
|
|||||||
- vulnerability-mappings
|
- vulnerability-mappings
|
||||||
- common-settings
|
- common-settings
|
||||||
- common-dynamic-mappings
|
- common-dynamic-mappings
|
||||||
|
- logs-redis.log@package
|
||||||
|
- logs-redis.log@custom
|
||||||
data_stream:
|
data_stream:
|
||||||
allow_custom_routing: false
|
allow_custom_routing: false
|
||||||
hidden: false
|
hidden: false
|
||||||
ignore_missing_component_templates: []
|
ignore_missing_component_templates:
|
||||||
|
- logs-redis.log@custom
|
||||||
index_patterns:
|
index_patterns:
|
||||||
- logs-redis.log*
|
- logs-redis.log*
|
||||||
priority: 501
|
priority: 501
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
{
|
||||||
|
"description": "zeek.ja4d",
|
||||||
|
"processors": [
|
||||||
|
{
|
||||||
|
"set": {
|
||||||
|
"field": "event.dataset",
|
||||||
|
"value": "ja4d"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"remove": {
|
||||||
|
"field": [
|
||||||
|
"host"
|
||||||
|
],
|
||||||
|
"ignore_failure": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"json": {
|
||||||
|
"field": "message",
|
||||||
|
"target_field": "message2",
|
||||||
|
"ignore_failure": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"rename": {
|
||||||
|
"field": "message2.ja4d",
|
||||||
|
"target_field": "hash.ja4d",
|
||||||
|
"ignore_missing": true,
|
||||||
|
"if": "ctx?.message2?.ja4d != null && ctx.message2.ja4d.length() > 0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"rename": {
|
||||||
|
"field": "message2.client_mac",
|
||||||
|
"target_field": "host.mac",
|
||||||
|
"ignore_missing": true,
|
||||||
|
"if": "ctx?.message2?.client_mac != null && ctx.message2.client_mac.length() > 0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"rename": {
|
||||||
|
"field": "message2.hostname",
|
||||||
|
"target_field": "host.hostname",
|
||||||
|
"ignore_missing": true,
|
||||||
|
"if": "ctx?.message2?.hostname != null && ctx.message2.hostname.length() > 0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"rename": {
|
||||||
|
"field": "message2.requested_ip",
|
||||||
|
"target_field": "dhcp.requested_address",
|
||||||
|
"ignore_missing": true,
|
||||||
|
"if": "ctx?.message2?.requested_ip != null && ctx.message2.requested_ip.length() > 0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"rename": {
|
||||||
|
"field": "message2.vendor_class_id",
|
||||||
|
"target_field": "zeek.ja4d.vendor_class_id",
|
||||||
|
"ignore_missing": true,
|
||||||
|
"if": "ctx?.message2?.vendor_class_id != null && ctx.message2.vendor_class_id.length() > 0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"pipeline": {
|
||||||
|
"name": "zeek.common"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -210,6 +210,26 @@ logrotate:
|
|||||||
- extension .log
|
- extension .log
|
||||||
- dateext
|
- dateext
|
||||||
- dateyesterday
|
- dateyesterday
|
||||||
|
/opt/so/log/salt/virtual_node_manager:
|
||||||
|
- daily
|
||||||
|
- rotate 14
|
||||||
|
- missingok
|
||||||
|
- copytruncate
|
||||||
|
- compress
|
||||||
|
- create
|
||||||
|
- extension .log
|
||||||
|
- dateext
|
||||||
|
- dateyesterday
|
||||||
|
/opt/so/log/salt/so-salt-cloud:
|
||||||
|
- daily
|
||||||
|
- rotate 14
|
||||||
|
- missingok
|
||||||
|
- copytruncate
|
||||||
|
- compress
|
||||||
|
- create
|
||||||
|
- extension .log
|
||||||
|
- dateext
|
||||||
|
- dateyesterday
|
||||||
/nsm/idh/*_x_log:
|
/nsm/idh/*_x_log:
|
||||||
- daily
|
- daily
|
||||||
- rotate 14
|
- rotate 14
|
||||||
|
|||||||
@@ -133,6 +133,20 @@ logrotate:
|
|||||||
multiline: True
|
multiline: True
|
||||||
global: True
|
global: True
|
||||||
forcedType: "[]string"
|
forcedType: "[]string"
|
||||||
|
"/opt/so/log/salt/virtual_node_manager":
|
||||||
|
description: List of logrotate options for this file.
|
||||||
|
title: /opt/so/log/salt/virtual_node_manager
|
||||||
|
advanced: True
|
||||||
|
multiline: True
|
||||||
|
global: True
|
||||||
|
forcedType: "[]string"
|
||||||
|
"/opt/so/log/salt/so-salt-cloud":
|
||||||
|
description: List of logrotate options for this file.
|
||||||
|
title: /opt/so/log/salt/so-salt-cloud
|
||||||
|
advanced: True
|
||||||
|
multiline: True
|
||||||
|
global: True
|
||||||
|
forcedType: "[]string"
|
||||||
"/nsm/idh/*_x_log":
|
"/nsm/idh/*_x_log":
|
||||||
description: List of logrotate options for this file.
|
description: List of logrotate options for this file.
|
||||||
title: /nsm/idh/*.log
|
title: /nsm/idh/*.log
|
||||||
|
|||||||
@@ -225,6 +225,7 @@ http {
|
|||||||
limit_req zone=auth_throttle burst={{ NGINXMERGED.config.throttle_login_burst }} nodelay;
|
limit_req zone=auth_throttle burst={{ NGINXMERGED.config.throttle_login_burst }} nodelay;
|
||||||
limit_req_status 429;
|
limit_req_status 429;
|
||||||
proxy_pass http://{{ GLOBALS.manager }}:4433;
|
proxy_pass http://{{ GLOBALS.manager }}:4433;
|
||||||
|
proxy_set_header Connection "Close";
|
||||||
proxy_read_timeout 90;
|
proxy_read_timeout 90;
|
||||||
proxy_connect_timeout 90;
|
proxy_connect_timeout 90;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
@@ -237,6 +238,7 @@ http {
|
|||||||
location ~ ^/auth/.*?(whoami|logout|settings|errors|webauthn.js) {
|
location ~ ^/auth/.*?(whoami|logout|settings|errors|webauthn.js) {
|
||||||
rewrite /auth/(.*) /$1 break;
|
rewrite /auth/(.*) /$1 break;
|
||||||
proxy_pass http://{{ GLOBALS.manager }}:4433;
|
proxy_pass http://{{ GLOBALS.manager }}:4433;
|
||||||
|
proxy_set_header Connection "Close";
|
||||||
proxy_read_timeout 90;
|
proxy_read_timeout 90;
|
||||||
proxy_connect_timeout 90;
|
proxy_connect_timeout 90;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
|
|||||||
@@ -46,10 +46,10 @@ postgresinitdir:
|
|||||||
- require:
|
- require:
|
||||||
- file: postgresconfdir
|
- file: postgresconfdir
|
||||||
|
|
||||||
postgresinitusers:
|
postgresinitdb:
|
||||||
file.managed:
|
file.managed:
|
||||||
- name: /opt/so/conf/postgres/init/init-users.sh
|
- name: /opt/so/conf/postgres/init/init-db.sh
|
||||||
- source: salt://postgres/files/init-users.sh
|
- source: salt://postgres/files/init-db.sh
|
||||||
- user: 939
|
- user: 939
|
||||||
- group: 939
|
- group: 939
|
||||||
- mode: 755
|
- mode: 755
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ so-postgres:
|
|||||||
- POSTGRES_DB=securityonion
|
- POSTGRES_DB=securityonion
|
||||||
# Passwords are delivered via mounted 0600 secret files, not plaintext env vars.
|
# Passwords are delivered via mounted 0600 secret files, not plaintext env vars.
|
||||||
# The upstream postgres image resolves POSTGRES_PASSWORD_FILE; entrypoint.sh and
|
# The upstream postgres image resolves POSTGRES_PASSWORD_FILE; entrypoint.sh and
|
||||||
# init-users.sh resolve SO_POSTGRES_PASS_FILE the same way.
|
# init-db.sh resolve SO_POSTGRES_PASS_FILE the same way.
|
||||||
- POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
|
- POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
|
||||||
- SO_POSTGRES_USER={{ SO_POSTGRES_USER }}
|
- SO_POSTGRES_USER={{ SO_POSTGRES_USER }}
|
||||||
- SO_POSTGRES_PASS_FILE=/run/secrets/so_postgres_pass
|
- SO_POSTGRES_PASS_FILE=/run/secrets/so_postgres_pass
|
||||||
@@ -46,7 +46,7 @@ so-postgres:
|
|||||||
- /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro
|
- /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro
|
||||||
- /opt/so/conf/postgres/pg_hba.conf:/conf/pg_hba.conf:ro
|
- /opt/so/conf/postgres/pg_hba.conf:/conf/pg_hba.conf:ro
|
||||||
- /opt/so/conf/postgres/secrets:/run/secrets:ro
|
- /opt/so/conf/postgres/secrets:/run/secrets:ro
|
||||||
- /opt/so/conf/postgres/init/init-users.sh:/docker-entrypoint-initdb.d/init-users.sh:ro
|
- /opt/so/conf/postgres/init/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
|
||||||
- /etc/pki/postgres.crt:/conf/postgres.crt:ro
|
- /etc/pki/postgres.crt:/conf/postgres.crt:ro
|
||||||
- /etc/pki/postgres.key:/conf/postgres.key:ro
|
- /etc/pki/postgres.key:/conf/postgres.key:ro
|
||||||
- /etc/pki/tls/certs/intca.crt:/conf/ca.crt:ro
|
- /etc/pki/tls/certs/intca.crt:/conf/ca.crt:ro
|
||||||
@@ -70,7 +70,7 @@ so-postgres:
|
|||||||
- watch:
|
- watch:
|
||||||
- file: postgresconf
|
- file: postgresconf
|
||||||
- file: postgreshba
|
- file: postgreshba
|
||||||
- file: postgresinitusers
|
- file: postgresinitdb
|
||||||
- file: postgres_super_secret
|
- file: postgres_super_secret
|
||||||
- file: postgres_app_secret
|
- file: postgres_app_secret
|
||||||
- x509: postgres_crt
|
- x509: postgres_crt
|
||||||
@@ -78,7 +78,7 @@ so-postgres:
|
|||||||
- require:
|
- require:
|
||||||
- file: postgresconf
|
- file: postgresconf
|
||||||
- file: postgreshba
|
- file: postgreshba
|
||||||
- file: postgresinitusers
|
- file: postgresinitdb
|
||||||
- file: postgres_super_secret
|
- file: postgres_super_secret
|
||||||
- file: postgres_app_secret
|
- file: postgres_app_secret
|
||||||
- x509: postgres_crt
|
- x509: postgres_crt
|
||||||
|
|||||||
@@ -39,17 +39,15 @@ postgres_wait_ready:
|
|||||||
- require:
|
- require:
|
||||||
- docker_container: so-postgres
|
- docker_container: so-postgres
|
||||||
|
|
||||||
# Ensure the shared Telegraf database exists. init-users.sh only runs on a
|
# Ensure the shared Telegraf database exists. init-db.sh only runs on a
|
||||||
# fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
|
# fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
|
||||||
# would otherwise never get so_telegraf.
|
# would otherwise never get so_telegraf.
|
||||||
postgres_create_telegraf_db:
|
postgres_create_telegraf_db:
|
||||||
cmd.run:
|
cmd.run:
|
||||||
- name: |
|
- name: /usr/sbin/so-telegraf-postgres create_db
|
||||||
if ! docker exec so-postgres psql -U postgres -tAc "SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
|
|
||||||
docker exec so-postgres psql -v ON_ERROR_STOP=1 -U postgres -c "CREATE DATABASE so_telegraf"
|
|
||||||
fi
|
|
||||||
- require:
|
- require:
|
||||||
- cmd: postgres_wait_ready
|
- cmd: postgres_wait_ready
|
||||||
|
- file: postgres_sbin
|
||||||
|
|
||||||
# Provision the shared group role and schema once. Every per-minion role is a
|
# Provision the shared group role and schema once. Every per-minion role is a
|
||||||
# member of so_telegraf, and each Telegraf connection does SET ROLE so_telegraf
|
# member of so_telegraf, and each Telegraf connection does SET ROLE so_telegraf
|
||||||
@@ -57,68 +55,26 @@ postgres_create_telegraf_db:
|
|||||||
# on first write are owned by the group role and every member can INSERT/SELECT.
|
# on first write are owned by the group role and every member can INSERT/SELECT.
|
||||||
postgres_telegraf_group_role:
|
postgres_telegraf_group_role:
|
||||||
cmd.run:
|
cmd.run:
|
||||||
- name: |
|
- name: /usr/sbin/so-telegraf-postgres group_role
|
||||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
|
||||||
DO $$
|
|
||||||
BEGIN
|
|
||||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
|
|
||||||
CREATE ROLE so_telegraf NOLOGIN;
|
|
||||||
END IF;
|
|
||||||
END
|
|
||||||
$$;
|
|
||||||
GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
|
|
||||||
CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
|
|
||||||
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
|
|
||||||
CREATE SCHEMA IF NOT EXISTS partman;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
|
|
||||||
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
|
||||||
-- Telegraf (running as so_telegraf) calls partman.create_parent()
|
|
||||||
-- on first write of each metric, which needs USAGE on the partman
|
|
||||||
-- schema, EXECUTE on its functions/procedures, and write access to
|
|
||||||
-- partman.part_config so it can register new partitioned parents.
|
|
||||||
GRANT USAGE, CREATE ON SCHEMA partman TO so_telegraf;
|
|
||||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA partman TO so_telegraf;
|
|
||||||
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO so_telegraf;
|
|
||||||
GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA partman TO so_telegraf;
|
|
||||||
-- partman creates per-parent template tables (partman.template_*) at
|
|
||||||
-- runtime; default privileges extend DML/sequence access to them.
|
|
||||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
|
||||||
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
|
|
||||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
|
||||||
GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
|
|
||||||
-- Hourly partman maintenance. cron.schedule is idempotent by jobname.
|
|
||||||
SELECT cron.schedule(
|
|
||||||
'telegraf-partman-maintenance',
|
|
||||||
'17 * * * *',
|
|
||||||
'CALL partman.run_maintenance_proc()'
|
|
||||||
);
|
|
||||||
EOSQL
|
|
||||||
- require:
|
- require:
|
||||||
- cmd: postgres_create_telegraf_db
|
- cmd: postgres_create_telegraf_db
|
||||||
|
- file: postgres_sbin
|
||||||
|
|
||||||
{% set creds = salt['pillar.get']('telegraf:postgres_creds', {}) %}
|
{% set creds = salt['pillar.get']('telegraf:postgres_creds', {}) %}
|
||||||
{% for mid, entry in creds.items() %}
|
{% for mid, entry in creds.items() %}
|
||||||
{% if entry.get('user') and entry.get('pass') %}
|
{% if entry.get('user') and entry.get('pass') %}
|
||||||
{% set u = entry.user %}
|
{% set u = entry.user %}
|
||||||
{% set p = entry.pass | replace("'", "''") %}
|
{% set p = entry.pass %}
|
||||||
|
|
||||||
postgres_telegraf_role_{{ u }}:
|
postgres_telegraf_role_{{ u }}:
|
||||||
cmd.run:
|
cmd.run:
|
||||||
- name: |
|
- name: /usr/sbin/so-telegraf-postgres user
|
||||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
- env:
|
||||||
DO $$
|
- ROLE_USER: {{ u | tojson }}
|
||||||
BEGIN
|
- ROLE_PASS: {{ p | tojson }}
|
||||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '{{ u }}') THEN
|
- hide_output: True
|
||||||
EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '{{ u }}', '{{ p }}');
|
|
||||||
ELSE
|
|
||||||
EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '{{ u }}', '{{ p }}');
|
|
||||||
END IF;
|
|
||||||
END
|
|
||||||
$$;
|
|
||||||
GRANT CONNECT ON DATABASE so_telegraf TO "{{ u }}";
|
|
||||||
GRANT so_telegraf TO "{{ u }}";
|
|
||||||
EOSQL
|
|
||||||
- require:
|
- require:
|
||||||
|
- file: postgres_sbin
|
||||||
- cmd: postgres_telegraf_group_role
|
- cmd: postgres_telegraf_group_role
|
||||||
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
@@ -130,21 +86,12 @@ postgres_telegraf_role_{{ u }}:
|
|||||||
{% set retention = salt['pillar.get']('postgres:telegraf:retention_days', 14) | int %}
|
{% set retention = salt['pillar.get']('postgres:telegraf:retention_days', 14) | int %}
|
||||||
postgres_telegraf_retention_reconcile:
|
postgres_telegraf_retention_reconcile:
|
||||||
cmd.run:
|
cmd.run:
|
||||||
- name: |
|
- name: /usr/sbin/so-telegraf-postgres retention
|
||||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
- env:
|
||||||
DO $$
|
- RETENTION_DAYS: {{ retention }}
|
||||||
BEGIN
|
|
||||||
IF EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman') THEN
|
|
||||||
UPDATE partman.part_config
|
|
||||||
SET retention = '{{ retention }} days',
|
|
||||||
retention_keep_table = false
|
|
||||||
WHERE parent_table LIKE 'telegraf.%';
|
|
||||||
END IF;
|
|
||||||
END
|
|
||||||
$$;
|
|
||||||
EOSQL
|
|
||||||
- require:
|
- require:
|
||||||
- cmd: postgres_telegraf_group_role
|
- cmd: postgres_telegraf_group_role
|
||||||
|
- file: postgres_sbin
|
||||||
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
|||||||
@@ -7,15 +7,29 @@
|
|||||||
|
|
||||||
. /usr/sbin/so-common
|
. /usr/sbin/so-common
|
||||||
|
|
||||||
|
# Without pipefail, a pipeline's exit status is gzip's. A failed pg_dumpall would
|
||||||
|
# otherwise be masked by a successful gzip, silently producing a valid .gz that
|
||||||
|
# holds a truncated dump.
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
# Backups contain role password hashes and full chat data; keep them 0600.
|
# Backups contain role password hashes and full chat data; keep them 0600.
|
||||||
umask 0077
|
umask 0077
|
||||||
|
|
||||||
TODAY=$(date '+%Y_%m_%d')
|
TODAY=$(date '+%Y_%m_%d')
|
||||||
BACKUPDIR=/nsm/backup
|
BACKUPDIR=/nsm/backup
|
||||||
BACKUPFILE="$BACKUPDIR/so-postgres-backup-$TODAY.sql.gz"
|
BACKUPFILE="$BACKUPDIR/so-postgres-backup-$TODAY.sql.gz"
|
||||||
|
TMPFILE="$BACKUPFILE.tmp"
|
||||||
MAXBACKUPS=7
|
MAXBACKUPS=7
|
||||||
|
LOGFILE=/opt/so/log/postgres/backup.log
|
||||||
|
|
||||||
mkdir -p $BACKUPDIR
|
log() {
|
||||||
|
echo "$(date '+%Y-%m-%d %H:%M:%S') $*" >> "$LOGFILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
mkdir -p "$BACKUPDIR"
|
||||||
|
|
||||||
|
# Remove any temp files left behind by a previously crashed run
|
||||||
|
rm -f "$BACKUPDIR"/so-postgres-backup-*.sql.gz.tmp
|
||||||
|
|
||||||
# Skip if already backed up today
|
# Skip if already backed up today
|
||||||
if [ -f "$BACKUPFILE" ]; then
|
if [ -f "$BACKUPFILE" ]; then
|
||||||
@@ -27,13 +41,33 @@ if ! docker ps --format '{{.Names}}' | grep -q '^so-postgres$'; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Dump all databases and roles, compress
|
# Always clean up the temp file on exit; the success path clears this trap
|
||||||
docker exec so-postgres pg_dumpall -U postgres | gzip > "$BACKUPFILE"
|
# after the atomic rename so the finished backup is not deleted.
|
||||||
|
trap 'rm -f "$TMPFILE"' EXIT
|
||||||
|
|
||||||
# Retention cleanup
|
# Dump all databases and roles, compress. Write to a temp file so the final
|
||||||
NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
|
# filename only ever appears for a complete, verified backup.
|
||||||
|
if ! docker exec so-postgres pg_dumpall -U postgres | gzip > "$TMPFILE"; then
|
||||||
|
log "ERROR: pg_dumpall/gzip failed; backup aborted"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Verify the compressed stream is intact before publishing it
|
||||||
|
if ! gzip -t "$TMPFILE"; then
|
||||||
|
log "ERROR: backup failed gzip integrity check; backup aborted"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Atomically publish the verified backup
|
||||||
|
mv "$TMPFILE" "$BACKUPFILE"
|
||||||
|
trap - EXIT
|
||||||
|
log "OK: wrote $BACKUPFILE"
|
||||||
|
|
||||||
|
# Retention cleanup (only reached after a successful backup). The glob is
|
||||||
|
# restricted to finished backups so an in-progress .tmp can never be counted.
|
||||||
|
NUMBACKUPS=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" | wc -l)
|
||||||
while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do
|
while [ "$NUMBACKUPS" -gt "$MAXBACKUPS" ]; do
|
||||||
OLDEST=$(find $BACKUPDIR -type f -name "so-postgres-backup*" -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
|
OLDEST=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" -printf '%T+ %p\n' | sort | head -n 1 | awk -F" " '{print $2}')
|
||||||
rm -f "$OLDEST"
|
rm -f "$OLDEST"
|
||||||
NUMBACKUPS=$(find $BACKUPDIR -type f -name "so-postgres-backup*" | wc -l)
|
NUMBACKUPS=$(find "$BACKUPDIR" -type f -name "so-postgres-backup-*.sql.gz" | wc -l)
|
||||||
done
|
done
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Provision Telegraf state inside the so-postgres container.
|
||||||
|
# Usage: so-telegraf-postgres <subcommand>
|
||||||
|
# create_db Ensure the so_telegraf database exists.
|
||||||
|
# group_role Provision the so_telegraf group role, telegraf/partman schemas,
|
||||||
|
# pg_partman, pg_cron, and the hourly partman maintenance job.
|
||||||
|
# user Create or update a per-minion login role granted to so_telegraf.
|
||||||
|
# Env: ROLE_USER, ROLE_PASS.
|
||||||
|
# retention Reconcile partman retention on telegraf parents.
|
||||||
|
# Env: RETENTION_DAYS.
|
||||||
|
|
||||||
|
cmd="${1:?subcommand required}"
|
||||||
|
|
||||||
|
case "$cmd" in
|
||||||
|
create_db)
|
||||||
|
if ! docker exec so-postgres psql -U postgres -tAc \
|
||||||
|
"SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1; then
|
||||||
|
docker exec so-postgres psql -v ON_ERROR_STOP=1 -U postgres \
|
||||||
|
-c "CREATE DATABASE so_telegraf"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
|
group_role)
|
||||||
|
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
|
||||||
|
CREATE ROLE so_telegraf NOLOGIN;
|
||||||
|
END IF;
|
||||||
|
END
|
||||||
|
$$;
|
||||||
|
GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
|
||||||
|
CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
|
||||||
|
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
|
||||||
|
CREATE SCHEMA IF NOT EXISTS partman;
|
||||||
|
CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
|
||||||
|
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
||||||
|
-- Telegraf (running as so_telegraf) calls partman.create_parent()
|
||||||
|
-- on first write of each metric, which needs USAGE on the partman
|
||||||
|
-- schema, EXECUTE on its functions/procedures, and write access to
|
||||||
|
-- partman.part_config so it can register new partitioned parents.
|
||||||
|
GRANT USAGE, CREATE ON SCHEMA partman TO so_telegraf;
|
||||||
|
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA partman TO so_telegraf;
|
||||||
|
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA partman TO so_telegraf;
|
||||||
|
GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA partman TO so_telegraf;
|
||||||
|
-- partman creates per-parent template tables (partman.template_*) at
|
||||||
|
-- runtime; default privileges extend DML/sequence access to them.
|
||||||
|
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||||
|
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO so_telegraf;
|
||||||
|
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||||
|
GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
|
||||||
|
-- Hourly partman maintenance. cron.schedule is idempotent by jobname.
|
||||||
|
SELECT cron.schedule(
|
||||||
|
'telegraf-partman-maintenance',
|
||||||
|
'17 * * * *',
|
||||||
|
'CALL partman.run_maintenance_proc()'
|
||||||
|
);
|
||||||
|
EOSQL
|
||||||
|
;;
|
||||||
|
|
||||||
|
user)
|
||||||
|
: "${ROLE_USER:?ROLE_USER is required}"
|
||||||
|
: "${ROLE_PASS:?ROLE_PASS is required}"
|
||||||
|
# psql does not substitute :vars inside dollar-quoted strings, so the
|
||||||
|
# conditional CREATE/ALTER is built outside any DO block and dispatched
|
||||||
|
# with \gexec. format() handles identifier/literal quoting.
|
||||||
|
docker exec -i so-postgres psql \
|
||||||
|
-v ON_ERROR_STOP=1 \
|
||||||
|
-v role_user="$ROLE_USER" \
|
||||||
|
-v role_pass="$ROLE_PASS" \
|
||||||
|
-U postgres -d so_telegraf <<'EOSQL'
|
||||||
|
SELECT format(
|
||||||
|
CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user')
|
||||||
|
THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L'
|
||||||
|
ELSE 'CREATE ROLE %I WITH LOGIN PASSWORD %L'
|
||||||
|
END,
|
||||||
|
:'role_user',
|
||||||
|
:'role_pass'
|
||||||
|
) \gexec
|
||||||
|
GRANT CONNECT ON DATABASE so_telegraf TO :"role_user";
|
||||||
|
GRANT so_telegraf TO :"role_user";
|
||||||
|
EOSQL
|
||||||
|
;;
|
||||||
|
|
||||||
|
retention)
|
||||||
|
: "${RETENTION_DAYS:?RETENTION_DAYS is required}"
|
||||||
|
# \gset + \if guards against a missing pg_partman without using a DO
|
||||||
|
# block (psql :var substitution doesn't reach into dollar-quoted code).
|
||||||
|
docker exec -i so-postgres psql \
|
||||||
|
-v ON_ERROR_STOP=1 \
|
||||||
|
-v retention_days="$RETENTION_DAYS" \
|
||||||
|
-U postgres -d so_telegraf <<'EOSQL'
|
||||||
|
SELECT CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_partman')
|
||||||
|
THEN 'true' ELSE 'false' END AS has_partman \gset
|
||||||
|
\if :has_partman
|
||||||
|
UPDATE partman.part_config
|
||||||
|
SET retention = :'retention_days' || ' days',
|
||||||
|
retention_keep_table = false
|
||||||
|
WHERE parent_table LIKE 'telegraf.%';
|
||||||
|
\endif
|
||||||
|
EOSQL
|
||||||
|
;;
|
||||||
|
|
||||||
|
*)
|
||||||
|
echo "Unknown subcommand: $cmd" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
@@ -261,7 +261,7 @@ strelka:
|
|||||||
priority: 5
|
priority: 5
|
||||||
options:
|
options:
|
||||||
limit: 1000
|
limit: 1000
|
||||||
'ScanLNK':
|
'ScanLnk':
|
||||||
- positive:
|
- positive:
|
||||||
flavors:
|
flavors:
|
||||||
- 'lnk_file'
|
- 'lnk_file'
|
||||||
|
|||||||
@@ -99,7 +99,7 @@ strelka:
|
|||||||
'ScanJpeg': *scannerOptions
|
'ScanJpeg': *scannerOptions
|
||||||
'ScanJson': *scannerOptions
|
'ScanJson': *scannerOptions
|
||||||
'ScanLibarchive': *scannerOptions
|
'ScanLibarchive': *scannerOptions
|
||||||
'ScanLNK': *scannerOptions
|
'ScanLnk': *scannerOptions
|
||||||
'ScanLsb': *scannerOptions
|
'ScanLsb': *scannerOptions
|
||||||
'ScanLzma': *scannerOptions
|
'ScanLzma': *scannerOptions
|
||||||
'ScanMacho': *scannerOptions
|
'ScanMacho': *scannerOptions
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
telegraf:
|
telegraf:
|
||||||
enabled: False
|
enabled: False
|
||||||
output: BOTH
|
output: INFLUXDB
|
||||||
config:
|
config:
|
||||||
interval: '30s'
|
interval: '30s'
|
||||||
metric_batch_size: 1000
|
metric_batch_size: 1000
|
||||||
|
|||||||
+27
-2
@@ -119,7 +119,7 @@ base:
|
|||||||
- kafka
|
- kafka
|
||||||
- pcap.cleanup
|
- pcap.cleanup
|
||||||
|
|
||||||
'*_manager or *_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False':
|
'*_manager and G@saltversion:{{saltversion}} and not I@node_data:False':
|
||||||
- match: compound
|
- match: compound
|
||||||
- salt.master
|
- salt.master
|
||||||
- registry
|
- registry
|
||||||
@@ -146,6 +146,32 @@ base:
|
|||||||
- stig
|
- stig
|
||||||
- kafka
|
- kafka
|
||||||
|
|
||||||
|
'*_managerhype and G@saltversion:{{saltversion}} and not I@node_data:False':
|
||||||
|
- match: compound
|
||||||
|
- salt.master
|
||||||
|
- registry
|
||||||
|
- nginx
|
||||||
|
- influxdb
|
||||||
|
- postgres
|
||||||
|
- strelka.manager
|
||||||
|
- soc
|
||||||
|
- kratos
|
||||||
|
- hydra
|
||||||
|
- firewall
|
||||||
|
- manager
|
||||||
|
- sensoroni
|
||||||
|
- telegraf
|
||||||
|
- backup.config_backup
|
||||||
|
- elasticsearch
|
||||||
|
- logstash
|
||||||
|
- redis
|
||||||
|
- elastic-fleet-package-registry
|
||||||
|
- kibana
|
||||||
|
- elastalert
|
||||||
|
- utility
|
||||||
|
- elasticfleet
|
||||||
|
- kafka
|
||||||
|
|
||||||
'*_managerhype and I@features:vrt and G@saltversion:{{saltversion}}':
|
'*_managerhype and I@features:vrt and G@saltversion:{{saltversion}}':
|
||||||
- match: compound
|
- match: compound
|
||||||
- manager.hypervisor
|
- manager.hypervisor
|
||||||
@@ -286,7 +312,6 @@ base:
|
|||||||
- libvirt
|
- libvirt
|
||||||
- libvirt.images
|
- libvirt.images
|
||||||
- elasticfleet.install_agent_grid
|
- elasticfleet.install_agent_grid
|
||||||
- stig
|
|
||||||
|
|
||||||
'*_desktop and G@saltversion:{{saltversion}}':
|
'*_desktop and G@saltversion:{{saltversion}}':
|
||||||
- sensoroni
|
- sensoroni
|
||||||
|
|||||||
Reference in New Issue
Block a user