mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-11-18 18:16:02 +01:00
Compare commits
113
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
180a4814f9 | ||
|
|
4dbea0e28d | ||
|
|
6d7be47a69 | ||
|
|
e51706acad | ||
|
|
9612b15fca | ||
|
|
0f5170e17e | ||
|
|
cac027d438 | ||
|
|
dea6932342 | ||
|
|
043b2185d6 | ||
|
|
e7bb00d19d | ||
|
|
8c3611ac72 | ||
|
|
5558c54632 | ||
|
|
9045aff88d | ||
|
|
cc9dd0a101 | ||
|
|
52f9fe3092 | ||
|
|
f492e87f20 | ||
|
|
01518ec25e | ||
|
|
5f3c1c84a4 | ||
|
|
70053e362c | ||
|
|
b01f01da82 | ||
|
|
03317ba5e4 | ||
|
|
9bc3ae6b26 | ||
|
|
40a30f6908 | ||
|
|
cc970892ed | ||
|
|
4e0558d2d3 | ||
|
|
22a4acbb5a | ||
|
|
1fd37251c9 | ||
|
|
4a5253f688 | ||
|
|
ec220572f0 | ||
|
|
9e8592e220 | ||
|
|
b7b00f8233 | ||
|
|
45d2b29a42 | ||
|
|
7b494439e7 | ||
|
|
9a8dc097b2 | ||
|
|
274da18929 | ||
|
|
99a8ad2d4b | ||
|
|
bd9cdeadef | ||
|
|
bdf01a5a5a | ||
|
|
1849c74cea | ||
|
|
021b069706 | ||
|
|
824d674eec | ||
|
|
d43791c63e | ||
|
|
4dc7f2d97c | ||
|
|
092be7f222 | ||
|
|
340e7928f6 | ||
|
|
f5ff40bd20 | ||
|
|
7c271ea9c1 | ||
|
|
6794af6f68 | ||
|
|
691bef2cf4 | ||
|
|
1bbaa213cb | ||
|
|
5342765ca8 | ||
|
|
02394acf69 | ||
|
|
d8f69f7457 | ||
|
|
11c8fe3093 | ||
|
|
d6fecc761a | ||
|
|
40e3dafbee | ||
|
|
1fb7e1bc79 | ||
|
|
aeb4e1c64c | ||
|
|
a7b4cdd813 | ||
|
|
a540e47672 | ||
|
|
6a83ea8f2f | ||
|
|
c584675259 | ||
|
|
466e959340 | ||
|
|
d2251d0cbc | ||
|
|
78013aa5bc | ||
|
|
69b2cb2b02 | ||
|
|
c13085ba71 | ||
|
|
da25083837 | ||
|
|
76fb1cc479 | ||
|
|
cfb8992a92 | ||
|
|
7121531336 | ||
|
|
ff2b46b73e | ||
|
|
b158a38099 | ||
|
|
26f5d07435 | ||
|
|
747abeed79 | ||
|
|
1bcf80dd10 | ||
|
|
e7d7676a90 | ||
|
|
2b6566d33b | ||
|
|
f5641647ab | ||
|
|
fc5e66e913 | ||
|
|
5f4fc58416 | ||
|
|
84b2b19698 | ||
|
|
7214f8770c | ||
|
|
0a0062b9ab | ||
|
|
5d88755429 | ||
|
|
ecc75ef65a | ||
|
|
037f6d3d4f | ||
|
|
0cc7e36af5 | ||
|
|
946e904434 | ||
|
|
b17b596a64 | ||
|
|
5a24fe8b61 | ||
|
|
fed0c5739d | ||
|
|
603949002b | ||
|
|
de63f95ab0 | ||
|
|
a1b76650fb | ||
|
|
0cd8e53832 | ||
|
|
0210ccfcc3 | ||
|
|
517076330a | ||
|
|
ad249782fc | ||
|
|
2eab084358 | ||
|
|
547d2a316b | ||
|
|
1c4eef4224 | ||
|
|
d352bc0384 | ||
|
|
3304aff62b | ||
|
|
7469d833d1 | ||
|
|
b11fc6257a | ||
|
|
d1114a0dae | ||
|
|
f4b301d71c | ||
|
|
c6e42131b2 | ||
|
|
337dddf596 | ||
|
|
f7dbfba178 | ||
|
|
cad18f5bfc | ||
|
|
0a628bb7e7 |
No files matched your search
@@ -14,6 +14,7 @@ body:
|
||||
- 3.2.0
|
||||
- 3.3.0
|
||||
- 3.4.0
|
||||
- 3.5.0
|
||||
- Other (please provide detail below)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
@@ -6,6 +6,7 @@ on:
|
||||
- "salt/sensoroni/files/analyzers/**"
|
||||
- "salt/manager/tools/sbin/**"
|
||||
- "salt/_beacons/**"
|
||||
- "salt/elastalert/files/modules/**"
|
||||
- "salt/telegraf/tools/sbin_jinja/**"
|
||||
- "salt/telegraf/defaults.yaml"
|
||||
- "salt/telegraf/soc_telegraf.yaml"
|
||||
@@ -18,7 +19,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.14"]
|
||||
python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons"]
|
||||
python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons", "salt/elastalert/files/modules/so"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
+11
-11
@@ -1,17 +1,17 @@
|
||||
### 3.3.0-20260911 ISO image released on 2026/09/11
|
||||
### 3.4.0-20261102 ISO image released on 2026/11/02
|
||||
|
||||
|
||||
### Download and Verify
|
||||
|
||||
3.3.0-20260911 ISO image:
|
||||
https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
|
||||
3.4.0-20261102 ISO image:
|
||||
https://download.securityonion.net/file/securityonion/securityonion-3.4.0-20261102.iso
|
||||
|
||||
MD5: 12B18433D3A2198A185892FF79CF638F
|
||||
SHA1: 2B3C2E1FA7A78ED1F956E7EDCC12E32593C14EEE
|
||||
SHA256: 0938C73B76CE30EC9E4394D312C79EA7CAC721B6818541697279A6221F7D870D
|
||||
MD5: FB0380AE996E175D6CC7E7A35D8CC178
|
||||
SHA1: D8960F1DA352710BCA209773BF0B3AE8791D0D1B
|
||||
SHA256: DFC4C87668836852A328EC7A0B755C8C1511FF0E43D423DEE23047A95F676D99
|
||||
|
||||
Signature for ISO image:
|
||||
https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
|
||||
https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.4.0-20261102.iso.sig
|
||||
|
||||
Signing key:
|
||||
https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/main/KEYS
|
||||
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/3/
|
||||
|
||||
Download the signature file for the ISO:
|
||||
```
|
||||
wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.3.0-20260911.iso.sig
|
||||
wget https://github.com/Security-Onion-Solutions/securityonion/raw/3/main/sigs/securityonion-3.4.0-20261102.iso.sig
|
||||
```
|
||||
|
||||
Download the ISO image:
|
||||
```
|
||||
wget https://download.securityonion.net/file/securityonion/securityonion-3.3.0-20260911.iso
|
||||
wget https://download.securityonion.net/file/securityonion/securityonion-3.4.0-20261102.iso
|
||||
```
|
||||
|
||||
Verify the downloaded ISO image using the signature file:
|
||||
```
|
||||
gpg --verify securityonion-3.3.0-20260911.iso.sig securityonion-3.3.0-20260911.iso
|
||||
gpg --verify securityonion-3.4.0-20261102.iso.sig securityonion-3.4.0-20261102.iso
|
||||
```
|
||||
|
||||
The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
|
||||
```
|
||||
gpg: Signature made Fri 11 Sep 2026 11:23:56 AM EDT using RSA key ID FE507013
|
||||
gpg: Signature made Sat 31 Oct 2026 09:27:50 AM EDT using RSA key ID FE507013
|
||||
gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
|
||||
gpg: WARNING: This key is not certified with a trusted signature!
|
||||
gpg: There is no indication that the signature belongs to the owner.
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG
|
||||
EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx
|
||||
IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw
|
||||
MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
|
||||
biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND
|
||||
IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci
|
||||
MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti
|
||||
sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O
|
||||
BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
|
||||
Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c
|
||||
3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J
|
||||
0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ==
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,15 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG
|
||||
EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo
|
||||
bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g
|
||||
RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ
|
||||
TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
|
||||
b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw
|
||||
djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0
|
||||
WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS
|
||||
fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB
|
||||
zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq
|
||||
hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB
|
||||
CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD
|
||||
+JbNR6iC8hZVdyR+EhCVBCyj
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,21 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG
|
||||
A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg
|
||||
SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw
|
||||
MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
|
||||
biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v
|
||||
dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ
|
||||
BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ
|
||||
HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH
|
||||
3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH
|
||||
GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c
|
||||
xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1
|
||||
aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq
|
||||
TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL
|
||||
BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87
|
||||
/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4
|
||||
kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG
|
||||
YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT
|
||||
+xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo
|
||||
WXzhriKi4gp6D/piq1JM4fHfyr6DDUI=
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,32 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIFcjCCA1qgAwIBAgIKLwq3aw3LSq8nWDANBgkqhkiG9w0BAQwFADBWMQswCQYD
|
||||
VQQGEwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJ
|
||||
bmMxHDAaBgNVBAMTE2VtU2lnbiBSb290IENBIC0gQzIwHhcNMTgwMjE4MTgzMDAw
|
||||
WhcNNDMwMjE4MTgzMDAwWjBWMQswCQYDVQQGEwJVUzETMBEGA1UECxMKZW1TaWdu
|
||||
IFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMxHDAaBgNVBAMTE2VtU2lnbiBSb290
|
||||
IENBIC0gQzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCMfX1lA+Tb
|
||||
mh9YInmRgOW97IVx4LUJf2DRZfs837Jrml+py64aVnYgWO4t6C78fgjfS7jX+c4T
|
||||
inIzEquWcI+zi0fd4Sc8NDf7JONp27VWX0qwUYqzLDRCt+s7zpLcfx1ky0zVIJj6
|
||||
L06uPyK3kIr9+YAsrVj+39utm6e2MBQsRNstSI3fCQYAGvoQTQ8fULauTqNWaYAk
|
||||
NYFe6HUHHQPp2u1Ua00odMXiD5oRFxLcDnGAcE1I/9E9mLCdkggXijYUmico7+Xw
|
||||
ZeFoPhva6eIJ5p03Lt3Du5W3EcHR0cJmmY1pyeA36JaXKWRNM9IRjYMVNCcp4jhB
|
||||
2tIYiZ+LVk8bwQ9/1c23txmv3u97taZlV22NF4ttS1qq3J+MOp0oGULBzpKfRx0q
|
||||
GVqbPukQNGAjOLIN8KDNQNzbR1iAl2d8H+MSoicBo4Aid8TjLWcNv48oCWL53ZrF
|
||||
BMTDjaIA6frG1t4IpbnHadA7qCJJe2qpJN6n2eQKAUn6UiQDHPsSqNBlcUhQ4Y/0
|
||||
Y0mU5rghm2OB9rXQS1Fb1JRCfJMNnJIm5AUB2+2RWzq5Tgz7SbSho8NsZk0UbQnF
|
||||
xciqQ9uoVTAsK14Sk9oG8Q3zfsM08cdPoRb0WlIZklR6mKD7L8nH/zfGu8PIJv94
|
||||
GGB9RZ9U4A69r3ePmy8MvrzfNxHKtH6svwIDAQABo0IwQDAdBgNVHQ4EFgQUs/eK
|
||||
pNYPiABZ6FEXT9V+7IYigZ0wDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
|
||||
Af8wDQYJKoZIhvcNAQEMBQADggIBADQlpiWM0cv2nZ0H5jVsBq0x2q62Q0LwqATs
|
||||
CFvyub7gxNCytRuoA8stmPOEu/lg8Igxj4FIjoyhIrWUVxyiLU7No4P+WjEUOwUT
|
||||
xIpkEOtvGUQ9fiOlcGHtIZDNBlZq7WpktXAxeV55RPPsor26p2FNAMRFfZQh0sLX
|
||||
hKgk8iulSSggqx8ezgPye63FaiYEi4c/dzRj3HOCnsZiwZZU02df5YpNFjxSwZvE
|
||||
41cjGpsrpWMfQFI2s53RbeXp47lSAxYE4NzjBFMe+EwFuEveBCJBEAH5rvYu3pi2
|
||||
orsJ424TqWEQV1tCsCkQz+Yq/Okal7yHAkKDeOXcP7oN4A+TdXc2pdqxuVCnBO0R
|
||||
mWz2JpGSSeJjiTk/OPwRsPNWtwG/KXL04o2ta3jiPpJuICVtWDAc9R3auBEgJl5r
|
||||
ShRmBdszG0LmzsHuZPCFSYC15RBDCOBsa8bDRJ8pBFU2Wi/CVXCACEuavgoveA4F
|
||||
a5bt38o0PWxsBP+MpocCdVtDMqzQhxy9IohKuXWAGresoIvKDg3xFk6rBOrjfVwJ
|
||||
elwi/xAisojHPJVQv9W1zVIoHp+EQg/4MQC21NbIX2RoioB+V3hK439b/w7deU8x
|
||||
2M8cl1OG0nPfbnARl5GPM7vJgi470jto4SeMg6HMAW3Egb56tQcNLwI9U8mZnNvR
|
||||
gUMrkAgL
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,22 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD
|
||||
VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU
|
||||
ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH
|
||||
MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO
|
||||
MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv
|
||||
Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN
|
||||
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz
|
||||
f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO
|
||||
8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq
|
||||
d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM
|
||||
tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt
|
||||
Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB
|
||||
o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD
|
||||
AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x
|
||||
PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM
|
||||
wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d
|
||||
GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH
|
||||
6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby
|
||||
RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx
|
||||
iN66zB+Afko=
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,32 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIFlTCCA32gAwIBAgILAIZNvw/jXtd9jtgwDQYJKoZIhvcNAQEMBQAwZzELMAkG
|
||||
A1UEBhMCSU4xEzARBgNVBAsTCmVtU2lnbiBQS0kxJTAjBgNVBAoTHGVNdWRocmEg
|
||||
VGVjaG5vbG9naWVzIExpbWl0ZWQxHDAaBgNVBAMTE2VtU2lnbiBSb290IENBIC0g
|
||||
RzIwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBnMQswCQYDVQQGEwJJ
|
||||
TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
|
||||
b2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBHMjCCAiIw
|
||||
DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMNwGIWW2kHfHK+sXTNwxF07K+IV
|
||||
ySTuyFM2r1v002wUfcdT+zs5OM5QbMYFFnedXQI6gCFLsjKrcaej48Zt37OyEb3i
|
||||
aPs7CsP4kAyTwzKH9aZe6gXYHrJq40/ZVMNcQVI2PcIp40B/SAN2gUZ+ZaUtIOvV
|
||||
jEx26/ebNaXRIsthlkOG/caB+QRwDw1tl7338Zlv0M2oTBUy4B3e7dGP5pgXH71M
|
||||
jqHPCoNo+xv9f0NTBT+hUDa8h8wUtcGQq9CDeJTpjWcD2bP2AMdVG6oVpMAUeUzo
|
||||
cCyglvtFdUMjggxBbw4qhau1HXPG8Ot9hwL7ZMi8tkTzrvUIxxb8G9LF/7kKeCE7
|
||||
tGZaVzDTnXuifl3msR4ErHsQ4P7lVu2AIjIAhrAXoedDidb7pMcf7TABdrYUT1Jo
|
||||
G/AiK+J9jO6GTjeADD4LMDSBZhHMuBK/PJ/g0kGBt+/C1L+/HURzQhJkMlRnM6Rv
|
||||
XoCtfKopSlns5trZmTi971Wjbn88QXP61lGpBCUPwCjs7rpOYvSUJtI+lcbF+37q
|
||||
kIqOXYkVT3cupDSpw+H89kFtj5GKY+Xny4LxY+3IvDIRiyd6ky1DPj713DI0yqve
|
||||
EpsIr3A0PdwuyUI7CS1jg0NnGFT6Xxyr0xB+VDt83FJYW8v16k2pbaQ4kVxA3aXd
|
||||
X9dZYyVR1S59KM75AgMBAAGjQjBAMB0GA1UdDgQWBBTt7E1FYRgo57MjKBEcTaUn
|
||||
DV7s9DAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
|
||||
AQwFAAOCAgEACFC/ilQg8KTCVBxFJW/sazomkS0kNYbEIZg4B3obqwsJ7SX98z8Z
|
||||
gfzBpz0nYClwwJjWbFN1R2zY8pCEot6/dgmA8Vbq0GxhwPM5YN/SZquNyRIxO3cU
|
||||
dlAcwf+vSezdVCf9wOzvSAF3q0a5ljvbdbNJNpfScQVp7UUd5sBsZk8jXO1KQ/go
|
||||
/Vf/GDPnrIFmxpAIGE3sgnO8lAv9FzUaAeuv7HWe47xN9J7+bQzF93yHuIXACPTL
|
||||
pQHhg2zMv5C7BAbuDHfbj1Cu294Z832yhSfBcziWGskOvl3es2EcHytbS9c9P+0z
|
||||
Mpka7zGC1FHrvLb/FoduH86TeZt0QjZ6pcplNzoaxDnDvzTJ6CC2Eny+qH/APFCu
|
||||
VUv5/wjwF+HPm8Pup2ARj9cEp92+0qcerfHacNq5hMeGZdbA/dzdUR/5z5zXdxAk
|
||||
nl8mcfGb0eMNSTXQmmB/i4AecNnr72uYjzlaXUGYN7Nrb6XouG0pnh0/BBtWWp0U
|
||||
ShIPpWEAqs7RJBj6+1ZUYXZ4ObrCw962DxhN2p19Hxw9LtuUUcLqqTPrFXYvwO4t
|
||||
ouj7KJnAkaTUfXGdEaFVtFig1EA30WzJY2X1vAQ7hVnniCjgaXAGqjsU6sklNM9n
|
||||
xDx5rFCCCEtj9Kh8UHjGK2QqgP5kwgttjOApQMaCoezMfK4KD7WpOXU=
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,32 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIFdjCCA16gAwIBAgIQCIYhdXKnxAzDuy7l1y1ubDANBgkqhkiG9w0BAQwFADBV
|
||||
MQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGlt
|
||||
aXRlZDEfMB0GA1UEAwwWZW1TaWduIFJvb3QgQ1MgQ0EgLSBHMjAeFw0yNDAyMDgw
|
||||
ODE1MzlaFw00OTAyMDcwODE1MzlaMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxl
|
||||
TXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9v
|
||||
dCBDUyBDQSAtIEcyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA4CVO
|
||||
xuBjdLJXpkrjosA+7ygTGaYzufUJrhQOeIZEuCqeCPI8hDqIu+MIqEqOlgDFetVM
|
||||
3Cor3CGBlGJzfzzlI31vyElvQq88jjCfo12Ghqcf+NAZDFduJk5+NgC0z3lfdiB6
|
||||
NNzxbfVkA+X6oDNNRioVUZsVFuIzT9Qt0Sh4KwQPRFBc/wI2yGpDuKHNsLehcECx
|
||||
Uw00m33xGYh9UkeL3reOmAaLYWZiO+Pd2vxPtrkXI4wyN4BMlESJ73uP0tyX9lq4
|
||||
M4yN2yh3i+ETI8/jzPjxHD7QFaVkCoS+uj2YmBa3mytlqlBgyOoKueNeotX8hfzB
|
||||
FNm962abpV4mN0SoC6hjkYv8IayseRPFv9VI0meQXg9OLRoNvBOqG5kw937FH0Li
|
||||
q3CIzKyvdyeYRT0LJdWA2iuB9l+4zzIfCwGyT4M4VDmJuPBhrTmjXKae6AUtm1eO
|
||||
q2f8gftA6J6AlbD0gRd/t8CrFssFNm+LYAEd36kWRBuolbaOfbSbEww84xNAgj4U
|
||||
SPwYjtNDsCkQjtgSOaDtDOmolN+Mbk528Y+LHMlbd4T3Xmp+VIWp5+70GhBDOA91
|
||||
hhtims+hHkXjRI2U7WrJLcTlcF8ykiymRbFKhwOBvTmtZNbHmDI7scU4wWUAsmW6
|
||||
tWaeeoM50KBl8ai28A6yBGYMO//03Ubvqiqst80CAwEAAaNCMEAwHQYDVR0OBBYE
|
||||
FOYcxqontIvyvYVkX6/0unKm0AF1MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E
|
||||
BTADAQH/MA0GCSqGSIb3DQEBDAUAA4ICAQCv5Asord7fdQMT30emwb+oLuDUxdmJ
|
||||
DQHnzxIXoNvymDTbNHF9LqjeXc+X+XcjIV8PTp4o2I9wyVppHmcaVgRjN/G9FxeN
|
||||
5tCQ97v5sdx4eAdPljK1xqg7Vhfmzcp+7rSjJG7Sn9gMlZx81kL3Oa2qZj+W5Fi6
|
||||
tN4vaR9OLEyvKKHDG/eFxS/qghD1d/Clkb7UGGjBcWJlaZTWVALMKG+QCxiMoELi
|
||||
oeKaoBbZoXooxfF0mWdkIl9RWNU4r9O/yt66LBE9rj49dX19L+rGg71/4bTi5ua+
|
||||
+Wz60XaOc2CdHJaBSols5hWEx/LBjS7UzfU3uJfnpQXVDi0US5sFzIHIlS6kjh7h
|
||||
YuF4+/9ZAl4Bx3w2TNN7pVm7I39pgYl7XN6KZRwfkAv4aRswPqhWbQfr3+qfcC93
|
||||
cgTyFy2RG5YE/OfmpSs6VtYKawqsdEODz/p2nhZfSDrVDnEPvERqahttY5+gaWNc
|
||||
2fbibgKMfuejTZaJ3yttPslVSko5SDs0xKU423swu34sb/fdISqtOu4NzPCZiHSG
|
||||
ladaoITmSA6UWFk4fdqaLBI+xSxY5Et3S1APp5r3j/l1KrWPNbb1/+ydHN+YLbh8
|
||||
hCBIecKdk6VbnnFRlulm0d2RSgpfCRrWLomN8JHCxCMVv/hBQXz3AXGM1ZlfYTyl
|
||||
E1BpDby0G2eEkw==
|
||||
-----END CERTIFICATE-----
|
||||
@@ -10,3 +10,4 @@ include:
|
||||
- ca.server
|
||||
{% endif %}
|
||||
- ca.trustca
|
||||
- ca.trust
|
||||
@@ -0,0 +1,12 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{# Containers mount the extracted directory rather than single files: update-ca-trust replaces
|
||||
files by rename, so a file bind mount would keep pointing at the old inode. #}
|
||||
{% set CATRUST = {
|
||||
'dir': '/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro',
|
||||
'java': '/etc/pki/ca-trust/extracted/java/cacerts',
|
||||
'pem': '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'
|
||||
} %}
|
||||
@@ -0,0 +1,37 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
ca_trust_anchor:
|
||||
file.managed:
|
||||
- name: /etc/pki/ca-trust/source/anchors/so-intca.crt
|
||||
- source: salt://ca/files/ca.crt
|
||||
- show_changes: False
|
||||
- makedirs: True
|
||||
|
||||
ca_trust_legacy_anchor:
|
||||
file.absent:
|
||||
- name: /etc/pki/ca-trust/source/anchors/ca.crt
|
||||
|
||||
ca_trust_legacy_bundles:
|
||||
file.absent:
|
||||
- name: /opt/so/conf/ca
|
||||
|
||||
# clean removes blocklist entries that are no longer shipped
|
||||
ca_trust_blocklist:
|
||||
file.recurse:
|
||||
- name: /etc/pki/ca-trust/source/blocklist
|
||||
- source: salt://ca/files/blocklist
|
||||
- clean: True
|
||||
- makedirs: True
|
||||
- file_mode: '0644'
|
||||
- dir_mode: '0755'
|
||||
|
||||
ca_trust_extract:
|
||||
cmd.run:
|
||||
- name: update-ca-trust extract
|
||||
- onchanges:
|
||||
- file: ca_trust_anchor
|
||||
- file: ca_trust_legacy_anchor
|
||||
- file: ca_trust_blocklist
|
||||
@@ -656,6 +656,108 @@ set_version() {
|
||||
fi
|
||||
}
|
||||
|
||||
so_telegraf_output() {
|
||||
local defaultsfile="${DEFAULT_SALT_DIR}/salt/telegraf/defaults.yaml"
|
||||
local output
|
||||
|
||||
output=$(lookup_pillar output telegraf) || return 1
|
||||
if [ -z "$output" ]; then
|
||||
if [ ! -f "$defaultsfile" ]; then
|
||||
echo "Could not find $defaultsfile" >&2
|
||||
return 1
|
||||
fi
|
||||
output=$(grep -E "^ output: " "$defaultsfile" | awk -F: '{print $2}' | tr -d '[:space:]')
|
||||
fi
|
||||
echo "${output^^}"
|
||||
}
|
||||
|
||||
so_telegraf_pg_check() {
|
||||
local app="$1"
|
||||
local databases role_ok
|
||||
|
||||
if ! docker_check_running postgres; then
|
||||
echo "so-postgres is not running. Run $app on the manager." >&2
|
||||
return 1
|
||||
fi
|
||||
if ! databases=$(docker exec so-postgres psql -U postgres -lqt 2>/dev/null); then
|
||||
echo "Unable to list the so-postgres databases." >&2
|
||||
return 1
|
||||
fi
|
||||
if ! echo "$databases" | cut -d\| -f1 | grep -qw so_telegraf; then
|
||||
echo "Database so_telegraf not found. Is telegraf.output set to POSTGRES or BOTH?" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null); then
|
||||
echo "Unable to query the so-postgres roles." >&2
|
||||
return 1
|
||||
fi
|
||||
if [ -z "$role_ok" ]; then
|
||||
echo "The so_maint_telegraf role is missing; run a highstate." >&2
|
||||
return 1
|
||||
elif [ "$role_ok" != t ]; then
|
||||
echo "The so_maint_telegraf role is misconfigured; run a highstate." >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
so_telegraf_psql() {
|
||||
local app="$1"
|
||||
local sql="$2"
|
||||
|
||||
docker exec -e PGOPTIONS="-c application_name=${app} -c default_transaction_read_only=on -c client_min_messages=error" \
|
||||
so-postgres psql -U so_maint_telegraf -d so_telegraf -v ON_ERROR_STOP=1 -At -F '|' -c "$sql"
|
||||
}
|
||||
|
||||
# Prints one stream|current|then line per data stream, in bytes summed across hosts. current is
|
||||
# each host's newest size in the past 4 hours; then is that host's newest size in the window
|
||||
# [now - start, now - stop), or its current size when it has none there, so a host missing from the
|
||||
# window adds no growth. Empty output means the window has no rows at all. Values over 15 digits
|
||||
# (1 PB) are skipped so the sums here and in the callers stay far below the 64-bit limit; such a
|
||||
# value can only come from a forged Telegraf writer.
|
||||
so_es_index_size_growth() {
|
||||
local app="$1"
|
||||
local start="$2"
|
||||
local stop="$3"
|
||||
local relkind
|
||||
|
||||
relkind=$(so_telegraf_psql "$app" "SELECT c.relkind FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'telegraf' AND c.relname = 'elasticsearch_index_size'") || return 1
|
||||
[[ "$relkind" == "r" || "$relkind" == "p" ]] || return 0
|
||||
|
||||
so_telegraf_psql "$app" "$(cat <<EOF
|
||||
WITH sizes AS (
|
||||
SELECT d.tag_id, d.time, f.key AS stream, f.value::bigint AS bytes
|
||||
FROM telegraf.elasticsearch_index_size d
|
||||
CROSS JOIN LATERAL pg_catalog.jsonb_each_text(d.fields) f
|
||||
WHERE d.time >= now() - interval '${start}'
|
||||
AND f.key ~ '^[^|"''[:space:][:cntrl:]]+$'
|
||||
AND pg_catalog.strpos(f.key, pg_catalog.chr(92)) = 0
|
||||
AND f.value ~ '^[0-9]{1,15}$'
|
||||
),
|
||||
cur AS (
|
||||
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
|
||||
FROM sizes
|
||||
WHERE time >= now() - interval '4 hours'
|
||||
ORDER BY tag_id, stream, time DESC
|
||||
),
|
||||
past AS (
|
||||
SELECT DISTINCT ON (tag_id, stream) tag_id, stream, bytes
|
||||
FROM sizes
|
||||
WHERE time < now() - interval '${stop}'
|
||||
ORDER BY tag_id, stream, time DESC
|
||||
)
|
||||
SELECT c.stream, sum(c.bytes), sum(coalesce(p.bytes, c.bytes))
|
||||
FROM cur c
|
||||
LEFT JOIN past p USING (tag_id, stream)
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM telegraf.elasticsearch_index_size
|
||||
WHERE time >= now() - interval '${start}' AND time < now() - interval '${stop}'
|
||||
)
|
||||
GROUP BY c.stream
|
||||
ORDER BY c.stream
|
||||
EOF
|
||||
)"
|
||||
}
|
||||
|
||||
status () {
|
||||
printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n"
|
||||
}
|
||||
|
||||
@@ -178,6 +178,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|respondError" # respondError is a function name, output via http middleware as standard request logging
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|GET /kibana/" # Ignore Kibana queries with triggered words
|
||||
fi
|
||||
|
||||
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
|
||||
@@ -243,6 +244,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
|
||||
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|push result unknown.*AuthenticationError" # so-push-drainer WARNING: salt-master restarted while a push was queued; the queued state run may still complete
|
||||
fi
|
||||
|
||||
RESULT=0
|
||||
|
||||
@@ -438,6 +438,10 @@ install_networkminer:
|
||||
pkg.latest:
|
||||
- name: securityonion-networkminer
|
||||
|
||||
install_so_crates:
|
||||
pkg.latest:
|
||||
- name: securityonion-so-crates
|
||||
|
||||
{% else %}
|
||||
|
||||
desktop_packages_os_fail:
|
||||
|
||||
@@ -1,18 +1,10 @@
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
|
||||
{# we only want this state to run it is CentOS #}
|
||||
{% if GLOBALS.os == 'OEL' %}
|
||||
|
||||
trusted_ca:
|
||||
file.managed:
|
||||
- name: /etc/pki/ca-trust/source/anchors/ca.crt
|
||||
- source: salt://ca/files/ca.crt
|
||||
|
||||
update_ca_certs:
|
||||
cmd.run:
|
||||
- name: update-ca-trust
|
||||
- onchanges:
|
||||
- file: trusted_ca
|
||||
include:
|
||||
- ca.trust
|
||||
|
||||
{% else %}
|
||||
|
||||
|
||||
@@ -103,7 +103,6 @@ docker:
|
||||
- 80:80
|
||||
- 443:443
|
||||
- 8443:8443
|
||||
- 7788:7788
|
||||
- 7789:7789
|
||||
custom_bind_mounts: []
|
||||
extra_hosts: []
|
||||
|
||||
@@ -10,7 +10,7 @@ elastalert:
|
||||
buffer_time:
|
||||
minutes: 10
|
||||
old_query_limit:
|
||||
minutes: 5
|
||||
minutes: 1440
|
||||
es_port: 9200
|
||||
es_conn_timeout: 55
|
||||
max_query_size: 5000
|
||||
|
||||
@@ -6,9 +6,11 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- elastalert.config
|
||||
- elastalert.sostatus
|
||||
|
||||
@@ -28,6 +30,7 @@ so-elastalert:
|
||||
- ipv4_address: {{ DOCKERMERGED.containers['so-elastalert'].ip }}
|
||||
- detach: True
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
|
||||
- /opt/so/log/elastalert:/var/log/elastalert:rw
|
||||
- /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
|
||||
@@ -46,8 +49,9 @@ so-elastalert:
|
||||
- {{ XTRAHOST }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
|
||||
- environment:
|
||||
- REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt
|
||||
{% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
|
||||
{% for XTRAENV in DOCKERMERGED.containers['so-elastalert'].extra_env %}
|
||||
- {{ XTRAENV }}
|
||||
{% endfor %}
|
||||
@@ -65,6 +69,7 @@ so-elastalert:
|
||||
- file: elastacustmodulesdir
|
||||
- file: elastaconf
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: elastaconf
|
||||
- onlyif:
|
||||
- "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 9" {# only run this state if elasticsearch is version 9 #}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
from datetime import datetime
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import types
|
||||
|
||||
# stand-ins when ElastAlert isn't installed (CI)
|
||||
try:
|
||||
import elastalert.alerts # noqa: F401
|
||||
except ImportError:
|
||||
|
||||
class Alerter:
|
||||
def __init__(self, rule):
|
||||
self.rule = rule
|
||||
|
||||
class DateTimeEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
return obj.isoformat() if hasattr(obj, 'isoformat') else json.JSONEncoder.default(self, obj)
|
||||
|
||||
class EAException(Exception):
|
||||
pass
|
||||
|
||||
def lookup_es_key(doc, term):
|
||||
for part in term.split('.'):
|
||||
if not isinstance(doc, dict) or part not in doc:
|
||||
return None
|
||||
doc = doc[part]
|
||||
return doc
|
||||
|
||||
def ts_to_dt(value):
|
||||
return value if isinstance(value, datetime) else datetime.fromisoformat(value)
|
||||
|
||||
def elasticsearch_client(conf):
|
||||
return None # tests set the alerter's client
|
||||
|
||||
alerts = types.ModuleType('elastalert.alerts')
|
||||
alerts.Alerter = Alerter
|
||||
alerts.DateTimeEncoder = DateTimeEncoder
|
||||
util = types.ModuleType('elastalert.util')
|
||||
util.EAException = EAException
|
||||
util.elastalert_logger = logging.getLogger('elastalert')
|
||||
util.lookup_es_key = lookup_es_key
|
||||
util.ts_to_dt = ts_to_dt
|
||||
util.elasticsearch_client = elasticsearch_client
|
||||
package = types.ModuleType('elastalert')
|
||||
package.alerts = alerts
|
||||
package.util = util
|
||||
sys.modules.update({'elastalert': package, 'elastalert.alerts': alerts, 'elastalert.util': util})
|
||||
|
||||
# stand-ins when elasticsearch-py isn't installed (CI)
|
||||
try:
|
||||
import elasticsearch.exceptions # noqa: F401
|
||||
except ImportError:
|
||||
|
||||
class ElasticsearchException(Exception):
|
||||
pass
|
||||
|
||||
class TransportError(ElasticsearchException):
|
||||
pass
|
||||
|
||||
class ConnectionError(TransportError):
|
||||
pass
|
||||
|
||||
class ConflictError(TransportError):
|
||||
pass
|
||||
|
||||
class RequestError(TransportError):
|
||||
pass
|
||||
|
||||
exceptions = types.ModuleType('elasticsearch.exceptions')
|
||||
for cls in (ElasticsearchException, TransportError, ConnectionError, ConflictError, RequestError):
|
||||
setattr(exceptions, cls.__name__, cls)
|
||||
es_package = types.ModuleType('elasticsearch')
|
||||
es_package.exceptions = exceptions
|
||||
sys.modules.update({'elasticsearch': es_package, 'elasticsearch.exceptions': exceptions})
|
||||
@@ -1,63 +1,269 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
from time import gmtime, strftime
|
||||
import requests,json
|
||||
from elastalert.alerts import Alerter
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import urllib3
|
||||
from elasticsearch.exceptions import ConflictError, ElasticsearchException, RequestError
|
||||
from elastalert.alerts import Alerter, DateTimeEncoder
|
||||
from elastalert.util import EAException, elastalert_logger, elasticsearch_client, lookup_es_key, ts_to_dt
|
||||
|
||||
# grid runs verify_certs: false; also quiets ElastAlert's own queries
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
ALERT_INDEX = 'logs-detections.alerts-so'
|
||||
# ES error text kept in logs and alerts
|
||||
ERROR_TEXT_LIMIT = 500
|
||||
# a match missing backend columns (window_start, count, @timestamp); the alert is still written
|
||||
MATCH_ERRORS = (KeyError, TypeError, ValueError)
|
||||
|
||||
|
||||
class SecurityOnionESAlerter(Alerter):
|
||||
"""
|
||||
Use matched data to create alerts in Elasticsearch.
|
||||
"""
|
||||
|
||||
required_options = set(['detection_title', 'sigma_level'])
|
||||
optional_fields = ['sigma_category', 'sigma_product', 'sigma_service']
|
||||
required_options = {'detection_title', 'sigma_level'}
|
||||
optional_fields = ['sigma_category', 'sigma_product', 'sigma_service', 'sigma_correlation']
|
||||
|
||||
# count column and default summary per type; stored alert data, so not localized
|
||||
CORRELATION_COUNTS = {
|
||||
'event_count': ('event_count', '%count% events'),
|
||||
'value_count': ('value_count', '%count% distinct values'),
|
||||
'temporal': ('event_type_count', '%count% correlated rules matched'),
|
||||
'value_sum': ('value_sum', 'total %count%'),
|
||||
'value_avg': ('value_avg', 'average %count%'),
|
||||
'value_percentile': ('value_percentile', 'percentile %count%'),
|
||||
'value_median': ('value_median', 'median %count%'),
|
||||
}
|
||||
PLACEHOLDER = re.compile(r'%([^%\s]+)%')
|
||||
# group-by fields copied into ECS related.*
|
||||
RELATED_USERS = {'user.name', 'winlog.event_data.TargetUserName', 'winlog.event_data.SubjectUserName'}
|
||||
RELATED_HOSTS = {'host.name', 'host.hostname', 'winlog.computer_name'}
|
||||
|
||||
def __init__(self, rule):
|
||||
super().__init__(rule)
|
||||
# uses the grid's TLS, auth and timeout settings
|
||||
self.es = elasticsearch_client(rule)
|
||||
|
||||
@property
|
||||
def is_correlation(self):
|
||||
return bool(self.rule.get('sigma_correlation'))
|
||||
|
||||
def query_keys(self):
|
||||
"""compound_query_key holds the list; query_key is flattened to a string."""
|
||||
if self.rule.get('compound_query_key'):
|
||||
return self.rule['compound_query_key']
|
||||
if self.rule.get('query_key'):
|
||||
return [self.rule['query_key']]
|
||||
return []
|
||||
|
||||
def alert_id(self, match):
|
||||
"""Stable id: window end + group values for correlations, source _id otherwise; random without one."""
|
||||
if self.is_correlation:
|
||||
# ungrouped rows have a hashed _id that changes with the count
|
||||
values = ''.join(f"|{lookup_es_key(match, k)}" for k in self.query_keys())
|
||||
key = f"{self.rule['detection_public_id']}|{ts_to_dt(match['@timestamp']).isoformat()}{values}"
|
||||
elif match.get('_id'):
|
||||
key = f"{self.rule['detection_public_id']}|{match['_id']}"
|
||||
else:
|
||||
return uuid.uuid4().hex
|
||||
|
||||
return hashlib.sha256(key.encode('utf-8')).hexdigest()
|
||||
|
||||
def group(self, match):
|
||||
"""Group-by values, joined like ElastAlert's realert key."""
|
||||
return ', '.join(str(lookup_es_key(match, k)) for k in self.query_keys())
|
||||
|
||||
def related_bucket(self, key):
|
||||
if key == 'ip' or key.endswith('.ip'):
|
||||
return 'ip'
|
||||
if key in self.RELATED_USERS or key.endswith('.user.name'):
|
||||
return 'user'
|
||||
if key in self.RELATED_HOSTS:
|
||||
return 'hosts'
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def valid_ip(value):
|
||||
try:
|
||||
ipaddress.ip_address(value)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
def related(self, match):
|
||||
"""ECS related.* from group-by values; skips invalid IPs."""
|
||||
related = {}
|
||||
for key in self.query_keys():
|
||||
bucket = self.related_bucket(key)
|
||||
if not bucket:
|
||||
continue
|
||||
# original spellings of a lowercased group
|
||||
value = lookup_es_key(match, f"{key}_spellings")
|
||||
if value is None:
|
||||
value = lookup_es_key(match, key)
|
||||
for v in value if isinstance(value, list) else [value]:
|
||||
if v is None or (bucket == 'ip' and not self.valid_ip(str(v))):
|
||||
continue
|
||||
# dict: ordered and deduped
|
||||
related.setdefault(bucket, {})[str(v)] = None
|
||||
return {bucket: list(values) for bucket, values in related.items()}
|
||||
|
||||
def event_data(self, match):
|
||||
"""The match minus the compound query_key field, which ES would map by its last part."""
|
||||
if not self.rule.get('compound_query_key'):
|
||||
return match
|
||||
return {k: v for k, v in match.items() if k != self.rule['query_key']}
|
||||
|
||||
@staticmethod
|
||||
def format_value(value):
|
||||
if isinstance(value, list):
|
||||
shown = ', '.join(str(v) for v in value[:3])
|
||||
return shown if len(value) <= 3 else f"{shown} and {len(value) - 3} more"
|
||||
return str(value)
|
||||
|
||||
@staticmethod
|
||||
def format_count(value):
|
||||
if isinstance(value, float) and not value.is_integer():
|
||||
return f"{value:,.2f}"
|
||||
if isinstance(value, (int, float)):
|
||||
return f"{int(value):,}"
|
||||
return str(value)
|
||||
|
||||
@staticmethod
|
||||
def format_duration(seconds):
|
||||
for unit, size in (('hour', 3600), ('minute', 60)):
|
||||
if seconds >= 2 * size:
|
||||
return f"{seconds // size} {unit}s"
|
||||
return f"{seconds} second{'' if seconds == 1 else 's'}"
|
||||
|
||||
def summary(self, match):
|
||||
"""One-line correlation summary."""
|
||||
column, label = self.CORRELATION_COUNTS.get(self.rule['sigma_correlation'], (None, '%count%'))
|
||||
start = ts_to_dt(match['window_start'])
|
||||
end = ts_to_dt(match['@timestamp'])
|
||||
values = {
|
||||
'count': self.format_count(match.get(column)),
|
||||
'start': start.strftime('%Y-%m-%d %H:%M:%S UTC'),
|
||||
'end': end.strftime('%Y-%m-%d %H:%M:%S UTC'),
|
||||
'duration': self.format_duration(int((end - start).total_seconds())),
|
||||
}
|
||||
|
||||
template = self.rule.get('summary_template')
|
||||
if not template:
|
||||
groups = ', '.join(f"{k} %{k}%" for k in self.query_keys())
|
||||
template = f"{label} for {groups} in %duration%" if groups else f"{label} in %duration%"
|
||||
|
||||
def fill(m):
|
||||
if m[1] in values:
|
||||
return values[m[1]]
|
||||
value = lookup_es_key(match, m[1])
|
||||
# unknown placeholders stay visible so typos show
|
||||
return m[0] if value is None else self.format_value(value)
|
||||
|
||||
return self.PLACEHOLDER.sub(fill, template)
|
||||
|
||||
def alert(self, matches):
|
||||
for match in matches:
|
||||
timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
|
||||
headers = {"Content-Type": "application/json"}
|
||||
try:
|
||||
alert_id = self.alert_id(match)
|
||||
except MATCH_ERRORS as e:
|
||||
elastalert_logger.warning("Writing alert for rule %s without a stable id, so a retry may duplicate it: %r",
|
||||
self.rule['detection_public_id'], e)
|
||||
alert_id = uuid.uuid4().hex
|
||||
try:
|
||||
self.write(alert_id, self.payload(match))
|
||||
except ElasticsearchException as e:
|
||||
# EAException makes ElastAlert retry
|
||||
raise EAException(f"Unable to write the alert to Elasticsearch: {str(e)[:ERROR_TEXT_LIMIT]}") from e
|
||||
|
||||
creds = None
|
||||
if 'es_username' in self.rule and 'es_password' in self.rule:
|
||||
creds = (self.rule['es_username'], self.rule['es_password'])
|
||||
def payload(self, match):
|
||||
rule_info = {
|
||||
"name": self.rule['detection_title'],
|
||||
"uuid": self.rule['detection_public_id']
|
||||
}
|
||||
|
||||
# Start building the rule dict
|
||||
rule_info = {
|
||||
"name": self.rule['detection_title'],
|
||||
"uuid": self.rule['detection_public_id']
|
||||
}
|
||||
# Add optional fields if they are present in the rule
|
||||
for field in self.optional_fields:
|
||||
rule_key = field.split('_')[-1] # Assumes field format "sigma_<key>"
|
||||
if field in self.rule:
|
||||
rule_info[rule_key] = self.rule[field]
|
||||
|
||||
# Add optional fields if they are present in the rule
|
||||
for field in self.optional_fields:
|
||||
rule_key = field.split('_')[-1] # Assumes field format "sigma_<key>"
|
||||
if field in self.rule:
|
||||
rule_info[rule_key] = self.rule[field]
|
||||
event_info = {
|
||||
"kind": "alert",
|
||||
"severity": self.rule['event.severity'],
|
||||
"module": self.rule['event.module'],
|
||||
"dataset": self.rule['event.dataset'],
|
||||
"severity_label": self.rule['sigma_level']
|
||||
}
|
||||
|
||||
# Construct the payload with the conditional rule_info
|
||||
payload = {
|
||||
"tags": "alert",
|
||||
"rule": rule_info,
|
||||
"event": {
|
||||
"severity": self.rule['event.severity'],
|
||||
"module": self.rule['event.module'],
|
||||
"dataset": self.rule['event.dataset'],
|
||||
"severity_label": self.rule['sigma_level']
|
||||
},
|
||||
"sigma_level": self.rule['sigma_level'],
|
||||
"event_data": match,
|
||||
"@timestamp": timestamp
|
||||
}
|
||||
url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/"
|
||||
requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
|
||||
payload = {
|
||||
"tags": ["alert"],
|
||||
"rule": rule_info,
|
||||
"event": event_info,
|
||||
"sigma_level": self.rule['sigma_level'],
|
||||
"event_data": self.event_data(match),
|
||||
"@timestamp": datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z')
|
||||
}
|
||||
|
||||
if self.is_correlation:
|
||||
keys = self.query_keys()
|
||||
try:
|
||||
# built before any is added, so a failure adds none
|
||||
reason = self.summary(match)
|
||||
labels = {"correlation_group_by": ', '.join(keys), "correlation_group": self.group(match)} if keys else None
|
||||
related = self.related(match)
|
||||
except MATCH_ERRORS as e:
|
||||
elastalert_logger.warning("Writing alert for rule %s without its correlation summary: %r",
|
||||
self.rule['detection_public_id'], e)
|
||||
else:
|
||||
payload["event"]["reason"] = reason
|
||||
if labels:
|
||||
payload["labels"] = labels
|
||||
if related:
|
||||
payload["related"] = related
|
||||
|
||||
return payload
|
||||
|
||||
def write(self, alert_id, payload):
|
||||
try:
|
||||
self.create(alert_id, payload)
|
||||
except RequestError as e:
|
||||
# mapping rejections come from event_data; retry it as text
|
||||
rejection = str(e)[:ERROR_TEXT_LIMIT]
|
||||
try:
|
||||
self.create(alert_id, self.without_event_data(payload, rejection))
|
||||
except RequestError as again:
|
||||
elastalert_logger.error("Dropping alert %s for rule %s, rejected by Elasticsearch even without its event data: %s; first rejection: %s",
|
||||
alert_id, self.rule['detection_public_id'], str(again)[:ERROR_TEXT_LIMIT], rejection)
|
||||
return
|
||||
elastalert_logger.warning("Stored alert %s for rule %s with its event data as text, rejected by Elasticsearch: %s",
|
||||
alert_id, self.rule['detection_public_id'], rejection)
|
||||
|
||||
def create(self, alert_id, payload):
|
||||
try:
|
||||
self.es.create(index=ALERT_INDEX, id=alert_id, body=payload)
|
||||
except ConflictError:
|
||||
pass # a repeat id is already stored
|
||||
|
||||
@staticmethod
|
||||
def without_event_data(payload, rejection):
|
||||
"""Moves event_data to event.original; the tag keeps Fleet's final pipeline from dropping it."""
|
||||
fallback = {k: v for k, v in payload.items() if k != 'event_data'}
|
||||
fallback['event'] = dict(payload['event'], original=json.dumps(payload['event_data'], cls=DateTimeEncoder))
|
||||
fallback['error'] = {'message': f"event_data rejected by Elasticsearch: {rejection}"}
|
||||
fallback['tags'] = payload['tags'] + ['preserve_original_event']
|
||||
return fallback
|
||||
|
||||
def get_info(self):
|
||||
return {'type': 'SecurityOnionESAlerter'}
|
||||
@@ -0,0 +1,260 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
import copy
|
||||
from datetime import datetime, timezone
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from elasticsearch.exceptions import ConflictError, ConnectionError, RequestError
|
||||
|
||||
spec = importlib.util.spec_from_file_location('securityonion_es', os.path.join(os.path.dirname(__file__), 'securityonion-es.py'))
|
||||
es = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(es)
|
||||
|
||||
BASE_RULE = {
|
||||
'detection_title': 'Many Failed Network Logons To One Host From One Source',
|
||||
'detection_public_id': '35a42db6-6629-45af-b8aa-e1fa33c28ef5',
|
||||
'sigma_level': 'medium',
|
||||
'sigma_correlation': 'event_count',
|
||||
'event.severity': 3,
|
||||
'event.module': 'sigma',
|
||||
'event.dataset': 'sigma.alert',
|
||||
'summary_template': '%count% failed network logons to %host.name% from %source.ip% in %duration%',
|
||||
}
|
||||
|
||||
PLAIN_RULE = {k: v for k, v in BASE_RULE.items() if k not in ('sigma_correlation', 'summary_template')}
|
||||
|
||||
|
||||
def correlation_match():
|
||||
return {
|
||||
'event_count': 3561,
|
||||
'window_start': '2026-09-30T18:05:10+00:00',
|
||||
'@timestamp': '2026-09-30T18:07:53+00:00',
|
||||
'host': {'name': 'host-01'},
|
||||
'source': {'ip': '192.0.2.10'},
|
||||
'_id': '6d1c',
|
||||
'num_hits': 1,
|
||||
'num_matches': 1,
|
||||
}
|
||||
|
||||
|
||||
class TestSecurityOnionESAlerter(unittest.TestCase):
|
||||
|
||||
def creates(self, rule, match, effects=None):
|
||||
"""Run alert(); return (body, id) of each create."""
|
||||
alerter = es.SecurityOnionESAlerter(rule)
|
||||
alerter.es = MagicMock()
|
||||
alerter.es.create.side_effect = effects
|
||||
alerter.alert([match])
|
||||
calls = alerter.es.create.call_args_list
|
||||
self.assertTrue(all(c.kwargs['index'] == 'logs-detections.alerts-so' for c in calls))
|
||||
return [(json.loads(json.dumps(c.kwargs['body'], cls=es.DateTimeEncoder)), c.kwargs['id']) for c in calls]
|
||||
|
||||
def send(self, rule, match):
|
||||
"""Run alert(); return the payload it wrote and its id."""
|
||||
(payload, alert_id), = self.creates(rule, match)
|
||||
return payload, alert_id
|
||||
|
||||
def test_compound_query_key_left_out_of_event_data(self):
|
||||
rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
|
||||
match = correlation_match()
|
||||
match['host.name,source.ip'] = 'host-01, 192.0.2.10'
|
||||
original = copy.deepcopy(match)
|
||||
|
||||
payload, _ = self.send(rule, match)
|
||||
|
||||
self.assertNotIn('host.name,source.ip', payload['event_data'])
|
||||
self.assertEqual(payload['event_data']['host'], {'name': 'host-01'})
|
||||
self.assertEqual(payload['event_data']['source'], {'ip': '192.0.2.10'})
|
||||
self.assertEqual(payload['rule'], {'name': BASE_RULE['detection_title'], 'uuid': BASE_RULE['detection_public_id'], 'correlation': 'event_count'})
|
||||
self.assertEqual(payload['labels'], {'correlation_group_by': 'host.name, source.ip', 'correlation_group': 'host-01, 192.0.2.10'})
|
||||
self.assertEqual(payload['related'], {'hosts': ['host-01'], 'ip': ['192.0.2.10']})
|
||||
self.assertEqual(payload['event']['kind'], 'alert')
|
||||
self.assertEqual(payload['event']['reason'], '3,561 failed network logons to host-01 from 192.0.2.10 in 2 minutes')
|
||||
# ElastAlert reuses the match
|
||||
self.assertEqual(match, original)
|
||||
|
||||
def test_single_query_key(self):
|
||||
rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
|
||||
match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
|
||||
'user': {'name': 'user@example.invalid'}}
|
||||
|
||||
payload, _ = self.send(rule, match)
|
||||
|
||||
self.assertEqual(payload['labels'], {'correlation_group_by': 'user.name', 'correlation_group': 'user@example.invalid'})
|
||||
self.assertEqual(payload['related'], {'user': ['user@example.invalid']})
|
||||
self.assertEqual(payload['event']['reason'], '3 failed SOC logins for user@example.invalid')
|
||||
self.assertEqual(payload['event_data'], match)
|
||||
|
||||
def test_related_buckets(self):
|
||||
rule = dict(BASE_RULE, compound_query_key=['winlog.event_data.TargetUserName', 'source.ip', 'dns.highest_registered_domain'],
|
||||
query_key='winlog.event_data.TargetUserName,source.ip,dns.highest_registered_domain')
|
||||
match = correlation_match()
|
||||
match.update({'winlog': {'event_data': {'TargetUserName': ['admin1', 'svc', 'admin1']}},
|
||||
'source': {'ip': 'not-an-ip'}, 'dns': {'highest_registered_domain': 'example.com'}})
|
||||
|
||||
payload, _ = self.send(rule, match)
|
||||
|
||||
# deduped; invalid IP skipped; domain stays in the group only
|
||||
self.assertEqual(payload['related'], {'user': ['admin1', 'svc']})
|
||||
self.assertEqual(payload['labels']['correlation_group'], "['admin1', 'svc', 'admin1'], not-an-ip, example.com")
|
||||
|
||||
payload, _ = self.send(dict(BASE_RULE, query_key='dns.highest_registered_domain'), match)
|
||||
|
||||
self.assertNotIn('related', payload)
|
||||
|
||||
def test_related_uses_original_spellings(self):
|
||||
rule = dict(BASE_RULE, query_key='user.name', summary_template=None)
|
||||
match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
|
||||
'user': {'name': 'admin', 'name_spellings': ['Admin', 'admin', 'ADMIN']}}
|
||||
|
||||
payload, _ = self.send(rule, match)
|
||||
|
||||
# the group shows the lowercased value; related.user finds every spelling
|
||||
self.assertEqual(payload['labels']['correlation_group'], 'admin')
|
||||
self.assertEqual(payload['related'], {'user': ['Admin', 'admin', 'ADMIN']})
|
||||
self.assertEqual(payload['event']['reason'], '3 events for user.name admin in 8 seconds')
|
||||
|
||||
def test_plain_rule_has_no_correlation_fields(self):
|
||||
# a query_key alone (e.g. from an override) isn't a correlation
|
||||
rule = dict(PLAIN_RULE, query_key='user.name')
|
||||
# ElastAlert parses @timestamp for EQL hits
|
||||
match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc',
|
||||
'process': {'name': 'whoami.exe'}, 'user': {'name': 'user'}}
|
||||
|
||||
payload, alert_id = self.send(rule, match)
|
||||
|
||||
alerter = es.SecurityOnionESAlerter(rule)
|
||||
self.assertEqual(payload['rule'], {'name': PLAIN_RULE['detection_title'], 'uuid': PLAIN_RULE['detection_public_id']})
|
||||
self.assertNotIn('labels', payload)
|
||||
self.assertNotIn('related', payload)
|
||||
self.assertNotIn('reason', payload['event'])
|
||||
self.assertEqual(payload['event']['kind'], 'alert')
|
||||
self.assertEqual(payload['event_data'], dict(match, **{'@timestamp': '2026-09-30T16:50:00+00:00'}))
|
||||
self.assertEqual(alert_id, alerter.alert_id(match))
|
||||
self.assertNotEqual(alerter.alert_id(match), alerter.alert_id(dict(match, _id='abd')))
|
||||
self.assertNotEqual(alerter.alert_id(match), es.SecurityOnionESAlerter(dict(rule, detection_public_id='other')).alert_id(match))
|
||||
# without an _id, never deduplicated
|
||||
self.assertNotEqual(alerter.alert_id({'_id': None}), alerter.alert_id({'_id': None}))
|
||||
|
||||
def test_ungrouped_correlation_id_ignores_row_hash(self):
|
||||
rule = dict(BASE_RULE, summary_template=None)
|
||||
first = {k: v for k, v in correlation_match().items() if k not in ('host', 'source')}
|
||||
# ES|QL hashes the row into _id, so a later count changes it
|
||||
later = dict(first, event_count=3600, _id='9f2a')
|
||||
|
||||
payload, alert_id = self.send(rule, first)
|
||||
|
||||
alerter = es.SecurityOnionESAlerter(rule)
|
||||
self.assertEqual(alerter.alert_id(first), alerter.alert_id(later))
|
||||
self.assertNotEqual(alerter.alert_id(first), alerter.alert_id(dict(first, **{'@timestamp': '2026-09-30T18:09:00+00:00'})))
|
||||
self.assertEqual(alert_id, alerter.alert_id(first))
|
||||
self.assertEqual(payload['event']['reason'], '3,561 events in 2 minutes')
|
||||
self.assertNotIn('labels', payload)
|
||||
|
||||
def test_temporal_count_column(self):
|
||||
rule = dict(BASE_RULE, sigma_correlation='temporal', summary_template=None)
|
||||
match = {'event_type_count': 2, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00'}
|
||||
|
||||
payload, _ = self.send(rule, match)
|
||||
|
||||
self.assertEqual(payload['event']['reason'], '2 correlated rules matched in 8 seconds')
|
||||
|
||||
def test_grouped_correlation_id_unchanged(self):
|
||||
"""Ids of alerts already written must not change."""
|
||||
rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
|
||||
key = f"{BASE_RULE['detection_public_id']}|2026-09-30T18:07:53+00:00|host-01|192.0.2.10"
|
||||
|
||||
self.assertEqual(es.SecurityOnionESAlerter(rule).alert_id(correlation_match()), es.hashlib.sha256(key.encode()).hexdigest())
|
||||
|
||||
def test_rejected_event_data_is_kept_as_text(self):
|
||||
rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
|
||||
match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
|
||||
'user': {'name': 'user@example.invalid'}}
|
||||
rejected = RequestError(400, 'document_parsing_exception', {})
|
||||
|
||||
(first, _), (second, _) = self.creates(rule, match, [rejected, None])
|
||||
|
||||
self.assertIn('event_data', first)
|
||||
self.assertNotIn('event_data', second)
|
||||
self.assertEqual(json.loads(second['event']['original']), match)
|
||||
self.assertEqual(second['tags'], ['alert', 'preserve_original_event'])
|
||||
self.assertEqual(first['tags'], ['alert'])
|
||||
self.assertTrue(second['error']['message'].startswith('event_data rejected by Elasticsearch: '))
|
||||
self.assertIn('document_parsing_exception', second['error']['message'])
|
||||
# everything else carries over
|
||||
self.assertEqual({k: v for k, v in second['event'].items() if k != 'original'}, first['event'])
|
||||
changed = ('event_data', 'event', 'error', 'tags')
|
||||
self.assertEqual({k: v for k, v in second.items() if k not in changed}, {k: v for k, v in first.items() if k not in changed})
|
||||
|
||||
def test_rejected_dates_are_kept_as_text(self):
|
||||
match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc'}
|
||||
|
||||
_, (second, _) = self.creates(PLAIN_RULE, match, [RequestError(400, 'document_parsing_exception', {}), None])
|
||||
|
||||
self.assertEqual(json.loads(second['event']['original']), {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'})
|
||||
|
||||
def test_rejected_twice_is_dropped_without_retry(self):
|
||||
rejected = RequestError(400, 'document_parsing_exception', {})
|
||||
match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
|
||||
|
||||
with self.assertLogs('elastalert', 'ERROR') as logs:
|
||||
self.assertEqual(len(self.creates(PLAIN_RULE, match, [rejected, rejected])), 2)
|
||||
|
||||
self.assertIn('Dropping alert', logs.output[0])
|
||||
|
||||
def test_write_failure_is_retried(self):
|
||||
match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
|
||||
|
||||
with self.assertRaisesRegex(es.EAException, 'Unable to write the alert to Elasticsearch'):
|
||||
self.creates(PLAIN_RULE, match, [ConnectionError('N/A', 'refused', None)])
|
||||
|
||||
def test_repeat_id_is_ignored(self):
|
||||
match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
|
||||
|
||||
self.assertEqual(len(self.creates(PLAIN_RULE, match, [ConflictError(409, 'version_conflict_engine_exception', {})])), 1)
|
||||
|
||||
def test_correlation_fields_are_optional(self):
|
||||
rule = dict(BASE_RULE, query_key='source.ip')
|
||||
# no window_start: the summary cannot be built
|
||||
match = {k: v for k, v in correlation_match().items() if k != 'window_start'}
|
||||
|
||||
with self.assertLogs('elastalert', 'WARNING') as logs:
|
||||
payload, _ = self.send(rule, match)
|
||||
|
||||
self.assertIn('without its correlation summary', logs.output[0])
|
||||
self.assertEqual(payload['event']['kind'], 'alert')
|
||||
self.assertNotIn('reason', payload['event'])
|
||||
self.assertNotIn('labels', payload)
|
||||
self.assertNotIn('related', payload)
|
||||
|
||||
def test_unstable_id_still_writes(self):
|
||||
# no @timestamp: the window end is unknown
|
||||
match = {k: v for k, v in correlation_match().items() if k not in ('@timestamp', 'window_start')}
|
||||
|
||||
with self.assertLogs('elastalert', 'WARNING') as logs:
|
||||
payload, alert_id = self.send(BASE_RULE, match)
|
||||
|
||||
self.assertIn('without a stable id', logs.output[0])
|
||||
self.assertEqual(len(alert_id), 32)
|
||||
self.assertEqual(payload['event_data'], match)
|
||||
|
||||
def test_summary_formats_values(self):
|
||||
rule = dict(BASE_RULE, sigma_correlation='value_avg', query_key='source.ip',
|
||||
summary_template='%count% for %source.ip% to %destination.port% %no.such.field%')
|
||||
match = {'value_avg': 2.5, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
|
||||
'source': {'ip': '192.0.2.10'}, 'destination': {'port': [22, 80, 443, 8080, 8443]}}
|
||||
|
||||
payload, _ = self.send(rule, match)
|
||||
|
||||
self.assertEqual(payload['event']['reason'], '2.50 for 192.0.2.10 to 22, 80, 443 and 2 more %no.such.field%')
|
||||
self.assertEqual(es.SecurityOnionESAlerter.format_count('n/a'), 'n/a')
|
||||
|
||||
def test_get_info(self):
|
||||
self.assertEqual(es.SecurityOnionESAlerter(PLAIN_RULE).get_info(), {'type': 'SecurityOnionESAlerter'})
|
||||
@@ -120,7 +120,7 @@ elastalert:
|
||||
helpLink: elastalert
|
||||
old_query_limit:
|
||||
minutes:
|
||||
description: Amount of time in minutes between queries to start at the most recently run query.
|
||||
description: How long ElastAlert can be down, in minutes, and still resume each rule where it stopped. After a longer outage, rules restart from now and skip the gap.
|
||||
global: True
|
||||
helpLink: elastalert
|
||||
es_conn_timeout:
|
||||
|
||||
@@ -6,9 +6,11 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- elastic-fleet-package-registry.config
|
||||
- elastic-fleet-package-registry.sostatus
|
||||
|
||||
@@ -34,8 +36,9 @@ so-elastic-fleet-package-registry:
|
||||
{% for BINDING in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].port_bindings %}
|
||||
- {{ BINDING }}
|
||||
{% endfor %}
|
||||
{% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
|
||||
- binds:
|
||||
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
|
||||
{% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
|
||||
{% for BIND in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
|
||||
- {{ BIND }}
|
||||
{% endfor %}
|
||||
@@ -52,6 +55,8 @@ so-elastic-fleet-package-registry:
|
||||
- {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
|
||||
wait_for_so-elastic-fleet-package-registry:
|
||||
http.wait_for_successful_query:
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
|
||||
include:
|
||||
@@ -37,6 +38,7 @@ so-elastic-agent:
|
||||
- {{ BINDING }}
|
||||
{% endfor %}
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro
|
||||
- /opt/so/log/elasticagent:/usr/share/elastic-agent/logs
|
||||
- /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
|
||||
@@ -65,6 +67,7 @@ so-elastic-agent:
|
||||
- file: create-elastic-agent-config
|
||||
- file: trusttheca
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: create-elastic-agent-config
|
||||
- file: trusttheca
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
|
||||
|
||||
@@ -63,6 +64,7 @@ so-elastic-fleet:
|
||||
- {{ BINDING }}
|
||||
{% endfor %}
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
|
||||
- /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
|
||||
- /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
|
||||
@@ -95,6 +97,7 @@ so-elastic-fleet:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: trusttheca
|
||||
- x509: etc_elasticfleet_key
|
||||
- x509: etc_elasticfleet_crt
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
'azure_metrics.monitor': 'azure.monitor',
|
||||
'azure_metrics.storage_account': 'azure.storage_account',
|
||||
'azure_openai.metrics': 'azure.open_ai',
|
||||
'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
|
||||
'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
|
||||
'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
|
||||
'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules',
|
||||
'kibana.node_actions': 'kibana.stack_monitoring.node_actions',
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
|
||||
# Move our new CA over so Elastic and Logstash can use SSL with the internal CA
|
||||
catrustdir:
|
||||
file.directory:
|
||||
- name: /opt/so/conf/ca
|
||||
- user: 939
|
||||
- group: 939
|
||||
- makedirs: True
|
||||
|
||||
{% if GLOBALS.is_manager %}
|
||||
# We have to add the Manager CA to the CA list
|
||||
catrustscript:
|
||||
cmd.script:
|
||||
- source: salt://elasticsearch/tools/sbin_jinja/so-catrust
|
||||
- template: jinja
|
||||
- cwd: /opt/so
|
||||
- defaults:
|
||||
GLOBALS: {{ GLOBALS }}
|
||||
{% endif %}
|
||||
|
||||
elasticsearch_cacerts:
|
||||
file.managed:
|
||||
- name: /opt/so/conf/ca/cacerts
|
||||
- source: salt://elasticsearch/cacerts
|
||||
- user: 939
|
||||
- group: 939
|
||||
|
||||
elasticsearch_capems:
|
||||
file.managed:
|
||||
- name: /opt/so/conf/ca/tls-ca-bundle.pem
|
||||
- source: salt://elasticsearch/tls-ca-bundle.pem
|
||||
- user: 939
|
||||
- group: 939
|
||||
|
||||
{% else %}
|
||||
|
||||
{{sls}}_state_not_allowed:
|
||||
test.fail_without_changes:
|
||||
- name: {{sls}}_state_not_allowed
|
||||
|
||||
{% endif %}
|
||||
@@ -107,6 +107,12 @@ so-elasticsearch-system-indices-patch:
|
||||
- file: so-elasticsearch-system-indices-patch-script
|
||||
{% endif %}
|
||||
|
||||
so-elasticsearch-ilm-policy-reload:
|
||||
file.absent:
|
||||
- name: /opt/so/state/esilm.txt
|
||||
- onchanges:
|
||||
- file: so-elasticsearch-ilm-policy-load-script
|
||||
|
||||
# heavynodes will only load ILM policies for SO managed indices. (Indicies defined in elasticsearch/defaults.yaml)
|
||||
so-elasticsearch-ilm-policy-load:
|
||||
cmd.run:
|
||||
@@ -115,8 +121,9 @@ so-elasticsearch-ilm-policy-load:
|
||||
- require:
|
||||
- docker_container: so-elasticsearch
|
||||
- file: so-elasticsearch-ilm-policy-load-script
|
||||
- onchanges:
|
||||
- file: so-elasticsearch-ilm-policy-load-script
|
||||
- http: wait_for_so-elasticsearch
|
||||
- file: so-elasticsearch-ilm-policy-reload
|
||||
- unless: test -f /opt/so/state/esilm.txt
|
||||
|
||||
so-elasticsearch-templates-reload:
|
||||
file.absent:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
elasticsearch:
|
||||
enabled: false
|
||||
esheap: '600m'
|
||||
version: 9.4.5
|
||||
version: 9.4.8
|
||||
index_clean: true
|
||||
data_retention_method: DLM
|
||||
vm:
|
||||
@@ -1160,6 +1160,7 @@ elasticsearch:
|
||||
- so-fleet_agent_id_verification-1
|
||||
- so-logs-mappings
|
||||
- so-logs-settings
|
||||
- detections-alerts-mappings
|
||||
data_stream:
|
||||
allow_custom_routing: false
|
||||
hidden: false
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
|
||||
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
|
||||
@@ -13,7 +14,6 @@
|
||||
|
||||
include:
|
||||
- ca
|
||||
- elasticsearch.ca
|
||||
- elasticsearch.ssl
|
||||
- elasticsearch.config
|
||||
- elasticsearch.sostatus
|
||||
@@ -63,15 +63,14 @@ so-elasticsearch:
|
||||
- {{ BINDING }}
|
||||
{% endfor %}
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /opt/so/conf/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
|
||||
- /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro
|
||||
- /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
|
||||
- /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
|
||||
- /opt/so/conf/ca/cacerts:/usr/share/elasticsearch/jdk/lib/security/cacerts:ro
|
||||
- /etc/pki/tls/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
|
||||
- /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
|
||||
- /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
|
||||
- /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
|
||||
- /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro
|
||||
- /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro
|
||||
{% if ELASTICSEARCHMERGED.config.path.get('repo', False) %}
|
||||
@@ -85,21 +84,19 @@ so-elasticsearch:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: trusttheca
|
||||
- x509: elasticsearch_crt
|
||||
- x509: elasticsearch_key
|
||||
- file: elasticsearch_cacerts
|
||||
- file: esyml
|
||||
- require:
|
||||
- file: trusttheca
|
||||
- x509: elasticsearch_crt
|
||||
- x509: elasticsearch_key
|
||||
- file: elasticsearch_cacerts
|
||||
- file: esyml
|
||||
- file: eslog4jfile
|
||||
- file: nsmesdir
|
||||
- file: eslogdir
|
||||
- file: elasticp12perms
|
||||
- cmd: auth_users_roles_inode
|
||||
- cmd: auth_users_inode
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"version": 3,
|
||||
"version": 4,
|
||||
"_meta": {
|
||||
"managed_by": "securityonion",
|
||||
"managed": true
|
||||
@@ -177,6 +177,32 @@
|
||||
"description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
|
||||
}
|
||||
},
|
||||
{
|
||||
"trim": {
|
||||
"if": "ctx.event?.dataset == 'system.auth'",
|
||||
"description": "Grok leaves a leading space on 'invalid user' names (elastic/integrations#12174) and, before 2.23.2, sudo padding",
|
||||
"field": "user.name",
|
||||
"ignore_missing": true,
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"trim": {
|
||||
"if": "ctx.event?.dataset == 'system.auth'",
|
||||
"description": "Appended from the untrimmed user.name",
|
||||
"field": "related.user",
|
||||
"ignore_missing": true,
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"script": {
|
||||
"if": "ctx.event?.dataset == 'system.auth' && ctx.related?.user instanceof List",
|
||||
"description": "Dedupe after trimming",
|
||||
"source": "ctx.related.user = new ArrayList(new LinkedHashSet(ctx.related.user));",
|
||||
"ignore_failure": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"script": {
|
||||
"description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",
|
||||
|
||||
@@ -38,10 +38,6 @@ elasticsearch_crt:
|
||||
- retry:
|
||||
attempts: 5
|
||||
interval: 30
|
||||
cmd.run:
|
||||
- name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:"
|
||||
- onchanges:
|
||||
- x509: /etc/pki/elasticsearch.key
|
||||
|
||||
elastickeyperms:
|
||||
file.managed:
|
||||
@@ -50,13 +46,6 @@ elastickeyperms:
|
||||
- mode: 640
|
||||
- group: 930
|
||||
|
||||
elasticp12perms:
|
||||
file.managed:
|
||||
- replace: False
|
||||
- name: /etc/pki/elasticsearch.p12
|
||||
- mode: 640
|
||||
- group: 930
|
||||
|
||||
{% else %}
|
||||
|
||||
{{sls}}_state_not_allowed:
|
||||
|
||||
@@ -11,7 +11,8 @@
|
||||
{% set DATA_RETENTION_METHOD = ELASTICSEARCHMERGED.data_retention_method %}
|
||||
|
||||
{% set PILLAR_GLOBAL_OVERRIDES = {} %}
|
||||
{% set ES_INDEX_PILLAR = salt['pillar.get']('elasticsearch:index_settings', {}) %}
|
||||
{# merge into a copy: pillar.get returns a live reference and this map pops/updates ES_INDEX_PILLAR #}
|
||||
{% set ES_INDEX_PILLAR = salt['defaults.merge'](salt['pillar.get']('elasticsearch:index_settings', {}), {}, in_place=False) %}
|
||||
{% if ES_INDEX_PILLAR.global_overrides is defined %}
|
||||
{% set PILLAR_GLOBAL_OVERRIDES = ES_INDEX_PILLAR.pop('global_overrides') %}
|
||||
{% endif %}
|
||||
|
||||
@@ -154,6 +154,20 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"runtime": {
|
||||
"so_detection.sigmaKind": {
|
||||
"type": "keyword",
|
||||
"script": {
|
||||
"source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); emit(c.startsWith('correlation:') || c.contains(NL + 'correlation:') ? 'correlation' : 'single');"
|
||||
}
|
||||
},
|
||||
"so_detection.sigmaCorrelationType": {
|
||||
"type": "keyword",
|
||||
"script": {
|
||||
"source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); int i = c.startsWith('correlation:') ? 0 : c.indexOf(NL + 'correlation:'); if (i < 0) { return; } i = c.indexOf(NL, i + 1); while (i >= 0 && i + 1 < c.length()) { int e = c.indexOf(NL, i + 1); String line = e < 0 ? c.substring(i + 1) : c.substring(i + 1, e); if (line.length() > 0 && !Character.isWhitespace(line.charAt(0)) && line.charAt(0) != (char) 35) { return; } String t = line.trim(); if (t.startsWith('type:')) { String v = t.substring(5).trim(); int h = v.indexOf(' ' + String.valueOf((char) 35)); if (h >= 0) { v = v.substring(0, h).trim(); } if (v.length() > 1 && (v.charAt(0) == (char) 39 || v.charAt(0) == (char) 34)) { v = v.substring(1, v.length() - 1); } emit(v); return; } i = e; }"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
{
|
||||
"template": {
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"tags": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"sigma_level": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"rule": {
|
||||
"properties": {
|
||||
"name": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"uuid": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"category": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"product": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"service": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"correlation": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"event": {
|
||||
"properties": {
|
||||
"severity": {
|
||||
"type": "long"
|
||||
},
|
||||
"severity_label": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"module": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"dataset": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"kind": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"reason": {
|
||||
"type": "match_only_text",
|
||||
"fields": {
|
||||
"keyword": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"original": {
|
||||
"type": "keyword",
|
||||
"index": false,
|
||||
"doc_values": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"event_data": {
|
||||
"properties": {
|
||||
"@timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"window_start": {
|
||||
"type": "date"
|
||||
},
|
||||
"event_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"value_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"event_type_count": {
|
||||
"type": "long"
|
||||
},
|
||||
"value_sum": {
|
||||
"type": "double"
|
||||
},
|
||||
"value_avg": {
|
||||
"type": "double"
|
||||
},
|
||||
"value_percentile": {
|
||||
"type": "double"
|
||||
},
|
||||
"value_median": {
|
||||
"type": "double"
|
||||
}
|
||||
}
|
||||
},
|
||||
"labels": {
|
||||
"properties": {
|
||||
"correlation_group_by": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"correlation_group": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"related": {
|
||||
"properties": {
|
||||
"ip": {
|
||||
"type": "ip"
|
||||
},
|
||||
"user": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
},
|
||||
"hosts": {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"error": {
|
||||
"properties": {
|
||||
"message": {
|
||||
"type": "match_only_text"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"description": "Fields written by the ElastAlert SecurityOnionESAlerter to logs-detections.alerts-so"
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,10 @@
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
# Reports per data stream Elasticsearch growth over the last 24 hours, 7 days and 30 days. The data
|
||||
# source follows telegraf.output: Postgres in POSTGRES mode, where history is limited to
|
||||
# postgres.telegraf.retention_days and the script must run on the manager; InfluxDB otherwise.
|
||||
|
||||
INFLUX_URL="https://localhost:8086/api/v2"
|
||||
|
||||
. /usr/sbin/so-common
|
||||
@@ -18,8 +22,6 @@ lookup_org_id() {
|
||||
echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id"
|
||||
}
|
||||
|
||||
ORG_ID=$(lookup_org_id)
|
||||
|
||||
run_flux_query() {
|
||||
local query=$1
|
||||
request "query?org=$ORG_ID" -H 'Accept:application/csv' -H 'Content-type:application/vnd.flux' -d "$query" -XPOST 2>/dev/null
|
||||
@@ -39,75 +41,137 @@ bytes_to_gb() {
|
||||
fi
|
||||
}
|
||||
|
||||
indexes_query='from(bucket: "telegraf/so_long_term")
|
||||
|> range(start: -7d)
|
||||
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|
||||
|> distinct(column: "_field")
|
||||
|> keep(columns: ["_field"])'
|
||||
influxdb_indices_growth() {
|
||||
ORG_ID=$(lookup_org_id)
|
||||
|
||||
indexes_result=$(run_flux_query "$indexes_query")
|
||||
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
|
||||
indexes_query='from(bucket: "telegraf/so_long_term")
|
||||
|> range(start: -7d)
|
||||
|> filter(fn: (r) => r._measurement == "elasticsearch_index_size")
|
||||
|> distinct(column: "_field")
|
||||
|> keep(columns: ["_field"])'
|
||||
|
||||
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
|
||||
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
|
||||
indexes_result=$(run_flux_query "$indexes_query")
|
||||
indexes=$(echo "$indexes_result" | tail -n +2 | cut -d',' -f4 | grep -v '^$' | grep -v '^_field$' | sed 's/\r$//' | sort -u)
|
||||
|
||||
for index in $indexes; do
|
||||
[[ -z "$index" ]] && continue
|
||||
current_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -4h)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
current_result=$(run_flux_query "$current_query")
|
||||
current_size=$(read_csv_result "$current_result")
|
||||
current_size=${current_size:-0}
|
||||
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
|
||||
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
|
||||
|
||||
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -25h, stop: -23h)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_24h_result=$(run_flux_query "$size_24h_query")
|
||||
size_24h_ago=$(read_csv_result "$size_24h_result")
|
||||
size_24h_ago=${size_24h_ago:-$current_size}
|
||||
for index in $indexes; do
|
||||
[[ -z "$index" ]] && continue
|
||||
current_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -4h)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
current_result=$(run_flux_query "$current_query")
|
||||
current_size=$(read_csv_result "$current_result")
|
||||
current_size=${current_size:-0}
|
||||
|
||||
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -7d8h, stop: -7d)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_7d_result=$(run_flux_query "$size_7d_query")
|
||||
size_7d_ago=$(read_csv_result "$size_7d_result")
|
||||
size_7d_ago=${size_7d_ago:-$current_size}
|
||||
size_24h_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -25h, stop: -23h)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_24h_result=$(run_flux_query "$size_24h_query")
|
||||
size_24h_ago=$(read_csv_result "$size_24h_result")
|
||||
size_24h_ago=${size_24h_ago:-$current_size}
|
||||
|
||||
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -30d8h, stop: -30d)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_30d_result=$(run_flux_query "$size_30d_query")
|
||||
size_30d_ago=$(read_csv_result "$size_30d_result")
|
||||
size_30d_ago=${size_30d_ago:-$current_size}
|
||||
size_7d_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -7d8h, stop: -7d)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_7d_result=$(run_flux_query "$size_7d_query")
|
||||
size_7d_ago=$(read_csv_result "$size_7d_result")
|
||||
size_7d_ago=${size_7d_ago:-$current_size}
|
||||
|
||||
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
|
||||
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
|
||||
size_30d_query="from(bucket: \"telegraf/so_long_term\")
|
||||
|> range(start: -30d8h, stop: -30d)
|
||||
|> filter(fn: (r) => r._measurement == \"elasticsearch_index_size\" and r._field == \"$index\")
|
||||
|> last()
|
||||
|> keep(columns: [\"_value\"])"
|
||||
size_30d_result=$(run_flux_query "$size_30d_query")
|
||||
size_30d_ago=$(read_csv_result "$size_30d_result")
|
||||
size_30d_ago=${size_30d_ago:-$current_size}
|
||||
|
||||
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
|
||||
# if an index was recently cleaned up by ilm it will result in a negative number for 'index growth'.
|
||||
growth_24h=$(( current_size > size_24h_ago ? current_size - size_24h_ago : 0 ))
|
||||
|
||||
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
|
||||
growth_7d=$(( current_size > size_7d_ago ? current_size - size_7d_ago : 0 ))
|
||||
|
||||
growth_24h_gb=$(bytes_to_gb "$growth_24h")
|
||||
growth_7d_gb=$(bytes_to_gb "$growth_7d")
|
||||
growth_30d_gb=$(bytes_to_gb "$growth_30d")
|
||||
growth_30d=$(( current_size > size_30d_ago ? current_size - size_30d_ago : 0 ))
|
||||
|
||||
# Only results for indices with atleast 1 metric above 0.00
|
||||
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
|
||||
printf "%020.2f|%-50s %15s %15s %15s\n" \
|
||||
"$growth_24h" \
|
||||
"$index" \
|
||||
"$growth_24h_gb" \
|
||||
"$growth_7d_gb" \
|
||||
"$growth_30d_gb"
|
||||
fi
|
||||
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
|
||||
growth_24h_gb=$(bytes_to_gb "$growth_24h")
|
||||
growth_7d_gb=$(bytes_to_gb "$growth_7d")
|
||||
growth_30d_gb=$(bytes_to_gb "$growth_30d")
|
||||
|
||||
# Only results for indices with atleast 1 metric above 0.00
|
||||
if [[ "$growth_24h_gb" != "0.00" ]] || [[ "$growth_7d_gb" != "0.00" ]] || [[ "$growth_30d_gb" != "0.00" ]]; then
|
||||
printf "%020.2f|%-50s %15s %15s %15s\n" \
|
||||
"$growth_24h" \
|
||||
"$index" \
|
||||
"$growth_24h_gb" \
|
||||
"$growth_7d_gb" \
|
||||
"$growth_30d_gb"
|
||||
fi
|
||||
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
|
||||
}
|
||||
|
||||
postgres_indices_growth() {
|
||||
local app="so-elasticsearch-indices-growth"
|
||||
local -a starts=("25 hours" "7 days 8 hours" "30 days 8 hours")
|
||||
local -a stops=("23 hours" "7 days" "30 days")
|
||||
local -A current past window_has_data
|
||||
local i rows stream current_size past_size growth growth_gb sort_key show
|
||||
local -a columns
|
||||
|
||||
for i in 0 1 2; do
|
||||
if ! rows=$(so_es_index_size_growth "$app" "${starts[i]}" "${stops[i]}"); then
|
||||
echo "Failed to query telegraf.elasticsearch_index_size in so_telegraf." >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ -z "$rows" ]] && continue
|
||||
window_has_data[$i]=1
|
||||
while IFS='|' read -r stream current_size past_size; do
|
||||
current[$stream]=$current_size
|
||||
past[$i,$stream]=$past_size
|
||||
done <<< "$rows"
|
||||
done
|
||||
|
||||
printf "%-50s %15s %15s %15s\n" "Index Name" "Last 24hr (GB)" "Last 7d (GB)" "Last 30d (GB)"
|
||||
printf "%-50s %15s %15s %15s\n" "$(printf '%.0s-' {1..50})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})" "$(printf '%.0s-' {1..15})"
|
||||
|
||||
for stream in "${!current[@]}"; do
|
||||
columns=()
|
||||
sort_key=0
|
||||
show=false
|
||||
for i in 0 1 2; do
|
||||
if [[ -z "${window_has_data[$i]}" ]]; then
|
||||
columns+=("n/a")
|
||||
continue
|
||||
fi
|
||||
current_size=${current[$stream]}
|
||||
past_size=${past[$i,$stream]-$current_size}
|
||||
growth=$(( current_size > past_size ? current_size - past_size : 0 ))
|
||||
growth_gb=$(bytes_to_gb "$growth")
|
||||
[[ "$growth_gb" != "0.00" ]] && show=true
|
||||
[[ $i -eq 0 ]] && sort_key=$growth
|
||||
columns+=("$growth_gb")
|
||||
done
|
||||
if [[ "$show" == true ]]; then
|
||||
printf "%020.2f|%-50s %15s %15s %15s\n" "$sort_key" "$stream" "${columns[@]}"
|
||||
fi
|
||||
done | sort -t'|' -k1,1nr | cut -d'|' -f2-
|
||||
}
|
||||
|
||||
if ! TG_OUTPUT=$(so_telegraf_output); then
|
||||
echo "Unable to read telegraf.output from pillar." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$TG_OUTPUT" == "POSTGRES" ]]; then
|
||||
so_telegraf_pg_check so-elasticsearch-indices-growth || exit 1
|
||||
postgres_indices_growth
|
||||
else
|
||||
influxdb_indices_growth
|
||||
fi
|
||||
@@ -58,6 +58,9 @@ Usage: $(basename "$0") [OPTIONS]
|
||||
|
||||
Estimate remaining days until Elasticsearch cluster reaches low watermark threshold.
|
||||
|
||||
Index growth history follows telegraf.output: in POSTGRES mode it is read from Postgres, limited to
|
||||
postgres.telegraf.retention_days, and the script must run on the manager; otherwise from InfluxDB.
|
||||
|
||||
OPTIONS:
|
||||
--json Output results in JSON format
|
||||
-v, --verbose Show additional output
|
||||
@@ -168,11 +171,25 @@ run_indices_growth() {
|
||||
fi
|
||||
}
|
||||
|
||||
fetch_postgres_total_bytes() {
|
||||
local total
|
||||
|
||||
total=$(so_es_index_size_growth so-elasticsearch-retention-estimate "$1" "$2") || return 1
|
||||
echo "$total" | awk -F'|' '{ sum += $3 } END { print sum + 0 }'
|
||||
}
|
||||
|
||||
fetch_total_bytes() {
|
||||
local start="$1"
|
||||
local stop="$2"
|
||||
local pg_start="$3"
|
||||
local pg_stop="$4"
|
||||
local range_line
|
||||
|
||||
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
|
||||
fetch_postgres_total_bytes "$pg_start" "$pg_stop"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ -n "$stop" ]; then
|
||||
range_line=" |> range(start: ${start}, stop: ${stop})"
|
||||
else
|
||||
@@ -208,8 +225,13 @@ fail() {
|
||||
|
||||
echo -e "\nDISCLAIMER: Script output is based on current data patterns, but are approximations solely intended to assist with getting a general ILM policy configured."
|
||||
|
||||
ORG_ID=$(lookup_org_id)
|
||||
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
|
||||
TG_OUTPUT=$(so_telegraf_output) || fail "Unable to read telegraf.output from pillar."
|
||||
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
|
||||
pg_check_error=$(so_telegraf_pg_check so-elasticsearch-retention-estimate 2>&1) || fail "$pg_check_error"
|
||||
else
|
||||
ORG_ID=$(lookup_org_id)
|
||||
[ -n "$ORG_ID" ] || fail "Unable to resolve InfluxDB org id"
|
||||
fi
|
||||
|
||||
cluster_storage_size=0
|
||||
indexed_storage_source="elasticsearch"
|
||||
@@ -221,10 +243,11 @@ if [ -n "$cluster_storage_size_output" ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# historical data from influxdb for growth calculation
|
||||
one_day_total=$(fetch_total_bytes "-25h" "-23h")
|
||||
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d")
|
||||
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d")
|
||||
# historical index sizes for growth calculation
|
||||
query_error="Failed to query telegraf.elasticsearch_index_size in so_telegraf."
|
||||
one_day_total=$(fetch_total_bytes "-25h" "-23h" "25 hours" "23 hours") || fail "$query_error"
|
||||
seven_day_total=$(fetch_total_bytes "-7d8h" "-7d" "7 days 8 hours" "7 days") || fail "$query_error"
|
||||
thirty_day_total=$(fetch_total_bytes "-30d8h" "-30d" "30 days 8 hours" "30 days") || fail "$query_error"
|
||||
|
||||
# available historical windows (prefer 30d/7d when available, to avoid using recent 24h traffic spike as true daily ingest rate)
|
||||
history_days=0
|
||||
@@ -244,7 +267,12 @@ elif [ "$one_day_total" -gt 0 ]; then
|
||||
historical_total=$one_day_total
|
||||
fi
|
||||
|
||||
[ "$history_days" -gt 0 ] || fail "Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
|
||||
if [ "$TG_OUTPUT" = "POSTGRES" ]; then
|
||||
history_unavailable="Historical Postgres data unavailable for growth calculation. Postgres keeps postgres.telegraf.retention_days days of history; if this is a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
|
||||
else
|
||||
history_unavailable="Historical InfluxDB data unavailable for growth calculation. If this a newer grid try re-running this script in a few days. Otherwise review /opt/so/log/telegraf/telegraf.log for errors with collecting required ES metrics."
|
||||
fi
|
||||
[ "$history_days" -gt 0 ] || fail "$history_unavailable"
|
||||
|
||||
# Daily growth rate
|
||||
growth_bytes=$(( cluster_storage_size - historical_total ))
|
||||
@@ -729,6 +757,7 @@ if [ "$should_trigger_recommendations" = true ]; then
|
||||
[ -n "$index" ] || continue
|
||||
|
||||
growth_24h_gb=$(echo "$line" | awk '{print $(NF-2)}')
|
||||
[[ "$growth_24h_gb" =~ ^[0-9]+(\.[0-9]+)?$ ]] || growth_24h_gb=0
|
||||
|
||||
creation_date_display=""
|
||||
retention_days=""
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
. /usr/sbin/so-common
|
||||
get_elastic_agent_vars
|
||||
|
||||
# Exit on errors, since all lines must succeed
|
||||
set -e
|
||||
|
||||
# Check to see if we have extracted the ca cert.
|
||||
if [ ! -f /opt/so/saltstack/local/salt/elasticsearch/cacerts ]; then
|
||||
docker run -v /etc/pki/ca.crt:/etc/ssl/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:$ELASTIC_AGENT_TARBALL_VERSION -keystore /usr/share/elasticsearch/jdk/lib/security/cacerts -alias SOSCA -import -file /etc/ssl/ca.crt -storepass changeit -noprompt
|
||||
# Make sure symbolic links are followed when copying from container
|
||||
docker cp -L so-elasticsearchca:/usr/share/elasticsearch/jdk/lib/security/cacerts /opt/so/saltstack/local/salt/elasticsearch/cacerts
|
||||
docker cp -L so-elasticsearchca:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
|
||||
docker rm so-elasticsearchca
|
||||
echo "" >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
|
||||
echo "sosca" >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
|
||||
cat /etc/pki/ca.crt >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
@@ -7,11 +7,17 @@
|
||||
. /usr/sbin/so-common
|
||||
|
||||
MAX_JOBS=${MAX_ILM_JOBS:-10}
|
||||
ILM_STATEFILE=/opt/so/state/esilm.txt
|
||||
rm -f "$ILM_STATEFILE"
|
||||
|
||||
check_elasticsearch_responsive
|
||||
|
||||
ILM_TMP_DIR=$(mktemp -d)
|
||||
trap 'rm -rf "$ILM_TMP_DIR"' EXIT
|
||||
# Lock used to serialize block writes so concurrent jobs never interleave their output.
|
||||
ILM_OUTPUT_LOCK=$(mktemp)
|
||||
ILM_FAIL_FILE=$(mktemp)
|
||||
trap 'rm -f "$ILM_OUTPUT_LOCK" "$ILM_FAIL_FILE"' EXIT
|
||||
ILM_OUTPUT_LOCK="$ILM_TMP_DIR/output.lock"
|
||||
ILM_FAIL_FILE="$ILM_TMP_DIR/failed"
|
||||
touch "$ILM_FAIL_FILE"
|
||||
|
||||
# Policies are loaded concurrently (up to MAX_JOBS at a time) for speed. Each policy's block is
|
||||
# printed the moment its curl returns, so output appears in COMPLETION ORDER, not the order
|
||||
@@ -20,13 +26,30 @@ echo "Loading ILM policies concurrently; output below appears in completion orde
|
||||
echo
|
||||
|
||||
put_policy() {
|
||||
local desc="$1" policyname="$2" data="$3" result rc=0
|
||||
if ! result=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L --fail \
|
||||
-X PUT "https://localhost:9200/_ilm/policy/${policyname}" \
|
||||
-H 'Content-Type: application/json' -d"${data}" 2>&1); then
|
||||
rc=1
|
||||
elif ! jq -e '.acknowledged == true' <<<"$result" >/dev/null 2>&1; then
|
||||
rc=1
|
||||
local desc="$1" policyname="$2" data="$3" rc=1 attempt out http_code body result reason tmpdata
|
||||
tmpdata=$(mktemp "$ILM_TMP_DIR/${policyname}.XXXXXX.json")
|
||||
printf '%s' "$data" >"$tmpdata"
|
||||
|
||||
for attempt in 1 2 3; do
|
||||
out=$(so-elasticsearch-query "_ilm/policy/${policyname}" -d@"${tmpdata}" -XPUT -w '\n%{http_code}' 2>/dev/null)
|
||||
http_code=${out##*$'\n'} body=${out%$'\n'*}
|
||||
|
||||
if [[ "$http_code" == 200 ]] && jq -e '.acknowledged == true' <<<"$body" >/dev/null 2>&1; then
|
||||
rc=0
|
||||
break
|
||||
fi
|
||||
# 429 = ES rejected/throttled (may succeed later); other 4xx will not
|
||||
[[ "$http_code" == 4?? && "$http_code" != 429 ]] && break
|
||||
(( attempt < 3 )) && sleep 3
|
||||
done
|
||||
|
||||
if (( rc == 0 )); then
|
||||
result='{"acknowledged":true}'
|
||||
else
|
||||
result=${body:-"HTTP ${http_code:-000}"}
|
||||
# flatten tabs/newlines so the TAB fail-file stays one record
|
||||
reason=$(jq -r '(if (.error|type) == "object" then (.error.reason // .error.type) elif (.error|type) == "string" then .error else empty end) // empty | tostring | gsub("[\t\r\n]+"; " ")' <<<"$body" 2>/dev/null)
|
||||
reason="HTTP ${http_code:-000}${reason:+: ${reason}}"
|
||||
fi
|
||||
|
||||
# curl above ran in parallel; serialize just this block write so concurrent jobs never interleave.
|
||||
@@ -34,7 +57,7 @@ put_policy() {
|
||||
flock 200
|
||||
printf 'Setting up %s policy...\n%s\n\n' "${desc}" "${result}"
|
||||
if (( rc != 0 )); then
|
||||
printf '%s\n' "${policyname}" >>"$ILM_FAIL_FILE"
|
||||
printf '%s\t%s\n' "${policyname}" "${reason}" >>"$ILM_FAIL_FILE"
|
||||
fi
|
||||
} 200>>"${ILM_OUTPUT_LOCK}"
|
||||
|
||||
@@ -82,10 +105,15 @@ wait || true
|
||||
|
||||
if [[ -s "$ILM_FAIL_FILE" ]]; then
|
||||
echo "ERROR: Failed to load ILM policy(s):"
|
||||
while read -r POLICY; do
|
||||
echo " - $POLICY"
|
||||
while IFS=$'\t' read -r POLICY REASON; do
|
||||
if [[ -n "$REASON" ]]; then
|
||||
echo " - ${POLICY}: ${REASON}"
|
||||
else
|
||||
echo " - ${POLICY}"
|
||||
fi
|
||||
done < "$ILM_FAIL_FILE"
|
||||
exit 1
|
||||
else
|
||||
echo "Successfully loaded all ILM policies."
|
||||
touch "$ILM_STATEFILE"
|
||||
fi
|
||||
@@ -118,10 +118,6 @@ firewall:
|
||||
tcp:
|
||||
- 5601
|
||||
udp: []
|
||||
localrules:
|
||||
tcp:
|
||||
- 7788
|
||||
udp: []
|
||||
nginx:
|
||||
tcp:
|
||||
- 80
|
||||
@@ -200,7 +196,6 @@ firewall:
|
||||
- postgres
|
||||
- elasticsearch_rest
|
||||
- elasticsearch_node
|
||||
- localrules
|
||||
sensor:
|
||||
portgroups:
|
||||
- beats_5044
|
||||
@@ -391,7 +386,6 @@ firewall:
|
||||
- elastic_agent_control
|
||||
- elastic_agent_data
|
||||
- elastic_agent_update
|
||||
- localrules
|
||||
- sensoroni
|
||||
fleet:
|
||||
portgroups:
|
||||
@@ -611,7 +605,6 @@ firewall:
|
||||
- elastic_agent_control
|
||||
- elastic_agent_data
|
||||
- elastic_agent_update
|
||||
- localrules
|
||||
- sensoroni
|
||||
fleet:
|
||||
portgroups:
|
||||
@@ -829,7 +822,6 @@ firewall:
|
||||
- elastic_agent_control
|
||||
- elastic_agent_data
|
||||
- elastic_agent_update
|
||||
- localrules
|
||||
- sensoroni
|
||||
fleet:
|
||||
portgroups:
|
||||
@@ -1058,7 +1050,6 @@ firewall:
|
||||
- elastic_agent_update
|
||||
- endgame
|
||||
- strelka_frontend
|
||||
- localrules
|
||||
fleet:
|
||||
portgroups:
|
||||
- elasticsearch_rest
|
||||
@@ -1518,7 +1509,6 @@ firewall:
|
||||
- elasticsearch_rest
|
||||
- elasticsearch_node
|
||||
- elastic_agent_control
|
||||
- localrules
|
||||
sensor:
|
||||
portgroups:
|
||||
- beats_5044
|
||||
|
||||
@@ -138,9 +138,6 @@ firewall:
|
||||
kibana:
|
||||
tcp: *tcpsettings
|
||||
udp: *udpsettings
|
||||
localrules:
|
||||
tcp: *tcpsettings
|
||||
udp: *udpsettings
|
||||
nginx:
|
||||
tcp: *tcpsettings
|
||||
udp: *udpsettings
|
||||
|
||||
@@ -13,9 +13,11 @@
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% if 'api' in salt['pillar.get']('features', []) %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- hydra.config
|
||||
- hydra.sostatus
|
||||
|
||||
@@ -29,6 +31,7 @@ so-hydra:
|
||||
- soauth:
|
||||
- ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /opt/so/conf/hydra/:/hydra-conf:ro
|
||||
- /opt/so/log/hydra/:/hydra-log:rw
|
||||
- /nsm/hydra/db:/hydra-data:rw
|
||||
@@ -60,6 +63,7 @@ so-hydra:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: hydraconfig
|
||||
- require:
|
||||
- file: hydraconfig
|
||||
|
||||
@@ -6,9 +6,11 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- idh.config
|
||||
- idh.sostatus
|
||||
|
||||
@@ -20,6 +22,7 @@ so-idh:
|
||||
- detach: True
|
||||
- network_mode: host
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /nsm/idh:/var/tmp:rw
|
||||
- /opt/so/conf/idh/http-skins:/opt/opencanary/http-skins:ro
|
||||
- /opt/so/conf/idh/opencanary.conf:/etc/opencanaryd/opencanary.conf:ro
|
||||
@@ -34,12 +37,13 @@ so-idh:
|
||||
- {{ XTRAHOST }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if DOCKERMERGED.containers['so-idh'].extra_env %}
|
||||
- environment:
|
||||
{% for XTRAENV in DOCKERMERGED.containers['so-idh'].extra_env %}
|
||||
- REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt
|
||||
{% if DOCKERMERGED.containers['so-idh'].extra_env %}
|
||||
{% for XTRAENV in DOCKERMERGED.containers['so-idh'].extra_env %}
|
||||
- {{ XTRAENV }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if DOCKERMERGED.containers['so-idh'].ulimits %}
|
||||
- ulimits:
|
||||
{% for ULIMIT in DOCKERMERGED.containers['so-idh'].ulimits %}
|
||||
@@ -47,6 +51,7 @@ so-idh:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: opencanary_config
|
||||
- require:
|
||||
- file: opencanary_config
|
||||
|
||||
@@ -6,11 +6,13 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
|
||||
{% set TOKEN = salt['pillar.get']('influxdb:token') %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- influxdb.ssl
|
||||
- influxdb.config
|
||||
- influxdb.sostatus
|
||||
@@ -38,6 +40,7 @@ so-influxdb:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- binds:
|
||||
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
|
||||
- /opt/so/log/influxdb/:/log:rw
|
||||
- /opt/so/conf/influxdb/config.yaml:/conf/config.yaml:ro
|
||||
- /opt/so/conf/influxdb/etc:/etc/influxdb2:rw
|
||||
@@ -66,6 +69,7 @@ so-influxdb:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: influxdbconf
|
||||
- x509: influxdb_key
|
||||
- x509: influxdb_crt
|
||||
|
||||
@@ -12,12 +12,14 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
|
||||
{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
|
||||
{% if 'gmd' in salt['pillar.get']('features', []) %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- kafka.ca
|
||||
- kafka.config
|
||||
- kafka.ssl
|
||||
@@ -51,6 +53,7 @@ so-kafka:
|
||||
- {{ BINDING }}
|
||||
{% endfor %}
|
||||
- binds:
|
||||
- {{ CATRUST.java }}:/opt/java/openjdk/lib/security/cacerts:ro
|
||||
- /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro
|
||||
- /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
|
||||
- /nsm/kafka/data/:/nsm/kafka/data/:rw
|
||||
@@ -61,6 +64,11 @@ so-kafka:
|
||||
{% if KAFKA_EXTERNAL_ACCESS %}
|
||||
- /opt/so/conf/kafka/kafka_server_jaas.conf:/opt/kafka/config/kafka_server_jaas.conf:ro
|
||||
{% endif %}
|
||||
{% if DOCKERMERGED.containers['so-kafka'].custom_bind_mounts %}
|
||||
{% for BIND in DOCKERMERGED.containers['so-kafka'].custom_bind_mounts %}
|
||||
- {{ BIND }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% if DOCKERMERGED.containers['so-kafka'].ulimits %}
|
||||
- ulimits:
|
||||
{% for ULIMIT in DOCKERMERGED.containers['so-kafka'].ulimits %}
|
||||
@@ -68,6 +76,7 @@ so-kafka:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
{% for sc in ['server', 'client'] %}
|
||||
- file: kafka_kraft_{{sc}}_properties
|
||||
{% endfor %}
|
||||
|
||||
@@ -22,7 +22,7 @@ kibana:
|
||||
- default
|
||||
- file
|
||||
migrations:
|
||||
discardCorruptObjects: "9.4.5"
|
||||
discardCorruptObjects: "9.4.8"
|
||||
telemetry:
|
||||
enabled: False
|
||||
xpack:
|
||||
|
||||
@@ -8,8 +8,10 @@
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- kibana.config
|
||||
- kibana.sostatus
|
||||
|
||||
@@ -27,6 +29,7 @@ so-kibana:
|
||||
- ELASTICSEARCH_HOST={{ GLOBALS.manager }}
|
||||
- ELASTICSEARCH_PORT=9200
|
||||
- MANAGER={{ GLOBALS.manager }}
|
||||
- NODE_OPTIONS=--use-openssl-ca
|
||||
{% if DOCKERMERGED.containers['so-kibana'].extra_env %}
|
||||
{% for XTRAENV in DOCKERMERGED.containers['so-kibana'].extra_env %}
|
||||
- {{ XTRAENV }}
|
||||
@@ -40,6 +43,7 @@ so-kibana:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /opt/so/conf/kibana/etc:/usr/share/kibana/config:rw
|
||||
- /opt/so/log/kibana:/var/log/kibana:rw
|
||||
- /opt/so/conf/kibana/customdashboards:/usr/share/kibana/custdashboards:ro
|
||||
@@ -60,6 +64,7 @@ so-kibana:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: kibanaconfig
|
||||
|
||||
wait_for_so-kibana:
|
||||
|
||||
@@ -7,8 +7,10 @@
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- kratos.config
|
||||
- kratos.sostatus
|
||||
|
||||
@@ -22,6 +24,7 @@ so-kratos:
|
||||
- soauth:
|
||||
- ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /opt/so/conf/kratos/:/kratos-conf:ro
|
||||
- /opt/so/log/kratos/:/kratos-log:rw
|
||||
- /nsm/kratos/db:/kratos-data:rw
|
||||
@@ -53,6 +56,7 @@ so-kratos:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: kratosschema
|
||||
- file: kratosconfig
|
||||
- require:
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% from 'logstash/map.jinja' import LOGSTASH_MERGED %}
|
||||
{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
|
||||
@@ -13,9 +14,6 @@
|
||||
|
||||
include:
|
||||
- ca
|
||||
{% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
|
||||
- elasticsearch.ca
|
||||
{% endif %}
|
||||
{# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #}
|
||||
{% if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %}
|
||||
- kafka.ca
|
||||
@@ -58,6 +56,8 @@ so-logstash:
|
||||
- {{ BINDING }}
|
||||
{% endfor %}
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- {{ CATRUST.java }}:/usr/share/logstash/jdk/lib/security/cacerts:ro
|
||||
- /opt/so/conf/elasticsearch/templates/:/templates/:ro
|
||||
- /opt/so/conf/logstash/etc/:/usr/share/logstash/config/:ro
|
||||
- /opt/so/conf/logstash/pipelines:/usr/share/logstash/pipelines:ro
|
||||
@@ -78,10 +78,6 @@ so-logstash:
|
||||
- /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
|
||||
- /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
|
||||
- /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
|
||||
{% endif %}
|
||||
{% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
|
||||
- /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
|
||||
- /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
|
||||
@@ -104,14 +100,11 @@ so-logstash:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: lsetcsync
|
||||
- file: lslog4j2
|
||||
- file: lspipelinesyml
|
||||
- file: trusttheca
|
||||
{% if GLOBALS.is_manager %}
|
||||
- file: elasticsearch_cacerts
|
||||
- file: elasticsearch_capems
|
||||
{% endif %}
|
||||
{% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
|
||||
- x509: etc_elasticfleet_logstash_crt
|
||||
- x509: etc_elasticfleet_logstash_key
|
||||
@@ -133,10 +126,6 @@ so-logstash:
|
||||
{% endif %}
|
||||
- require:
|
||||
- file: trusttheca
|
||||
{% if GLOBALS.is_manager %}
|
||||
- file: elasticsearch_cacerts
|
||||
- file: elasticsearch_capems
|
||||
{% endif %}
|
||||
{% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
|
||||
- x509: etc_elasticfleet_logstash_crt
|
||||
- x509: etc_elasticfleet_logstash_key
|
||||
|
||||
@@ -16,6 +16,7 @@ For each intent file under /opt/so/state/push_pending/*.json whose last_touch
|
||||
is older than debounce_seconds, this script:
|
||||
* concatenates the actions lists from every ready intent
|
||||
* dedupes by (state or __highstate__, tgt, tgt_type)
|
||||
* drops actions whose target matches no accepted minion (e.g. a role with no nodes yet)
|
||||
* dispatches a single `salt-run state.orchestrate orch.push_batch --async`
|
||||
with the deduped actions list passed as pillar kwargs
|
||||
* deletes the contributed intent files on successful dispatch
|
||||
@@ -38,10 +39,13 @@ import sys
|
||||
import time
|
||||
|
||||
import salt.client
|
||||
import salt.config
|
||||
import salt.utils.minions
|
||||
|
||||
PENDING_DIR = '/opt/so/state/push_pending'
|
||||
LOCK_FILE = os.path.join(PENDING_DIR, '.lock')
|
||||
LOG_FILE = '/opt/so/log/salt/so-push-drainer.log'
|
||||
MASTER_CONFIG = '/etc/salt/master'
|
||||
|
||||
DISPATCHED_DIR = '/opt/so/state/push_dispatched'
|
||||
|
||||
@@ -53,6 +57,9 @@ RESULT_MAX_AGE = 7200
|
||||
RESULT_CHECKS_PER_PASS = 5
|
||||
TEXT_LIMIT = 500
|
||||
|
||||
# Lead-in salt puts on the comment of an orchestration step that raised.
|
||||
STEP_RAISED = 'An exception occurred in this state:'
|
||||
|
||||
# salt-run --async reports the jid only in a log line (stderr by default).
|
||||
JID_RE = re.compile(r'salt/run/(\d{20})')
|
||||
|
||||
@@ -110,6 +117,29 @@ def _dedupe_actions(actions):
|
||||
return deduped
|
||||
|
||||
|
||||
def _targeted(actions, log):
|
||||
"""Drops actions whose target matches no accepted minion (e.g. a role with no nodes yet), which
|
||||
the orchestration would otherwise report as failed with 'No minions responded'."""
|
||||
try:
|
||||
ckminions = salt.utils.minions.CkMinions(salt.config.master_config(MASTER_CONFIG))
|
||||
except Exception:
|
||||
log.exception('cannot load master config; dispatching without checking targets')
|
||||
return actions
|
||||
targeted = []
|
||||
for action in actions:
|
||||
try:
|
||||
matched = ckminions.check_minions(action['tgt'], action.get('tgt_type', 'compound')).get('minions')
|
||||
except Exception:
|
||||
log.exception('cannot resolve tgt=%s; dispatching it anyway', action.get('tgt'))
|
||||
matched = True
|
||||
if matched:
|
||||
targeted.append(action)
|
||||
else:
|
||||
log.info('skipping %s: no minions match tgt=%s',
|
||||
'highstate' if action.get('highstate') else action.get('state'), action.get('tgt'))
|
||||
return targeted
|
||||
|
||||
|
||||
def _dispatch(actions, log):
|
||||
pillar_arg = json.dumps({'actions': actions})
|
||||
cmd = [
|
||||
@@ -242,6 +272,24 @@ def _orch_failures(ret):
|
||||
return failures
|
||||
|
||||
|
||||
def _failed_steps(ret):
|
||||
steps = []
|
||||
for job in ret.values() if isinstance(ret, dict) else []:
|
||||
job_ret = job.get('return') if isinstance(job, dict) else None
|
||||
data = job_ret.get('data') if isinstance(job_ret, dict) else None
|
||||
for group in data.values() if isinstance(data, dict) else []:
|
||||
if isinstance(group, dict):
|
||||
steps.extend(step for step in group.values() if isinstance(step, dict) and step.get('result') is False)
|
||||
return steps
|
||||
|
||||
|
||||
def _result_unknown(ret):
|
||||
# A step that raised (e.g. salt-master restarted while it waited on a queued state run)
|
||||
# never collected the minion's return, so the state run may still have completed.
|
||||
steps = _failed_steps(ret)
|
||||
return bool(steps) and all(str(step.get('comment', '')).startswith(STEP_RAISED) for step in steps)
|
||||
|
||||
|
||||
def _recheck_delay(age):
|
||||
return min(RESULT_RECHECK_MAX, max(RESULT_CHECK_DELAY, age / 4))
|
||||
|
||||
@@ -253,9 +301,15 @@ def _check_dispatched(log, now):
|
||||
if not isinstance(record, dict) or not record.get('jid'):
|
||||
_unlink(path, log)
|
||||
continue
|
||||
age = now - record.get('dispatched_at', 0)
|
||||
last_check = record.get('checked_at', record.get('dispatched_at', 0))
|
||||
if now - last_check >= _recheck_delay(age):
|
||||
try:
|
||||
age = now - record.get('dispatched_at', 0)
|
||||
last_check = record.get('checked_at', record.get('dispatched_at', 0))
|
||||
is_due = now - last_check >= _recheck_delay(age)
|
||||
except TypeError:
|
||||
log.warning('malformed record for jid=%s; no longer tracking', record.get('jid'))
|
||||
_unlink(path, log)
|
||||
continue
|
||||
if is_due:
|
||||
due.append((last_check, path, record, age))
|
||||
# Least recently checked first, so pushes that are still running can't starve finished ones.
|
||||
for _, path, record, age in sorted(due, key=lambda item: item[:2])[:RESULT_CHECKS_PER_PASS]:
|
||||
@@ -283,7 +337,12 @@ def _report_result(record, age, log):
|
||||
return True
|
||||
return False
|
||||
failures = _orch_failures(ret)
|
||||
if failures:
|
||||
if failures and _result_unknown(ret):
|
||||
# so-log-check excludes this message by its text.
|
||||
log.warning('push result unknown jid=%s paths=%s; the orchestration lost track of the state run, which may '
|
||||
'still have completed (if not, the change will be applied at the next scheduled highstate): %s',
|
||||
jid, paths, ' | '.join(failures))
|
||||
elif failures:
|
||||
log.error('push failed jid=%s paths=%s; change will be applied at the next scheduled highstate: %s',
|
||||
jid, paths, ' | '.join(failures))
|
||||
else:
|
||||
@@ -382,12 +441,16 @@ def main():
|
||||
log.info('action: %s tgt=%s', 'highstate' if action.get('highstate') else action.get('state'),
|
||||
action.get('tgt'))
|
||||
|
||||
jid = _dispatch(deduped, log)
|
||||
if jid is None:
|
||||
log.warning('dispatch failed; leaving intent files in place for retry')
|
||||
return 1
|
||||
if jid:
|
||||
_record_dispatch(jid, deduped, all_paths[:20], log)
|
||||
targeted = _targeted(deduped, log)
|
||||
if targeted:
|
||||
jid = _dispatch(targeted, log)
|
||||
if jid is None:
|
||||
log.warning('dispatch failed; leaving intent files in place for retry')
|
||||
return 1
|
||||
if jid:
|
||||
_record_dispatch(jid, targeted, all_paths[:20], log)
|
||||
else:
|
||||
log.info('no action matches a minion; nothing to dispatch')
|
||||
|
||||
for path, _ in ready:
|
||||
try:
|
||||
|
||||
@@ -22,10 +22,11 @@ _loader = SourceFileLoader('so_push_drainer', SCRIPT)
|
||||
_spec = importlib.util.spec_from_loader('so_push_drainer', _loader)
|
||||
drainer = importlib.util.module_from_spec(_spec)
|
||||
|
||||
# salt is not installed where these tests run; the drainer only needs salt.client.Caller.
|
||||
# salt is not installed where these tests run; the drainer only needs salt.client.Caller and CkMinions.
|
||||
# Mocked only while the drainer loads: run from the repo root, 'salt' is this repo's salt/ directory.
|
||||
_salt = MagicMock()
|
||||
with patch.dict(sys.modules, {'salt': _salt, 'salt.client': _salt.client}):
|
||||
with patch.dict(sys.modules, {'salt': _salt, 'salt.client': _salt.client, 'salt.config': _salt.config,
|
||||
'salt.utils': _salt.utils, 'salt.utils.minions': _salt.utils.minions}):
|
||||
_loader.exec_module(drainer)
|
||||
|
||||
MASTER = 'manager.localdomain_master'
|
||||
@@ -76,6 +77,17 @@ STATE_FAIL_RET = _orch_ret({
|
||||
},
|
||||
}, success=False)
|
||||
|
||||
RAISED = ('An exception occurred in this state: Traceback (most recent call last):\n'
|
||||
' File "salt/client/__init__.py", line 1934, in pub\n'
|
||||
' raise AuthenticationError(err_msg)\n'
|
||||
'salt.exceptions.AuthenticationError: Authentication error occurred.\n')
|
||||
|
||||
RAISED_RET = _orch_ret(dict(REFRESH_STEP, **{
|
||||
'salt_|-apply_hydra_1_|-apply_hydra_1_|-state': {
|
||||
'__id__': 'apply_hydra_1', 'result': False, 'changes': {}, 'comment': RAISED,
|
||||
},
|
||||
}), success=False)
|
||||
|
||||
SUCCESS_RET = _orch_ret(dict(REFRESH_STEP, **{
|
||||
'salt_|-apply_telegraf_1_|-apply_telegraf_1_|-state': {
|
||||
'__id__': 'apply_telegraf_1', 'result': True,
|
||||
@@ -294,6 +306,15 @@ class TestResults(DrainerTestCase):
|
||||
ret = {MASTER: {'return': 'Exception occurred', 'success': False}}
|
||||
self.assertEqual(drainer._orch_failures(ret), ['orchestration reported failure: Exception occurred'])
|
||||
|
||||
def test_result_unknown(self):
|
||||
self.assertTrue(drainer._result_unknown(RAISED_RET))
|
||||
for ret in (CONFLICT_RET, STATE_FAIL_RET, SUCCESS_RET, ['No minions matched'], {MASTER: 'odd'},
|
||||
{MASTER: {'return': {'data': {MASTER: ['Rendering SLS failed']}}, 'success': False}}):
|
||||
self.assertFalse(drainer._result_unknown(ret), ret)
|
||||
mixed = _orch_ret(dict(RAISED_RET[MASTER]['return']['data'][MASTER],
|
||||
**CONFLICT_RET[MASTER]['return']['data'][MASTER]), success=False)
|
||||
self.assertFalse(drainer._result_unknown(mixed))
|
||||
|
||||
def record(self, jid, age, now):
|
||||
return self.write_json(self.dispatched, jid + '.json', {
|
||||
'jid': jid, 'dispatched_at': now - age, 'actions': [], 'paths': ['audit:' + jid],
|
||||
@@ -306,6 +327,7 @@ class TestResults(DrainerTestCase):
|
||||
'2_ok': SUCCESS_RET,
|
||||
'3_pending': {},
|
||||
'4_expired': None,
|
||||
'6_unknown': RAISED_RET,
|
||||
}
|
||||
young = self.record('0_young', 5, now)
|
||||
paths = {jid: self.record(jid, 60, now) for jid in results}
|
||||
@@ -317,13 +339,16 @@ class TestResults(DrainerTestCase):
|
||||
self.assertTrue(os.path.exists(young))
|
||||
with open(paths['3_pending']) as f:
|
||||
self.assertEqual(json.load(f)['checked_at'], now)
|
||||
for jid in ('1_failed', '2_ok', '4_expired'):
|
||||
for jid in ('1_failed', '2_ok', '4_expired', '6_unknown'):
|
||||
self.assertFalse(os.path.exists(paths[jid]), jid)
|
||||
self.assertFalse(os.path.exists(bad))
|
||||
self.assertIn('push failed jid=1_failed', self.logged('error'))
|
||||
self.assertIn('is running as PID 372218', self.logged('error'))
|
||||
self.assertIn('push succeeded jid=2_ok', self.logged('info'))
|
||||
self.assertIn('no result for jid=4_expired', self.logged('warning'))
|
||||
self.assertIn('push result unknown jid=6_unknown', self.logged('warning'))
|
||||
self.assertIn('AuthenticationError: Authentication error occurred.', self.logged('warning'))
|
||||
self.assertNotIn('6_unknown', self.logged('error'))
|
||||
|
||||
def test_check_dispatched_survives_bad_result(self):
|
||||
now = time.time()
|
||||
@@ -344,6 +369,22 @@ class TestResults(DrainerTestCase):
|
||||
self.assertIn('jid=1_bad', self.log.exception.call_args[0][0] % self.log.exception.call_args[0][1:])
|
||||
self.assertIn('push succeeded jid=2_ok', self.logged('info'))
|
||||
|
||||
def test_check_dispatched_drops_malformed_record(self):
|
||||
now = time.time()
|
||||
bad_dispatched = self.write_json(self.dispatched, '1_bad.json', {'jid': '1_bad', 'dispatched_at': 'soon'})
|
||||
bad_checked = self.write_json(self.dispatched, '2_bad.json', {
|
||||
'jid': '2_bad', 'dispatched_at': now - 60, 'checked_at': None,
|
||||
})
|
||||
good = self.record('3_ok', 60, now)
|
||||
with patch.object(drainer, '_lookup_jid', return_value=SUCCESS_RET) as lookup:
|
||||
drainer._check_dispatched(self.log, now)
|
||||
for path in (bad_dispatched, bad_checked, good):
|
||||
self.assertFalse(os.path.exists(path), path)
|
||||
self.assertEqual([c.args[0] for c in lookup.call_args_list], ['3_ok'])
|
||||
self.assertIn('malformed record for jid=1_bad', self.logged('warning'))
|
||||
self.assertIn('malformed record for jid=2_bad', self.logged('warning'))
|
||||
self.assertIn('push succeeded jid=3_ok', self.logged('info'))
|
||||
|
||||
def test_recheck_delay(self):
|
||||
self.assertEqual(drainer._recheck_delay(10), drainer.RESULT_CHECK_DELAY)
|
||||
self.assertEqual(drainer._recheck_delay(400), 100)
|
||||
@@ -381,6 +422,49 @@ class TestResults(DrainerTestCase):
|
||||
self.assertIn('push succeeded jid=2_done', self.logged('info'))
|
||||
|
||||
|
||||
class TestTargeted(DrainerTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.ckminions = MagicMock()
|
||||
patcher = patch.object(drainer.salt.utils.minions, 'CkMinions', return_value=self.ckminions)
|
||||
self.CkMinions = patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def test_drops_unmatched(self):
|
||||
matches = {'G@role:so-sensor': ['sensor_sensor'], 'manager_manager': ['manager_manager']}
|
||||
self.ckminions.check_minions.side_effect = lambda tgt, tgt_type: {'minions': matches.get(tgt, [])}
|
||||
actions = [
|
||||
{'state': 'strelka', 'tgt': 'G@role:so-sensor'},
|
||||
{'state': 'suricata', 'tgt': 'G@role:so-heavynode'},
|
||||
{'highstate': True, 'tgt': 'G@role:so-idh'},
|
||||
{'state': 'soc', 'tgt': 'manager_manager', 'tgt_type': 'list'},
|
||||
]
|
||||
self.assertEqual(drainer._targeted(actions, self.log), [actions[0], actions[3]])
|
||||
self.CkMinions.assert_called_once()
|
||||
self.assertEqual([c.args for c in self.ckminions.check_minions.call_args_list], [
|
||||
('G@role:so-sensor', 'compound'), ('G@role:so-heavynode', 'compound'),
|
||||
('G@role:so-idh', 'compound'), ('manager_manager', 'list'),
|
||||
])
|
||||
info = self.logged('info')
|
||||
self.assertIn('skipping suricata: no minions match tgt=G@role:so-heavynode', info)
|
||||
self.assertIn('skipping highstate: no minions match tgt=G@role:so-idh', info)
|
||||
self.assertNotIn('strelka', info)
|
||||
|
||||
def test_master_config_error_keeps_actions(self):
|
||||
actions = [{'state': 'soc', 'tgt': '*'}]
|
||||
with patch.object(drainer.salt.config, 'master_config', side_effect=OSError('no config')):
|
||||
self.assertEqual(drainer._targeted(actions, self.log), actions)
|
||||
self.log.exception.assert_called_once()
|
||||
self.CkMinions.assert_not_called()
|
||||
|
||||
def test_check_error_keeps_action(self):
|
||||
actions = [{'state': 'soc', 'tgt': 'G@role:'}]
|
||||
self.ckminions.check_minions.side_effect = RuntimeError('bad target')
|
||||
self.assertEqual(drainer._targeted(actions, self.log), actions)
|
||||
self.log.exception.assert_called_once()
|
||||
|
||||
|
||||
class TestMain(DrainerTestCase):
|
||||
|
||||
def setUp(self):
|
||||
@@ -390,6 +474,7 @@ class TestMain(DrainerTestCase):
|
||||
('_make_logger', {'return_value': self.log}),
|
||||
('_load_push_cfg', {'side_effect': lambda: self.cfg}),
|
||||
('_check_dispatched', {}),
|
||||
('_targeted', {'side_effect': lambda actions, log: actions}),
|
||||
):
|
||||
patcher = patch.object(drainer, name, **kwargs)
|
||||
setattr(self, name, patcher.start())
|
||||
@@ -472,6 +557,29 @@ class TestMain(DrainerTestCase):
|
||||
record.assert_not_called()
|
||||
self.log.exception.assert_called_once()
|
||||
|
||||
def test_untargeted_not_dispatched(self):
|
||||
path = self.intent('strelka.json', actions=[{'state': 'strelka', 'tgt': 'G@role:so-sensor'}])
|
||||
self._targeted.side_effect = None
|
||||
self._targeted.return_value = []
|
||||
with patch.object(drainer, '_dispatch') as dispatch, \
|
||||
patch.object(drainer, '_record_dispatch') as record:
|
||||
self.assertEqual(drainer.main(), 0)
|
||||
dispatch.assert_not_called()
|
||||
record.assert_not_called()
|
||||
self.assertFalse(os.path.exists(path))
|
||||
self.assertIn('no action matches a minion; nothing to dispatch', self.logged('info'))
|
||||
|
||||
def test_dispatches_only_targeted(self):
|
||||
kept = {'state': 'soc', 'tgt': '*'}
|
||||
self.intent('mixed.json', actions=[kept, {'state': 'strelka', 'tgt': 'G@role:so-sensor'}])
|
||||
self._targeted.side_effect = None
|
||||
self._targeted.return_value = [kept]
|
||||
with patch.object(drainer, '_dispatch', return_value=JID) as dispatch, \
|
||||
patch.object(drainer, '_record_dispatch') as record:
|
||||
self.assertEqual(drainer.main(), 0)
|
||||
self.assertEqual(dispatch.call_args[0][0], [kept])
|
||||
self.assertEqual(record.call_args[0][1], [kept])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -20,6 +20,7 @@ def showUsage(args):
|
||||
print(' append - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
|
||||
print(' appendlistobject - Append an object to a yaml list key. Requires KEY and JSON_OBJECT args.', file=sys.stderr)
|
||||
print(' removelistitem - Remove a list item from a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.', file=sys.stderr)
|
||||
print(' purgelistitem - Remove every occurrence of a list item from all lists nested under a yaml key, if it exists. Requires KEY and LISTITEM args.', file=sys.stderr)
|
||||
print(' replacelistobject - Replace a list object based on a condition. Requires KEY, CONDITION_FIELD, CONDITION_VALUE, and JSON_OBJECT args.', file=sys.stderr)
|
||||
print(' add - Add a new key and set its value. Fails if key already exists. Requires KEY and VALUE args.', file=sys.stderr)
|
||||
print(' get [-r] - Displays (to stdout) the value stored in the given key. Requires KEY arg. Use -r for raw output without YAML formatting.', file=sys.stderr)
|
||||
@@ -199,6 +200,33 @@ def removelistitem(args):
|
||||
return 0
|
||||
|
||||
|
||||
def purgeListItem(content, listItem):
|
||||
if isinstance(content, dict):
|
||||
for value in content.values():
|
||||
purgeListItem(value, listItem)
|
||||
elif isinstance(content, list):
|
||||
content[:] = [item for item in content if item != listItem]
|
||||
for item in content:
|
||||
purgeListItem(item, listItem)
|
||||
|
||||
|
||||
def purgelistitem(args):
|
||||
if len(args) != 3:
|
||||
print('Missing filename, key arg, or list item to purge', file=sys.stderr)
|
||||
showUsage(None)
|
||||
return 1
|
||||
|
||||
filename = args[0]
|
||||
key = args[1]
|
||||
listItem = args[2]
|
||||
|
||||
content = loadYaml(filename)
|
||||
purgeListItem(getKeyValue(content, key), convertType(listItem))
|
||||
writeYaml(filename, content)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def replaceListObjectByCondition(content, key, conditionField, conditionValue, newObject):
|
||||
pieces = key.split(".", 1)
|
||||
if len(pieces) > 1:
|
||||
@@ -291,6 +319,8 @@ def add(args):
|
||||
|
||||
|
||||
def removeKey(content, key):
|
||||
if not isinstance(content, dict):
|
||||
return
|
||||
pieces = key.split(".", 1)
|
||||
if len(pieces) > 1:
|
||||
if pieces[0] in content:
|
||||
@@ -334,6 +364,8 @@ def replace(args):
|
||||
|
||||
|
||||
def getKeyValue(content, key):
|
||||
if not isinstance(content, dict):
|
||||
return None
|
||||
pieces = key.split(".", 1)
|
||||
if len(pieces) > 1 and pieces[0] in content:
|
||||
return getKeyValue(content[pieces[0]], pieces[1])
|
||||
@@ -385,6 +417,7 @@ def main():
|
||||
"append": append,
|
||||
"appendlistobject": appendlistobject,
|
||||
"removelistitem": removelistitem,
|
||||
"purgelistitem": purgelistitem,
|
||||
"replacelistobject": replacelistobject,
|
||||
"get": get,
|
||||
"remove": remove,
|
||||
|
||||
@@ -128,6 +128,22 @@ class TestRemove(unittest.TestCase):
|
||||
sysmock.assert_called_once_with(1)
|
||||
self.assertIn("Missing filename or key arg\n", mock_stderr.getvalue())
|
||||
|
||||
def test_remove_null_parent(self):
|
||||
filename = "/tmp/so-yaml_test-remove.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("key1: null\n")
|
||||
file.close()
|
||||
|
||||
result = soyaml.remove([filename, "key1.child1.deep1"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1: null\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_append_missing_arg(self):
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_stderr:
|
||||
@@ -565,6 +581,19 @@ class TestRemove(unittest.TestCase):
|
||||
self.assertEqual(result, 2)
|
||||
self.assertEqual("", mock_stdout.getvalue())
|
||||
|
||||
def test_get_null_parent(self):
|
||||
with patch('sys.stdout', new=StringIO()) as mock_stdout:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_stderr:
|
||||
filename = "/tmp/so-yaml_test-get.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("key1: null\n")
|
||||
file.close()
|
||||
|
||||
result = soyaml.get([filename, "key1.child1"])
|
||||
self.assertEqual(result, 2)
|
||||
self.assertEqual("", mock_stdout.getvalue())
|
||||
self.assertIn("Key 'key1.child1' not found by so-yaml.py", mock_stderr.getvalue())
|
||||
|
||||
def test_get_empty_file(self):
|
||||
with patch('sys.stdout', new=StringIO()) as mock_stdout:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_stderr:
|
||||
@@ -709,6 +738,255 @@ class TestRemoveListItem(unittest.TestCase):
|
||||
self.assertEqual("The existing value for the given key is not a list. No action was taken on the file.\n", mock_stderr.getvalue())
|
||||
|
||||
|
||||
class TestPurgeListItem(unittest.TestCase):
|
||||
|
||||
def test_purgelistitem_missing_arg(self):
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
with patch('sys.stderr', new=StringIO()) as mock_stderr:
|
||||
sys.argv = ["cmd", "help"]
|
||||
soyaml.purgelistitem(["file", "key"])
|
||||
sysmock.assert_called()
|
||||
self.assertIn("Missing filename, key arg, or list item to purge", mock_stderr.getvalue())
|
||||
|
||||
def test_purgelistitem(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [a,b,c], key2: false}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- c\nkey2: false\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_nested_lists(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: [a,b,c], child2: { deep1: [b,d], deep2: 45 } }, key2: [b,e]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1:\n - a\n - c\n child2:\n deep1:\n - d\n deep2: 45\nkey2:\n- b\n- e\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_duplicates(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [b,a,b,c,b]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_list_of_objects(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [{name: item1, values: [a,b]}, b, {name: item2, values: [b,c]}]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- name: item1\n values:\n - a\n- name: item2\n values:\n - c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_typed(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [1, '1', 2]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "1"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- '1'\n- 2\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_item_not_in_list(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: [a,b,c] }}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "d"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1:\n - a\n - b\n - c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_key_noexist(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [a,b,c]}")
|
||||
file.close()
|
||||
|
||||
result = soyaml.purgelistitem([filename, "key2.child1", "b"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- b\n- c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_key_noexist_deep(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: [a,b,c] }}")
|
||||
file.close()
|
||||
|
||||
result = soyaml.purgelistitem([filename, "key1.child2.deep1", "b"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1:\n - a\n - b\n - c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_scalar_and_null_values(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: b, child2: null, child3: [b] }}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1: b\n child2: null\n child3: []\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_lists_of_lists(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [[a,b], [b,[b,c]], b]}")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "b"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- - a\n- - - c\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_file_object(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: [{name: x}, a, {name: y}, {name: x}]}")
|
||||
file.close()
|
||||
|
||||
itemfile = "/tmp/so-yaml_test-purgelistitem-item.yaml"
|
||||
file = open(itemfile, "w")
|
||||
file.write("name: x\n")
|
||||
file.close()
|
||||
|
||||
soyaml.purgelistitem([filename, "key1", "file:" + itemfile])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- name: y\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_main(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: [a,b,c] }, key2: [b]}")
|
||||
file.close()
|
||||
|
||||
with patch('sys.exit', new=MagicMock()) as sysmock:
|
||||
sys.argv = ["cmd", "purgelistitem", filename, "key1", "b"]
|
||||
soyaml.main()
|
||||
sysmock.assert_called_once_with(0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child1:\n - a\n - c\nkey2:\n- b\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_with_remove(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("{key1: { child1: { deep1: [a] }, child2: [b] }, key2: { group1: { items: [x,child1,y] }, group2: { items: [child1] } }}")
|
||||
file.close()
|
||||
|
||||
soyaml.remove([filename, "key1.child1"])
|
||||
soyaml.purgelistitem([filename, "key2", "child1"])
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n child2:\n - b\nkey2:\n group1:\n items:\n - x\n - y\n group2:\n items: []\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_null_parent(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("key1: null\n")
|
||||
file.close()
|
||||
|
||||
result = soyaml.purgelistitem([filename, "key1.child1", "b"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1: null\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_purgelistitem_non_dict_parent(self):
|
||||
filename = "/tmp/so-yaml_test-purgelistitem.yaml"
|
||||
file = open(filename, "w")
|
||||
file.write("key1:\n- a\n- b\n")
|
||||
file.close()
|
||||
|
||||
result = soyaml.purgelistitem([filename, "key1.child1", "b"])
|
||||
self.assertEqual(result, 0)
|
||||
|
||||
file = open(filename, "r")
|
||||
actual = file.read()
|
||||
file.close()
|
||||
|
||||
expected = "key1:\n- a\n- b\n"
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
|
||||
class TestAppendListObject(unittest.TestCase):
|
||||
|
||||
def test_appendlistobject_missing_arg(self):
|
||||
|
||||
+82
-112
@@ -5,7 +5,6 @@
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
|
||||
. /usr/sbin/so-common
|
||||
. /usr/sbin/so-image-common
|
||||
|
||||
@@ -17,7 +16,9 @@ INSTALLEDVERSION=$(cat /etc/soversion)
|
||||
# not finish its post-upgrade steps; its contents are the resume point. It is read
|
||||
# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
|
||||
# stamps /etc/soversion from the pillar.
|
||||
|
||||
POSTVERSION_FILE=/etc/sopostversion
|
||||
|
||||
if [ -f "$POSTVERSION_FILE" ]; then
|
||||
POSTVERSION=$(cat "$POSTVERSION_FILE")
|
||||
else
|
||||
@@ -121,9 +122,6 @@ check_err() {
|
||||
161)
|
||||
echo 'Required intermediate Elasticsearch upgrade not complete'
|
||||
;;
|
||||
162)
|
||||
echo 'One or more Elastic Agent nodes do not support the x86-64-v3 CPU instruction set'
|
||||
;;
|
||||
170)
|
||||
echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
|
||||
echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
|
||||
@@ -351,83 +349,6 @@ check_cluster_health() {
|
||||
exit 0
|
||||
}
|
||||
|
||||
no_soup_for_you() {
|
||||
echo ""
|
||||
echo "No soup for you!"
|
||||
exit 162
|
||||
}
|
||||
|
||||
check_cpu_compatibility() {
|
||||
# Roles running a container built from the so-elastic-agent image; mirrors the
|
||||
# elasticagent and elasticfleet entries in salt/reactor/pillar_push_map.yaml.
|
||||
local cpu_target='G@role:so-heavynode or G@role:so-eval or G@role:so-fleet or G@role:so-import or G@role:so-manager or G@role:so-managerhype or G@role:so-managersearch or G@role:so-standalone'
|
||||
local expected_nodes cpu_results node result confirm
|
||||
local -a unsupported=() offline=()
|
||||
|
||||
echo "Checking that Elastic Agent nodes support the x86-64-v3 CPU instruction set now required by Elastic."
|
||||
|
||||
if [[ "$SKIP_CPU_CHECK" == "true" ]]; then
|
||||
printf "\nSkipping the x86-64-v3 CPU check because --skip-cpu-check was specified.\n\n"
|
||||
return
|
||||
fi
|
||||
|
||||
# Nodes that never answer are absent from the results, so diff against who should have.
|
||||
expected_nodes=$(salt -C "$cpu_target" --preview-target --out=json 2>/dev/null | jq -r '.[]?') || true
|
||||
if [[ -z "$expected_nodes" ]]; then
|
||||
printf "\nCould not determine which nodes run the Elastic Agent, so the x86-64-v3 CPU check cannot run.\n"
|
||||
no_soup_for_you
|
||||
fi
|
||||
|
||||
cpu_results=$(salt -t 30 -C "$cpu_target" cmd.run "/lib64/ld-linux-x86-64.so.2 --help | grep x86-64-v3" --out=json 2>/dev/null) || true
|
||||
|
||||
while IFS= read -r node; do
|
||||
[[ -z "$node" ]] && continue
|
||||
result=$(jq -r --arg node "$node" '.[$node] // empty' <<< "$cpu_results" 2>/dev/null)
|
||||
if [[ -z "$result" || "$result" == *"did not return"* ]]; then
|
||||
offline+=("$node")
|
||||
elif [[ "$result" != *"x86-64-v3 (supported"* ]]; then
|
||||
# glibc appends "(supported, searched)" only when supported; the open paren keeps
|
||||
# this from matching a future "(unsupported".
|
||||
unsupported+=("$node")
|
||||
fi
|
||||
done <<< "$expected_nodes"
|
||||
|
||||
if [[ ${#unsupported[@]} -eq 0 && ${#offline[@]} -eq 0 ]]; then
|
||||
printf "\nAll Elastic Agent nodes support x86-64-v3. We can proceed with SOUP.\n\n"
|
||||
return
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ ${#unsupported[@]} -gt 0 ]]; then
|
||||
echo "The following node(s) do NOT support the x86-64-v3 CPU instruction set:"
|
||||
printf ' %s\n' "${unsupported[@]}"
|
||||
echo ""
|
||||
echo "Upstream Elastic now builds its binaries for x86-64-v3, so these nodes can no"
|
||||
echo "longer run Elastic. Upgrading them WILL BREAK them."
|
||||
echo ""
|
||||
fi
|
||||
if [[ ${#offline[@]} -gt 0 ]]; then
|
||||
echo "The following node(s) did not respond and could not be checked:"
|
||||
printf ' %s\n' "${offline[@]}"
|
||||
echo ""
|
||||
echo "These nodes are offline, so we cannot confirm they support x86-64-v3, which"
|
||||
echo "upstream Elastic now requires."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [[ -n $UNATTENDED ]]; then
|
||||
echo "Unattended mode cannot prompt for an override. Re-run soup interactively, or pass --skip-cpu-check to bypass this check."
|
||||
no_soup_for_you
|
||||
fi
|
||||
|
||||
read -rp "Type 'override' to continue anyway, or press Enter to exit: " confirm
|
||||
if [[ "${confirm,,}" == "override" ]]; then
|
||||
printf "\nOverride accepted. Continuing at your own risk.\n\n"
|
||||
else
|
||||
no_soup_for_you
|
||||
fi
|
||||
}
|
||||
|
||||
check_fleet_server() {
|
||||
echo "Checking that Elastic Fleet Server is responding."
|
||||
# Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
|
||||
@@ -607,6 +528,7 @@ preupgrade_changes() {
|
||||
[[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
|
||||
[[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
|
||||
[[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
|
||||
[[ "$INSTALLEDVERSION" == "3.4.0" ]] && up_to_3.5.0
|
||||
true
|
||||
}
|
||||
|
||||
@@ -626,6 +548,7 @@ postupgrade_changes() {
|
||||
[[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
|
||||
[[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
|
||||
[[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
|
||||
[[ "$POSTVERSION" == "3.4.0" ]] && post_to_3.5.0
|
||||
# All applicable post-upgrade steps completed; clear the resume marker.
|
||||
rm -f "$POSTVERSION_FILE"
|
||||
true
|
||||
@@ -1131,9 +1054,6 @@ post_to_3.2.0() {
|
||||
|
||||
### 3.3.0 Scripts ###
|
||||
up_to_3.3.0() {
|
||||
# download 9.4.5 elastic agent packages
|
||||
determine_elastic_agent_upgrade
|
||||
|
||||
# remove existing (patched) elasticsearch index template to match integration naming change
|
||||
if ! remove_elasticsearch_index_template "so-logs-sentinel_one_cloud_funnel.login" "sentinel_one_cloud_funnel.login changed to sentinel_one_cloud_funnel.logins"; then
|
||||
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the so-logs-sentinel_one_cloud_funnel.login index template. This step can be performed manually using the following command:")
|
||||
@@ -1148,28 +1068,38 @@ telegraf_repair() {
|
||||
# when there is something to repair, so everyone else is left alone.
|
||||
local repair=/usr/sbin/so-telegraf-repair
|
||||
[[ -x "$repair" ]] || return 0
|
||||
docker ps --format '{{.Names}}' | grep -qx so-postgres || return 0
|
||||
local state
|
||||
state=$(docker inspect -f '{{.State.Running}} {{.State.Restarting}}' so-postgres 2>/dev/null) || return 0
|
||||
if [[ "$state" != "true false" ]]; then
|
||||
echo " so-postgres is not running; skipping the Telegraf partition check."
|
||||
FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres is not running, so Telegraf partitions were not checked. If /nsm is full, follow \"Telegraf Partition Repair\" at:")
|
||||
FINAL_MESSAGE_QUEUE+=(" - https://securityonion.net/docs/postgresql")
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Checking Telegraf metric partitions."
|
||||
local status=0
|
||||
"$repair" --check >> "$SOUP_LOG" 2>&1 || status=$?
|
||||
local out status=0
|
||||
out=$("$repair" --check 2>&1) || status=$?
|
||||
echo "$out" >> "$SOUP_LOG"
|
||||
case "$status" in
|
||||
0) echo " Telegraf partitions are healthy; nothing to repair." ;;
|
||||
1) echo " Repairing stalled Telegraf partitions."
|
||||
"$repair" --yes \
|
||||
|| echo " warning: so-telegraf-repair failed; run it manually" >&2 ;;
|
||||
*) echo " Skipping; Telegraf is not storing metrics in Postgres on this host." ;;
|
||||
if ! "$repair" --yes; then
|
||||
echo " warning: so-telegraf-repair failed; run it manually" >&2
|
||||
FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partition repair failed; Postgres metrics will keep filling /nsm until it is repaired:")
|
||||
FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair")
|
||||
fi ;;
|
||||
2) echo " Skipping: $(tail -n 1 <<<"$out")" ;;
|
||||
*) echo " warning: could not check Telegraf partitions" >&2
|
||||
FINAL_MESSAGE_QUEUE+=("WARNING: Telegraf partitions could not be checked during the upgrade; run:")
|
||||
FINAL_MESSAGE_QUEUE+=(" - sudo so-telegraf-repair --check") ;;
|
||||
esac
|
||||
}
|
||||
|
||||
post_to_3.3.0() {
|
||||
# Recollate again since some internal DBs were excluded during 3.2.0 soup
|
||||
recollate_postgres
|
||||
|
||||
# Generate 9.4.5 elastic agent installers
|
||||
echo "Regenerating Elastic Agent Installers"
|
||||
/sbin/so-elastic-agent-gen-installers
|
||||
|
||||
|
||||
telegraf_repair
|
||||
|
||||
set_postversion 3.3.0
|
||||
@@ -1178,6 +1108,9 @@ post_to_3.3.0() {
|
||||
|
||||
### 3.4.0 Scripts ###
|
||||
up_to_3.4.0() {
|
||||
# download 9.4.8 elastic agent packages
|
||||
determine_elastic_agent_upgrade
|
||||
|
||||
set_soauth_range
|
||||
|
||||
echo "Removing so-kratos, so-hydra and so-soc so they are recreated on the soauth network."
|
||||
@@ -1255,14 +1188,63 @@ valid_soauth_range() {
|
||||
}
|
||||
|
||||
post_to_3.4.0() {
|
||||
# Generate 9.4.8 elastic agent installers
|
||||
echo "Regenerating Elastic Agent Installers"
|
||||
/sbin/so-elastic-agent-gen-installers
|
||||
|
||||
for idx in "metrics-logstash.node-default" "metrics-logstash.stack_monitoring.node-default"; do
|
||||
rollover_index "$idx"
|
||||
done
|
||||
|
||||
add_detection_runtime_fields
|
||||
|
||||
remove_frozen_ca_bundles
|
||||
|
||||
set_postversion 3.4.0
|
||||
}
|
||||
|
||||
remove_frozen_ca_bundles() {
|
||||
rm -f /opt/so/saltstack/local/salt/elasticsearch/cacerts /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem >> $SOUP_LOG 2>&1
|
||||
}
|
||||
|
||||
add_detection_runtime_fields() {
|
||||
local template=$DEFAULT_SALT_DIR/salt/elasticsearch/templates/component/so/detection-mappings.json
|
||||
local runtime
|
||||
|
||||
runtime=$(jq -c '{runtime: .template.mappings.runtime}' "$template")
|
||||
if ! so-elasticsearch-query so-detection/_mapping -XPUT -d "$runtime" --retry 3 --retry-delay 5 --fail >> $SOUP_LOG 2>&1; then
|
||||
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to add the Sigma search fields to the so-detection index. Searches on so_detection.sigmaKind will match nothing until they are added:")
|
||||
FINAL_MESSAGE_QUEUE+=(" - sudo so-elasticsearch-query so-detection/_mapping -XPUT -d \"\$(jq -c '{runtime: .template.mappings.runtime}' $template)\"")
|
||||
fi
|
||||
}
|
||||
### 3.4.0 End ###
|
||||
|
||||
### 3.5.0 Scripts ###
|
||||
up_to_3.5.0() {
|
||||
remove_localrules_portgroup
|
||||
|
||||
INSTALLEDVERSION=3.5.0
|
||||
}
|
||||
|
||||
remove_localrules_portgroup() {
|
||||
local pillar_file
|
||||
|
||||
for pillar_file in /opt/so/saltstack/local/pillar/firewall/{soc,adv}_firewall.sls; do
|
||||
grep -qs localrules "$pillar_file" || continue
|
||||
so-yaml.py get "$pillar_file" firewall 2>/dev/null | grep -q '\<localrules\>' || continue
|
||||
echo "Removing the localrules portgroup from $pillar_file."
|
||||
so-yaml.py remove "$pillar_file" firewall.portgroups.localrules
|
||||
so-yaml.py purgelistitem "$pillar_file" firewall.role localrules
|
||||
done
|
||||
}
|
||||
|
||||
post_to_3.5.0() {
|
||||
telegraf_repair
|
||||
|
||||
set_postversion 3.5.0
|
||||
}
|
||||
### 3.5.0 End ###
|
||||
|
||||
|
||||
repo_sync() {
|
||||
echo "Sync the local repo."
|
||||
@@ -1535,9 +1517,10 @@ verify_es_version_compatibility() {
|
||||
["8.18.4"]="8.18.6 8.18.8 9.0.8"
|
||||
["8.18.6"]="8.18.8 9.0.8"
|
||||
["8.18.8"]="9.0.8"
|
||||
["9.0.8"]="9.3.3 9.3.7 9.4.5"
|
||||
["9.3.3"]="9.3.7 9.4.5"
|
||||
["9.3.7"]="9.4.5"
|
||||
["9.0.8"]="9.3.3 9.3.7 9.4.5 9.4.8"
|
||||
["9.3.3"]="9.3.7 9.4.5 9.4.8"
|
||||
["9.3.7"]="9.4.5 9.4.8"
|
||||
["9.4.5"]="9.4.8"
|
||||
)
|
||||
|
||||
# Elasticsearch MUST upgrade through these versions
|
||||
@@ -2148,8 +2131,6 @@ main() {
|
||||
echo "Let's see if we need to update Security Onion."
|
||||
upgrade_check
|
||||
|
||||
check_cpu_compatibility
|
||||
|
||||
upgrade_space
|
||||
|
||||
echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
|
||||
@@ -2428,17 +2409,6 @@ fi
|
||||
echo "### soup has been served at $(date) ###"
|
||||
}
|
||||
|
||||
SKIP_CPU_CHECK=false
|
||||
declare -a SOUP_ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--skip-cpu-check" ]]; then
|
||||
SKIP_CPU_CHECK=true
|
||||
else
|
||||
SOUP_ARGS+=("$arg")
|
||||
fi
|
||||
done
|
||||
set -- "${SOUP_ARGS[@]}"
|
||||
|
||||
while getopts ":b:f:y" opt; do
|
||||
case ${opt} in
|
||||
b )
|
||||
@@ -2462,7 +2432,7 @@ while getopts ":b:f:y" opt; do
|
||||
ISOLOC="$OPTARG"
|
||||
;;
|
||||
\? )
|
||||
echo "Usage: soup [-b] [-y] [-f <iso location>] [--skip-cpu-check]"
|
||||
echo "Usage: soup [-b] [-y] [-f <iso location>]"
|
||||
exit 1
|
||||
;;
|
||||
: )
|
||||
|
||||
+18
-18
@@ -6,31 +6,29 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% from 'nginx/map.jinja' import NGINXMERGED %}
|
||||
|
||||
include:
|
||||
- nginx.ssl
|
||||
- nginx.config
|
||||
- nginx.sostatus
|
||||
|
||||
{% if GLOBALS.role != 'so-fleet' %}
|
||||
{% set container_config = 'so-nginx' %}
|
||||
make-rule-dir-nginx:
|
||||
file.directory:
|
||||
- name: /nsm/rules
|
||||
- user: socore
|
||||
- group: socore
|
||||
- recurse:
|
||||
- user
|
||||
- group
|
||||
- show_changes: False
|
||||
|
||||
{% else %}
|
||||
{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #}
|
||||
{% set container_config = 'so-nginx-fleet-node' %}
|
||||
{% endif %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- nginx.ssl
|
||||
- nginx.config
|
||||
- nginx.sostatus
|
||||
|
||||
fleet_beats_dir_nginx:
|
||||
file.directory:
|
||||
- name: /nsm/elastic-fleet/artifacts/beats
|
||||
- makedirs: True
|
||||
- show_changes: False
|
||||
|
||||
so-nginx:
|
||||
docker_container.running:
|
||||
- image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}
|
||||
@@ -47,13 +45,14 @@ so-nginx:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- binds:
|
||||
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
|
||||
- /opt/so/conf/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- /opt/so/log/nginx/:/var/log/nginx:rw
|
||||
- /opt/so/tmp/nginx/:/var/lib/nginx:rw
|
||||
- /opt/so/tmp/nginx/:/run:rw
|
||||
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
|
||||
- /nsm/elastic-fleet/artifacts/:/opt/socore/html/artifacts
|
||||
- /nsm/elastic-fleet/artifacts/beats/:/opt/socore/elastic-artifacts/artifacts/beats:ro
|
||||
{% if GLOBALS.is_manager %}
|
||||
- /nsm/elastic-fleet/so_agent-installers/:/opt/socore/html/packages
|
||||
- /etc/pki/managerssl.crt:/etc/pki/nginx/server.crt:ro
|
||||
- /etc/pki/managerssl.key:/etc/pki/nginx/server.key:ro
|
||||
# ATT&CK Navigator binds
|
||||
@@ -61,7 +60,6 @@ so-nginx:
|
||||
- /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
|
||||
- /nsm/repo:/opt/socore/html/repo:ro
|
||||
- /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
|
||||
- /nsm/rules:/nsm/rules:ro
|
||||
{% if NGINXMERGED.external_suricata %}
|
||||
- /opt/so/rules/nids/suri:/surirules:ro
|
||||
{% endif %}
|
||||
@@ -89,6 +87,7 @@ so-nginx:
|
||||
- {{ BINDING }}
|
||||
{% endfor %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: nginxconf
|
||||
- file: nginxconfdir
|
||||
{% if GLOBALS.is_manager %}
|
||||
@@ -102,6 +101,7 @@ so-nginx:
|
||||
{% endif %}
|
||||
- require:
|
||||
- file: nginxconf
|
||||
- file: fleet_beats_dir_nginx
|
||||
{% if GLOBALS.is_manager %}
|
||||
{% if NGINXMERGED.ssl.replace_cert %}
|
||||
- file: managerssl_key
|
||||
|
||||
+16
-50
@@ -4,6 +4,8 @@
|
||||
{%- set role = grains.id.split('_') | last %}
|
||||
{%- set influxpass = salt['pillar.get']('secrets:influx_pass') %}
|
||||
{%- set influxauth = ('so:' + influxpass) | base64_encode %}
|
||||
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
|
||||
{%- set full_server_name = all_names | unique | join(' ') %}
|
||||
|
||||
worker_processes auto;
|
||||
error_log /var/log/nginx/error.log;
|
||||
@@ -39,52 +41,31 @@ http {
|
||||
|
||||
include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
{%- if role in ['fleet'] %}
|
||||
|
||||
server {
|
||||
listen 8443;
|
||||
server_name {{ GLOBALS.hostname }};
|
||||
root /opt/socore/html;
|
||||
location /artifacts/ {
|
||||
try_files $uri =206;
|
||||
proxy_read_timeout 90;
|
||||
proxy_connect_timeout 90;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Proxy "";
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}
|
||||
|
||||
{%- endif %}
|
||||
|
||||
{%- if GLOBALS.is_manager %}
|
||||
{%- set all_names = [GLOBALS.hostname, GLOBALS.url_base] + NGINXMERGED.ssl.alt_names %}
|
||||
{%- set full_server_name = all_names | unique | join(' ') %}
|
||||
|
||||
server {
|
||||
listen 80 default_server;
|
||||
server_name _;
|
||||
return 307 https://{{ GLOBALS.url_base }}$request_uri;
|
||||
}
|
||||
|
||||
{%- endif %}
|
||||
{%- if GLOBALS.is_manager or role == 'fleet' %}
|
||||
server {
|
||||
listen 8443;
|
||||
{%- if GLOBALS.is_manager %}
|
||||
server_name {{ full_server_name }};
|
||||
root /opt/socore/html;
|
||||
location /artifacts/ {
|
||||
try_files $uri =206;
|
||||
proxy_read_timeout 90;
|
||||
proxy_connect_timeout 90;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header Proxy "";
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
{%- else %}
|
||||
server_name {{ GLOBALS.hostname }};
|
||||
{%- endif %}
|
||||
root /opt/socore/elastic-artifacts;
|
||||
location /artifacts/beats/ {
|
||||
try_files $uri =404;
|
||||
}
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
|
||||
{%- endif %}
|
||||
{%- if GLOBALS.is_manager %}
|
||||
server {
|
||||
listen 443 ssl default_server;
|
||||
http2 on;
|
||||
@@ -111,21 +92,6 @@ http {
|
||||
{%- endif %}
|
||||
|
||||
{%- if GLOBALS.is_manager %}
|
||||
|
||||
server {
|
||||
listen 7788;
|
||||
server_name {{ full_server_name }};
|
||||
root /nsm/rules;
|
||||
location / {
|
||||
allow all;
|
||||
sendfile on;
|
||||
sendfile_max_chunk 1m;
|
||||
autoindex on;
|
||||
autoindex_exact_size off;
|
||||
autoindex_format html;
|
||||
autoindex_localtime on;
|
||||
}
|
||||
}
|
||||
{%- if NGINXMERGED.external_suricata %}
|
||||
server {
|
||||
listen 7789 ssl;
|
||||
|
||||
@@ -17,3 +17,4 @@ postgres:
|
||||
log_min_messages: 'warning'
|
||||
shared_preload_libraries: pg_cron
|
||||
cron.database_name: postgres
|
||||
cron.host: /run/postgresql
|
||||
@@ -6,10 +6,12 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- postgres.auth
|
||||
- postgres.ssl
|
||||
- postgres.config
|
||||
@@ -42,6 +44,7 @@ so-postgres:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /opt/so/log/postgres/:/log:rw
|
||||
- /nsm/postgres:/var/lib/postgresql/data:rw
|
||||
- /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro
|
||||
@@ -69,6 +72,7 @@ so-postgres:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: postgresconf
|
||||
- file: postgreshba
|
||||
- file: postgresinitdb
|
||||
|
||||
@@ -87,3 +87,9 @@ postgres:
|
||||
global: True
|
||||
advanced: True
|
||||
helpLink: postgres
|
||||
cron.host:
|
||||
description: Where pg_cron connects to run scheduled jobs such as Telegraf partition maintenance. Must be the container's PostgreSQL socket directory, /run/postgresql, because pg_cron connects without a password and only local socket connections are trusted. Any other value makes every scheduled job fail. Changing it restarts PostgreSQL.
|
||||
readonly: True
|
||||
global: True
|
||||
advanced: True
|
||||
helpLink: postgres
|
||||
@@ -52,7 +52,7 @@ if [ -n "$FILTER_HOST" ] && ! [[ "$FILTER_HOST" =~ $HOST_RE ]]; then
|
||||
fi
|
||||
|
||||
so_psql() {
|
||||
docker exec so-postgres psql -U postgres -d so_telegraf -At -F $'\t' "$@"
|
||||
docker exec -e PGOPTIONS='-c application_name=so-stats-show' so-postgres psql -U so_maint_telegraf -d so_telegraf -At -F $'\t' "$@"
|
||||
}
|
||||
|
||||
if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 | grep -qw so_telegraf; then
|
||||
@@ -60,6 +60,15 @@ if ! docker exec so-postgres psql -U postgres -lqt 2>/dev/null | cut -d\| -f1 |
|
||||
exit 2
|
||||
fi
|
||||
|
||||
role_ok=$(docker exec so-postgres psql -U postgres -tAc "SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls) FROM pg_roles WHERE rolname = 'so_maint_telegraf'" 2>/dev/null)
|
||||
if [ -z "$role_ok" ]; then
|
||||
echo "The so_maint_telegraf role is missing; run a highstate."
|
||||
exit 1
|
||||
elif [ "$role_ok" != t ]; then
|
||||
echo "The so_maint_telegraf role is misconfigured; run a highstate."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
table_exists() {
|
||||
local table="$1"
|
||||
[ -n "$(so_psql -c "SELECT 1 FROM information_schema.tables WHERE table_schema='${SCHEMA}' AND table_name='${table}' LIMIT 1;")" ]
|
||||
|
||||
@@ -4,15 +4,18 @@ set -e
|
||||
# Provision Telegraf state inside the so-postgres container.
|
||||
# Usage: so-telegraf-postgres <subcommand>
|
||||
# create_db Ensure the so_telegraf database exists.
|
||||
# group_role Provision the so_telegraf group role, telegraf/partman schemas,
|
||||
# pg_partman, the so_admin maintenance routines, and the hourly
|
||||
# pg_cron maintenance job.
|
||||
# group_role Provision the so_telegraf group role, the so_maint_telegraf
|
||||
# maintenance login role, telegraf/partman schemas, pg_partman,
|
||||
# the so_admin maintenance routines, and the hourly pg_cron
|
||||
# maintenance job.
|
||||
# user Create or update a per-minion login role granted to so_telegraf.
|
||||
# Env: ROLE_USER, ROLE_PASS.
|
||||
# retention Reconcile partman retention and premake on telegraf parents.
|
||||
# Env: RETENTION_DAYS.
|
||||
# maintenance Drain default partitions and run partman maintenance.
|
||||
# check Report partition health. Non-zero if any parent is unhealthy.
|
||||
# maintenance Drain default partitions and run partman maintenance, as
|
||||
# so_maint_telegraf.
|
||||
# check Report partition health, as so_maint_telegraf. Non-zero if any
|
||||
# parent is unhealthy.
|
||||
#
|
||||
# A default partition holding rows for a day blocks creating that day's child,
|
||||
# so maintenance drains defaults before calling partman. Use so-telegraf-repair
|
||||
@@ -20,6 +23,10 @@ set -e
|
||||
|
||||
cmd="${1:?subcommand required}"
|
||||
|
||||
# Long enough for an hourly maintenance run to release its locks, short enough
|
||||
# that soup and admins are not left hanging.
|
||||
LOCK_TIMEOUT=60s
|
||||
|
||||
case "$cmd" in
|
||||
create_db)
|
||||
if ! docker exec so-postgres psql -U postgres -tAc \
|
||||
@@ -30,7 +37,9 @@ case "$cmd" in
|
||||
;;
|
||||
|
||||
group_role)
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v lock_timeout="$LOCK_TIMEOUT" \
|
||||
-U postgres -d so_telegraf <<'EOSQL'
|
||||
SET lock_timeout = :'lock_timeout';
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_telegraf') THEN
|
||||
@@ -38,6 +47,17 @@ BEGIN
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'so_maint_telegraf') THEN
|
||||
CREATE ROLE so_maint_telegraf;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
ALTER ROLE so_maint_telegraf LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION NOBYPASSRLS PASSWORD NULL;
|
||||
GRANT so_telegraf TO so_maint_telegraf;
|
||||
ALTER ROLE so_maint_telegraf SET role = so_telegraf;
|
||||
ALTER ROLE so_maint_telegraf SET search_path = pg_catalog;
|
||||
GRANT CONNECT ON DATABASE so_telegraf TO so_telegraf;
|
||||
CREATE SCHEMA IF NOT EXISTS telegraf AUTHORIZATION so_telegraf;
|
||||
GRANT USAGE, CREATE ON SCHEMA telegraf TO so_telegraf;
|
||||
@@ -58,52 +78,90 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA partman
|
||||
GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO so_telegraf;
|
||||
|
||||
-- pg_cron runs these as postgres, so they must not sit in a schema any
|
||||
-- Telegraf role can create objects in.
|
||||
-- Owned by postgres so no Telegraf role can create objects here; the routines
|
||||
-- run with the caller's privileges.
|
||||
CREATE SCHEMA IF NOT EXISTS so_admin AUTHORIZATION postgres;
|
||||
REVOKE ALL ON SCHEMA so_admin FROM PUBLIC;
|
||||
|
||||
-- Resolved through pg_inherits, never by name, so a view or a table that only
|
||||
-- carries the _default name is never read or written.
|
||||
CREATE OR REPLACE FUNCTION so_admin.telegraf_defaults()
|
||||
RETURNS TABLE (
|
||||
parent_table regclass,
|
||||
default_part regclass,
|
||||
has_rows boolean,
|
||||
default_size bigint
|
||||
)
|
||||
LANGUAGE plpgsql
|
||||
STABLE
|
||||
SECURITY INVOKER
|
||||
AS $func$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT c.oid::regclass AS pt
|
||||
FROM partman.part_config pc
|
||||
JOIN pg_catalog.pg_class c ON c.oid = to_regclass(pc.parent_table)
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
AND c.relkind = 'p'
|
||||
ORDER BY pc.parent_table
|
||||
LOOP
|
||||
parent_table := r.pt;
|
||||
|
||||
SELECT c.oid::regclass INTO default_part
|
||||
FROM pg_catalog.pg_inherits i
|
||||
JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt
|
||||
AND c.relkind = 'r' AND c.relispartition
|
||||
AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT';
|
||||
|
||||
IF default_part IS NULL THEN
|
||||
has_rows := false;
|
||||
default_size := NULL;
|
||||
ELSE
|
||||
EXECUTE format('SELECT EXISTS (SELECT 1 FROM %s LIMIT 1)', default_part)
|
||||
INTO has_rows;
|
||||
default_size := pg_catalog.pg_total_relation_size(default_part);
|
||||
END IF;
|
||||
|
||||
RETURN NEXT;
|
||||
END LOOP;
|
||||
END;
|
||||
$func$;
|
||||
|
||||
CREATE OR REPLACE PROCEDURE so_admin.telegraf_maintenance()
|
||||
LANGUAGE plpgsql
|
||||
SECURITY INVOKER
|
||||
-- No SET clause: it would forbid the COMMITs below, so so_maint_telegraf pins search_path instead.
|
||||
AS $proc$
|
||||
DECLARE
|
||||
r record;
|
||||
v_default text;
|
||||
v_rows bigint;
|
||||
r record;
|
||||
BEGIN
|
||||
-- No per-parent EXCEPTION handler: partition_data_proc commits internally,
|
||||
-- and COMMIT is illegal while a subtransaction is active. A failing parent
|
||||
-- aborts the run and the next pass retries.
|
||||
FOR r IN
|
||||
SELECT parent_table, retention
|
||||
FROM partman.part_config
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
ORDER BY parent_table
|
||||
SELECT pc.parent_table, pc.retention, d.default_part, d.default_size
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
WHERE d.default_part IS NOT NULL AND d.has_rows
|
||||
ORDER BY pc.parent_table
|
||||
LOOP
|
||||
v_default := format('%I.%I',
|
||||
split_part(r.parent_table, '.', 1),
|
||||
split_part(r.parent_table, '.', 2) || '_default');
|
||||
|
||||
CONTINUE WHEN to_regclass(v_default) IS NULL;
|
||||
|
||||
EXECUTE format('SELECT count(*) FROM %s', v_default) INTO v_rows;
|
||||
CONTINUE WHEN v_rows = 0;
|
||||
|
||||
RAISE WARNING 'so_admin.telegraf_maintenance: % rows stranded in %, draining',
|
||||
v_rows, v_default;
|
||||
RAISE WARNING 'so_admin.telegraf_maintenance: % holds rows (%), draining',
|
||||
r.default_part, pg_size_pretty(r.default_size);
|
||||
|
||||
-- Cheaper to delete expired rows than to repartition and then drop them.
|
||||
IF r.retention IS NOT NULL THEN
|
||||
EXECUTE format('DELETE FROM %s WHERE "time" < now() - %L::interval',
|
||||
v_default, r.retention);
|
||||
r.default_part, r.retention);
|
||||
COMMIT;
|
||||
END IF;
|
||||
|
||||
-- Bounded so a large backlog drains across several runs.
|
||||
CALL partman.partition_data_proc(
|
||||
p_parent_table := r.parent_table,
|
||||
p_loop_count := 200,
|
||||
p_source_table := v_default
|
||||
p_loop_count := 200
|
||||
);
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
@@ -112,58 +170,75 @@ BEGIN
|
||||
END;
|
||||
$proc$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION so_admin.telegraf_partition_status()
|
||||
DROP FUNCTION IF EXISTS so_admin.telegraf_partition_status();
|
||||
CREATE FUNCTION so_admin.telegraf_partition_status()
|
||||
RETURNS TABLE (
|
||||
parent_table text,
|
||||
oldest_child date,
|
||||
newest_child date,
|
||||
days_ahead int,
|
||||
retention text,
|
||||
default_rows bigint,
|
||||
default_size text
|
||||
parent_table text,
|
||||
oldest_child date,
|
||||
newest_child date,
|
||||
days_ahead int,
|
||||
retention text,
|
||||
default_has_rows boolean,
|
||||
default_size text
|
||||
)
|
||||
LANGUAGE plpgsql
|
||||
STABLE
|
||||
AS $func$
|
||||
DECLARE
|
||||
r record;
|
||||
v_default regclass;
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT pc.parent_table AS pt, pc.retention AS ret
|
||||
FROM partman.part_config pc
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
ORDER BY pc.parent_table
|
||||
SELECT d.parent_table AS pt, d.has_rows, d.default_size AS size, pc.retention AS ret
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
ORDER BY d.parent_table::text
|
||||
LOOP
|
||||
parent_table := r.pt;
|
||||
parent_table := r.pt::text;
|
||||
retention := r.ret;
|
||||
|
||||
SELECT min(d), max(d) INTO oldest_child, newest_child
|
||||
SELECT min(x), max(x) INTO oldest_child, newest_child
|
||||
FROM (
|
||||
SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS d
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt::regclass
|
||||
AND pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
SELECT to_date(substring(c.relname FROM '_p(\d{8})$'), 'YYYYMMDD') AS x
|
||||
FROM pg_catalog.pg_inherits i
|
||||
JOIN pg_catalog.pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = r.pt
|
||||
AND pg_catalog.pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
) s;
|
||||
|
||||
days_ahead := newest_child - current_date;
|
||||
|
||||
v_default := to_regclass(format('%I.%I',
|
||||
split_part(r.pt, '.', 1),
|
||||
split_part(r.pt, '.', 2) || '_default'));
|
||||
IF v_default IS NULL THEN
|
||||
default_rows := NULL;
|
||||
default_size := NULL;
|
||||
ELSE
|
||||
EXECUTE format('SELECT count(*) FROM %s', v_default::text) INTO default_rows;
|
||||
default_size := pg_size_pretty(pg_total_relation_size(v_default));
|
||||
END IF;
|
||||
days_ahead := newest_child - current_date;
|
||||
default_has_rows := r.has_rows;
|
||||
default_size := pg_size_pretty(r.size);
|
||||
|
||||
RETURN NEXT;
|
||||
END LOOP;
|
||||
END;
|
||||
$func$;
|
||||
|
||||
GRANT USAGE ON SCHEMA so_admin TO so_telegraf;
|
||||
GRANT EXECUTE ON FUNCTION so_admin.telegraf_defaults() TO so_telegraf;
|
||||
GRANT EXECUTE ON PROCEDURE so_admin.telegraf_maintenance() TO so_telegraf;
|
||||
GRANT EXECUTE ON FUNCTION so_admin.telegraf_partition_status() TO so_telegraf;
|
||||
|
||||
-- Partitions created by a superuser session (an older repair) belong to
|
||||
-- postgres; partman acting as so_telegraf could not drop them.
|
||||
DO $$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT n.nspname, c.relname
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE c.relkind IN ('r', 'p')
|
||||
AND (n.nspname = 'telegraf'
|
||||
OR (n.nspname = 'partman' AND c.relname LIKE 'template\_telegraf\_%'))
|
||||
AND c.relowner <> 'so_telegraf'::regrole
|
||||
LOOP
|
||||
EXECUTE format('ALTER TABLE %I.%I OWNER TO so_telegraf', r.nspname, r.relname);
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
|
||||
-- Drop the registration older releases left in so_telegraf.
|
||||
SELECT CASE
|
||||
WHEN current_setting('cron.database_name', true) IS DISTINCT FROM current_database()
|
||||
@@ -175,20 +250,69 @@ DROP EXTENSION pg_cron CASCADE;
|
||||
\endif
|
||||
EOSQL
|
||||
|
||||
stalled=false
|
||||
probe_failed=false
|
||||
if out=$(docker exec -e PGOPTIONS="-c lock_timeout=$LOCK_TIMEOUT" so-postgres \
|
||||
psql -U so_maint_telegraf -d so_telegraf -tAc \
|
||||
"SELECT coalesce(bool_or(has_rows), false) FROM so_admin.telegraf_defaults()"); then
|
||||
case "$out" in
|
||||
t) stalled=true ;;
|
||||
f) stalled=false ;;
|
||||
*) probe_failed=true ;;
|
||||
esac
|
||||
else
|
||||
probe_failed=true
|
||||
fi
|
||||
if $probe_failed; then
|
||||
echo "ERROR: could not check Telegraf default partitions; the maintenance job was not changed." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Guarded on the live GUC so applying this before the postgresql.conf change
|
||||
# has restarted the container skips rather than failing.
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d postgres <<'EOSQL'
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -v stalled="$stalled" \
|
||||
-U postgres -d postgres <<'EOSQL'
|
||||
SELECT CASE WHEN current_setting('cron.database_name', true) = current_database()
|
||||
THEN 'true' ELSE 'false' END AS cron_here \gset
|
||||
\if :cron_here
|
||||
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
||||
-- cron.schedule_in_database is idempotent by jobname.
|
||||
-- A highstate reconciles the job's definition but never its active flag; only so-telegraf-repair turns it on.
|
||||
BEGIN;
|
||||
SELECT coalesce(max(jobid)::text, '') AS jobid,
|
||||
count(*) > 0 AS job_exists,
|
||||
coalesce(bool_and(active), true) AS job_active
|
||||
FROM cron.job
|
||||
WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset
|
||||
SELECT cron.unschedule(jobid) FROM cron.job
|
||||
WHERE jobname = 'telegraf-partman-maintenance' AND username <> 'so_maint_telegraf';
|
||||
\if :job_exists
|
||||
SELECT cron.alter_job(:jobid,
|
||||
schedule := '17 * * * *',
|
||||
command := 'CALL so_admin.telegraf_maintenance()',
|
||||
database := 'so_telegraf');
|
||||
\else
|
||||
-- Registered inactive on a stalled grid so nothing drains the default
|
||||
-- partitions before so-telegraf-repair truncates them.
|
||||
SELECT cron.schedule_in_database(
|
||||
'telegraf-partman-maintenance',
|
||||
'17 * * * *',
|
||||
'CALL so_admin.telegraf_maintenance()',
|
||||
'so_telegraf'
|
||||
'so_telegraf',
|
||||
'so_maint_telegraf',
|
||||
NOT :'stalled'::boolean
|
||||
);
|
||||
\endif
|
||||
UPDATE cron.job SET nodename = current_setting('cron.host')
|
||||
WHERE jobname = 'telegraf-partman-maintenance'
|
||||
AND nodename IS DISTINCT FROM current_setting('cron.host');
|
||||
COMMIT;
|
||||
SELECT coalesce(bool_and(active), true) AS job_active
|
||||
FROM cron.job
|
||||
WHERE jobname = 'telegraf-partman-maintenance' AND username = 'so_maint_telegraf' \gset
|
||||
\if :job_active
|
||||
\else
|
||||
\echo 'WARNING: the Telegraf maintenance job is inactive; run ''sudo so-telegraf-repair''.'
|
||||
\endif
|
||||
\else
|
||||
\echo 'pg_cron metadata database is not `postgres` yet; skipping job registration.'
|
||||
\endif
|
||||
@@ -241,13 +365,15 @@ SET retention = :'retention_days' || ' days',
|
||||
retention_keep_table = false,
|
||||
premake = 7,
|
||||
infinite_time_partitions = true
|
||||
WHERE parent_table LIKE 'telegraf.%';
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
AND EXISTS (SELECT 1 FROM pg_catalog.pg_class c
|
||||
WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p');
|
||||
\endif
|
||||
EOSQL
|
||||
;;
|
||||
|
||||
maintenance)
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U postgres -d so_telegraf <<'EOSQL'
|
||||
docker exec -i so-postgres psql -v ON_ERROR_STOP=1 -U so_maint_telegraf -d so_telegraf <<'EOSQL'
|
||||
SELECT CASE WHEN to_regproc('so_admin.telegraf_maintenance') IS NOT NULL
|
||||
THEN 'true' ELSE 'false' END AS has_proc \gset
|
||||
\if :has_proc
|
||||
@@ -259,7 +385,7 @@ EOSQL
|
||||
;;
|
||||
|
||||
check)
|
||||
docker exec -i so-postgres psql -U postgres -d so_telegraf <<'EOSQL'
|
||||
docker exec -i so-postgres psql -U so_maint_telegraf -d so_telegraf <<'EOSQL'
|
||||
\pset border 2
|
||||
SELECT * FROM so_admin.telegraf_partition_status();
|
||||
EOSQL
|
||||
@@ -278,9 +404,9 @@ LIMIT 5;
|
||||
\echo 'pg_cron is not installed in this database.'
|
||||
\endif
|
||||
EOSQL
|
||||
unhealthy=$(docker exec so-postgres psql -U postgres -d so_telegraf -tAc \
|
||||
unhealthy=$(docker exec so-postgres psql -U so_maint_telegraf -d so_telegraf -tAc \
|
||||
"SELECT count(*) FROM so_admin.telegraf_partition_status()
|
||||
WHERE coalesce(default_rows, 0) > 0 OR coalesce(days_ahead, -1) < 1")
|
||||
WHERE default_has_rows OR coalesce(days_ahead, -1) < 1")
|
||||
if [ "${unhealthy:-1}" != "0" ]; then
|
||||
echo "so-telegraf-postgres check: $unhealthy telegraf parent(s) unhealthy" >&2
|
||||
exit 1
|
||||
|
||||
@@ -11,26 +11,38 @@
|
||||
# maintenance once. Healthy grids are reported and left alone.
|
||||
#
|
||||
# Usage: so-telegraf-repair [--check] [--yes] [--no-restart]
|
||||
# --check Report health and change nothing.
|
||||
# --check Report health and change nothing. Also reports a missing
|
||||
# so_maint_telegraf role and a maintenance job that is missing
|
||||
# or inactive.
|
||||
# --yes Skip the confirmation prompt (for soup and other automation).
|
||||
# --no-restart Never restart so-postgres, even if pg_cron's launcher is dead.
|
||||
#
|
||||
# Repair mode (without --check) registers the so_maint_telegraf role and the
|
||||
# maintenance job if they are missing, pauses the job while it repairs, and
|
||||
# always leaves it active afterwards. Data work runs as so_maint_telegraf.
|
||||
#
|
||||
# Exit status:
|
||||
# 0 healthy, or repair completed
|
||||
# 1 repair is needed (--check only)
|
||||
# 2 cannot run here: so-postgres, so_telegraf or pg_partman is missing
|
||||
# 1 repair is needed (--check), or the repair failed after it started
|
||||
# 2 skipped: Telegraf output is not Postgres, or so-postgres, so_telegraf or pg_partman is missing
|
||||
# 3 could not check, for example a connection error
|
||||
|
||||
set -e
|
||||
set -eE
|
||||
|
||||
# Matches p_premake in telegraf.conf's create_parent template.
|
||||
PREMAKE=7
|
||||
JOB_NAME=telegraf-partman-maintenance
|
||||
MAINT_ROLE=so_maint_telegraf
|
||||
# Long enough for an hourly maintenance run to release its locks, short enough
|
||||
# that soup and admins are not left hanging.
|
||||
LOCK_TIMEOUT=60s
|
||||
PG_OPTS="-c lock_timeout=$LOCK_TIMEOUT -c application_name=so-telegraf-repair"
|
||||
|
||||
CHECK_ONLY=false
|
||||
ASSUME_YES=false
|
||||
NO_RESTART=false
|
||||
|
||||
usage() { sed -n '/^# Usage:/,/^# 2 /p' "$0" | sed 's/^# \?//'; }
|
||||
usage() { sed -n '/^# Usage:/,/^# 3 /p' "$0" | sed 's/^# \?//'; }
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -38,75 +50,170 @@ while [[ $# -gt 0 ]]; do
|
||||
--yes|-y) ASSUME_YES=true ;;
|
||||
--no-restart) NO_RESTART=true ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;;
|
||||
*) echo "Unknown option: $1" >&2; usage >&2; exit 3 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
skip() { echo "$*"; exit 2; }
|
||||
# Strips C0 (except tab and newline), DEL and C1 characters, so names and
|
||||
# messages written with Telegraf credentials can't drive the terminal.
|
||||
sanitise() { LC_ALL=C tr -d '\000-\010\013-\037\177' | LC_ALL=C sed 's/\xc2[\x80-\x9f]//g'; }
|
||||
|
||||
psql_tg() { docker exec -i so-postgres psql -U postgres -d so_telegraf "$@"; }
|
||||
psql_pg() { docker exec -i so-postgres psql -U postgres -d postgres "$@"; }
|
||||
ERR_FILE=$(mktemp)
|
||||
ERR_STATUS=3
|
||||
ERR_PREFIX="Could not check Telegraf partitions"
|
||||
job_id=
|
||||
restore_active=
|
||||
|
||||
on_err() {
|
||||
# Command substitutions inherit this trap; only the top level reports.
|
||||
[[ $BASH_SUBSHELL -eq 0 ]] || exit "$ERR_STATUS"
|
||||
local line
|
||||
line=$(tail -n 1 "$ERR_FILE" | sanitise)
|
||||
echo "$ERR_PREFIX: ${line:-command failed}" >&2
|
||||
exit "$ERR_STATUS"
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
if [[ -n "$restore_active" ]]; then
|
||||
psql_cron -tAc "SELECT cron.alter_job($job_id, active := $restore_active)" >/dev/null || true
|
||||
fi
|
||||
rm -f "$ERR_FILE"
|
||||
}
|
||||
|
||||
trap on_err ERR
|
||||
trap on_exit EXIT
|
||||
|
||||
finish() { trap - ERR; exit "$1"; }
|
||||
skip() { echo "$*"; finish 2; }
|
||||
|
||||
# Keeps the last stderr line for on_err while still showing all of it.
|
||||
capture() {
|
||||
local rc=0
|
||||
: >"$ERR_FILE"
|
||||
"$@" 2>"$ERR_FILE" || rc=$?
|
||||
sanitise <"$ERR_FILE" >&2
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
psql_tg() {
|
||||
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U "$MAINT_ROLE" -d so_telegraf "$@"
|
||||
}
|
||||
psql_pg() {
|
||||
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d postgres "$@"
|
||||
}
|
||||
psql_cron() {
|
||||
capture docker exec -i -e PGOPTIONS="$PG_OPTS" so-postgres psql -U postgres -d "$cron_db" "$@"
|
||||
}
|
||||
|
||||
# A part_config row counts only while its parent is a partitioned table.
|
||||
TELEGRAF_PARENTS="parent_table LIKE 'telegraf.%'
|
||||
AND EXISTS (SELECT 1 FROM pg_class c
|
||||
WHERE c.oid = to_regclass(parent_table) AND c.relkind = 'p')"
|
||||
|
||||
# query_to_xml so the per-table row counts need no helper function installed.
|
||||
REPORT="
|
||||
WITH parents AS (
|
||||
SELECT pc.parent_table,
|
||||
pc.premake,
|
||||
split_part(pc.parent_table, '.', 1) AS sch,
|
||||
split_part(pc.parent_table, '.', 2) AS tbl
|
||||
FROM partman.part_config pc
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
), children AS (
|
||||
SELECT p.parent_table,
|
||||
max(to_date(substring(c.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
|
||||
FROM parents p
|
||||
JOIN pg_class pt ON pt.oid = p.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = pt.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE pg_get_expr(c.relpartbound, c.oid) <> 'DEFAULT'
|
||||
GROUP BY p.parent_table
|
||||
), defaults AS (
|
||||
SELECT p.parent_table,
|
||||
p.premake,
|
||||
format('%I.%I', p.sch, p.tbl || '_default') AS default_table,
|
||||
to_regclass(format('%I.%I', p.sch, p.tbl || '_default')) AS default_oid
|
||||
FROM parents p
|
||||
)
|
||||
SELECT d.parent_table,
|
||||
c.newest_child,
|
||||
(c.newest_child - current_date) AS days_ahead,
|
||||
d.premake,
|
||||
CASE WHEN d.default_oid IS NULL THEN NULL ELSE
|
||||
(xpath('/row/cnt/text()',
|
||||
query_to_xml(format('SELECT count(*) AS cnt FROM %s', d.default_table),
|
||||
false, true, '')))[1]::text::bigint
|
||||
END AS default_rows,
|
||||
CASE WHEN d.default_oid IS NULL THEN NULL
|
||||
ELSE pg_size_pretty(pg_total_relation_size(d.default_oid)) END AS default_size
|
||||
FROM defaults d
|
||||
LEFT JOIN children c ON c.parent_table = d.parent_table
|
||||
ORDER BY 1
|
||||
pc.premake,
|
||||
d.has_rows AS default_has_rows,
|
||||
pg_size_pretty(d.default_size) AS default_size
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
JOIN pg_class p ON p.oid = d.parent_table AND p.relkind = 'p'
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT max(to_date(substring(ch.relname FROM '_p(\d{8})\$'), 'YYYYMMDD')) AS newest_child
|
||||
FROM pg_inherits i
|
||||
JOIN pg_class ch ON ch.oid = i.inhrelid
|
||||
WHERE i.inhparent = d.parent_table
|
||||
AND pg_get_expr(ch.relpartbound, ch.oid) <> 'DEFAULT'
|
||||
) c ON true
|
||||
ORDER BY d.parent_table::text
|
||||
"
|
||||
|
||||
docker ps --format '{{.Names}}' | grep -qx so-postgres \
|
||||
print_report() {
|
||||
local report
|
||||
report=$(psql_tg -c "$REPORT")
|
||||
sanitise <<<"$report"
|
||||
}
|
||||
|
||||
has_cron() {
|
||||
[[ -n "$cron_db" ]] || return 0
|
||||
psql_cron -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_cron'"
|
||||
}
|
||||
|
||||
lookup_job() {
|
||||
local cron
|
||||
cron=$(has_cron)
|
||||
[[ -n "$cron" ]] || return 0
|
||||
psql_cron -tA -F '|' -c \
|
||||
"SELECT jobid, active FROM cron.job
|
||||
WHERE jobname = '$JOB_NAME' AND username = '$MAINT_ROLE'
|
||||
ORDER BY jobid LIMIT 1"
|
||||
}
|
||||
|
||||
output=$(capture salt-call pillar.get telegraf:output --out=newline_values_only)
|
||||
output=$(tr -d '[:space:]' <<<"$output" | tr '[:lower:]' '[:upper:]')
|
||||
output=${output:-INFLUXDB}
|
||||
case "$output" in
|
||||
POSTGRES|BOTH) ;;
|
||||
*) skip "Telegraf output is $(sanitise <<<"$output"); not checking Postgres partitions." ;;
|
||||
esac
|
||||
|
||||
containers=$(capture docker ps --format '{{.Names}}')
|
||||
grep -qx so-postgres <<<"$containers" \
|
||||
|| skip "so-postgres is not running; nothing to repair."
|
||||
docker exec so-postgres psql -U postgres -tAc \
|
||||
"SELECT 1 FROM pg_database WHERE datname='so_telegraf'" | grep -q 1 \
|
||||
has_db=$(psql_pg -tAc "SELECT 1 FROM pg_database WHERE datname = 'so_telegraf'")
|
||||
[[ -n "$has_db" ]] \
|
||||
|| skip "The so_telegraf database does not exist; Telegraf is not writing to Postgres."
|
||||
psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname='pg_partman'" | grep -q 1 \
|
||||
|
||||
# so_telegraf before the postgres state lands, postgres after.
|
||||
cron_db=$(psql_pg -tAc "SELECT current_setting('cron.database_name', true)")
|
||||
cron_db=$(tr -d '[:space:]' <<<"$cron_db")
|
||||
|
||||
role_ok=$(psql_pg -tAc \
|
||||
"SELECT rolcanlogin AND NOT (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls)
|
||||
FROM pg_roles WHERE rolname = '$MAINT_ROLE'")
|
||||
job=
|
||||
[[ "$cron_db" == postgres ]] && job=$(lookup_job)
|
||||
|
||||
if [[ "$role_ok" != t ]] && $CHECK_ONLY; then
|
||||
echo "Repair is needed:"
|
||||
if [[ -z "$role_ok" ]]; then
|
||||
echo " * maintenance role $MAINT_ROLE is missing"
|
||||
else
|
||||
echo " * maintenance role $MAINT_ROLE is misconfigured"
|
||||
fi
|
||||
echo
|
||||
echo "Re-run without --check to repair."
|
||||
finish 1
|
||||
fi
|
||||
|
||||
if ! $CHECK_ONLY && [[ "$role_ok" != t || ( "$cron_db" == postgres && -z "$job" ) ]]; then
|
||||
echo "Registering the maintenance role and job."
|
||||
if ! /usr/sbin/so-telegraf-postgres group_role; then
|
||||
echo "Could not register the maintenance job; run a highstate and try again." >&2
|
||||
finish 3
|
||||
fi
|
||||
[[ "$cron_db" == postgres ]] && job=$(lookup_job)
|
||||
fi
|
||||
|
||||
has_partman=$(psql_tg -tAc "SELECT 1 FROM pg_extension WHERE extname = 'pg_partman'")
|
||||
[[ -n "$has_partman" ]] \
|
||||
|| skip "pg_partman is not installed in so_telegraf; nothing to repair."
|
||||
|
||||
parents=$(psql_tg -tAc \
|
||||
"SELECT count(*) FROM partman.part_config WHERE parent_table LIKE 'telegraf.%'")
|
||||
stranded=$(psql_tg -tAc "SELECT coalesce(sum(default_rows), 0) FROM ( $REPORT ) t")
|
||||
behind=$(psql_tg -tAc \
|
||||
"SELECT count(*) FROM ( $REPORT ) t WHERE coalesce(days_ahead, -1) < 1")
|
||||
"SELECT count(*) FROM partman.part_config WHERE $TELEGRAF_PARENTS")
|
||||
counts=$(psql_tg -tAc \
|
||||
"SELECT count(*) FILTER (WHERE default_has_rows) || '|' ||
|
||||
count(*) FILTER (WHERE coalesce(days_ahead, -1) < 1)
|
||||
FROM ( $REPORT ) t")
|
||||
IFS='|' read -r stranded behind <<<"$counts"
|
||||
# premake < 7, or infinite_time_partitions off: without the latter partman
|
||||
# refuses to premake forward across the gap the stall left behind.
|
||||
misconfigured=$(psql_tg -tAc \
|
||||
"SELECT count(*) FROM partman.part_config
|
||||
WHERE parent_table LIKE 'telegraf.%'
|
||||
WHERE $TELEGRAF_PARENTS
|
||||
AND (premake < $PREMAKE OR NOT infinite_time_partitions)")
|
||||
|
||||
# Both columns are matched because which one carries the launcher's name varies
|
||||
@@ -115,16 +222,24 @@ launcher=$(psql_pg -tAc \
|
||||
"SELECT count(*) FROM pg_stat_activity
|
||||
WHERE backend_type ILIKE '%pg_cron%' OR application_name ILIKE '%pg_cron%'")
|
||||
|
||||
# so_telegraf before the postgres state lands, postgres after.
|
||||
cron_db=$(docker exec so-postgres psql -U postgres -tAc \
|
||||
"SELECT current_setting('cron.database_name', true)" | tr -d '[:space:]')
|
||||
last_run=never
|
||||
if [[ -n "$cron_db" ]]; then
|
||||
last_run=$(docker exec so-postgres psql -U postgres -d "$cron_db" -tAc \
|
||||
"SELECT coalesce(max(d.start_time)::text, 'never')
|
||||
cron_installed=$(has_cron)
|
||||
if [[ -n "$cron_installed" ]]; then
|
||||
run=$(psql_cron -tAc \
|
||||
"SELECT d.status || ' at ' || d.start_time
|
||||
FROM cron.job_run_details d JOIN cron.job j USING (jobid)
|
||||
WHERE j.jobname = '$JOB_NAME'" 2>/dev/null | tr -d '[:space:]' || echo unknown)
|
||||
[[ -n "$last_run" ]] || last_run=never
|
||||
WHERE j.jobname = '$JOB_NAME'
|
||||
ORDER BY d.start_time DESC LIMIT 1")
|
||||
if [[ -n "$run" ]]; then
|
||||
message=$(psql_cron -tAc \
|
||||
"SELECT coalesce(d.return_message, '')
|
||||
FROM cron.job_run_details d JOIN cron.job j USING (jobid)
|
||||
WHERE j.jobname = '$JOB_NAME'
|
||||
ORDER BY d.start_time DESC LIMIT 1")
|
||||
message=$(sanitise <<<"$message")
|
||||
message=${message//$'\n'/ }
|
||||
last_run="$(sanitise <<<"$run"): ${message:0:200}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# A grid that has never written a metric has nothing to recover, and an empty
|
||||
@@ -132,43 +247,67 @@ fi
|
||||
restart_needed=false
|
||||
[[ "$launcher" -eq 0 && "$parents" -gt 0 && -n "$cron_db" ]] && restart_needed=true
|
||||
|
||||
# group_role registers the job only in the postgres database.
|
||||
job_inactive=false
|
||||
if [[ "$cron_db" == postgres ]]; then
|
||||
[[ -z "$job" || "${job##*|}" == f ]] && job_inactive=true
|
||||
fi
|
||||
|
||||
repair_needed=false
|
||||
[[ "$stranded" -gt 0 ]] && repair_needed=true
|
||||
[[ "$behind" -gt 0 ]] && repair_needed=true
|
||||
[[ "$misconfigured" -gt 0 ]] && repair_needed=true
|
||||
$restart_needed && repair_needed=true
|
||||
$job_inactive && repair_needed=true
|
||||
|
||||
echo "Telegraf partition status:"
|
||||
psql_tg -c "$REPORT"
|
||||
echo "Rows stranded in default partitions: $stranded"
|
||||
echo "pg_cron metadata database: ${cron_db:-unset}"
|
||||
print_report
|
||||
echo "Default partitions with rows: $stranded"
|
||||
echo "pg_cron metadata database: $(sanitise <<<"${cron_db:-unset}")"
|
||||
[[ "$cron_db" == postgres ]] \
|
||||
|| echo "pg_cron metadata database is $(sanitise <<<"${cron_db:-unset}"); not checking the maintenance job."
|
||||
echo "pg_cron launcher running: $([[ "$launcher" -gt 0 ]] && echo yes || echo no)"
|
||||
echo "Last $JOB_NAME run: $last_run"
|
||||
echo
|
||||
|
||||
if ! $repair_needed; then
|
||||
echo "Telegraf partitions are healthy. Nothing to do."
|
||||
exit 0
|
||||
finish 0
|
||||
fi
|
||||
|
||||
if $CHECK_ONLY; then
|
||||
echo "Repair is needed:"
|
||||
[[ "$stranded" -gt 0 ]] && echo " * $stranded row(s) stranded in default partitions"
|
||||
[[ "$stranded" -gt 0 ]] && echo " * $stranded default partition(s) hold rows"
|
||||
[[ "$behind" -gt 0 ]] && echo " * $behind parent(s) with no partition for the current window"
|
||||
[[ "$misconfigured" -gt 0 ]] && echo " * $misconfigured parent(s) with stale partman settings"
|
||||
$restart_needed && echo " * pg_cron's launcher is dead; maintenance is not running at all"
|
||||
$job_inactive && echo " * the maintenance job is missing or inactive"
|
||||
echo
|
||||
echo "Re-run without --check to repair."
|
||||
exit 1
|
||||
finish 1
|
||||
fi
|
||||
|
||||
if [[ "$stranded" -gt 0 ]] && ! $ASSUME_YES; then
|
||||
echo "This will permanently discard the $stranded stranded row(s) above."
|
||||
echo "This will permanently discard the rows in the $stranded default partition(s) above."
|
||||
$restart_needed && ! $NO_RESTART && \
|
||||
echo "so-postgres will also be restarted, which briefly interrupts SOC."
|
||||
[[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; exit 2; }
|
||||
[[ -t 0 ]] || { echo "Not a terminal; re-run with --yes to confirm." >&2; finish 1; }
|
||||
read -r -p "Continue? [y/N] " answer
|
||||
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
|
||||
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; finish 0; }
|
||||
fi
|
||||
|
||||
ERR_STATUS=1
|
||||
ERR_PREFIX="Telegraf partition repair failed"
|
||||
|
||||
if [[ "$cron_db" == postgres ]]; then
|
||||
if [[ -z "$job" ]]; then
|
||||
echo "Could not register the maintenance job; run a highstate and try again." >&2
|
||||
finish 3
|
||||
fi
|
||||
job_id=${job%%|*}
|
||||
[[ "${job##*|}" == t ]] && restore_active=true || restore_active=false
|
||||
echo "Pausing the $JOB_NAME job during the repair."
|
||||
psql_cron -tAc "SELECT cron.alter_job($job_id, active := false)" >/dev/null
|
||||
fi
|
||||
|
||||
if [[ "$misconfigured" -gt 0 ]]; then
|
||||
@@ -178,7 +317,7 @@ if [[ "$misconfigured" -gt 0 ]]; then
|
||||
"UPDATE partman.part_config
|
||||
SET premake = GREATEST(premake, $PREMAKE),
|
||||
infinite_time_partitions = true
|
||||
WHERE parent_table LIKE 'telegraf.%'"
|
||||
WHERE $TELEGRAF_PARENTS"
|
||||
fi
|
||||
|
||||
if [[ "$stranded" -gt 0 ]]; then
|
||||
@@ -186,27 +325,24 @@ if [[ "$stranded" -gt 0 ]]; then
|
||||
# One transaction: Telegraf is still writing, so a default emptied without
|
||||
# its partition in place is refilled before maintenance can attach one.
|
||||
psql_tg -v ON_ERROR_STOP=1 <<'EOSQL'
|
||||
BEGIN;
|
||||
DO $$
|
||||
DECLARE
|
||||
r record;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT pc.parent_table,
|
||||
format('%I.%I', n.nspname, c.relname) AS default_table
|
||||
FROM partman.part_config pc
|
||||
JOIN pg_class p ON p.oid = pc.parent_table::regclass
|
||||
JOIN pg_inherits i ON i.inhparent = p.oid
|
||||
JOIN pg_class c ON c.oid = i.inhrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE pc.parent_table LIKE 'telegraf.%'
|
||||
AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'
|
||||
SELECT pc.parent_table, d.default_part
|
||||
FROM so_admin.telegraf_defaults() d
|
||||
JOIN partman.part_config pc ON to_regclass(pc.parent_table) = d.parent_table
|
||||
WHERE d.default_part IS NOT NULL
|
||||
LOOP
|
||||
EXECUTE format('TRUNCATE TABLE %s', r.default_table);
|
||||
EXECUTE format('TRUNCATE TABLE %s', r.default_part);
|
||||
PERFORM partman.create_partition_time(
|
||||
r.parent_table, ARRAY[date_trunc('day', now())]::timestamptz[]);
|
||||
END LOOP;
|
||||
END
|
||||
$$;
|
||||
COMMIT;
|
||||
EOSQL
|
||||
fi
|
||||
|
||||
@@ -222,7 +358,7 @@ if $restart_needed; then
|
||||
sleep 2
|
||||
done
|
||||
docker exec so-postgres pg_isready -U postgres -q \
|
||||
|| { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; exit 1; }
|
||||
|| { echo "so-postgres did not come back; check 'docker logs so-postgres'." >&2; finish 1; }
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -238,8 +374,14 @@ CALL partman.run_maintenance_proc();
|
||||
\endif
|
||||
EOSQL
|
||||
|
||||
if [[ -n "$job_id" ]]; then
|
||||
psql_cron -tAc "SELECT cron.alter_job($job_id, active := true)" >/dev/null
|
||||
restore_active=
|
||||
echo "The $JOB_NAME job is active."
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Telegraf partition status after repair:"
|
||||
psql_tg -c "$REPORT"
|
||||
print_report
|
||||
echo "The $JOB_NAME job runs hourly at :17. Confirm it fired with:"
|
||||
echo " so-telegraf-repair --check"
|
||||
@@ -3,7 +3,7 @@
|
||||
#
|
||||
# One pillar directory can map to multiple (state, tgt) actions.
|
||||
# tgt is a raw salt compound expression. tgt_type is always "compound".
|
||||
# Per-action `batch` / `batch_wait` override the orch defaults (25% / 15s).
|
||||
# Per-action `batch` / `batch_wait` override the orch defaults (10% / 15s).
|
||||
# An action with `highstate: True` triggers state.highstate instead of
|
||||
# state.apply -- see salt/orch/push_batch.sls.
|
||||
#
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
# Two branches, keyed on node_id:
|
||||
# A) node_id populated -> the change is scoped to that one minion. Look up the
|
||||
# app in pillar_push_map.yaml and write an intent that runs the app's mapped
|
||||
# state(s) targeted to just that node.
|
||||
# state(s) on just that node, and only if the node is in the app's target.
|
||||
# B) node_id empty -> grid-wide app change. Look up the app in
|
||||
# pillar_push_map.yaml and write an intent with the entry's actions as-is.
|
||||
#
|
||||
@@ -116,15 +116,17 @@ def _app_from_setting(setting_id):
|
||||
|
||||
|
||||
def _node_actions(entry, node_id):
|
||||
# Copy the app's mapped actions but retarget each one to the single node.
|
||||
# Copy the app's mapped actions but narrow each one to the single node.
|
||||
# Preserves the state/highstate selection and any batch/batch_wait overrides.
|
||||
# The app's own target is kept so the state is not applied to a node that does not
|
||||
# run the app (e.g. an elasticsearch setting saved for a sensor fails to render there).
|
||||
actions = []
|
||||
for action in entry:
|
||||
if not isinstance(action, dict):
|
||||
continue
|
||||
node_action = dict(action)
|
||||
node_action['tgt'] = node_id
|
||||
node_action['tgt_type'] = 'glob'
|
||||
node_action['tgt'] = 'L@{} and ( {} )'.format(node_id, action.get('tgt', '*'))
|
||||
node_action['tgt_type'] = 'compound'
|
||||
actions.append(node_action)
|
||||
return actions
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
|
||||
include:
|
||||
- ca
|
||||
@@ -28,6 +29,7 @@ so-redis:
|
||||
- {{ BINDING }}
|
||||
{% endfor %}
|
||||
- binds:
|
||||
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
|
||||
- /opt/so/log/redis:/var/log/redis:rw
|
||||
- /opt/so/conf/redis/etc/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
- /opt/so/conf/redis/working:/redis:rw
|
||||
@@ -60,6 +62,7 @@ so-redis:
|
||||
{% endif %}
|
||||
- entrypoint: "redis-server /usr/local/etc/redis/redis.conf"
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: trusttheca
|
||||
- x509: redis_crt
|
||||
- x509: redis_key
|
||||
|
||||
@@ -6,9 +6,11 @@
|
||||
{% from 'allowed_states.map.jinja' import allowed_states %}
|
||||
{% if sls.split('.')[0] in allowed_states %}
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- registry.ssl
|
||||
- registry.config
|
||||
- registry.sostatus
|
||||
@@ -29,6 +31,7 @@ so-dockerregistry:
|
||||
- {{ BINDING }}
|
||||
{% endfor %}
|
||||
- binds:
|
||||
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
|
||||
- /opt/so/conf/docker-registry/etc/config.yml:/etc/distribution/config.yml:ro
|
||||
- /opt/so/conf/docker-registry:/var/lib/registry:rw
|
||||
- /nsm/docker-registry/docker:/var/lib/registry/docker:rw
|
||||
@@ -63,6 +66,7 @@ so-dockerregistry:
|
||||
attempts: 5
|
||||
interval: 30
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- x509: registry_crt
|
||||
- x509: registry_key
|
||||
- require:
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -3,6 +3,31 @@
|
||||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% set ONEDIR_PACKAGES = {'relenv': '0.22.27', 'pip': '26.2.2', 'certifi': '2026.10.19'} %}
|
||||
|
||||
{% if grains.saltversion == '3006.19' %}
|
||||
onedir_package:
|
||||
file.recurse:
|
||||
- name: /opt/so/conf/salt/module_packages/onedir
|
||||
- source: salt://salt/module_packages/onedir
|
||||
- clean: True
|
||||
- makedirs: True
|
||||
|
||||
# Backport of salt 3006.29 (PR 70466): certifi 2026.10.19 and pip 26.2.2 drop the compromised
|
||||
# eMudhra/emSign root. relenv must move with pip: 0.22.2's runtime wrapper hardcodes pre-26.2 pip
|
||||
# install() signatures, so a lone pip upgrade breaks every later pip run in the onedir. Installing
|
||||
# all three in one invocation under the stock pip avoids ever being in that state.
|
||||
onedir_python_module_install:
|
||||
cmd.run:
|
||||
- name: /opt/saltstack/salt/bin/python3.10 -m pip install --no-index --find-links=/opt/so/conf/salt/module_packages/onedir/ --upgrade --root-user-action=ignore {% for pkg, ver in ONEDIR_PACKAGES.items() %}{{ pkg }}=={{ ver }} {% endfor %}
|
||||
- unless: /opt/saltstack/salt/bin/python3.10 -c 'import {{ ONEDIR_PACKAGES.keys() | join(", ") }}; assert {% for pkg, ver in ONEDIR_PACKAGES.items() %}{{ pkg }}.__version__ == "{{ ver }}"{% if not loop.last %} and {% endif %}{% endfor %}'
|
||||
- require:
|
||||
- file: onedir_package
|
||||
- require_in:
|
||||
- cmd: docker_python_module_install
|
||||
- failhard: True
|
||||
{% endif %}
|
||||
|
||||
docker_module_package:
|
||||
file.recurse:
|
||||
- name: /opt/so/conf/salt/module_packages/docker
|
||||
|
||||
@@ -4,10 +4,12 @@
|
||||
# Elastic License 2.0.
|
||||
|
||||
{% from 'vars/globals.map.jinja' import GLOBALS %}
|
||||
{% from 'ca/trust.map.jinja' import CATRUST %}
|
||||
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
|
||||
|
||||
|
||||
include:
|
||||
- ca.trust
|
||||
- sensoroni.config
|
||||
- sensoroni.sostatus
|
||||
|
||||
@@ -17,6 +19,7 @@ so-sensoroni:
|
||||
- restart_policy: unless-stopped
|
||||
- network_mode: host
|
||||
- binds:
|
||||
- {{ CATRUST.dir }}
|
||||
- /nsm/import:/nsm/import:rw
|
||||
- /nsm/pcapout:/nsm/pcapout:rw
|
||||
- /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro
|
||||
@@ -48,6 +51,7 @@ so-sensoroni:
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
- watch:
|
||||
- cmd: ca_trust_extract
|
||||
- file: /opt/so/conf/sensoroni/sensoroni.json
|
||||
- require:
|
||||
- file: sensoroniagentconf
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
requests>=2.31.0
|
||||
pyyaml>=6.0
|
||||
urllib3>=2.1.0
|
||||
urllib3>=2.1.0
|
||||
certifi>=2026.10.23
|
||||
idna>=3.20
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -1,2 +1,4 @@
|
||||
requests>=2.31.0
|
||||
pyyaml>=6.0
|
||||
certifi>=2026.10.23
|
||||
idna>=3.20
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -1,2 +1,4 @@
|
||||
requests>=2.31.0
|
||||
pyyaml>=6.0
|
||||
certifi>=2026.10.23
|
||||
idna>=3.20
|
||||
BIN
Binary file not shown.
Loaded 100 of 264 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user