Compare commits

..
Author SHA1 Message Date
Mike Reeves 3e5a934ff8 Merge pull request #16238 from Security-Onion-Solutions/hotfix/3.3.0
Hotfix/3.3.0
2026-09-11 16:38:51 -04:00
155 changed files with 501 additions and 4810 deletions

No files matched your search

-1
View File
@@ -13,7 +13,6 @@ body:
- 3.1.0
- 3.2.0
- 3.3.0
- 3.4.0
- Other (please provide detail below)
validations:
required: true
+1 -27
View File
@@ -6,10 +6,6 @@ on:
- "salt/sensoroni/files/analyzers/**"
- "salt/manager/tools/sbin/**"
- "salt/_beacons/**"
- "salt/elastalert/files/modules/**"
- "salt/telegraf/tools/sbin_jinja/**"
- "salt/telegraf/defaults.yaml"
- "salt/telegraf/soc_telegraf.yaml"
jobs:
build:
@@ -19,7 +15,7 @@ jobs:
fail-fast: false
matrix:
python-version: ["3.14"]
python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons", "salt/elastalert/files/modules/so"]
python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin", "salt/_beacons"]
steps:
- uses: actions/checkout@v3
@@ -38,25 +34,3 @@ jobs:
- name: Test with pytest
run: |
PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini
telegraf-collector:
# so-container-stats is a jinja template rather than an importable module, so it gets its
# own job: the test renders it the way salt does, then drives it with a faked docker engine
# and cgroup tree. No container runtime is needed.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: "3.14"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install flake8 pytest jinja2 pyyaml
- name: Lint with flake8
run: |
flake8 salt/telegraf/tools/sbin_jinja/so-container-stats_test.py --config=pytest.ini
- name: Test with pytest
run: |
pytest salt/telegraf/tools/sbin_jinja/so-container-stats_test.py -v
+1 -1
View File
@@ -1 +1 @@
20260911
+1 -1
View File
@@ -1 +1 @@
3.4.0
3.3.0
@@ -1,14 +0,0 @@
-----BEGIN CERTIFICATE-----
MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG
EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx
IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw
MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND
IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci
MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti
sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O
BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c
3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J
0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ==
-----END CERTIFICATE-----
@@ -1,15 +0,0 @@
-----BEGIN CERTIFICATE-----
MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG
EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo
bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g
RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ
TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw
djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0
WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS
fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB
zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq
hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB
CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD
+JbNR6iC8hZVdyR+EhCVBCyj
-----END CERTIFICATE-----
@@ -1,21 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG
A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg
SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw
MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v
dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ
BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ
HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH
3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH
GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c
xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1
aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq
TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL
BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87
/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4
kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG
YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT
+xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo
WXzhriKi4gp6D/piq1JM4fHfyr6DDUI=
-----END CERTIFICATE-----
@@ -1,32 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIFcjCCA1qgAwIBAgIKLwq3aw3LSq8nWDANBgkqhkiG9w0BAQwFADBWMQswCQYD
VQQGEwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJ
bmMxHDAaBgNVBAMTE2VtU2lnbiBSb290IENBIC0gQzIwHhcNMTgwMjE4MTgzMDAw
WhcNNDMwMjE4MTgzMDAwWjBWMQswCQYDVQQGEwJVUzETMBEGA1UECxMKZW1TaWdu
IFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMxHDAaBgNVBAMTE2VtU2lnbiBSb290
IENBIC0gQzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCMfX1lA+Tb
mh9YInmRgOW97IVx4LUJf2DRZfs837Jrml+py64aVnYgWO4t6C78fgjfS7jX+c4T
inIzEquWcI+zi0fd4Sc8NDf7JONp27VWX0qwUYqzLDRCt+s7zpLcfx1ky0zVIJj6
L06uPyK3kIr9+YAsrVj+39utm6e2MBQsRNstSI3fCQYAGvoQTQ8fULauTqNWaYAk
NYFe6HUHHQPp2u1Ua00odMXiD5oRFxLcDnGAcE1I/9E9mLCdkggXijYUmico7+Xw
ZeFoPhva6eIJ5p03Lt3Du5W3EcHR0cJmmY1pyeA36JaXKWRNM9IRjYMVNCcp4jhB
2tIYiZ+LVk8bwQ9/1c23txmv3u97taZlV22NF4ttS1qq3J+MOp0oGULBzpKfRx0q
GVqbPukQNGAjOLIN8KDNQNzbR1iAl2d8H+MSoicBo4Aid8TjLWcNv48oCWL53ZrF
BMTDjaIA6frG1t4IpbnHadA7qCJJe2qpJN6n2eQKAUn6UiQDHPsSqNBlcUhQ4Y/0
Y0mU5rghm2OB9rXQS1Fb1JRCfJMNnJIm5AUB2+2RWzq5Tgz7SbSho8NsZk0UbQnF
xciqQ9uoVTAsK14Sk9oG8Q3zfsM08cdPoRb0WlIZklR6mKD7L8nH/zfGu8PIJv94
GGB9RZ9U4A69r3ePmy8MvrzfNxHKtH6svwIDAQABo0IwQDAdBgNVHQ4EFgQUs/eK
pNYPiABZ6FEXT9V+7IYigZ0wDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
Af8wDQYJKoZIhvcNAQEMBQADggIBADQlpiWM0cv2nZ0H5jVsBq0x2q62Q0LwqATs
CFvyub7gxNCytRuoA8stmPOEu/lg8Igxj4FIjoyhIrWUVxyiLU7No4P+WjEUOwUT
xIpkEOtvGUQ9fiOlcGHtIZDNBlZq7WpktXAxeV55RPPsor26p2FNAMRFfZQh0sLX
hKgk8iulSSggqx8ezgPye63FaiYEi4c/dzRj3HOCnsZiwZZU02df5YpNFjxSwZvE
41cjGpsrpWMfQFI2s53RbeXp47lSAxYE4NzjBFMe+EwFuEveBCJBEAH5rvYu3pi2
orsJ424TqWEQV1tCsCkQz+Yq/Okal7yHAkKDeOXcP7oN4A+TdXc2pdqxuVCnBO0R
mWz2JpGSSeJjiTk/OPwRsPNWtwG/KXL04o2ta3jiPpJuICVtWDAc9R3auBEgJl5r
ShRmBdszG0LmzsHuZPCFSYC15RBDCOBsa8bDRJ8pBFU2Wi/CVXCACEuavgoveA4F
a5bt38o0PWxsBP+MpocCdVtDMqzQhxy9IohKuXWAGresoIvKDg3xFk6rBOrjfVwJ
elwi/xAisojHPJVQv9W1zVIoHp+EQg/4MQC21NbIX2RoioB+V3hK439b/w7deU8x
2M8cl1OG0nPfbnARl5GPM7vJgi470jto4SeMg6HMAW3Egb56tQcNLwI9U8mZnNvR
gUMrkAgL
-----END CERTIFICATE-----
@@ -1,22 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD
VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU
ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH
MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO
MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv
Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz
f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO
8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq
d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM
tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt
Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB
o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD
AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x
PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM
wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d
GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH
6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby
RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx
iN66zB+Afko=
-----END CERTIFICATE-----
@@ -1,32 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIFlTCCA32gAwIBAgILAIZNvw/jXtd9jtgwDQYJKoZIhvcNAQEMBQAwZzELMAkG
A1UEBhMCSU4xEzARBgNVBAsTCmVtU2lnbiBQS0kxJTAjBgNVBAoTHGVNdWRocmEg
VGVjaG5vbG9naWVzIExpbWl0ZWQxHDAaBgNVBAMTE2VtU2lnbiBSb290IENBIC0g
RzIwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBnMQswCQYDVQQGEwJJ
TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
b2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBHMjCCAiIw
DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMNwGIWW2kHfHK+sXTNwxF07K+IV
ySTuyFM2r1v002wUfcdT+zs5OM5QbMYFFnedXQI6gCFLsjKrcaej48Zt37OyEb3i
aPs7CsP4kAyTwzKH9aZe6gXYHrJq40/ZVMNcQVI2PcIp40B/SAN2gUZ+ZaUtIOvV
jEx26/ebNaXRIsthlkOG/caB+QRwDw1tl7338Zlv0M2oTBUy4B3e7dGP5pgXH71M
jqHPCoNo+xv9f0NTBT+hUDa8h8wUtcGQq9CDeJTpjWcD2bP2AMdVG6oVpMAUeUzo
cCyglvtFdUMjggxBbw4qhau1HXPG8Ot9hwL7ZMi8tkTzrvUIxxb8G9LF/7kKeCE7
tGZaVzDTnXuifl3msR4ErHsQ4P7lVu2AIjIAhrAXoedDidb7pMcf7TABdrYUT1Jo
G/AiK+J9jO6GTjeADD4LMDSBZhHMuBK/PJ/g0kGBt+/C1L+/HURzQhJkMlRnM6Rv
XoCtfKopSlns5trZmTi971Wjbn88QXP61lGpBCUPwCjs7rpOYvSUJtI+lcbF+37q
kIqOXYkVT3cupDSpw+H89kFtj5GKY+Xny4LxY+3IvDIRiyd6ky1DPj713DI0yqve
EpsIr3A0PdwuyUI7CS1jg0NnGFT6Xxyr0xB+VDt83FJYW8v16k2pbaQ4kVxA3aXd
X9dZYyVR1S59KM75AgMBAAGjQjBAMB0GA1UdDgQWBBTt7E1FYRgo57MjKBEcTaUn
DV7s9DAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
AQwFAAOCAgEACFC/ilQg8KTCVBxFJW/sazomkS0kNYbEIZg4B3obqwsJ7SX98z8Z
gfzBpz0nYClwwJjWbFN1R2zY8pCEot6/dgmA8Vbq0GxhwPM5YN/SZquNyRIxO3cU
dlAcwf+vSezdVCf9wOzvSAF3q0a5ljvbdbNJNpfScQVp7UUd5sBsZk8jXO1KQ/go
/Vf/GDPnrIFmxpAIGE3sgnO8lAv9FzUaAeuv7HWe47xN9J7+bQzF93yHuIXACPTL
pQHhg2zMv5C7BAbuDHfbj1Cu294Z832yhSfBcziWGskOvl3es2EcHytbS9c9P+0z
Mpka7zGC1FHrvLb/FoduH86TeZt0QjZ6pcplNzoaxDnDvzTJ6CC2Eny+qH/APFCu
VUv5/wjwF+HPm8Pup2ARj9cEp92+0qcerfHacNq5hMeGZdbA/dzdUR/5z5zXdxAk
nl8mcfGb0eMNSTXQmmB/i4AecNnr72uYjzlaXUGYN7Nrb6XouG0pnh0/BBtWWp0U
ShIPpWEAqs7RJBj6+1ZUYXZ4ObrCw962DxhN2p19Hxw9LtuUUcLqqTPrFXYvwO4t
ouj7KJnAkaTUfXGdEaFVtFig1EA30WzJY2X1vAQ7hVnniCjgaXAGqjsU6sklNM9n
xDx5rFCCCEtj9Kh8UHjGK2QqgP5kwgttjOApQMaCoezMfK4KD7WpOXU=
-----END CERTIFICATE-----
@@ -1,32 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIFdjCCA16gAwIBAgIQCIYhdXKnxAzDuy7l1y1ubDANBgkqhkiG9w0BAQwFADBV
MQswCQYDVQQGEwJJTjElMCMGA1UECgwcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGlt
aXRlZDEfMB0GA1UEAwwWZW1TaWduIFJvb3QgQ1MgQ0EgLSBHMjAeFw0yNDAyMDgw
ODE1MzlaFw00OTAyMDcwODE1MzlaMFUxCzAJBgNVBAYTAklOMSUwIwYDVQQKDBxl
TXVkaHJhIFRlY2hub2xvZ2llcyBMaW1pdGVkMR8wHQYDVQQDDBZlbVNpZ24gUm9v
dCBDUyBDQSAtIEcyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA4CVO
xuBjdLJXpkrjosA+7ygTGaYzufUJrhQOeIZEuCqeCPI8hDqIu+MIqEqOlgDFetVM
3Cor3CGBlGJzfzzlI31vyElvQq88jjCfo12Ghqcf+NAZDFduJk5+NgC0z3lfdiB6
NNzxbfVkA+X6oDNNRioVUZsVFuIzT9Qt0Sh4KwQPRFBc/wI2yGpDuKHNsLehcECx
Uw00m33xGYh9UkeL3reOmAaLYWZiO+Pd2vxPtrkXI4wyN4BMlESJ73uP0tyX9lq4
M4yN2yh3i+ETI8/jzPjxHD7QFaVkCoS+uj2YmBa3mytlqlBgyOoKueNeotX8hfzB
FNm962abpV4mN0SoC6hjkYv8IayseRPFv9VI0meQXg9OLRoNvBOqG5kw937FH0Li
q3CIzKyvdyeYRT0LJdWA2iuB9l+4zzIfCwGyT4M4VDmJuPBhrTmjXKae6AUtm1eO
q2f8gftA6J6AlbD0gRd/t8CrFssFNm+LYAEd36kWRBuolbaOfbSbEww84xNAgj4U
SPwYjtNDsCkQjtgSOaDtDOmolN+Mbk528Y+LHMlbd4T3Xmp+VIWp5+70GhBDOA91
hhtims+hHkXjRI2U7WrJLcTlcF8ykiymRbFKhwOBvTmtZNbHmDI7scU4wWUAsmW6
tWaeeoM50KBl8ai28A6yBGYMO//03Ubvqiqst80CAwEAAaNCMEAwHQYDVR0OBBYE
FOYcxqontIvyvYVkX6/0unKm0AF1MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E
BTADAQH/MA0GCSqGSIb3DQEBDAUAA4ICAQCv5Asord7fdQMT30emwb+oLuDUxdmJ
DQHnzxIXoNvymDTbNHF9LqjeXc+X+XcjIV8PTp4o2I9wyVppHmcaVgRjN/G9FxeN
5tCQ97v5sdx4eAdPljK1xqg7Vhfmzcp+7rSjJG7Sn9gMlZx81kL3Oa2qZj+W5Fi6
tN4vaR9OLEyvKKHDG/eFxS/qghD1d/Clkb7UGGjBcWJlaZTWVALMKG+QCxiMoELi
oeKaoBbZoXooxfF0mWdkIl9RWNU4r9O/yt66LBE9rj49dX19L+rGg71/4bTi5ua+
+Wz60XaOc2CdHJaBSols5hWEx/LBjS7UzfU3uJfnpQXVDi0US5sFzIHIlS6kjh7h
YuF4+/9ZAl4Bx3w2TNN7pVm7I39pgYl7XN6KZRwfkAv4aRswPqhWbQfr3+qfcC93
cgTyFy2RG5YE/OfmpSs6VtYKawqsdEODz/p2nhZfSDrVDnEPvERqahttY5+gaWNc
2fbibgKMfuejTZaJ3yttPslVSko5SDs0xKU423swu34sb/fdISqtOu4NzPCZiHSG
ladaoITmSA6UWFk4fdqaLBI+xSxY5Et3S1APp5r3j/l1KrWPNbb1/+ydHN+YLbh8
hCBIecKdk6VbnnFRlulm0d2RSgpfCRrWLomN8JHCxCMVv/hBQXz3AXGM1ZlfYTyl
E1BpDby0G2eEkw==
-----END CERTIFICATE-----
-1
View File
@@ -10,4 +10,3 @@ include:
- ca.server
{% endif %}
- ca.trustca
- ca.trust
-12
View File
@@ -1,12 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{# Containers mount the extracted directory rather than single files: update-ca-trust replaces
files by rename, so a file bind mount would keep pointing at the old inode. #}
{% set CATRUST = {
'dir': '/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro',
'java': '/etc/pki/ca-trust/extracted/java/cacerts',
'pem': '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'
} %}
-37
View File
@@ -1,37 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
ca_trust_anchor:
file.managed:
- name: /etc/pki/ca-trust/source/anchors/so-intca.crt
- source: salt://ca/files/ca.crt
- show_changes: False
- makedirs: True
ca_trust_legacy_anchor:
file.absent:
- name: /etc/pki/ca-trust/source/anchors/ca.crt
ca_trust_legacy_bundles:
file.absent:
- name: /opt/so/conf/ca
# clean removes blocklist entries that are no longer shipped
ca_trust_blocklist:
file.recurse:
- name: /etc/pki/ca-trust/source/blocklist
- source: salt://ca/files/blocklist
- clean: True
- makedirs: True
- file_mode: '0644'
- dir_mode: '0755'
ca_trust_extract:
cmd.run:
- name: update-ca-trust extract
- onchanges:
- file: ca_trust_anchor
- file: ca_trust_legacy_anchor
- file: ca_trust_blocklist
+5 -20
View File
@@ -117,25 +117,14 @@ elastic_curl_config:
{% endif %}
# A non-root owner here can chmod the directory and replace any script in it, including
# the root-owned ones. 555 is the mode the filesystem RPM ships; root ignores it anyway.
usr_sbin_perms:
file.directory:
- name: /usr/sbin
- user: root
- group: root
- mode: 555
common_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://common/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- show_changes: False
- require:
- file: usr_sbin_perms
{% if GLOBALS.role == 'so-heavynode' %}
- exclude_pat:
- so-pcap-import
@@ -170,8 +159,8 @@ common_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://common/tools/sbin_jinja
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- template: jinja
- show_changes: False
@@ -184,8 +173,6 @@ so-status_script:
file.managed:
- name: /usr/sbin/so-status
- source: salt://common/tools/sbin/so-status
- user: root
- group: root
- mode: 755
{% if GLOBALS.is_sensor %}
@@ -217,11 +204,9 @@ sostatus_log:
- replace: False
# Install sostatus check cron. This is used to populate Grid.
# telegraf reads status.log on the same minute boundary this runs, so write aside and rename
# rather than truncating the file it is reading
so-status_check_cron:
cron.present:
- name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log.tmp 2>&1; mv -f /opt/so/log/sostatus/status.log.tmp /opt/so/log/sostatus/status.log'
- name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log 2>&1'
- identifier: so-status_check_cron
- user: root
- minute: '*/1'
+14 -42
View File
@@ -18,61 +18,47 @@ copy_so-common_common_tools_sbin:
- name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
- source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-image-common_common_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common
- source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_soup_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-firewall_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-yaml_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-repo-sync_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
- user: root
- group: root
- mode: 755
- preserve: True
copy_bootstrap-salt_manager_tools_sbin:
file.copy:
- name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh
- source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
- user: root
- group: root
- mode: 644
- preserve: True
# This section is used to put the new script in place so that it can be called during soup.
# It is faster than calling the states that normally manage them to put them in place.
@@ -81,60 +67,46 @@ copy_so-common_sbin:
- name: /usr/sbin/so-common
- source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-image-common_sbin:
file.copy:
- name: /usr/sbin/so-image-common
- source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_soup_sbin:
file.copy:
- name: /usr/sbin/soup
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-firewall_sbin:
file.copy:
- name: /usr/sbin/so-firewall
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-yaml_sbin:
file.copy:
- name: /usr/sbin/so-yaml.py
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_so-repo-sync_sbin:
file.copy:
- name: /usr/sbin/so-repo-sync
- source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
copy_bootstrap-salt_sbin:
file.copy:
- name: /usr/sbin/bootstrap-salt.sh
- source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
- force: True
- user: root
- group: root
- mode: 755
- preserve: True
+1 -2
View File
@@ -240,8 +240,7 @@ copy_new_files() {
cd $UPDATE_DIR
rsync -a salt $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
rsync -a pillar $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
# Root-executed code; SOC only needs to read it. Local dirs stay socore-owned.
chown -R root:root $DEFAULT_SALT_DIR/
chown -R socore:socore $DEFAULT_SALT_DIR/
cd /tmp
}
+6 -19
View File
@@ -9,7 +9,6 @@ import sys
import subprocess
import os
import json
import tempfile
sys.path.append('/opt/saltstack/salt/lib/python3.10/site-packages/')
import salt.config
@@ -18,21 +17,6 @@ import salt.loader
__opts__ = salt.config.minion_config('/etc/salt/minion')
__grains__ = salt.loader.grains(__opts__)
def write_atomic(path, value):
# telegraf reads these files on its own schedule; replacing them by rename means it never
# reads a truncated file and reports an empty value as if it were real
directory = os.path.dirname(path)
handle, temp = tempfile.mkstemp(dir=directory)
try:
with os.fdopen(handle, 'w') as f:
f.write(str(value))
os.chmod(temp, 0o644)
os.replace(temp, path)
except Exception:
os.path.exists(temp) and os.unlink(temp)
raise
def check_needs_restarted():
osfam = __grains__['os_family']
val = '0'
@@ -50,7 +34,8 @@ def check_needs_restarted():
else:
fail("Unsupported OS")
write_atomic(outfile, val)
with open(outfile, 'w') as f:
f.write(val)
def check_for_fps():
feat = 'fps'
@@ -71,7 +56,8 @@ def check_for_fps():
# Unknown, so assume 0
fps = 0
write_atomic('/opt/so/log/sostatus/fps_enabled', fps)
with open('/opt/so/log/sostatus/fps_enabled', 'w') as f:
f.write(str(fps))
def check_for_lks():
feat = 'Lks'
@@ -94,7 +80,8 @@ def check_for_lks():
lks = 1
if lks:
break
write_atomic('/opt/so/log/sostatus/lks_enabled', lks)
with open('/opt/so/log/sostatus/lks_enabled', 'w') as f:
f.write(str(lks))
def fail(msg):
print(msg, file=sys.stderr)
+1 -3
View File
@@ -177,8 +177,6 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unexpected authorization header" # expected WARN log lines indicating invalid auth header
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Missing ory_kratos_session cookie" # expected WARN log lines indicating invalid auth header
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Static assets preprocessor only supports GET and HEAD requests" # expected WARN log lines indicating invalid auth header
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|respondError" # respondError is a function name, output via http middleware as standard request logging
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|GET /kibana/" # Ignore Kibana queries with triggered words
fi
if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -242,7 +240,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal" # docker container getting recycled
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-.*user so_kibana lacks the required permissions" # Known issue with integrations starting transform jobs that are explicitly not allowed to start as a system user
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # so-dockerregistry logs a tag lookup miss during image copy; not tied to one docker version
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown" # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Could not index event to Elasticsearch.*\"version\" => \"9.0.8\"" # Expected during Elastic upgrade temporarily, as policies referencing older pipelines are updated
fi
+47 -78
View File
@@ -8,37 +8,21 @@
# Elastic License 2.0.
SENSOR_DIR="${SENSOR_DIR:-/nsm}"
SENSOR_DIR='/nsm'
CRIT_DISK_USAGE=90
LOG="${LOG:-/opt/so/log/sensor_clean.log}"
LOCK="${LOCK:-/var/tmp/so-sensor-clean.lock}"
MAX_PASSES=100
CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
LOG="/opt/so/log/sensor_clean.log"
TODAY=$(date -u "+%Y-%m-%d")
ZEEK_LOGS="$SENSOR_DIR/zeek/logs"
STRELKA_FILES="$SENSOR_DIR/strelka/processed"
SURICATA_LOGS="$SENSOR_DIR/suricata"
PCAPS="$SENSOR_DIR/pcapout"
log() {
echo "$(date) - $*" >>"$LOG"
}
disk_usage() {
df -P "$SENSOR_DIR" | tail -1 | awk '{print $5}' | tr -d %
}
disk_avail() {
df -P "$SENSOR_DIR" | tail -1 | awk '{print $4}'
}
# sets REMOVED=1 if anything was actually deleted
clean() {
## find the oldest Zeek logs directory
OLDEST_DIR=$(ls "$ZEEK_LOGS" 2>/dev/null | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
if [ -n "$OLDEST_DIR" ]; then
log "Removing directory: $ZEEK_LOGS/$OLDEST_DIR"
rm -rf "$ZEEK_LOGS/$OLDEST_DIR"
REMOVED=1
OLDEST_DIR=$(ls /nsm/zeek/logs/ | grep -v "current" | grep -v "stats" | grep -v "packetloss" | grep -v "zeek_clean" | sort | head -n 1)
if [ -z "$OLDEST_DIR" -o "$OLDEST_DIR" == ".." -o "$OLDEST_DIR" == "." ]; then
echo "$(date) - No old Zeek logs available to clean up in /nsm/zeek/logs/" >>$LOG
#exit 0
else
echo "$(date) - Removing directory: /nsm/zeek/logs/$OLDEST_DIR" >>$LOG
rm -rf /nsm/zeek/logs/"$OLDEST_DIR"
fi
## Remarking for now, as we are moving extracted files to /nsm/strelka/processed
@@ -59,73 +43,58 @@ clean() {
#fi
## Clean up Zeek extracted files processed by Strelka
OLDEST_STRELKA=$(find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
if [ -n "$OLDEST_STRELKA" ]; then
STRELKA_FILES='/nsm/strelka/processed'
OLDEST_STRELKA=$(find $STRELKA_FILES -type f -printf '%T+ %p\n' | sort -n | head -n 1)
if [ -z "$OLDEST_STRELKA" -o "$OLDEST_STRELKA" == ".." -o "$OLDEST_STRELKA" == "." ]; then
echo "$(date) - No old files available to clean up in $STRELKA_FILES" >>$LOG
else
OLDEST_STRELKA_DATE=$(echo $OLDEST_STRELKA | awk '{print $1}' | cut -d+ -f1)
log "Removing extracted files for $OLDEST_STRELKA_DATE"
REMOVED=1
find "$STRELKA_FILES" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
log "Removing file: $FILE"
OLDEST_STRELKA_FILE=$(echo $OLDEST_STRELKA | awk '{print $2}')
echo "$(date) - Removing extracted files for $OLDEST_STRELKA_DATE" >>$LOG
find $STRELKA_FILES -type f -printf '%T+ %p\n' | grep $OLDEST_STRELKA_DATE | awk '{print $2}' | while read FILE; do
echo "$(date) - Removing file: $FILE" >>$LOG
rm -f "$FILE"
done
fi
## Clean up Suricata log files
OLDEST_SURICATA=$(find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
if [ -n "$OLDEST_SURICATA" ]; then
SURICATA_LOGS='/nsm/suricata'
OLDEST_SURICATA=$(find $SURICATA_LOGS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
if [[ -z "$OLDEST_SURICATA" ]] || [[ "$OLDEST_SURICATA" == ".." ]] || [[ "$OLDEST_SURICATA" == "." ]]; then
echo "$(date) - No old files available to clean up in $SURICATA_LOGS" >>$LOG
else
OLDEST_SURICATA_DATE=$(echo $OLDEST_SURICATA | awk '{print $1}' | cut -d+ -f1)
log "Removing logs for $OLDEST_SURICATA_DATE"
REMOVED=1
find "$SURICATA_LOGS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
log "Removing file: $FILE"
OLDEST_SURICATA_FILE=$(echo $OLDEST_SURICATA | awk '{print $2}')
echo "$(date) - Removing logs for $OLDEST_SURICATA_DATE" >>$LOG
find $SURICATA_LOGS -type f -printf '%T+ %p\n' | grep $OLDEST_SURICATA_DATE | awk '{print $2}' | while read FILE; do
echo "$(date) - Removing file: $FILE" >>$LOG
rm -f "$FILE"
done
fi
## Clean up extracted pcaps
OLDEST_PCAP=$(find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | sort -n | head -n 1)
if [ -n "$OLDEST_PCAP" ]; then
PCAPS='/nsm/pcapout'
OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then
echo "$(date) - No old files available to clean up in $PCAPS" >>$LOG
else
OLDEST_PCAP_DATE=$(echo $OLDEST_PCAP | awk '{print $1}' | cut -d+ -f1)
log "Removing extracted files for $OLDEST_PCAP_DATE"
REMOVED=1
find "$PCAPS" -type f -printf '%T+ %p\n' 2>/dev/null | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
log "Removing file: $FILE"
OLDEST_PCAP_FILE=$(echo $OLDEST_PCAP | awk '{print $2}')
echo "$(date) - Removing extracted files for $OLDEST_PCAP_DATE" >>$LOG
find $PCAPS -type f -printf '%T+ %p\n' | grep $OLDEST_PCAP_DATE | awk '{print $2}' | while read FILE; do
echo "$(date) - Removing file: $FILE" >>$LOG
rm -f "$FILE"
done
fi
}
# Only one instance at a time; the lock is the fd, so it releases on any exit
exec 9>"$LOCK" || exit 1
if ! flock -n 9; then
log "another so-sensor-clean is already running (lock $LOCK held); exiting"
exit 0
# Check to see if we are already running
NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-sensor-clean")
[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING sensor clean script processes running...exiting." >>$LOG && exit 0
if [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; then
while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
clean
CUR_USAGE=$(df -P $SENSOR_DIR | tail -1 | awk '{print $5}' | tr -d %)
done
fi
CUR_USAGE=$(disk_usage)
[ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ] || exit 0
log "$SENSOR_DIR at ${CUR_USAGE}% (threshold ${CRIT_DISK_USAGE}%); starting cleanup"
PASS=0
while [ "$CUR_USAGE" -gt "$CRIT_DISK_USAGE" ]; do
PASS=$((PASS + 1))
if [ "$PASS" -gt "$MAX_PASSES" ]; then
log "stopping after $MAX_PASSES passes; $SENSOR_DIR still at ${CUR_USAGE}%"
break
fi
REMOVED=0
BEFORE=$(disk_avail)
clean
CUR_USAGE=$(disk_usage)
if [ "$REMOVED" -eq 0 ]; then
log "nothing left to remove in $ZEEK_LOGS, $STRELKA_FILES, $SURICATA_LOGS, $PCAPS; $SENSOR_DIR still at ${CUR_USAGE}% - space is consumed outside of NSM cleanup scope"
break
fi
if [ "$(disk_avail)" -le "$BEFORE" ]; then
log "pass $PASS freed no space; $SENSOR_DIR still at ${CUR_USAGE}% - stopping until next run"
break
fi
done
+1 -3
View File
@@ -125,6 +125,4 @@ else
RAIDSTATUS=1
fi
# telegraf reads this file; write aside and rename so it never sees a half-written file
echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log.tmp
mv -f /opt/so/log/raid/status.log.tmp /opt/so/log/raid/status.log
echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log
+11 -3
View File
@@ -1,10 +1,18 @@
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{# we only want this state to run it is CentOS #}
{% if GLOBALS.os == 'OEL' %}
include:
- ca.trust
trusted_ca:
file.managed:
- name: /etc/pki/ca-trust/source/anchors/ca.crt
- source: salt://ca/files/ca.crt
update_ca_certs:
cmd.run:
- name: update-ca-trust
- onchanges:
- file: trusted_ca
{% else %}
+2 -9
View File
@@ -1,12 +1,6 @@
docker:
range: '172.17.1.0/24'
gateway: '172.17.1.1'
networks:
sobridge: {}
soauth:
range: '172.17.2.0/24'
gateway: '172.17.2.1'
manager_only: True
ulimits:
- name: nofile
soft: 1048576
@@ -64,18 +58,18 @@ docker:
ulimits: []
'so-kratos':
final_octet: 28
networks: ['soauth']
port_bindings:
- 0.0.0.0:4433:4433
- 0.0.0.0:4434:4434
custom_bind_mounts: []
extra_hosts: []
extra_env: []
ulimits: []
'so-hydra':
final_octet: 30
networks: ['soauth']
port_bindings:
- 0.0.0.0:4444:4444
- 0.0.0.0:4445:4445
custom_bind_mounts: []
extra_hosts: []
extra_env: []
@@ -134,7 +128,6 @@ docker:
ulimits: []
'so-soc':
final_octet: 34
networks: ['sobridge', 'soauth']
port_bindings:
- 0.0.0.0:9822:9822
custom_bind_mounts: []
+3 -21
View File
@@ -1,26 +1,8 @@
{% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
{% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
{% if DOCKERMERGED.networks.sobridge is not mapping %}
{% do DOCKERMERGED.networks.update({'sobridge': {}}) %}
{% endif %}
{% do DOCKERMERGED.networks['sobridge'].update({'range': DOCKERMERGED.range, 'gateway': DOCKERMERGED.gateway}) %}
{% for netname, net in DOCKERMERGED.networks.items() %}
{% set RANGESPLIT = net.range.split('.') %}
{% do net.update({'prefix': RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.'}) %}
{% endfor %}
{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
{% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}
{% for container, vals in DOCKERMERGED.containers.items() %}
{% set CONTAINER_NETS = vals.get('networks', ['sobridge']) %}
{% set IPS = {} %}
{% for netname in CONTAINER_NETS %}
{% do IPS.update({netname: DOCKERMERGED.networks[netname].prefix ~ vals.final_octet}) %}
{% endfor %}
{% do DOCKERMERGED.containers[container].update({
'networks': CONTAINER_NETS,
'ips': IPS,
'network': CONTAINER_NETS[0],
'ip': IPS[CONTAINER_NETS[0]]
}) %}
{% do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
{% endfor %}
+10 -14
View File
@@ -18,10 +18,10 @@ dockergroup:
dockerheldpackages:
pkg.installed:
- pkgs:
- containerd.io: 2.3.6-1.el9
- docker-ce: 3:29.8.1-1.el9
- docker-ce-cli: 1:29.8.1-1.el9
- docker-ce-rootless-extras: 29.8.1-1.el9
- containerd.io: 2.2.1-1.el9
- docker-ce: 3:29.2.1-1.el9
- docker-ce-cli: 1:29.2.1-1.el9
- docker-ce-rootless-extras: 29.2.1-1.el9
- hold: True
- update_holds: True
@@ -71,19 +71,15 @@ dockerreserveports:
- source: salt://common/files/99-reserved-ports.conf
- name: /etc/sysctl.d/99-reserved-ports.conf
{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
sos_docker_net_{{ NETNAME }}:
sos_docker_net:
docker_network.present:
- name: {{ NETNAME }}
- subnet: {{ NETWORK.range }}
- gateway: {{ NETWORK.gateway }}
- name: sobridge
- subnet: {{ DOCKERMERGED.range }}
- gateway: {{ DOCKERMERGED.gateway }}
- options:
com.docker.network.bridge.name: '{{ NETNAME }}'
com.docker.network.bridge.name: 'sobridge'
com.docker.network.driver.mtu: '1500'
com.docker.network.bridge.enable_ip_masquerade: 'true'
com.docker.network.bridge.enable_icc: 'true'
com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
- unless: ip l | grep {{ NETNAME }}
{% endif %}
{% endfor %}
- unless: ip l | grep sobridge
-44
View File
@@ -7,40 +7,6 @@ docker:
description: Default docker IP range for containers.
helpLink: docker
advanced: True
networks:
sobridge:
description: |
The default docker network, carrying most containers. Its range and gateway are taken
from the docker.range and docker.gateway settings above rather than set here.
helpLink: docker
readonly: True
advanced: True
global: True
soauth:
range:
description: |
IP range for the soauth docker network, an isolated network for the authentication
services, so that the Kratos and Hydra admin APIs are only reachable from the
containers placed on it.
helpLink: docker
readonly: True
advanced: True
global: True
gateway:
description: Gateway for the soauth docker network.
helpLink: docker
readonly: True
advanced: True
global: True
manager_only:
description: |
Limits the soauth network to grid members running the authentication containers,
instead of creating it on every node.
helpLink: docker
readonly: True
advanced: True
global: True
forcedType: bool
ulimits:
description: |
Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
@@ -68,16 +34,6 @@ docker:
readonly: True
advanced: True
global: True
networks:
description: |
Docker networks this container is attached to. The first entry is the container's
primary network and determines the address its published ports are forwarded to.
Defaults to sobridge when unset.
helpLink: docker
readonly: True
advanced: True
global: True
forcedType: "[]string"
port_bindings:
description: List of port bindings for the container.
helpLink: docker
+2 -3
View File
@@ -21,7 +21,6 @@ elastalert:
- gid: 933
- home: /opt/so/conf/elastalert
- createhome: False
- shell: /sbin/nologin
elastalogdir:
file.directory:
@@ -34,8 +33,8 @@ elastalert_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://elastalert/tools/sbin
- user: root
- group: root
- user: 933
- group: 939
- file_mode: 755
#elastalert_sbin_jinja:
+1 -1
View File
@@ -10,7 +10,7 @@ elastalert:
buffer_time:
minutes: 10
old_query_limit:
minutes: 1440
minutes: 5
es_port: 9200
es_conn_timeout: 55
max_query_size: 5000
+1 -6
View File
@@ -6,11 +6,9 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
include:
- ca.trust
- elastalert.config
- elastalert.sostatus
@@ -30,7 +28,6 @@ so-elastalert:
- ipv4_address: {{ DOCKERMERGED.containers['so-elastalert'].ip }}
- detach: True
- binds:
- {{ CATRUST.dir }}
- /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
- /opt/so/log/elastalert:/var/log/elastalert:rw
- /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
@@ -49,9 +46,8 @@ so-elastalert:
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
- environment:
- REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt
{% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
- environment:
{% for XTRAENV in DOCKERMERGED.containers['so-elastalert'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
@@ -69,7 +65,6 @@ so-elastalert:
- file: elastacustmodulesdir
- file: elastaconf
- watch:
- cmd: ca_trust_extract
- file: elastaconf
- onlyif:
- "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 9" {# only run this state if elasticsearch is version 9 #}
@@ -1,80 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
from datetime import datetime
import json
import logging
import sys
import types
# stand-ins when ElastAlert isn't installed (CI)
try:
import elastalert.alerts # noqa: F401
except ImportError:
class Alerter:
def __init__(self, rule):
self.rule = rule
class DateTimeEncoder(json.JSONEncoder):
def default(self, obj):
return obj.isoformat() if hasattr(obj, 'isoformat') else json.JSONEncoder.default(self, obj)
class EAException(Exception):
pass
def lookup_es_key(doc, term):
for part in term.split('.'):
if not isinstance(doc, dict) or part not in doc:
return None
doc = doc[part]
return doc
def ts_to_dt(value):
return value if isinstance(value, datetime) else datetime.fromisoformat(value)
def elasticsearch_client(conf):
return None # tests set the alerter's client
alerts = types.ModuleType('elastalert.alerts')
alerts.Alerter = Alerter
alerts.DateTimeEncoder = DateTimeEncoder
util = types.ModuleType('elastalert.util')
util.EAException = EAException
util.elastalert_logger = logging.getLogger('elastalert')
util.lookup_es_key = lookup_es_key
util.ts_to_dt = ts_to_dt
util.elasticsearch_client = elasticsearch_client
package = types.ModuleType('elastalert')
package.alerts = alerts
package.util = util
sys.modules.update({'elastalert': package, 'elastalert.alerts': alerts, 'elastalert.util': util})
# stand-ins when elasticsearch-py isn't installed (CI)
try:
import elasticsearch.exceptions # noqa: F401
except ImportError:
class ElasticsearchException(Exception):
pass
class TransportError(ElasticsearchException):
pass
class ConnectionError(TransportError):
pass
class ConflictError(TransportError):
pass
class RequestError(TransportError):
pass
exceptions = types.ModuleType('elasticsearch.exceptions')
for cls in (ElasticsearchException, TransportError, ConnectionError, ConflictError, RequestError):
setattr(exceptions, cls.__name__, cls)
es_package = types.ModuleType('elasticsearch')
es_package.exceptions = exceptions
sys.modules.update({'elasticsearch': es_package, 'elasticsearch.exceptions': exceptions})
@@ -1,269 +1,63 @@
# -*- coding: utf-8 -*-
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
from datetime import datetime, timezone
import hashlib
import ipaddress
import json
import re
import uuid
from time import gmtime, strftime
import requests,json
from elastalert.alerts import Alerter
import urllib3
from elasticsearch.exceptions import ConflictError, ElasticsearchException, RequestError
from elastalert.alerts import Alerter, DateTimeEncoder
from elastalert.util import EAException, elastalert_logger, elasticsearch_client, lookup_es_key, ts_to_dt
# grid runs verify_certs: false; also quiets ElastAlert's own queries
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
ALERT_INDEX = 'logs-detections.alerts-so'
# ES error text kept in logs and alerts
ERROR_TEXT_LIMIT = 500
# a match missing backend columns (window_start, count, @timestamp); the alert is still written
MATCH_ERRORS = (KeyError, TypeError, ValueError)
class SecurityOnionESAlerter(Alerter):
"""
Use matched data to create alerts in Elasticsearch.
"""
required_options = {'detection_title', 'sigma_level'}
optional_fields = ['sigma_category', 'sigma_product', 'sigma_service', 'sigma_correlation']
# count column and default summary per type; stored alert data, so not localized
CORRELATION_COUNTS = {
'event_count': ('event_count', '%count% events'),
'value_count': ('value_count', '%count% distinct values'),
'temporal': ('event_type_count', '%count% correlated rules matched'),
'value_sum': ('value_sum', 'total %count%'),
'value_avg': ('value_avg', 'average %count%'),
'value_percentile': ('value_percentile', 'percentile %count%'),
'value_median': ('value_median', 'median %count%'),
}
PLACEHOLDER = re.compile(r'%([^%\s]+)%')
# group-by fields copied into ECS related.*
RELATED_USERS = {'user.name', 'winlog.event_data.TargetUserName', 'winlog.event_data.SubjectUserName'}
RELATED_HOSTS = {'host.name', 'host.hostname', 'winlog.computer_name'}
def __init__(self, rule):
super().__init__(rule)
# uses the grid's TLS, auth and timeout settings
self.es = elasticsearch_client(rule)
@property
def is_correlation(self):
return bool(self.rule.get('sigma_correlation'))
def query_keys(self):
"""compound_query_key holds the list; query_key is flattened to a string."""
if self.rule.get('compound_query_key'):
return self.rule['compound_query_key']
if self.rule.get('query_key'):
return [self.rule['query_key']]
return []
def alert_id(self, match):
"""Stable id: window end + group values for correlations, source _id otherwise; random without one."""
if self.is_correlation:
# ungrouped rows have a hashed _id that changes with the count
values = ''.join(f"|{lookup_es_key(match, k)}" for k in self.query_keys())
key = f"{self.rule['detection_public_id']}|{ts_to_dt(match['@timestamp']).isoformat()}{values}"
elif match.get('_id'):
key = f"{self.rule['detection_public_id']}|{match['_id']}"
else:
return uuid.uuid4().hex
return hashlib.sha256(key.encode('utf-8')).hexdigest()
def group(self, match):
"""Group-by values, joined like ElastAlert's realert key."""
return ', '.join(str(lookup_es_key(match, k)) for k in self.query_keys())
def related_bucket(self, key):
if key == 'ip' or key.endswith('.ip'):
return 'ip'
if key in self.RELATED_USERS or key.endswith('.user.name'):
return 'user'
if key in self.RELATED_HOSTS:
return 'hosts'
return None
@staticmethod
def valid_ip(value):
try:
ipaddress.ip_address(value)
return True
except ValueError:
return False
def related(self, match):
"""ECS related.* from group-by values; skips invalid IPs."""
related = {}
for key in self.query_keys():
bucket = self.related_bucket(key)
if not bucket:
continue
# original spellings of a lowercased group
value = lookup_es_key(match, f"{key}_spellings")
if value is None:
value = lookup_es_key(match, key)
for v in value if isinstance(value, list) else [value]:
if v is None or (bucket == 'ip' and not self.valid_ip(str(v))):
continue
# dict: ordered and deduped
related.setdefault(bucket, {})[str(v)] = None
return {bucket: list(values) for bucket, values in related.items()}
def event_data(self, match):
"""The match minus the compound query_key field, which ES would map by its last part."""
if not self.rule.get('compound_query_key'):
return match
return {k: v for k, v in match.items() if k != self.rule['query_key']}
@staticmethod
def format_value(value):
if isinstance(value, list):
shown = ', '.join(str(v) for v in value[:3])
return shown if len(value) <= 3 else f"{shown} and {len(value) - 3} more"
return str(value)
@staticmethod
def format_count(value):
if isinstance(value, float) and not value.is_integer():
return f"{value:,.2f}"
if isinstance(value, (int, float)):
return f"{int(value):,}"
return str(value)
@staticmethod
def format_duration(seconds):
for unit, size in (('hour', 3600), ('minute', 60)):
if seconds >= 2 * size:
return f"{seconds // size} {unit}s"
return f"{seconds} second{'' if seconds == 1 else 's'}"
def summary(self, match):
"""One-line correlation summary."""
column, label = self.CORRELATION_COUNTS.get(self.rule['sigma_correlation'], (None, '%count%'))
start = ts_to_dt(match['window_start'])
end = ts_to_dt(match['@timestamp'])
values = {
'count': self.format_count(match.get(column)),
'start': start.strftime('%Y-%m-%d %H:%M:%S UTC'),
'end': end.strftime('%Y-%m-%d %H:%M:%S UTC'),
'duration': self.format_duration(int((end - start).total_seconds())),
}
template = self.rule.get('summary_template')
if not template:
groups = ', '.join(f"{k} %{k}%" for k in self.query_keys())
template = f"{label} for {groups} in %duration%" if groups else f"{label} in %duration%"
def fill(m):
if m[1] in values:
return values[m[1]]
value = lookup_es_key(match, m[1])
# unknown placeholders stay visible so typos show
return m[0] if value is None else self.format_value(value)
return self.PLACEHOLDER.sub(fill, template)
required_options = set(['detection_title', 'sigma_level'])
optional_fields = ['sigma_category', 'sigma_product', 'sigma_service']
def alert(self, matches):
for match in matches:
try:
alert_id = self.alert_id(match)
except MATCH_ERRORS as e:
elastalert_logger.warning("Writing alert for rule %s without a stable id, so a retry may duplicate it: %r",
self.rule['detection_public_id'], e)
alert_id = uuid.uuid4().hex
try:
self.write(alert_id, self.payload(match))
except ElasticsearchException as e:
# EAException makes ElastAlert retry
raise EAException(f"Unable to write the alert to Elasticsearch: {str(e)[:ERROR_TEXT_LIMIT]}") from e
timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
headers = {"Content-Type": "application/json"}
def payload(self, match):
rule_info = {
"name": self.rule['detection_title'],
"uuid": self.rule['detection_public_id']
}
creds = None
if 'es_username' in self.rule and 'es_password' in self.rule:
creds = (self.rule['es_username'], self.rule['es_password'])
# Add optional fields if they are present in the rule
for field in self.optional_fields:
rule_key = field.split('_')[-1] # Assumes field format "sigma_<key>"
if field in self.rule:
rule_info[rule_key] = self.rule[field]
# Start building the rule dict
rule_info = {
"name": self.rule['detection_title'],
"uuid": self.rule['detection_public_id']
}
event_info = {
"kind": "alert",
"severity": self.rule['event.severity'],
"module": self.rule['event.module'],
"dataset": self.rule['event.dataset'],
"severity_label": self.rule['sigma_level']
}
# Add optional fields if they are present in the rule
for field in self.optional_fields:
rule_key = field.split('_')[-1] # Assumes field format "sigma_<key>"
if field in self.rule:
rule_info[rule_key] = self.rule[field]
payload = {
"tags": ["alert"],
"rule": rule_info,
"event": event_info,
"sigma_level": self.rule['sigma_level'],
"event_data": self.event_data(match),
"@timestamp": datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z')
}
if self.is_correlation:
keys = self.query_keys()
try:
# built before any is added, so a failure adds none
reason = self.summary(match)
labels = {"correlation_group_by": ', '.join(keys), "correlation_group": self.group(match)} if keys else None
related = self.related(match)
except MATCH_ERRORS as e:
elastalert_logger.warning("Writing alert for rule %s without its correlation summary: %r",
self.rule['detection_public_id'], e)
else:
payload["event"]["reason"] = reason
if labels:
payload["labels"] = labels
if related:
payload["related"] = related
return payload
def write(self, alert_id, payload):
try:
self.create(alert_id, payload)
except RequestError as e:
# mapping rejections come from event_data; retry it as text
rejection = str(e)[:ERROR_TEXT_LIMIT]
try:
self.create(alert_id, self.without_event_data(payload, rejection))
except RequestError as again:
elastalert_logger.error("Dropping alert %s for rule %s, rejected by Elasticsearch even without its event data: %s; first rejection: %s",
alert_id, self.rule['detection_public_id'], str(again)[:ERROR_TEXT_LIMIT], rejection)
return
elastalert_logger.warning("Stored alert %s for rule %s with its event data as text, rejected by Elasticsearch: %s",
alert_id, self.rule['detection_public_id'], rejection)
def create(self, alert_id, payload):
try:
self.es.create(index=ALERT_INDEX, id=alert_id, body=payload)
except ConflictError:
pass # a repeat id is already stored
@staticmethod
def without_event_data(payload, rejection):
"""Moves event_data to event.original; the tag keeps Fleet's final pipeline from dropping it."""
fallback = {k: v for k, v in payload.items() if k != 'event_data'}
fallback['event'] = dict(payload['event'], original=json.dumps(payload['event_data'], cls=DateTimeEncoder))
fallback['error'] = {'message': f"event_data rejected by Elasticsearch: {rejection}"}
fallback['tags'] = payload['tags'] + ['preserve_original_event']
return fallback
# Construct the payload with the conditional rule_info
payload = {
"tags": "alert",
"rule": rule_info,
"event": {
"severity": self.rule['event.severity'],
"module": self.rule['event.module'],
"dataset": self.rule['event.dataset'],
"severity_label": self.rule['sigma_level']
},
"sigma_level": self.rule['sigma_level'],
"event_data": match,
"@timestamp": timestamp
}
url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/"
requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
def get_info(self):
return {'type': 'SecurityOnionESAlerter'}
@@ -1,260 +0,0 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
import copy
from datetime import datetime, timezone
import importlib.util
import json
import os
import unittest
from unittest.mock import MagicMock
from elasticsearch.exceptions import ConflictError, ConnectionError, RequestError
spec = importlib.util.spec_from_file_location('securityonion_es', os.path.join(os.path.dirname(__file__), 'securityonion-es.py'))
es = importlib.util.module_from_spec(spec)
spec.loader.exec_module(es)
BASE_RULE = {
'detection_title': 'Many Failed Network Logons To One Host From One Source',
'detection_public_id': '35a42db6-6629-45af-b8aa-e1fa33c28ef5',
'sigma_level': 'medium',
'sigma_correlation': 'event_count',
'event.severity': 3,
'event.module': 'sigma',
'event.dataset': 'sigma.alert',
'summary_template': '%count% failed network logons to %host.name% from %source.ip% in %duration%',
}
PLAIN_RULE = {k: v for k, v in BASE_RULE.items() if k not in ('sigma_correlation', 'summary_template')}
def correlation_match():
return {
'event_count': 3561,
'window_start': '2026-09-30T18:05:10+00:00',
'@timestamp': '2026-09-30T18:07:53+00:00',
'host': {'name': 'host-01'},
'source': {'ip': '192.0.2.10'},
'_id': '6d1c',
'num_hits': 1,
'num_matches': 1,
}
class TestSecurityOnionESAlerter(unittest.TestCase):
def creates(self, rule, match, effects=None):
"""Run alert(); return (body, id) of each create."""
alerter = es.SecurityOnionESAlerter(rule)
alerter.es = MagicMock()
alerter.es.create.side_effect = effects
alerter.alert([match])
calls = alerter.es.create.call_args_list
self.assertTrue(all(c.kwargs['index'] == 'logs-detections.alerts-so' for c in calls))
return [(json.loads(json.dumps(c.kwargs['body'], cls=es.DateTimeEncoder)), c.kwargs['id']) for c in calls]
def send(self, rule, match):
"""Run alert(); return the payload it wrote and its id."""
(payload, alert_id), = self.creates(rule, match)
return payload, alert_id
def test_compound_query_key_left_out_of_event_data(self):
rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
match = correlation_match()
match['host.name,source.ip'] = 'host-01, 192.0.2.10'
original = copy.deepcopy(match)
payload, _ = self.send(rule, match)
self.assertNotIn('host.name,source.ip', payload['event_data'])
self.assertEqual(payload['event_data']['host'], {'name': 'host-01'})
self.assertEqual(payload['event_data']['source'], {'ip': '192.0.2.10'})
self.assertEqual(payload['rule'], {'name': BASE_RULE['detection_title'], 'uuid': BASE_RULE['detection_public_id'], 'correlation': 'event_count'})
self.assertEqual(payload['labels'], {'correlation_group_by': 'host.name, source.ip', 'correlation_group': 'host-01, 192.0.2.10'})
self.assertEqual(payload['related'], {'hosts': ['host-01'], 'ip': ['192.0.2.10']})
self.assertEqual(payload['event']['kind'], 'alert')
self.assertEqual(payload['event']['reason'], '3,561 failed network logons to host-01 from 192.0.2.10 in 2 minutes')
# ElastAlert reuses the match
self.assertEqual(match, original)
def test_single_query_key(self):
rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
'user': {'name': 'user@example.invalid'}}
payload, _ = self.send(rule, match)
self.assertEqual(payload['labels'], {'correlation_group_by': 'user.name', 'correlation_group': 'user@example.invalid'})
self.assertEqual(payload['related'], {'user': ['user@example.invalid']})
self.assertEqual(payload['event']['reason'], '3 failed SOC logins for user@example.invalid')
self.assertEqual(payload['event_data'], match)
def test_related_buckets(self):
rule = dict(BASE_RULE, compound_query_key=['winlog.event_data.TargetUserName', 'source.ip', 'dns.highest_registered_domain'],
query_key='winlog.event_data.TargetUserName,source.ip,dns.highest_registered_domain')
match = correlation_match()
match.update({'winlog': {'event_data': {'TargetUserName': ['admin1', 'svc', 'admin1']}},
'source': {'ip': 'not-an-ip'}, 'dns': {'highest_registered_domain': 'example.com'}})
payload, _ = self.send(rule, match)
# deduped; invalid IP skipped; domain stays in the group only
self.assertEqual(payload['related'], {'user': ['admin1', 'svc']})
self.assertEqual(payload['labels']['correlation_group'], "['admin1', 'svc', 'admin1'], not-an-ip, example.com")
payload, _ = self.send(dict(BASE_RULE, query_key='dns.highest_registered_domain'), match)
self.assertNotIn('related', payload)
def test_related_uses_original_spellings(self):
rule = dict(BASE_RULE, query_key='user.name', summary_template=None)
match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
'user': {'name': 'admin', 'name_spellings': ['Admin', 'admin', 'ADMIN']}}
payload, _ = self.send(rule, match)
# the group shows the lowercased value; related.user finds every spelling
self.assertEqual(payload['labels']['correlation_group'], 'admin')
self.assertEqual(payload['related'], {'user': ['Admin', 'admin', 'ADMIN']})
self.assertEqual(payload['event']['reason'], '3 events for user.name admin in 8 seconds')
def test_plain_rule_has_no_correlation_fields(self):
# a query_key alone (e.g. from an override) isn't a correlation
rule = dict(PLAIN_RULE, query_key='user.name')
# ElastAlert parses @timestamp for EQL hits
match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc',
'process': {'name': 'whoami.exe'}, 'user': {'name': 'user'}}
payload, alert_id = self.send(rule, match)
alerter = es.SecurityOnionESAlerter(rule)
self.assertEqual(payload['rule'], {'name': PLAIN_RULE['detection_title'], 'uuid': PLAIN_RULE['detection_public_id']})
self.assertNotIn('labels', payload)
self.assertNotIn('related', payload)
self.assertNotIn('reason', payload['event'])
self.assertEqual(payload['event']['kind'], 'alert')
self.assertEqual(payload['event_data'], dict(match, **{'@timestamp': '2026-09-30T16:50:00+00:00'}))
self.assertEqual(alert_id, alerter.alert_id(match))
self.assertNotEqual(alerter.alert_id(match), alerter.alert_id(dict(match, _id='abd')))
self.assertNotEqual(alerter.alert_id(match), es.SecurityOnionESAlerter(dict(rule, detection_public_id='other')).alert_id(match))
# without an _id, never deduplicated
self.assertNotEqual(alerter.alert_id({'_id': None}), alerter.alert_id({'_id': None}))
def test_ungrouped_correlation_id_ignores_row_hash(self):
rule = dict(BASE_RULE, summary_template=None)
first = {k: v for k, v in correlation_match().items() if k not in ('host', 'source')}
# ES|QL hashes the row into _id, so a later count changes it
later = dict(first, event_count=3600, _id='9f2a')
payload, alert_id = self.send(rule, first)
alerter = es.SecurityOnionESAlerter(rule)
self.assertEqual(alerter.alert_id(first), alerter.alert_id(later))
self.assertNotEqual(alerter.alert_id(first), alerter.alert_id(dict(first, **{'@timestamp': '2026-09-30T18:09:00+00:00'})))
self.assertEqual(alert_id, alerter.alert_id(first))
self.assertEqual(payload['event']['reason'], '3,561 events in 2 minutes')
self.assertNotIn('labels', payload)
def test_temporal_count_column(self):
rule = dict(BASE_RULE, sigma_correlation='temporal', summary_template=None)
match = {'event_type_count': 2, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00'}
payload, _ = self.send(rule, match)
self.assertEqual(payload['event']['reason'], '2 correlated rules matched in 8 seconds')
def test_grouped_correlation_id_unchanged(self):
"""Ids of alerts already written must not change."""
rule = dict(BASE_RULE, compound_query_key=['host.name', 'source.ip'], query_key='host.name,source.ip')
key = f"{BASE_RULE['detection_public_id']}|2026-09-30T18:07:53+00:00|host-01|192.0.2.10"
self.assertEqual(es.SecurityOnionESAlerter(rule).alert_id(correlation_match()), es.hashlib.sha256(key.encode()).hexdigest())
def test_rejected_event_data_is_kept_as_text(self):
rule = dict(BASE_RULE, query_key='user.name', summary_template='%count% failed SOC logins for %user.name%')
match = {'event_count': 3, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
'user': {'name': 'user@example.invalid'}}
rejected = RequestError(400, 'document_parsing_exception', {})
(first, _), (second, _) = self.creates(rule, match, [rejected, None])
self.assertIn('event_data', first)
self.assertNotIn('event_data', second)
self.assertEqual(json.loads(second['event']['original']), match)
self.assertEqual(second['tags'], ['alert', 'preserve_original_event'])
self.assertEqual(first['tags'], ['alert'])
self.assertTrue(second['error']['message'].startswith('event_data rejected by Elasticsearch: '))
self.assertIn('document_parsing_exception', second['error']['message'])
# everything else carries over
self.assertEqual({k: v for k, v in second['event'].items() if k != 'original'}, first['event'])
changed = ('event_data', 'event', 'error', 'tags')
self.assertEqual({k: v for k, v in second.items() if k not in changed}, {k: v for k, v in first.items() if k not in changed})
def test_rejected_dates_are_kept_as_text(self):
match = {'@timestamp': datetime(2026, 9, 30, 16, 50, tzinfo=timezone.utc), '_id': 'abc'}
_, (second, _) = self.creates(PLAIN_RULE, match, [RequestError(400, 'document_parsing_exception', {}), None])
self.assertEqual(json.loads(second['event']['original']), {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'})
def test_rejected_twice_is_dropped_without_retry(self):
rejected = RequestError(400, 'document_parsing_exception', {})
match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
with self.assertLogs('elastalert', 'ERROR') as logs:
self.assertEqual(len(self.creates(PLAIN_RULE, match, [rejected, rejected])), 2)
self.assertIn('Dropping alert', logs.output[0])
def test_write_failure_is_retried(self):
match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
with self.assertRaisesRegex(es.EAException, 'Unable to write the alert to Elasticsearch'):
self.creates(PLAIN_RULE, match, [ConnectionError('N/A', 'refused', None)])
def test_repeat_id_is_ignored(self):
match = {'@timestamp': '2026-09-30T16:50:00+00:00', '_id': 'abc'}
self.assertEqual(len(self.creates(PLAIN_RULE, match, [ConflictError(409, 'version_conflict_engine_exception', {})])), 1)
def test_correlation_fields_are_optional(self):
rule = dict(BASE_RULE, query_key='source.ip')
# no window_start: the summary cannot be built
match = {k: v for k, v in correlation_match().items() if k != 'window_start'}
with self.assertLogs('elastalert', 'WARNING') as logs:
payload, _ = self.send(rule, match)
self.assertIn('without its correlation summary', logs.output[0])
self.assertEqual(payload['event']['kind'], 'alert')
self.assertNotIn('reason', payload['event'])
self.assertNotIn('labels', payload)
self.assertNotIn('related', payload)
def test_unstable_id_still_writes(self):
# no @timestamp: the window end is unknown
match = {k: v for k, v in correlation_match().items() if k not in ('@timestamp', 'window_start')}
with self.assertLogs('elastalert', 'WARNING') as logs:
payload, alert_id = self.send(BASE_RULE, match)
self.assertIn('without a stable id', logs.output[0])
self.assertEqual(len(alert_id), 32)
self.assertEqual(payload['event_data'], match)
def test_summary_formats_values(self):
rule = dict(BASE_RULE, sigma_correlation='value_avg', query_key='source.ip',
summary_template='%count% for %source.ip% to %destination.port% %no.such.field%')
match = {'value_avg': 2.5, 'window_start': '2026-09-30T16:49:52+00:00', '@timestamp': '2026-09-30T16:50:00+00:00',
'source': {'ip': '192.0.2.10'}, 'destination': {'port': [22, 80, 443, 8080, 8443]}}
payload, _ = self.send(rule, match)
self.assertEqual(payload['event']['reason'], '2.50 for 192.0.2.10 to 22, 80, 443 and 2 more %no.such.field%')
self.assertEqual(es.SecurityOnionESAlerter.format_count('n/a'), 'n/a')
def test_get_info(self):
self.assertEqual(es.SecurityOnionESAlerter(PLAIN_RULE).get_info(), {'type': 'SecurityOnionESAlerter'})
+1 -1
View File
@@ -120,7 +120,7 @@ elastalert:
helpLink: elastalert
old_query_limit:
minutes:
description: How long ElastAlert can be down, in minutes, and still resume each rule where it stopped. After a longer outage, rules restart from now and skip the gap.
description: Amount of time in minutes between queries to start at the most recently run query.
global: True
helpLink: elastalert
es_conn_timeout:
@@ -19,7 +19,6 @@ elastic-agent-pr:
- gid: 948
- home: /opt/so/conf/elastic-fleet-pr
- createhome: False
- shell: /sbin/nologin
{% else %}
@@ -6,11 +6,9 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
include:
- ca.trust
- elastic-fleet-package-registry.config
- elastic-fleet-package-registry.sostatus
@@ -36,9 +34,8 @@ so-elastic-fleet-package-registry:
{% for BINDING in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].port_bindings %}
- {{ BINDING }}
{% endfor %}
- binds:
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
{% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
- binds:
{% for BIND in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
- {{ BIND }}
{% endfor %}
@@ -55,8 +52,6 @@ so-elastic-fleet-package-registry:
- {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
wait_for_so-elastic-fleet-package-registry:
http.wait_for_successful_query:
+2 -3
View File
@@ -20,7 +20,6 @@ elastic-agent:
- gid: 949
- home: /opt/so/conf/elastic-agent
- createhome: False
- shell: /sbin/nologin
elasticagentconfdir:
file.directory:
@@ -40,8 +39,8 @@ elasticagent_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://elasticagent/tools/sbin_jinja
- user: root
- group: root
- user: 949
- group: 939
- file_mode: 755
- template: jinja
-3
View File
@@ -6,7 +6,6 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
include:
@@ -38,7 +37,6 @@ so-elastic-agent:
- {{ BINDING }}
{% endfor %}
- binds:
- {{ CATRUST.dir }}
- /opt/so/conf/elastic-agent/elastic-agent.yml:/usr/share/elastic-agent/elastic-agent.yml:ro
- /opt/so/log/elasticagent:/usr/share/elastic-agent/logs
- /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
@@ -67,7 +65,6 @@ so-elastic-agent:
- file: create-elastic-agent-config
- file: trusttheca
- watch:
- cmd: ca_trust_extract
- file: create-elastic-agent-config
- file: trusttheca
+6 -7
View File
@@ -26,14 +26,13 @@ elastic-fleet:
- gid: 947
- home: /opt/so/conf/elastic-fleet
- createhome: False
- shell: /sbin/nologin
elasticfleet_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://elasticfleet/tools/sbin
- user: root
- group: root
- user: 947
- group: 939
- file_mode: 755
- show_changes: False
@@ -41,8 +40,8 @@ elasticfleet_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://elasticfleet/tools/sbin_jinja
- user: root
- group: root
- user: 947
- group: 939
- file_mode: 755
- template: jinja
- exclude_pat:
@@ -82,8 +81,8 @@ eapackageupgrade:
file.managed:
- name: /usr/sbin/so-elastic-fleet-package-upgrade
- source: salt://elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade
- user: root
- group: root
- user: 947
- group: 939
- mode: 755
- template: jinja
-3
View File
@@ -6,7 +6,6 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
@@ -64,7 +63,6 @@ so-elastic-fleet:
- {{ BINDING }}
{% endfor %}
- binds:
- {{ CATRUST.dir }}
- /etc/pki/elasticfleet-server.crt:/etc/pki/elasticfleet-server.crt:ro
- /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
- /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
@@ -97,7 +95,6 @@ so-elastic-fleet:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: trusttheca
- x509: etc_elasticfleet_key
- x509: etc_elasticfleet_crt
@@ -29,7 +29,7 @@
"\\.gz$"
],
"include_files": [],
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- add_fields:\n target: event\n fields:\n dataset: windows.forwarded\n module: windows\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: import.evtx\n- if:\n equals:\n winlog.channel: 'Security'\n then: \n - add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n- if:\n equals:\n winlog.channel: 'Windows PowerShell'\n then: \n - add_fields:\n target: event\n fields:\n dataset: windows.powershell\n module: windows\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: event\n fields:\n dataset: system.application\n module: system\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: event\n fields:\n dataset: system.system\n module: system\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: import",
"processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- decode_json_fields:\n fields: [\"message\"]\n target: \"\"\n- drop_fields:\n fields: [\"host\"]\n ignore_missing: true\n- add_fields:\n target: data_stream\n fields:\n type: logs\n dataset: system.security\n- add_fields:\n target: event\n fields:\n dataset: system.security\n module: system\n imported: true\n- add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.security-2.22.3\n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.sysmon_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.sysmon_operational\n module: windows\n imported: true\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.sysmon_operational-3.9.0\n- if:\n equals:\n winlog.channel: 'Application'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.application\n - add_fields:\n target: event\n fields:\n dataset: system.application\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.application-2.22.3\n- if:\n equals:\n winlog.channel: 'System'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: system.system\n - add_fields:\n target: event\n fields:\n dataset: system.system\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-system.system-2.22.3\n \n- if:\n equals:\n winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n then: \n - add_fields:\n target: data_stream\n fields:\n dataset: windows.powershell_operational\n - add_fields:\n target: event\n fields:\n dataset: windows.powershell_operational\n module: windows\n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: logs-windows.powershell_operational-3.9.0\n- add_fields:\n target: data_stream\n fields:\n dataset: import",
"tags": [
"import"
],
+2 -2
View File
@@ -14,8 +14,8 @@ so-elastic-agent-install:
file.managed:
- name: /usr/sbin/so-elastic-agent-install
- source: salt://elasticfleet/tools/sbin/so-elastic-agent-install
- user: root
- group: root
- user: 947
- group: 939
- mode: 755
- show_changes: False
@@ -30,6 +30,8 @@
'azure_metrics.monitor': 'azure.monitor',
'azure_metrics.storage_account': 'azure.storage_account',
'azure_openai.metrics': 'azure.open_ai',
'beat.state': 'beats.stack_monitoring.state',
'beat.stats': 'beats.stack_monitoring.stats',
'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
@@ -38,8 +40,7 @@
'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
'kibana.stats': 'kibana.stack_monitoring.stats',
'kibana.status': 'kibana.stack_monitoring.status',
'logstash.node': 'logstash.stack_monitoring.node',
'logstash.node_cel': 'logstash.node',
'logstash.node_cel': 'logstash.stack_monitoring.node',
'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
'synthetics.browser': 'synthetics-browser',
'synthetics.browser_network': 'synthetics-browser.network',
@@ -30,56 +30,6 @@ fleet_api() {
curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
}
elastic_fleet_require_agent_policy() {
local AGENT_POLICY=$1
local POLICY_JSON
if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY") || [ -z "$POLICY_JSON" ]; then
echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
return 1
fi
if ! jq -e '.item.package_policies | type == "array"' <<<"$POLICY_JSON" >/dev/null 2>&1; then
echo "Error: Agent policy '$AGENT_POLICY' was not found or is not visible in the current Kibana space." >&2
return 1
fi
echo "$POLICY_JSON"
}
# Print the single active enrollment token for POLICY_ID.
# Exit 1: retryable (API failure, invalid response, no active token)
# Exit 2: multiple active tokens - Shouldn't get into this state without manual intervention
elastic_fleet_active_enrollment_token() {
local POLICY_ID=$1
local RESP TOKEN_COUNT API_KEY
if ! RESP=$(fleet_api "enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
echo "Error: Failed to retrieve enrollment tokens for agent policy '$POLICY_ID'." >&2
return 1
fi
if ! jq -e '.list' <<<"$RESP" >/dev/null 2>&1; then
echo "Error: Invalid enrollment token response for agent policy '$POLICY_ID'." >&2
return 1
fi
TOKEN_COUNT=$(jq --arg pid "$POLICY_ID" '[.list[] | select(.policy_id == $pid and .active == true)] | length' <<<"$RESP")
if [ "${TOKEN_COUNT:-0}" -eq 0 ]; then
echo "Error: No active enrollment token found for agent policy '$POLICY_ID'." >&2
return 1
fi
if [ "$TOKEN_COUNT" -gt 1 ]; then
echo "Error: Found $TOKEN_COUNT active enrollment tokens for agent policy '$POLICY_ID'; expected exactly one." >&2
return 2
fi
API_KEY=$(jq -r --arg pid "$POLICY_ID" '.list[] | select(.policy_id == $pid and .active == true) | .api_key' <<<"$RESP")
echo "$API_KEY"
}
# Max number of concurrent Fleet write jobs (create/update). Override via env if needed.
MAX_FLEET_JOBS=${MAX_FLEET_JOBS:-10}
@@ -112,7 +62,15 @@ elastic_fleet_load_integrations_dir() {
i=0
# Fetch the agent policy a single time; we look up integration ids locally below.
if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
if ! POLICY_JSON=$(fleet_api "agent_policies/$AGENT_POLICY"); then
echo "Error: Failed to retrieve agent policy '$AGENT_POLICY'."
rm -f "$FAIL_FILE"
rm -rf "$OUT_DIR"
return 1
fi
if ! jq -e '.item.package_policies' <<<"$POLICY_JSON" >/dev/null 2>&1; then
echo "Error: Invalid agent policy response for '$AGENT_POLICY'."
rm -f "$FAIL_FILE"
rm -rf "$OUT_DIR"
return 1
@@ -166,15 +124,9 @@ elastic_fleet_integration_check() {
JSON_STRING=$2
NAME=$(jq -r .name "$JSON_STRING")
INTEGRATION_ID=""
NAME=$(jq -r .name $JSON_STRING)
local POLICY_JSON
if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
return 1
fi
INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
}
@@ -196,16 +148,7 @@ elastic_fleet_integration_remove() {
NAME=$2
local POLICY_JSON
if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$AGENT_POLICY"); then
return 1
fi
INTEGRATION_ID=$(jq -r --arg n "$NAME" '.item.package_policies[]? | select(.name==$n) | .id' <<<"$POLICY_JSON")
if [ -z "$INTEGRATION_ID" ]; then
echo "Error: Integration '$NAME' was not found in agent policy '$AGENT_POLICY'." >&2
return 1
fi
INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$AGENT_POLICY" | jq -r '.item.package_policies[] | select(.name=="'"$NAME"'") | .id')
JSON_STRING=$( jq -n \
--arg INTEGRATIONID "$INTEGRATION_ID" \
@@ -13,10 +13,7 @@ ERROR=false
for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/elastic-defend/*.json
do
printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
if ! elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"; then
ERROR=true
continue
fi
elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
if [ -n "$INTEGRATION_ID" ]; then
printf "\n\nIntegration $NAME exists - Upgrading integration policy\n"
if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
@@ -7,35 +7,20 @@
. /usr/sbin/so-elastic-fleet-common
# Get all the fleet policies
if ! json_output=$(fleet_api "agent_policies" -H 'kbn-xsrf: true'); then
echo "Error: Failed to retrieve Fleet agent policies." >&2
exit 1
fi
if ! jq -e '.items' <<<"$json_output" >/dev/null 2>&1; then
echo "Error: Invalid Fleet agent policies response." >&2
exit 1
fi
json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true')
# Extract the IDs that start with "FleetServer_"
POLICY=$(jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id' <<<"$json_output")
POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id')
# Iterate over each ID in the POLICY variable
for POLICYNAME in $POLICY; do
printf "\nUpdating Policy: $POLICYNAME\n"
if ! POLICY_JSON=$(elastic_fleet_require_agent_policy "$POLICYNAME"); then
exit 1
fi
INTEGRATION_ID=$(jq -r '.item.package_policies[]? | select(.package.name == "fleet_server") | .id' <<<"$POLICY_JSON")
if [ -z "$INTEGRATION_ID" ]; then
echo "Error: fleet_server integration was not found in agent policy '$POLICYNAME'." >&2
exit 1
fi
# First get the Integration ID
INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$POLICYNAME" | jq -r '.item.package_policies[] | select(.package.name == "fleet_server") | .id')
# Modify the default integration policy to update the policy_id and an with the correct naming
UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
.policy_id = $policy_id |
.name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
@@ -22,19 +22,12 @@ NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
for i in {1..30}
do
ENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial")
TOKEN_RC=$?
if [ "$TOKEN_RC" -eq 2 ]; then
exit 1
fi
ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r '.item.host_urls[]' | paste -sd ',')
if [[ -n "$FLEETHOST" ]] && [[ -n "$ENROLLMENTOKEN" ]]; then
break
fi
sleep 10
if [[ $FLEETHOST ]] && [[ $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
done
if [[ -z "$FLEETHOST" ]] || [[ -z "$ENROLLMENTOKEN" ]]; then
if [[ -z $FLEETHOST ]] || [[ -z $ENROLLMENTOKEN ]]; then
printf "\nFleet Host URL, Enrollment Token or Elastic Version empty - exiting..."
printf "\nFleet Host: $FLEETHOST, Enrollment Token: $ENROLLMENTOKEN\n"
exit 1
@@ -74,25 +67,19 @@ for GOOS in "${GOTARGETOS[@]}"; do
GOARCH="amd64"
if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
if ! docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
docker run -e CGO_ENABLED=0 -e GOOS=$GOOS -e GOARCH=$GOARCH \
--mount type=bind,source=/etc/pki/tls/certs/,target=/workspace/files/cert/ \
--mount type=bind,source=/nsm/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ \
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH}; then
printf "\n### ERROR: Failed to generate $GOOS/$GOARCH installer. Exiting...\n"
exit 1
fi
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHostURLsList=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN" -o /output/so-elastic-agent_${GOOS}_${GOARCH}
printf "\n### $GOOS/$GOARCH Installer Generated...\n"
done
printf "\n\n### Generating MSI...\n"
cp /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so-elastic-agent_windows_amd64.exe
if ! docker run \
docker run \
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/,target=/output/ -w /output \
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs; then
printf "\n### ERROR: Failed to generate MSI. Exiting...\n"
exit 1
fi
{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
printf "\n### MSI Generated...\n"
# Verify installers were created
@@ -202,9 +202,26 @@ fi
### Finalization ###
# Query for Enrollment Tokens for default policies
ENDPOINTSENROLLMENTOKEN=$(elastic_fleet_active_enrollment_token "endpoints-initial") || exit 1
GRIDNODESENROLLMENTOKENGENERAL=$(elastic_fleet_active_enrollment_token "so-grid-nodes_general") || exit 1
GRIDNODESENROLLMENTOKENHEAVY=$(elastic_fleet_active_enrollment_token "so-grid-nodes_heavy") || exit 1
if ENDPOINTSENROLLMENTOKEN_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
ENDPOINTSENROLLMENTOKEN=$(echo "$ENDPOINTSENROLLMENTOKEN_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
else
echo -e "\nFailed to query for Endpoints enrollment token"
exit 1
fi
if GRIDNODESENROLLMENTOKENGENERAL_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
GRIDNODESENROLLMENTOKENGENERAL=$(echo "$GRIDNODESENROLLMENTOKENGENERAL_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_general")) | .api_key')
else
echo -e "\nFailed to query for Grid nodes - General enrollment token"
exit 1
fi
if GRIDNODESENROLLMENTOKENHEAVY_RAW=$(fleet_api "enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'); then
GRIDNODESENROLLMENTOKENHEAVY=$(echo "$GRIDNODESENROLLMENTOKENHEAVY_RAW" | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes_heavy")) | .api_key')
else
echo -e "\nFailed to query for Grid nodes - Heavy enrollment token"
exit 1
fi
# Store needed data in minion pillar
pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls
+49
View File
@@ -0,0 +1,49 @@
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states or sls in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
# Move our new CA over so Elastic and Logstash can use SSL with the internal CA
catrustdir:
file.directory:
- name: /opt/so/conf/ca
- user: 939
- group: 939
- makedirs: True
{% if GLOBALS.is_manager %}
# We have to add the Manager CA to the CA list
catrustscript:
cmd.script:
- source: salt://elasticsearch/tools/sbin_jinja/so-catrust
- template: jinja
- cwd: /opt/so
- defaults:
GLOBALS: {{ GLOBALS }}
{% endif %}
elasticsearch_cacerts:
file.managed:
- name: /opt/so/conf/ca/cacerts
- source: salt://elasticsearch/cacerts
- user: 939
- group: 939
elasticsearch_capems:
file.managed:
- name: /opt/so/conf/ca/tls-ca-bundle.pem
- source: salt://elasticsearch/tls-ca-bundle.pem
- user: 939
- group: 939
{% else %}
{{sls}}_state_not_allowed:
test.fail_without_changes:
- name: {{sls}}_state_not_allowed
{% endif %}
+10 -11
View File
@@ -32,14 +32,13 @@ elasticsearch:
- gid: 930
- home: /opt/so/conf/elasticsearch
- createhome: False
- shell: /sbin/nologin
elasticsearch_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://elasticsearch/tools/sbin
- user: root
- group: root
- user: 930
- group: 939
- file_mode: 755
- exclude_pat:
- so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
@@ -50,8 +49,8 @@ so-elasticsearch-system-indices-patch-script:
file.managed:
- name: /usr/sbin/so-elasticsearch-system-indices-patch
- source: salt://elasticsearch/tools/sbin/so-elasticsearch-system-indices-patch
- user: root
- group: root
- user: 930
- group: 939
- mode: 755
- show_changes: False
@@ -59,8 +58,8 @@ elasticsearch_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://elasticsearch/tools/sbin_jinja
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- template: jinja
- exclude_pat:
@@ -73,8 +72,8 @@ so-elasticsearch-ilm-policy-load-script:
file.managed:
- name: /usr/sbin/so-elasticsearch-ilm-policy-load
- source: salt://elasticsearch/tools/sbin_jinja/so-elasticsearch-ilm-policy-load
- user: root
- group: root
- user: 930
- group: 939
- mode: 754
- template: jinja
- defaults:
@@ -85,8 +84,8 @@ so-elasticsearch-pipelines-script:
file.managed:
- name: /usr/sbin/so-elasticsearch-pipelines
- source: salt://elasticsearch/tools/sbin/so-elasticsearch-pipelines
- user: root
- group: root
- user: 930
- group: 939
- mode: 754
- show_changes: False
+1 -7
View File
@@ -1,7 +1,7 @@
elasticsearch:
enabled: false
esheap: '600m'
version: 9.4.8
version: 9.4.5
index_clean: true
data_retention_method: DLM
vm:
@@ -1160,7 +1160,6 @@ elasticsearch:
- so-fleet_agent_id_verification-1
- so-logs-mappings
- so-logs-settings
- detections-alerts-mappings
data_stream:
allow_custom_routing: false
hidden: false
@@ -3310,7 +3309,6 @@ elasticsearch:
composed_of:
- event-mappings
- logs-system.security@package
- so-fleet_system.security_caseless-1
- logs-system.security@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -4177,7 +4175,6 @@ elasticsearch:
index_template:
composed_of:
- logs-windows.forwarded@package
- so-fleet_process_caseless-1
- logs-windows.forwarded@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -4227,7 +4224,6 @@ elasticsearch:
index_template:
composed_of:
- logs-windows.powershell@package
- so-fleet_process_caseless-1
- logs-windows.powershell@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -4277,7 +4273,6 @@ elasticsearch:
index_template:
composed_of:
- logs-windows.powershell_operational@package
- so-fleet_process_caseless-1
- logs-windows.powershell_operational@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
@@ -4327,7 +4322,6 @@ elasticsearch:
index_template:
composed_of:
- logs-windows.sysmon_operational@package
- so-fleet_process_caseless-1
- logs-windows.sysmon_operational@custom
- so-fleet_integrations.ip_mappings-1
- so-fleet_globals-1
+6 -3
View File
@@ -6,7 +6,6 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
@@ -14,6 +13,7 @@
include:
- ca
- elasticsearch.ca
- elasticsearch.ssl
- elasticsearch.config
- elasticsearch.sostatus
@@ -63,14 +63,15 @@ so-elasticsearch:
- {{ BINDING }}
{% endfor %}
- binds:
- {{ CATRUST.dir }}
- /opt/so/conf/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
- /opt/so/conf/elasticsearch/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro
- /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
- /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
- /opt/so/conf/ca/cacerts:/usr/share/elasticsearch/jdk/lib/security/cacerts:ro
- /etc/pki/tls/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
- /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
- /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
- /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
- /opt/so/conf/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles:ro
- /opt/so/conf/elasticsearch/users:/usr/share/elasticsearch/config/users:ro
{% if ELASTICSEARCHMERGED.config.path.get('repo', False) %}
@@ -84,19 +85,21 @@ so-elasticsearch:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: trusttheca
- x509: elasticsearch_crt
- x509: elasticsearch_key
- file: elasticsearch_cacerts
- file: esyml
- require:
- file: trusttheca
- x509: elasticsearch_crt
- x509: elasticsearch_key
- file: elasticsearch_cacerts
- file: esyml
- file: eslog4jfile
- file: nsmesdir
- file: eslogdir
- file: elasticp12perms
- cmd: auth_users_roles_inode
- cmd: auth_users_inode
+3 -29
View File
@@ -1,5 +1,5 @@
{
"version": 4,
"version": 3,
"_meta": {
"managed_by": "securityonion",
"managed": true
@@ -99,7 +99,7 @@
},
{
"set": {
"if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
"if": "ctx.tags != null && ctx.tags.contains('import')",
"override": true,
"field": "data_stream.dataset",
"value": "import"
@@ -107,7 +107,7 @@
},
{
"set": {
"if": "ctx.tags != null && ctx.tags.contains('import') && ctx._index != null && ctx._index.startsWith('logs-import-')",
"if": "ctx.tags != null && ctx.tags.contains('import')",
"override": true,
"field": "data_stream.namespace",
"value": "so"
@@ -177,32 +177,6 @@
"description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
}
},
{
"trim": {
"if": "ctx.event?.dataset == 'system.auth'",
"description": "Grok leaves a leading space on 'invalid user' names (elastic/integrations#12174) and, before 2.23.2, sudo padding",
"field": "user.name",
"ignore_missing": true,
"ignore_failure": true
}
},
{
"trim": {
"if": "ctx.event?.dataset == 'system.auth'",
"description": "Appended from the untrimmed user.name",
"field": "related.user",
"ignore_missing": true,
"ignore_failure": true
}
},
{
"script": {
"if": "ctx.event?.dataset == 'system.auth' && ctx.related?.user instanceof List",
"description": "Dedupe after trimming",
"source": "ctx.related.user = new ArrayList(new LinkedHashSet(ctx.related.user));",
"ignore_failure": true
}
},
{
"script": {
"description": "Snapshot event.ingested into _tmp.event_ingested_pre_fleet before .fleet_final_pipeline-1 overwrites it with ES ingest time",
@@ -1,31 +0,0 @@
{
"description" : "import.evtx: normalize imported EVTX and reroute to logs-<dataset>-import",
"processors" : [
{ "script": {
"description": "Host from the event, not the importing node",
"lang": "painless",
"source": "Map host = ['os': ['type': 'windows', 'family': 'windows', 'platform': 'windows']]; def cn = ctx.winlog?.computer_name; if (cn != null && cn.toString().length() > 0) { String name = cn.toString(); int dot = name.indexOf('.'); if (dot > 0) { name = name.substring(0, dot); } host.put('hostname', name); host.put('name', name.toLowerCase()); } ctx.host = host;"
} },
{ "script": {
"description": "String event IDs, as Winlogbeat sends",
"lang": "painless",
"source": "if (ctx.winlog?.event_id != null) { ctx.winlog.event_id = ctx.winlog.event_id.toString(); } if (ctx.event?.code != null) { ctx.event.code = ctx.event.code.toString(); }"
} },
{ "script": {
"description": "Unnamed <Data> to param1..N, as Winlogbeat",
"lang": "painless",
"if": "ctx.winlog?.event_data?.Data instanceof Map && ctx.winlog.event_data.Data['#text'] != null",
"source": "def t = ctx.winlog.event_data.Data['#text']; List vals = t instanceof List ? t : [t]; for (int i = 0; i < vals.size(); i++) { ctx.winlog.event_data['param' + (i + 1)] = vals.get(i); } ctx.winlog.event_data.remove('Data');"
} },
{ "script": {
"description": "String values and LF line endings, as Winlogbeat",
"lang": "painless",
"if": "ctx.winlog?.event_data instanceof Map || ctx.winlog?.user_data instanceof Map",
"source": "String lf = String.valueOf((char) 10); String crlf = String.valueOf((char) 13) + lf; for (def key : ['event_data', 'user_data']) { def m = ctx.winlog[key]; if (!(m instanceof Map)) { continue; } for (def e : m.entrySet()) { def v = e.getValue(); if (v instanceof String) { e.setValue(v.replace(crlf, lf)); } else if (v instanceof Number || v instanceof Boolean) { e.setValue(v.toString()); } } }"
} },
{ "set": { "description": "event.kind, as Winlogbeat", "field": "event.kind", "value": "event", "override": false } },
{ "set": { "field": "data_stream.dataset", "copy_from": "event.dataset", "override": true, "ignore_empty_value": true } },
{ "set": { "field": "data_stream.namespace", "value": "import", "override": true } },
{ "reroute": { "dataset": "{{data_stream.dataset}}", "namespace": "{{data_stream.namespace}}" } }
]
}
@@ -5,8 +5,7 @@
{ "rename": { "field": "message2.proto", "target_field": "network.transport", "ignore_missing": true } },
{ "rename": { "field": "message2.app_proto", "target_field": "network.protocol", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.filename", "target_field": "file.name", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.gaps", "target_field": "suricata.fileinfo.gaps", "ignore_missing": true } },
{ "set": { "if": "ctx.suricata?.fileinfo?.gaps == false", "field": "file.bytes.missing", "value": 0 } },
{ "rename": { "field": "message2.fileinfo.gaps", "target_field": "file.bytes.missing", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.magic", "target_field": "file.mime_type", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.md5", "target_field": "hash.md5", "ignore_missing": true } },
{ "rename": { "field": "message2.fileinfo.sha1", "target_field": "hash.sha1", "ignore_missing": true } },
+11
View File
@@ -38,6 +38,10 @@ elasticsearch_crt:
- retry:
attempts: 5
interval: 30
cmd.run:
- name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:"
- onchanges:
- x509: /etc/pki/elasticsearch.key
elastickeyperms:
file.managed:
@@ -46,6 +50,13 @@ elastickeyperms:
- mode: 640
- group: 930
elasticp12perms:
file.managed:
- replace: False
- name: /etc/pki/elasticsearch.p12
- mode: 640
- group: 930
{% else %}
{{sls}}_state_not_allowed:
@@ -1,123 +0,0 @@
{
"_meta": {
"managed_by": "security_onion",
"managed": true,
"description": "Adds .caseless for Lucene queries. Restates each field's package type and .text."
},
"template": {
"mappings": {
"properties": {
"process": {
"properties": {
"executable": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"name": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"command_line": {
"type": "wildcard",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"parent": {
"properties": {
"executable": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"name": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
},
"command_line": {
"type": "wildcard",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
}
}
}
}
},
"file": {
"properties": {
"path": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
},
"text": {
"type": "match_only_text"
}
}
}
}
}
}
}
}
}
@@ -1,80 +0,0 @@
{
"_meta": {
"managed_by": "security_onion",
"managed": true,
"description": "Adds .caseless for Lucene queries. Keeps each field's existing keyword type."
},
"template": {
"mappings": {
"properties": {
"process": {
"properties": {
"command_line": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
},
"parent": {
"properties": {
"executable": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
},
"name": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
},
"command_line": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
}
}
}
}
},
"file": {
"properties": {
"path": {
"type": "keyword",
"ignore_above": 1024,
"fields": {
"caseless": {
"type": "keyword",
"ignore_above": 1024,
"normalizer": "lowercase"
}
}
}
}
}
}
}
}
}
@@ -154,20 +154,6 @@
}
}
}
},
"runtime": {
"so_detection.sigmaKind": {
"type": "keyword",
"script": {
"source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); emit(c.startsWith('correlation:') || c.contains(NL + 'correlation:') ? 'correlation' : 'single');"
}
},
"so_detection.sigmaCorrelationType": {
"type": "keyword",
"script": {
"source": "if (doc['so_detection.language'].size() == 0 || doc['so_detection.language'].value != 'sigma') { return; } def d = params._source.so_detection; if (d == null || d.content == null) { return; } String c = d.content; String NL = String.valueOf((char) 10); int i = c.startsWith('correlation:') ? 0 : c.indexOf(NL + 'correlation:'); if (i < 0) { return; } i = c.indexOf(NL, i + 1); while (i >= 0 && i + 1 < c.length()) { int e = c.indexOf(NL, i + 1); String line = e < 0 ? c.substring(i + 1) : c.substring(i + 1, e); if (line.length() > 0 && !Character.isWhitespace(line.charAt(0)) && line.charAt(0) != (char) 35) { return; } String t = line.trim(); if (t.startsWith('type:')) { String v = t.substring(5).trim(); int h = v.indexOf(' ' + String.valueOf((char) 35)); if (h >= 0) { v = v.substring(0, h).trim(); } if (v.length() > 1 && (v.charAt(0) == (char) 39 || v.charAt(0) == (char) 34)) { v = v.substring(1, v.length() - 1); } emit(v); return; } i = e; }"
}
}
}
}
},
@@ -1,149 +0,0 @@
{
"template": {
"mappings": {
"properties": {
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"sigma_level": {
"ignore_above": 1024,
"type": "keyword"
},
"rule": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"uuid": {
"ignore_above": 1024,
"type": "keyword"
},
"category": {
"ignore_above": 1024,
"type": "keyword"
},
"product": {
"ignore_above": 1024,
"type": "keyword"
},
"service": {
"ignore_above": 1024,
"type": "keyword"
},
"correlation": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"event": {
"properties": {
"severity": {
"type": "long"
},
"severity_label": {
"ignore_above": 1024,
"type": "keyword"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"dataset": {
"ignore_above": 1024,
"type": "keyword"
},
"kind": {
"ignore_above": 1024,
"type": "keyword"
},
"reason": {
"type": "match_only_text",
"fields": {
"keyword": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"original": {
"type": "keyword",
"index": false,
"doc_values": false
}
}
},
"event_data": {
"properties": {
"@timestamp": {
"type": "date"
},
"window_start": {
"type": "date"
},
"event_count": {
"type": "long"
},
"value_count": {
"type": "long"
},
"event_type_count": {
"type": "long"
},
"value_sum": {
"type": "double"
},
"value_avg": {
"type": "double"
},
"value_percentile": {
"type": "double"
},
"value_median": {
"type": "double"
}
}
},
"labels": {
"properties": {
"correlation_group_by": {
"ignore_above": 1024,
"type": "keyword"
},
"correlation_group": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"related": {
"properties": {
"ip": {
"type": "ip"
},
"user": {
"ignore_above": 1024,
"type": "keyword"
},
"hosts": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"properties": {
"message": {
"type": "match_only_text"
}
}
}
}
}
},
"_meta": {
"description": "Fields written by the ElastAlert SecurityOnionESAlerter to logs-detections.alerts-so"
}
}
@@ -0,0 +1,26 @@
#!/bin/bash
#
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
get_elastic_agent_vars
# Exit on errors, since all lines must succeed
set -e
# Check to see if we have extracted the ca cert.
if [ ! -f /opt/so/saltstack/local/salt/elasticsearch/cacerts ]; then
docker run -v /etc/pki/ca.crt:/etc/ssl/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:$ELASTIC_AGENT_TARBALL_VERSION -keystore /usr/share/elasticsearch/jdk/lib/security/cacerts -alias SOSCA -import -file /etc/ssl/ca.crt -storepass changeit -noprompt
# Make sure symbolic links are followed when copying from container
docker cp -L so-elasticsearchca:/usr/share/elasticsearch/jdk/lib/security/cacerts /opt/so/saltstack/local/salt/elasticsearch/cacerts
docker cp -L so-elasticsearchca:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
docker rm so-elasticsearchca
echo "" >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
echo "sosca" >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
cat /etc/pki/ca.crt >> /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
else
exit 0
fi
+14 -33
View File
@@ -4,19 +4,11 @@
{%- set role = GLOBALS.role.split('-')[1] %}
{%- from 'firewall/containers.map.jinja' import NODE_CONTAINERS %}
{%- set NODE_NETWORKS = [] %}
{%- for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
{%- if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
{%- do NODE_NETWORKS.append(NETNAME) %}
{%- endif %}
{%- endfor %}
{%- set PR = [] %}
{%- set D1 = [] %}
{%- set D2 = [] %}
{%- for container in NODE_CONTAINERS %}
{%- set IP = DOCKERMERGED.containers[container].ip %}
{%- set BRIDGE = DOCKERMERGED.containers[container].network %}
{%- if DOCKERMERGED.containers[container].port_bindings is defined %}
{%- for binding in DOCKERMERGED.containers[container].port_bindings %}
{#- cant split int so we convert to string #}
@@ -43,11 +35,11 @@
{%- endif %}
{%- do PR.append("-A POSTROUTING -s " ~ DOCKERMERGED.containers[container].ip ~ "/32 -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j MASQUERADE") %}
{%- if bindip | length and bindip != '0.0.0.0' %}
{%- do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
{%- do D1.append("-A DOCKER -d " ~ bindip ~ "/32 ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
{%- else %}
{%- do D1.append("-A DOCKER ! -i " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
{%- do D1.append("-A DOCKER ! -i sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ hostPort ~ " -j DNAT --to-destination " ~ DOCKERMERGED.containers[container].ip ~ ":" ~ containerPort) %}
{%- endif %}
{%- do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i " ~ BRIDGE ~ " -o " ~ BRIDGE ~ " -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
{%- do D2.append("-A DOCKER -d " ~ DOCKERMERGED.containers[container].ip ~ "/32 ! -i sobridge -o sobridge -p " ~ proto ~ " -m " ~ proto ~ " --dport " ~ containerPort ~ " -j ACCEPT") %}
{%- endfor %}
{%- endif %}
{%- endfor %}
@@ -60,15 +52,11 @@
:DOCKER - [0:0]
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
{%- for NETNAME in NODE_NETWORKS %}
-A POSTROUTING -s {{ DOCKERMERGED.networks[NETNAME].range }} ! -o {{ NETNAME }} -j MASQUERADE
{%- endfor %}
-A POSTROUTING -s {{DOCKERMERGED.range}} ! -o sobridge -j MASQUERADE
{%- for rule in PR %}
{{ rule }}
{%- endfor %}
{%- for NETNAME in NODE_NETWORKS %}
-A DOCKER -i {{ NETNAME }} -j RETURN
{%- endfor %}
-A DOCKER -i sobridge -j RETURN
{%- for rule in D1 %}
{{ rule }}
{%- endfor %}
@@ -109,12 +97,10 @@ COMMIT
{%- endif %}
-A FORWARD -j DOCKER-USER
-A FORWARD -j DOCKER-ISOLATION-STAGE-1
{%- for NETNAME in NODE_NETWORKS %}
-A FORWARD -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -o {{ NETNAME }} -j DOCKER
-A FORWARD -i {{ NETNAME }} ! -o {{ NETNAME }} -j ACCEPT
-A FORWARD -i {{ NETNAME }} -o {{ NETNAME }} -j ACCEPT
{%- endfor %}
-A FORWARD -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -o sobridge -j DOCKER
-A FORWARD -i sobridge ! -o sobridge -j ACCEPT
-A FORWARD -i sobridge -o sobridge -j ACCEPT
-A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i lo -j ACCEPT
-A FORWARD -m conntrack --ctstate INVALID -j DROP
@@ -126,18 +112,13 @@ COMMIT
{%- for rule in D2 %}
{{ rule }}
{%- endfor %}
{% for NETNAME in NODE_NETWORKS %}
-A DOCKER-ISOLATION-STAGE-1 -i {{ NETNAME }} ! -o {{ NETNAME }} -j DOCKER-ISOLATION-STAGE-2
{%- endfor %}
-A DOCKER-ISOLATION-STAGE-1 -i sobridge ! -o sobridge -j DOCKER-ISOLATION-STAGE-2
-A DOCKER-ISOLATION-STAGE-1 -j RETURN
{%- for NETNAME in NODE_NETWORKS %}
-A DOCKER-ISOLATION-STAGE-2 -o {{ NETNAME }} -j DROP
{%- endfor %}
-A DOCKER-ISOLATION-STAGE-2 -o sobridge -j DROP
-A DOCKER-ISOLATION-STAGE-2 -j RETURN
{%- for NETNAME in NODE_NETWORKS %}
-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A DOCKER-USER ! -i {{ NETNAME }} -o {{ NETNAME }} -j LOGGING
{%- endfor %}
-A DOCKER-USER ! -i sobridge -o sobridge -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A DOCKER-USER ! -i sobridge -o sobridge -j LOGGING
-A DOCKER-USER -j RETURN
-A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables-dropped: "
-A LOGGING -j DROP
+2 -6
View File
@@ -4,12 +4,8 @@
{# add our ip to self #}
{% do FIREWALL_DEFAULT.firewall.hostgroups.self.append(GLOBALS.node_ip) %}
{# add dockernet ranges #}
{% for NETNAME, NETWORK in DOCKERMERGED.networks.items() %}
{% if not NETWORK.get('manager_only') or GLOBALS.get('is_manager', False) %}
{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(NETWORK.range) %}
{% endif %}
{% endfor %}
{# add dockernet range #}
{% do FIREWALL_DEFAULT.firewall.hostgroups.dockernet.append(DOCKERMERGED.range) %}
{% if GLOBALS.role == 'so-idh' %}
{% from 'idh/opencanary_config.map.jinja' import IDH_PORTGROUPS %}
+3 -7
View File
@@ -13,11 +13,9 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% if 'api' in salt['pillar.get']('features', []) %}
include:
- ca.trust
- hydra.config
- hydra.sostatus
@@ -28,10 +26,9 @@ so-hydra:
- hostname: hydra
- name: so-hydra
- networks:
- soauth:
- ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}
- sobridge:
- ipv4_address: {{ DOCKERMERGED.containers['so-hydra'].ip }}
- binds:
- {{ CATRUST.dir }}
- /opt/so/conf/hydra/:/hydra-conf:ro
- /opt/so/log/hydra/:/hydra-log:rw
- /nsm/hydra/db:/hydra-data:rw
@@ -63,7 +60,6 @@ so-hydra:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: hydraconfig
- require:
- file: hydraconfig
@@ -77,7 +73,7 @@ delete_so-hydra_so-status.disabled:
wait_for_hydra:
http.wait_for_successful_query:
- name: 'http://{{ DOCKERMERGED.containers['so-hydra'].ips['soauth'] }}:4444/health/alive'
- name: 'http://{{ GLOBALS.manager }}:4444/health/alive'
- ssl: True
- verify_ssl: False
- status:
-4
View File
@@ -21,16 +21,12 @@ hypervisor_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://hypervisor/tools/sbin
- user: root
- group: root
- file_mode: 744
hypervisor_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://hypervisor/tools/sbin_jinja
- user: root
- group: root
- template: jinja
- file_mode: 744
+2 -2
View File
@@ -86,8 +86,8 @@ idh_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://idh/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
#idh_sbin_jinja:
+4 -9
View File
@@ -6,11 +6,9 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
include:
- ca.trust
- idh.config
- idh.sostatus
@@ -22,7 +20,6 @@ so-idh:
- detach: True
- network_mode: host
- binds:
- {{ CATRUST.dir }}
- /nsm/idh:/var/tmp:rw
- /opt/so/conf/idh/http-skins:/opt/opencanary/http-skins:ro
- /opt/so/conf/idh/opencanary.conf:/etc/opencanaryd/opencanary.conf:ro
@@ -37,13 +34,12 @@ so-idh:
- {{ XTRAHOST }}
{% endfor %}
{% endif %}
{% if DOCKERMERGED.containers['so-idh'].extra_env %}
- environment:
- REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt
{% if DOCKERMERGED.containers['so-idh'].extra_env %}
{% for XTRAENV in DOCKERMERGED.containers['so-idh'].extra_env %}
{% for XTRAENV in DOCKERMERGED.containers['so-idh'].extra_env %}
- {{ XTRAENV }}
{% endfor %}
{% endif %}
{% endfor %}
{% endif %}
{% if DOCKERMERGED.containers['so-idh'].ulimits %}
- ulimits:
{% for ULIMIT in DOCKERMERGED.containers['so-idh'].ulimits %}
@@ -51,7 +47,6 @@ so-idh:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: opencanary_config
- require:
- file: opencanary_config
+2 -2
View File
@@ -41,8 +41,8 @@ influxdb_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://influxdb/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
#influxdb_sbin_jinja:
+1 -7
View File
@@ -6,13 +6,11 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
{% set TOKEN = salt['pillar.get']('influxdb:token') %}
include:
- ca.trust
- influxdb.ssl
- influxdb.config
- influxdb.sostatus
@@ -40,7 +38,6 @@ so-influxdb:
{% endfor %}
{% endif %}
- binds:
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
- /opt/so/log/influxdb/:/log:rw
- /opt/so/conf/influxdb/config.yaml:/conf/config.yaml:ro
- /opt/so/conf/influxdb/etc:/etc/influxdb2:rw
@@ -69,7 +66,6 @@ so-influxdb:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: influxdbconf
- x509: influxdb_key
- x509: influxdb_crt
@@ -98,11 +94,9 @@ metrics_link_file:
- docker_container: so-influxdb
# Install cron job to determine size of influxdb for telegraf
# telegraf reads this while the cron rewrites it, so write aside and rename rather than
# truncating in place. tgraflogdir recurses ownership, so the temp file is chowned to match
get_influxdb_size:
cron.present:
- name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log.tmp 2>&1; chown 939:939 /opt/so/log/telegraf/influxdb_size.log.tmp; mv -f /opt/so/log/telegraf/influxdb_size.log.tmp /opt/so/log/telegraf/influxdb_size.log'
- name: 'du -s -k /nsm/influxdb | cut -f1 > /opt/so/log/telegraf/influxdb_size.log 2>&1'
- identifier: get_influxdb_size
- user: root
- minute: '*/1'
+4 -5
View File
@@ -21,7 +21,6 @@ kafka_user:
- gid: 960
- home: /opt/so/conf/kafka
- createhome: False
- shell: /sbin/nologin
kafka_home_dir:
file.absent:
@@ -31,16 +30,16 @@ kafka_sbin_tools:
file.recurse:
- name: /usr/sbin
- source: salt://kafka/tools/sbin
- user: root
- group: root
- user: 960
- group: 960
- file_mode: 755
kafka_sbin_jinja_tools:
file.recurse:
- name: /usr/sbin
- source: salt://kafka/tools/sbin_jinja
- user: root
- group: root
- user: 960
- group: 960
- file_mode: 755
- template: jinja
- defaults:
-4
View File
@@ -12,14 +12,12 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% set KAFKANODES = salt['pillar.get']('kafka:nodes') %}
{% set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
{% if 'gmd' in salt['pillar.get']('features', []) %}
include:
- ca.trust
- kafka.ca
- kafka.config
- kafka.ssl
@@ -53,7 +51,6 @@ so-kafka:
- {{ BINDING }}
{% endfor %}
- binds:
- {{ CATRUST.java }}:/opt/java/openjdk/lib/security/cacerts:ro
- /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro
- /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
- /nsm/kafka/data/:/nsm/kafka/data/:rw
@@ -71,7 +68,6 @@ so-kafka:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
{% for sc in ['server', 'client'] %}
- file: kafka_kraft_{{sc}}_properties
{% endfor %}
+4 -5
View File
@@ -22,7 +22,6 @@ kibana:
- gid: 932
- home: /opt/so/conf/kibana
- createhome: False
- shell: /sbin/nologin
# Drop the correct nginx config based on role
@@ -37,16 +36,16 @@ kibana_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://kibana/tools/sbin
- user: root
- group: root
- user: 932
- group: 939
- file_mode: 755
kibana_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://kibana/tools/sbin_jinja
- user: root
- group: root
- user: 932
- group: 939
- file_mode: 755
- template: jinja
- defaults:
+1 -1
View File
@@ -22,7 +22,7 @@ kibana:
- default
- file
migrations:
discardCorruptObjects: "9.4.8"
discardCorruptObjects: "9.4.5"
telemetry:
enabled: False
xpack:
-5
View File
@@ -8,10 +8,8 @@
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
include:
- ca.trust
- kibana.config
- kibana.sostatus
@@ -29,7 +27,6 @@ so-kibana:
- ELASTICSEARCH_HOST={{ GLOBALS.manager }}
- ELASTICSEARCH_PORT=9200
- MANAGER={{ GLOBALS.manager }}
- NODE_OPTIONS=--use-openssl-ca
{% if DOCKERMERGED.containers['so-kibana'].extra_env %}
{% for XTRAENV in DOCKERMERGED.containers['so-kibana'].extra_env %}
- {{ XTRAENV }}
@@ -43,7 +40,6 @@ so-kibana:
{% endfor %}
{% endif %}
- binds:
- {{ CATRUST.dir }}
- /opt/so/conf/kibana/etc:/usr/share/kibana/config:rw
- /opt/so/log/kibana:/var/log/kibana:rw
- /opt/so/conf/kibana/customdashboards:/usr/share/kibana/custdashboards:ro
@@ -64,7 +60,6 @@ so-kibana:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: kibanaconfig
wait_for_so-kibana:
-1
View File
@@ -27,7 +27,6 @@ kratos:
- uid: 928
- gid: 928
- home: /opt/so/conf/kratos
- shell: /sbin/nologin
kratosdir:
file.directory:
+3 -7
View File
@@ -7,10 +7,8 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
include:
- ca.trust
- kratos.config
- kratos.sostatus
@@ -21,10 +19,9 @@ so-kratos:
- hostname: kratos
- name: so-kratos
- networks:
- soauth:
- ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}
- sobridge:
- ipv4_address: {{ DOCKERMERGED.containers['so-kratos'].ip }}
- binds:
- {{ CATRUST.dir }}
- /opt/so/conf/kratos/:/kratos-conf:ro
- /opt/so/log/kratos/:/kratos-log:rw
- /nsm/kratos/db:/kratos-data:rw
@@ -56,7 +53,6 @@ so-kratos:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: kratosschema
- file: kratosconfig
- require:
@@ -75,7 +71,7 @@ delete_so-kratos_so-status.disabled:
wait_for_kratos:
http.wait_for_successful_query:
- name: 'http://{{ DOCKERMERGED.containers['so-kratos'].ips['soauth'] }}:4434/'
- name: 'http://{{ GLOBALS.manager }}:4434/'
- ssl: True
- verify_ssl: False
- status:
-2
View File
@@ -6,8 +6,6 @@ so-fix-salt-ldap_script:
file.managed:
- name: /usr/sbin/so-fix-salt-ldap.py
- source: salt://libvirt/64962/scripts/so-fix-salt-ldap.py
- user: root
- group: root
- mode: 744
fix-salt-ldap:
+2 -3
View File
@@ -35,14 +35,13 @@ logstash:
- uid: 931
- gid: 931
- home: /opt/so/conf/logstash
- shell: /sbin/nologin
logstash_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://logstash/tools/sbin
- user: root
- group: root
- user: 931
- group: 939
- file_mode: 755
#logstash_sbin_jinja:
+15 -4
View File
@@ -6,7 +6,6 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'logstash/map.jinja' import LOGSTASH_MERGED %}
{% from 'logstash/map.jinja' import LOGSTASH_NODES %}
@@ -14,6 +13,9 @@
include:
- ca
{% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
- elasticsearch.ca
{% endif %}
{# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #}
{% if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managerhype', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %}
- kafka.ca
@@ -56,8 +58,6 @@ so-logstash:
- {{ BINDING }}
{% endfor %}
- binds:
- {{ CATRUST.dir }}
- {{ CATRUST.java }}:/usr/share/logstash/jdk/lib/security/cacerts:ro
- /opt/so/conf/elasticsearch/templates/:/templates/:ro
- /opt/so/conf/logstash/etc/:/usr/share/logstash/config/:ro
- /opt/so/conf/logstash/pipelines:/usr/share/logstash/pipelines:ro
@@ -78,6 +78,10 @@ so-logstash:
- /etc/pki/filebeat.p8:/usr/share/logstash/filebeat.key:ro
{% endif %}
{% endif %}
{% if GLOBALS.role not in ['so-receiver','so-fleet'] %}
- /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro
- /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro
{% endif %}
{% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone', 'so-searchnode'] %}
- /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro
- /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro
@@ -100,11 +104,14 @@ so-logstash:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: lsetcsync
- file: lslog4j2
- file: lspipelinesyml
- file: trusttheca
{% if GLOBALS.is_manager %}
- file: elasticsearch_cacerts
- file: elasticsearch_capems
{% endif %}
{% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
- x509: etc_elasticfleet_logstash_crt
- x509: etc_elasticfleet_logstash_key
@@ -126,6 +133,10 @@ so-logstash:
{% endif %}
- require:
- file: trusttheca
{% if GLOBALS.is_manager %}
- file: elasticsearch_cacerts
- file: elasticsearch_capems
{% endif %}
{% if GLOBALS.is_manager or GLOBALS.role in ['so-fleet', 'so-heavynode', 'so-receiver'] %}
- x509: etc_elasticfleet_logstash_crt
- x509: etc_elasticfleet_logstash_key
+8 -12
View File
@@ -113,8 +113,8 @@ manager_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://manager/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- exclude_pat:
- "*_test.py"
@@ -124,8 +124,8 @@ manager_sbin_jinja:
file.recurse:
- name: /usr/sbin/
- source: salt://manager/tools/sbin_jinja/
- user: root
- group: root
- user: socore
- group: socore
- file_mode: 755
- template: jinja
- show_changes: False
@@ -166,7 +166,7 @@ so-repo-sync:
so_fleetagent_status:
cron.present:
- name: '/usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log.tmp 2>&1; mv -f /opt/so/log/agents/agentstatus.log.tmp /opt/so/log/agents/agentstatus.log'
- name: /usr/sbin/so-elasticagent-status > /opt/so/log/agents/agentstatus.log 2>&1
- identifier: so_fleetagent_status
- user: root
- minute: '*/5'
@@ -190,15 +190,11 @@ so_fleetagent_monitor:
- month: '*'
- dayweek: '*'
# This tree is the source of every root-executed script (/usr/sbin, reactors, _runners,
# engines, salt-relay.sh). SOC mounts /opt/so/saltstack rw as uid 939 but only writes
# under local/. Do not add dir_mode/file_mode here -- SOC reads default/ and 750/640
# would break its config load.
root_own_saltstack_default:
socore_own_saltstack_default:
file.directory:
- name: /opt/so/saltstack/default
- user: root
- group: root
- user: socore
- group: socore
- recurse:
- user
- group
+8 -13
View File
@@ -106,8 +106,7 @@ while [[ $# -gt 0 ]]; do
esac
done
hydraContainer=${HYDRA_CONTAINER:-so-hydra}
hydraUrl=${HYDRA_URL:-http://localhost:4445}
hydraUrl=${HYDRA_URL:-http://127.0.0.1:4445}
socRolesFile=${SOC_ROLES_FILE:-/opt/so/conf/soc/soc_clients_roles}
soUID=${SOCORE_UID:-939}
soGID=${SOCORE_GID:-939}
@@ -125,10 +124,6 @@ function fail() {
exit 1
}
function hydraCurl() {
docker exec "$hydraContainer" curl "$@"
}
function require() {
cmd=$1
which "$1" 2>&1 > /dev/null
@@ -138,8 +133,8 @@ function require() {
# Verify this environment is capable of running this script
function verifyEnvironment() {
require "jq"
require "docker"
response=$(hydraCurl -Ss -L ${hydraUrl}/health/alive)
require "curl"
response=$(curl -Ss -L ${hydraUrl}/health/alive)
[[ "$response" != '{"status":"ok"}' ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
}
@@ -169,7 +164,7 @@ function ensureRoleFileExists() {
}
function listClients() {
response=$(hydraCurl -Ss -L -f ${hydraUrl}/admin/clients)
response=$(curl -Ss -L -f ${hydraUrl}/admin/clients)
[[ $? != 0 ]] && fail "Unable to communicate with Hydra"
clientIds=$(echo "${response}" | jq -r ".[] | .client_id" | sort)
@@ -256,7 +251,7 @@ function createClient() {
EOF
)
response=$(hydraCurl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
response=$(curl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
if [[ $? != 0 ]]; then
error=$(echo $response | jq .error)
fail "Failed to submit request to Hydra: $error"
@@ -288,7 +283,7 @@ function update() {
EOF
)
response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
if [[ $? != 0 ]]; then
error=$(echo $response | jq .error)
fail "Failed to submit request to Hydra: $error"
@@ -310,7 +305,7 @@ function generateSecret() {
EOF
)
response=$(hydraCurl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
if [[ $? != 0 ]]; then
error=$(echo $response | jq .error)
fail "Failed to submit request to Hydra: $error"
@@ -322,7 +317,7 @@ function deleteClient() {
[[ ${identityId} == "" ]] && fail "Client not found"
response=$(hydraCurl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
response=$(curl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
if [[ $? != 0 ]]; then
error=$(echo $response | jq .error)
fail "Failed to submit request to Hydra: $error"
+6 -84
View File
@@ -121,14 +121,8 @@ for i in "$@"; do
esac
done
if [[ -n "$MINION_ID" && ! "$MINION_ID" =~ ^[A-Za-z0-9._-]{1,253}$ ]]; then
echo "Invalid minion id: $MINION_ID"
log "ERROR" "Invalid minion id: $MINION_ID"
exit 1
fi
readonly PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
readonly ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
function getinstallinfo() {
log "INFO" "Getting install info for minion $MINION_ID"
@@ -139,23 +133,10 @@ function getinstallinfo() {
return 1
fi
# install.txt is controlled by the minion; only accept known keys and never eval or export them
local line key
while IFS= read -r line; do
[[ "$line" == *=* ]] || continue
key=${line%%=*}
case "$key" in
MAINIP|MNIC|NODE_DESCRIPTION|ES_HEAP_SIZE|PATCHSCHEDULENAME|INTERFACE|NODETYPE|CORECOUNT|LSHOSTNAME|LSHEAP|CPUCORES|IDH_MGTRESTRICT|IDH_SERVICES)
printf -v "$key" '%s' "${line#*=}"
;;
*)
log "WARN" "Ignoring unexpected install var from $MINION_ID: ${key:0:64}"
;;
esac
done <<< "$INSTALLVARS"
if [[ "$NODE_DESCRIPTION" == \'*\' ]]; then
NODE_DESCRIPTION=${NODE_DESCRIPTION:1:-1}
while read -r var; do export "$var"; done <<< "$INSTALLVARS"
if [ $? -ne 0 ]; then
log "ERROR" "Failed to source install variables"
return 1
fi
log "INFO" "Fetched install info for $MINION_ID (node type: ${NODETYPE:-unset})"
@@ -195,12 +176,6 @@ function pcapspace() {
fi
fi
# Must be checked before arithmetic expansion, which evaluates array subscripts
if [[ ! "$SPACESIZE" =~ ^[0-9]+$ ]]; then
log "ERROR" "Invalid disk size for $MINION_ID: ${SPACESIZE:0:64}"
return 1
fi
local s=$(( $SPACESIZE / 1000000 ))
local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))
@@ -1075,57 +1050,6 @@ function updateMineAndApplyStates() {
fi
}
# Values end up in a Jinja-rendered pillar and in bash, and may come from the minion
function validate_minion_vars() {
local error_msg=""
# Inline rather than valid_ip4: so-common is not installed yet when setup runs -o=setup
local octet='(25[0-5]|2[0-4][0-9]|1?[0-9]?[0-9])'
local ip4_re="^($octet\.){3}$octet$"
case "$NODETYPE" in
EVAL|STANDALONE|MANAGER|MANAGERSEARCH|MANAGERHYPE|IMPORT)
# Manager pillars also rewrite the CA pillar, so never accept them from a remote node
[[ "$OPERATION" == "setup" ]] || error_msg="Node type $NODETYPE can only be configured during setup"
;;
FLEET|IDH|HEAVYNODE|SENSOR|SEARCHNODE|RECEIVER|HYPERVISOR|DESKTOP)
;;
*)
error_msg="Invalid node type: ${NODETYPE:0:64}"
;;
esac
if [[ -z "$error_msg" ]]; then
if [[ ! "$MAINIP" =~ $ip4_re ]]; then
error_msg="Invalid MAINIP: ${MAINIP:0:64}"
elif [[ ! "$MNIC" =~ ^[A-Za-z0-9._-]*$ ]]; then
error_msg="Invalid MNIC: ${MNIC:0:64}"
elif [[ ! "$INTERFACE" =~ ^[A-Za-z0-9._-]*$ ]]; then
error_msg="Invalid INTERFACE: ${INTERFACE:0:64}"
elif [[ ! "$LSHOSTNAME" =~ ^[A-Za-z0-9._-]*$ ]]; then
error_msg="Invalid LSHOSTNAME: ${LSHOSTNAME:0:64}"
elif [[ ! "$ES_HEAP_SIZE" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
error_msg="Invalid ES_HEAP_SIZE: ${ES_HEAP_SIZE:0:64}"
elif [[ ! "$LSHEAP" =~ ^([0-9]+[kKmMgG]?)?$ ]]; then
error_msg="Invalid LSHEAP: ${LSHEAP:0:64}"
elif [[ ! "$CORECOUNT" =~ ^[0-9]*$ ]]; then
error_msg="Invalid CORECOUNT: ${CORECOUNT:0:64}"
elif [[ ! "$CPUCORES" =~ ^[0-9]*$ ]]; then
error_msg="Invalid CPUCORES: ${CPUCORES:0:64}"
elif [[ ! "$IDH_MGTRESTRICT" =~ ^(True|False)?$ ]]; then
error_msg="Invalid IDH_MGTRESTRICT: ${IDH_MGTRESTRICT:0:64}"
fi
fi
if [[ -n "$error_msg" ]]; then
log "ERROR" "$error_msg"
echo "$error_msg"
return 1
fi
# Free text; removing braces is enough to prevent any Jinja delimiter
NODE_DESCRIPTION=${NODE_DESCRIPTION//[\{\}[:cntrl:]]/}
}
function setupMinionFiles() {
log "INFO" "Setting up minion files for $MINION_ID (pillar: $PILLARFILE)"
@@ -1137,8 +1061,6 @@ function setupMinionFiles() {
return 1
fi
validate_minion_vars || return 1
# Create the base minion files
create_minion_files || return 1
+2 -2
View File
@@ -124,8 +124,8 @@ copy_new_files() {
rsync -a salt $default_salt_dir/
rsync -a pillar $default_salt_dir/
chown -R root:root $default_salt_dir/salt
chown -R root:root $default_salt_dir/pillar
chown -R socore:socore $default_salt_dir/salt
chown -R socore:socore $default_salt_dir/pillar
chmod 755 $default_salt_dir/pillar/firewall/addfirewall.sh
rm -rf /tmp/sogh
+11 -16
View File
@@ -129,8 +129,7 @@ while [[ $# -gt 0 ]]; do
esac
done
kratosContainer=${KRATOS_CONTAINER:-so-kratos}
kratosUrl=${KRATOS_URL:-http://localhost:4434/admin}
kratosUrl=${KRATOS_URL:-http://127.0.0.1:4434/admin}
databasePath=${KRATOS_DB_PATH:-/nsm/kratos/db/db.sqlite}
databaseTimeout=${KRATOS_DB_TIMEOUT:-5000}
bcryptRounds=${BCRYPT_ROUNDS:-12}
@@ -155,10 +154,6 @@ function fail() {
exit 1
}
function kratosCurl() {
docker exec "$kratosContainer" curl "$@"
}
function require() {
cmd=$1
which "$1" 2>&1 > /dev/null
@@ -169,18 +164,18 @@ function require() {
function verifyEnvironment() {
require "htpasswd"
require "jq"
require "docker"
require "curl"
require "openssl"
require "sqlite3"
[[ ! -f $databasePath ]] && fail "Unable to find database file; specify path via KRATOS_DB_PATH environment variable"
response=$(kratosCurl -Ss -L ${kratosUrl}/)
response=$(curl -Ss -L ${kratosUrl}/)
[[ "$response" != "404 page not found" ]] && fail "Unable to communicate with Kratos; specify URL via KRATOS_URL environment variable"
}
function findIdByEmail() {
email=${1,,}
response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
response=$(curl -Ss -L ${kratosUrl}/identities)
identityId=$(echo "${response}" | jq -r ".[] | select(.verifiable_addresses[0].value == \"$email\") | .id")
echo $identityId
}
@@ -421,7 +416,7 @@ function syncAll() {
}
function listUsers() {
response=$(kratosCurl -Ss -L ${kratosUrl}/identities)
response=$(curl -Ss -L ${kratosUrl}/identities)
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
users=$(echo "${response}" | jq -r ".[] | .verifiable_addresses[0].value" | sort)
@@ -500,7 +495,7 @@ function createUser() {
EOF
)
response=$(kratosCurl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
response=$(curl -Ss -L ${kratosUrl}/identities -d "$addUserJson")
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
identityId=$(echo "${response}" | jq -r ".id")
@@ -523,7 +518,7 @@ function updateStatus() {
identityId=$(findIdByEmail "$email")
[[ ${identityId} == "" ]] && fail "User not found"
response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
schemaId=$(echo "$response" | jq -r .schema_id)
@@ -536,7 +531,7 @@ function updateStatus() {
state="inactive"
fi
body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
[[ $? != 0 ]] && fail "Unable to update user"
}
@@ -555,7 +550,7 @@ function updateUserProfile() {
identityId=$(findIdByEmail "$email")
[[ ${identityId} == "" ]] && fail "User not found"
response=$(kratosCurl -Ss -L "${kratosUrl}/identities/$identityId")
response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
schemaId=$(echo "$response" | jq -r .schema_id)
@@ -564,7 +559,7 @@ function updateUserProfile() {
traitBlock="{\"email\":\"$email\",\"firstName\":\"$firstName\",\"lastName\":\"$lastName\",\"note\":\"$note\"}"
body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
response=$(kratosCurl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
response=$(curl -fSsL -XPUT -H "Content-Type: application/json" "${kratosUrl}/identities/$identityId" -d "$body")
[[ $? != 0 ]] && fail "Unable to update user"
}
@@ -574,7 +569,7 @@ function deleteUser() {
identityId=$(findIdByEmail "$email")
[[ ${identityId} == "" ]] && fail "User not found"
response=$(kratosCurl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
response=$(curl -Ss -XDELETE -L "${kratosUrl}/identities/$identityId")
[[ $? != 0 ]] && fail "Unable to communicate with Kratos"
rolesTmpFile="${socRolesFile}.tmp"
+1 -2
View File
@@ -42,8 +42,7 @@ def loadYaml(filename):
try:
with open(filename, "r") as file:
content = file.read()
loaded = yaml.safe_load(content)
return loaded if loaded is not None else {}
return yaml.safe_load(content)
except FileNotFoundError:
print(f"File not found: {filename}", file=sys.stderr)
sys.exit(1)
-97
View File
@@ -95,20 +95,6 @@ class TestRemove(unittest.TestCase):
expected = "key1:\n child1: 123\n child2:\n deep2: ab\nkey2: false\n"
self.assertEqual(actual, expected)
def test_remove_empty_file(self):
filename = "/tmp/so-yaml_test-remove-empty.yaml"
file = open(filename, "w")
file.close()
code = soyaml.remove([filename, "key1"])
self.assertEqual(code, 0)
file = open(filename, "r")
actual = file.read()
file.close()
self.assertEqual(actual, "{}\n")
def test_remove_missing_args(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -308,36 +294,6 @@ class TestRemove(unittest.TestCase):
expected = "key1:\n child1: 123\n child2:\n deep1: 45\n deep2: d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
self.assertEqual(actual, expected)
def test_add_empty_file(self):
filename = "/tmp/so-yaml_test-add-empty.yaml"
file = open(filename, "w")
file.close()
code = soyaml.add([filename, "telegraf.output", "BOTH"])
self.assertEqual(code, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "telegraf:\n output: BOTH\n"
self.assertEqual(actual, expected)
def test_add_empty_file_simple(self):
filename = "/tmp/so-yaml_test-add-empty-simple.yaml"
file = open(filename, "w")
file.close()
code = soyaml.add([filename, "telegraf", "BOTH"])
self.assertEqual(code, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "telegraf: BOTH\n"
self.assertEqual(actual, expected)
def test_replace_missing_arg(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -390,21 +346,6 @@ class TestRemove(unittest.TestCase):
expected = "key1:\n child1: 123\n child2:\n deep1: 46\nkey2: false\nkey3:\n- e\n- f\n- g\n"
self.assertEqual(actual, expected)
def test_replace_empty_file(self):
filename = "/tmp/so-yaml_test-replace-empty.yaml"
file = open(filename, "w")
file.close()
code = soyaml.replace([filename, "telegraf.output", "BOTH"])
self.assertEqual(code, 0)
file = open(filename, "r")
actual = file.read()
file.close()
expected = "telegraf:\n output: BOTH\n"
self.assertEqual(actual, expected)
def test_convert(self):
self.assertEqual(soyaml.convertType("foo"), "foo")
self.assertEqual(soyaml.convertType("foo.bar"), "foo.bar")
@@ -565,18 +506,6 @@ class TestRemove(unittest.TestCase):
self.assertEqual(result, 2)
self.assertEqual("", mock_stdout.getvalue())
def test_get_empty_file(self):
with patch('sys.stdout', new=StringIO()) as mock_stdout:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
filename = "/tmp/so-yaml_test-get-empty.yaml"
file = open(filename, "w")
file.close()
result = soyaml.get([filename, "telegraf.output"])
self.assertEqual(result, 2)
self.assertEqual("", mock_stdout.getvalue())
self.assertIn("Key 'telegraf.output' not found by so-yaml.py", mock_stderr.getvalue())
def test_get_usage(self):
with patch('sys.exit', new=MagicMock()) as sysmock:
with patch('sys.stderr', new=StringIO()) as mock_stderr:
@@ -1062,29 +991,3 @@ class TestLoadYaml(unittest.TestCase):
soyaml.loadYaml("/tmp/so-yaml_test-unreadable.yaml")
sysmock.assert_called_with(1)
self.assertIn("Error reading file", mock_stderr.getvalue())
def test_load_yaml_empty_file(self):
filename = "/tmp/so-yaml_test-load-empty.yaml"
file = open(filename, "w")
file.close()
result = soyaml.loadYaml(filename)
self.assertEqual(result, {})
def test_load_yaml_whitespace_only(self):
filename = "/tmp/so-yaml_test-load-whitespace.yaml"
file = open(filename, "w")
file.write(" \n\n \n")
file.close()
result = soyaml.loadYaml(filename)
self.assertEqual(result, {})
def test_load_yaml_comments_only(self):
filename = "/tmp/so-yaml_test-load-comments.yaml"
file = open(filename, "w")
file.write("# Just a comment\n# Another comment\n")
file.close()
result = soyaml.loadYaml(filename)
self.assertEqual(result, {})
+11 -121
View File
@@ -28,7 +28,6 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
# percentage like "25%"). Empty means so-soup-grid-highstate uses the salt:auto_apply:batch
# pillar default.
BATCHSIZE=
DEFAULT_DOCKER_RANGE='172.17.1.0/24'
SOUP_LOG=/root/soup.log
SOUP_DEBUG_LOG=/root/soup-debug.log
WHATWOULDYOUSAYYAHDOHERE=soup
@@ -606,7 +605,6 @@ preupgrade_changes() {
[[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0
[[ "$INSTALLEDVERSION" == "3.1.0" ]] && up_to_3.2.0
[[ "$INSTALLEDVERSION" == "3.2.0" ]] && up_to_3.3.0
[[ "$INSTALLEDVERSION" == "3.3.0" ]] && up_to_3.4.0
true
}
@@ -625,7 +623,6 @@ postupgrade_changes() {
[[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
[[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
[[ "$POSTVERSION" == "3.2.0" ]] && post_to_3.3.0
[[ "$POSTVERSION" == "3.3.0" ]] && post_to_3.4.0
# All applicable post-upgrade steps completed; clear the resume marker.
rm -f "$POSTVERSION_FILE"
true
@@ -1131,6 +1128,9 @@ post_to_3.2.0() {
### 3.3.0 Scripts ###
up_to_3.3.0() {
# download 9.4.5 elastic agent packages
determine_elastic_agent_upgrade
# remove existing (patched) elasticsearch index template to match integration naming change
if ! remove_elasticsearch_index_template "so-logs-sentinel_one_cloud_funnel.login" "sentinel_one_cloud_funnel.login changed to sentinel_one_cloud_funnel.logins"; then
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the so-logs-sentinel_one_cloud_funnel.login index template. This step can be performed manually using the following command:")
@@ -1162,126 +1162,17 @@ telegraf_repair() {
post_to_3.3.0() {
# Recollate again since some internal DBs were excluded during 3.2.0 soup
recollate_postgres
# Generate 9.4.5 elastic agent installers
echo "Regenerating Elastic Agent Installers"
/sbin/so-elastic-agent-gen-installers
telegraf_repair
set_postversion 3.3.0
}
### 3.3.0 End ###
### 3.4.0 Scripts ###
up_to_3.4.0() {
# download 9.4.8 elastic agent packages
determine_elastic_agent_upgrade
set_soauth_range
echo "Removing so-kratos, so-hydra and so-soc so they are recreated on the soauth network."
docker rm -f so-kratos so-hydra so-soc >> $SOUP_LOG 2>&1
for template in so-metrics-logstash.node so-metrics-logstash.stack_monitoring.node; do
if ! remove_elasticsearch_index_template "$template" "logstash node and node_cel index patterns reversed"; then
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to automatically remove the $template index template. Addon integration templates may fail to load until it is removed:")
FINAL_MESSAGE_QUEUE+=(" - sudo so-elasticsearch-query _index_template/$template -XDELETE && so-checkin")
fi
done
INSTALLEDVERSION=3.4.0
}
set_soauth_range() {
local pillar_file=/opt/so/saltstack/local/pillar/docker/soc_docker.sls
local current_range suggested authnet authgw input
[[ -f "$pillar_file" ]] || return 0
current_range=$(so-yaml.py get -r "$pillar_file" docker.range 2>/dev/null) || return 0
# A default range gets the 172.17.2.0/24 from docker/defaults.yaml, same as a fresh
# install, so there is nothing to ask about.
[[ -n "$current_range" && "$current_range" != "$DEFAULT_DOCKER_RANGE" ]] || return 0
if so-yaml.py get -r "$pillar_file" docker.networks.soauth.range >/dev/null 2>&1; then
return 0
fi
suggested=$(echo "${current_range%%/*}" | awk -F'.' '{ printf "%s.%s.%s.%s", $1, $2, ($3 + 1) % 256, $4 }')
if [[ -z $UNATTENDED ]]; then
echo ""
echo "This grid uses a custom Docker range ($current_range). The authentication"
echo "services are moving to their own isolated network, which needs a second /24"
echo "that does not overlap it."
echo ""
while :; do
read -rp "Enter the network without the /24 suffix, or press Enter for ${suggested}: " input
[[ -z "$input" ]] && input="$suggested"
if valid_soauth_range "$input" "$current_range"; then
authnet="$input"
break
fi
echo "That range must be a valid IPv4 network, must not be within 172.17.0.0/24, and must not overlap ${current_range}."
done
else
if ! valid_soauth_range "$suggested" "$current_range"; then
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to pick a range for the authentication network alongside $current_range. Set it manually before the next highstate:")
FINAL_MESSAGE_QUEUE+=(" - so-yaml.py add $pillar_file docker.networks.soauth.range <network>/24")
FINAL_MESSAGE_QUEUE+=(" - so-yaml.py add $pillar_file docker.networks.soauth.gateway <gateway>")
return 0
fi
authnet="$suggested"
FINAL_MESSAGE_QUEUE+=("NOTE: The authentication services moved to an isolated Docker network and were assigned ${authnet}/24.")
FINAL_MESSAGE_QUEUE+=(" - If that conflicts with your environment, update docker.networks.soauth in $pillar_file and run so-checkin.")
fi
authgw=$(echo "$authnet" | awk -F'.' '{print $1,$2,$3,1}' OFS='.')
echo "Assigning the authentication network the range ${authnet}/24."
so-yaml.py add "$pillar_file" docker.networks.soauth.range "${authnet}/24" >> $SOUP_LOG 2>&1
so-yaml.py add "$pillar_file" docker.networks.soauth.gateway "$authgw" >> $SOUP_LOG 2>&1
}
valid_soauth_range() {
local candidate=$1 docker_range=$2
valid_ip4 "$candidate" || return 1
[[ $candidate =~ ^172\.17\.0\. ]] && return 1
[[ "${candidate}/24" == "$docker_range" ]] && return 1
return 0
}
post_to_3.4.0() {
# Generate 9.4.8 elastic agent installers
echo "Regenerating Elastic Agent Installers"
/sbin/so-elastic-agent-gen-installers
for idx in "metrics-logstash.node-default" "metrics-logstash.stack_monitoring.node-default"; do
rollover_index "$idx"
done
add_detection_runtime_fields
remove_frozen_ca_bundles
set_postversion 3.4.0
}
remove_frozen_ca_bundles() {
rm -f /opt/so/saltstack/local/salt/elasticsearch/cacerts /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem >> $SOUP_LOG 2>&1
}
add_detection_runtime_fields() {
local template=$DEFAULT_SALT_DIR/salt/elasticsearch/templates/component/so/detection-mappings.json
local runtime
runtime=$(jq -c '{runtime: .template.mappings.runtime}' "$template")
if ! so-elasticsearch-query so-detection/_mapping -XPUT -d "$runtime" --retry 3 --retry-delay 5 --fail >> $SOUP_LOG 2>&1; then
FINAL_MESSAGE_QUEUE+=("WARNING: Unable to add the Sigma search fields to the so-detection index. Searches on so_detection.sigmaKind will match nothing until they are added:")
FINAL_MESSAGE_QUEUE+=(" - sudo so-elasticsearch-query so-detection/_mapping -XPUT -d \"\$(jq -c '{runtime: .template.mappings.runtime}' $template)\"")
fi
}
### 3.4.0 End ###
repo_sync() {
echo "Sync the local repo."
@@ -1554,10 +1445,9 @@ verify_es_version_compatibility() {
["8.18.4"]="8.18.6 8.18.8 9.0.8"
["8.18.6"]="8.18.8 9.0.8"
["8.18.8"]="9.0.8"
["9.0.8"]="9.3.3 9.3.7 9.4.5 9.4.8"
["9.3.3"]="9.3.7 9.4.5 9.4.8"
["9.3.7"]="9.4.5 9.4.8"
["9.4.5"]="9.4.8"
["9.0.8"]="9.3.3 9.3.7 9.4.5"
["9.3.3"]="9.3.7 9.4.5"
["9.3.7"]="9.4.5"
)
# Elasticsearch MUST upgrade through these versions
+2 -2
View File
@@ -57,8 +57,8 @@ nginx_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://nginx/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
#nginx_sbin_jinja:
-4
View File
@@ -6,12 +6,10 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'nginx/map.jinja' import NGINXMERGED %}
include:
- ca.trust
- nginx.ssl
- nginx.config
- nginx.sostatus
@@ -49,7 +47,6 @@ so-nginx:
{% endfor %}
{% endif %}
- binds:
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
- /opt/so/conf/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- /opt/so/log/nginx/:/var/log/nginx:rw
- /opt/so/tmp/nginx/:/var/lib/nginx:rw
@@ -92,7 +89,6 @@ so-nginx:
- {{ BINDING }}
{% endfor %}
- watch:
- cmd: ca_trust_extract
- file: nginxconf
- file: nginxconfdir
{% if GLOBALS.is_manager %}
+1 -1
View File
@@ -183,7 +183,7 @@ http {
ssl_prefer_server_ciphers on;
ssl_protocols TLSv1.2 TLSv1.3;
location ~* (^/login|^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
location ~* (^/login/.*|^/js/.*|^/css/.*|^/images/.*|^/pages/.*|^/docs/.*) {
proxy_pass http://{{ GLOBALS.manager }}:9822;
proxy_read_timeout 90;
proxy_connect_timeout 90;
-4
View File
@@ -6,12 +6,10 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% set SO_POSTGRES_USER = salt['pillar.get']('postgres:auth:users:so_postgres_user:user', 'so_postgres') %}
include:
- ca.trust
- postgres.auth
- postgres.ssl
- postgres.config
@@ -44,7 +42,6 @@ so-postgres:
{% endfor %}
{% endif %}
- binds:
- {{ CATRUST.dir }}
- /opt/so/log/postgres/:/log:rw
- /nsm/postgres:/var/lib/postgresql/data:rw
- /opt/so/conf/postgres/postgresql.conf:/conf/postgresql.conf:ro
@@ -72,7 +69,6 @@ so-postgres:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: postgresconf
- file: postgreshba
- file: postgresinitdb
+4 -4
View File
@@ -50,16 +50,16 @@ redis_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://redis/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
redis_sbin_jinja:
file.recurse:
- name: /usr/sbin
- source: salt://redis/tools/sbin_jinja
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
- template: jinja
-3
View File
@@ -7,7 +7,6 @@
{% if sls.split('.')[0] in allowed_states %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
include:
- ca
@@ -29,7 +28,6 @@ so-redis:
- {{ BINDING }}
{% endfor %}
- binds:
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
- /opt/so/log/redis:/var/log/redis:rw
- /opt/so/conf/redis/etc/redis.conf:/usr/local/etc/redis/redis.conf:ro
- /opt/so/conf/redis/working:/redis:rw
@@ -62,7 +60,6 @@ so-redis:
{% endif %}
- entrypoint: "redis-server /usr/local/etc/redis/redis.conf"
- watch:
- cmd: ca_trust_extract
- file: trusttheca
- x509: redis_crt
- x509: redis_key
-4
View File
@@ -6,11 +6,9 @@
{% from 'allowed_states.map.jinja' import allowed_states %}
{% if sls.split('.')[0] in allowed_states %}
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
include:
- ca.trust
- registry.ssl
- registry.config
- registry.sostatus
@@ -31,7 +29,6 @@ so-dockerregistry:
- {{ BINDING }}
{% endfor %}
- binds:
- {{ CATRUST.pem }}:/etc/ssl/certs/ca-certificates.crt:ro
- /opt/so/conf/docker-registry/etc/config.yml:/etc/distribution/config.yml:ro
- /opt/so/conf/docker-registry:/var/lib/registry:rw
- /nsm/docker-registry/docker:/var/lib/registry/docker:rw
@@ -66,7 +63,6 @@ so-dockerregistry:
attempts: 5
interval: 30
- watch:
- cmd: ca_trust_extract
- x509: registry_crt
- x509: registry_key
- require:
+1 -1
View File
@@ -9,7 +9,7 @@
'epel-testing.repo',
'saltstack.repo',
'salt-latest.repo',
'wazuh.repo',
'wazuh.repo'
'Rocky-Base.repo',
'Rocky-CR.repo',
'Rocky-Debuginfo.repo',
+2 -4
View File
@@ -3,8 +3,6 @@ salt_bootstrap:
file.managed:
- name: /usr/sbin/bootstrap-salt.sh
- source: salt://salt/scripts/bootstrap-salt.sh
- user: root
- group: root
- mode: 755
- show_changes: False
@@ -12,6 +10,6 @@ salt_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://salt/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
-2
View File
@@ -35,8 +35,6 @@ combine_bond_script:
file.managed:
- name: /usr/sbin/so-combine-bond
- source: salt://sensor/tools/sbin_jinja/so-combine-bond
- user: root
- group: root
- mode: 755
- template: jinja
- defaults:
+2 -2
View File
@@ -64,8 +64,8 @@ sensoroni_sbin:
file.recurse:
- name: /usr/sbin
- source: salt://sensoroni/tools/sbin
- user: root
- group: root
- user: 939
- group: 939
- file_mode: 755
#sensoroni_sbin_jinja:
-4
View File
@@ -4,12 +4,10 @@
# Elastic License 2.0.
{% from 'vars/globals.map.jinja' import GLOBALS %}
{% from 'ca/trust.map.jinja' import CATRUST %}
{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
include:
- ca.trust
- sensoroni.config
- sensoroni.sostatus
@@ -19,7 +17,6 @@ so-sensoroni:
- restart_policy: unless-stopped
- network_mode: host
- binds:
- {{ CATRUST.dir }}
- /nsm/import:/nsm/import:rw
- /nsm/pcapout:/nsm/pcapout:rw
- /opt/so/conf/sensoroni/sensoroni.json:/opt/sensoroni/sensoroni.json:ro
@@ -51,7 +48,6 @@ so-sensoroni:
{% endfor %}
{% endif %}
- watch:
- cmd: ca_trust_extract
- file: /opt/so/conf/sensoroni/sensoroni.json
- require:
- file: sensoroniagentconf
@@ -1,3 +1,2 @@
requests>=2.34.0
whoisit>=4.0.5
anyio>=4.15.1
requests>=2.31.0
whoisit>=2.7.0
Loaded 100 of 155 files, more files were not shown because too many files have changed in this diff. Show more