From 42038984275e14f7fa7d72b9d7ab8c1af5b89932 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 28 Nov 2018 20:44:56 -0500 Subject: [PATCH] Logstash Module - Rework it --- .../{files => etc}/beats-template.json | 0 .../logstash/{files => etc}/log4j2.properties | 0 .../logstash-ossec-template.json | 0 .../{files => etc}/logstash-template.json | 0 salt/logstash/{files => etc}/logstash.yml | 0 salt/logstash/init.sls | 49 ++++++++++++++----- so-setup-network.sh | 2 +- 7 files changed, 38 insertions(+), 13 deletions(-) rename salt/logstash/{files => etc}/beats-template.json (100%) rename salt/logstash/{files => etc}/log4j2.properties (100%) rename salt/logstash/{files => etc}/logstash-ossec-template.json (100%) rename salt/logstash/{files => etc}/logstash-template.json (100%) rename salt/logstash/{files => etc}/logstash.yml (100%) diff --git a/salt/logstash/files/beats-template.json b/salt/logstash/etc/beats-template.json similarity index 100% rename from salt/logstash/files/beats-template.json rename to salt/logstash/etc/beats-template.json diff --git a/salt/logstash/files/log4j2.properties b/salt/logstash/etc/log4j2.properties similarity index 100% rename from salt/logstash/files/log4j2.properties rename to salt/logstash/etc/log4j2.properties diff --git a/salt/logstash/files/logstash-ossec-template.json b/salt/logstash/etc/logstash-ossec-template.json similarity index 100% rename from salt/logstash/files/logstash-ossec-template.json rename to salt/logstash/etc/logstash-ossec-template.json diff --git a/salt/logstash/files/logstash-template.json b/salt/logstash/etc/logstash-template.json similarity index 100% rename from salt/logstash/files/logstash-template.json rename to salt/logstash/etc/logstash-template.json diff --git a/salt/logstash/files/logstash.yml b/salt/logstash/etc/logstash.yml similarity index 100% rename from salt/logstash/files/logstash.yml rename to salt/logstash/etc/logstash.yml diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 7a7935935..6ba4dba75 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -21,13 +21,11 @@ {% elif grains['role'] == 'so-node' %} {% set lsheap = salt['pillar.get']('node:lsheap', '') %} -{% set lsaccessip = salt['pillar.get']('node:lsaccessip', '') %} {% set nodetype = salt['pillar.get']('node:node_type', 'storage') %} {% elif grains['role'] == 'so-master' %} {% set lsheap = salt['pillar.get']('master:lsheap', '') %} -{% set lsaccessip = salt['pillar.get']('master:lsaccessip', '') %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:domainstats', '0') %} {% set nodetype = salt['grains.get']('role', '') %} @@ -35,7 +33,6 @@ {% elif grains['role'] == 'so-eval' %} {% set lsheap = salt['pillar.get']('master:lsheap', '') %} -{% set lsaccessip = salt['pillar.get']('master:lsaccessip', '') %} {% set freq = salt['pillar.get']('master:freq', '0') %} {% set dstats = salt['pillar.get']('master:domainstats', '0') %} {% set nodetype = salt['grains.get']('role', '') %} @@ -63,13 +60,26 @@ lscustdir: - group: 939 - makedirs: True +lsdyndir: + file.directory: + - name: /opt/so/conf/logstash/dynamic + - user: 931 + - group: 939 + - makedirs: True + +lsetcdir: + file.directory: + - name: /opt/so/conf/logstash/etc + - user: 931 + - group: 939 + - makedirs: True + lscustparserdir: file.directory: - name: /opt/so/conf/logstash/custom/parsers - user: 931 - group: 939 - makedirs: True - - template: jinja lscusttemplatedir: file.directory: @@ -79,14 +89,29 @@ lscusttemplatedir: - makedirs: True # Copy down all the configs including custom - TODO add watch restart -lssync: +lsetcsync: file.recurse: - - name: /opt/so/conf/logstash - - source: salt://logstash/files + - name: /opt/so/conf/logstash/etc + - source: salt://logstash/etc - user: 931 - group: 939 - template: jinja +lssync: + file.recurse: + - name: /opt/so/conf/logstash/dynamic + - source: salt://logstash/files/dynamic + - user: 931 + - group: 939 + - template: jinja + +lscustsync: + file.recurse: + - name: /opt/so/conf/logstash/custom + - source: salt://logstash/files/custom + - user: 931 + - group: 939 + # Copy the config file for enabled logstash plugins/parsers lsconfsync: file.managed: @@ -139,11 +164,11 @@ so-logstash: - 0.0.0.0:6053:6053 - 0.0.0.0:9600:9600 - binds: - - /opt/so/conf/logstash/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro - - /opt/so/conf/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml:ro - - /opt/so/conf/logstash/logstash-template.json:/logstash-template.json:ro - - /opt/so/conf/logstash/logstash-ossec-template.json:/logstash-ossec-template.json:ro - - /opt/so/conf/logstash/beats-template.json:/beats-template.json:ro + - /opt/so/conf/logstash/etc/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro + - /opt/so/conf/logstash/etc/logstash.yml:/usr/share/logstash/config/logstash.yml:ro + - /opt/so/conf/logstash/etc/logstash-template.json:/logstash-template.json:ro + - /opt/so/conf/logstash/etc/logstash-ossec-template.json:/logstash-ossec-template.json:ro + - /opt/so/conf/logstash/etc/beats-template.json:/beats-template.json:ro - /opt/so/conf/logstash/custom:/usr/share/logstash/pipeline.custom:ro - /opt/so/conf/logstash/rulesets:/usr/share/logstash/rulesets:ro - /opt/so/conf/logstash/dynamic:/usr/share/logstash/pipeline.dynamic diff --git a/so-setup-network.sh b/so-setup-network.sh index 06d98102b..0b098e70d 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1534,7 +1534,7 @@ if (whiptail_you_sure); then else NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE - LSPIPELINEWORKERS=1 + LSPIPELINEWORKERS=$CPUCORES LSPIPELINEBATCH=125 LSINPUTTHREADS=1 LSINPUTBATCHCOUNT=125