From b69dfd9b25031f067dc8448959f676b697385da0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 5 Feb 2020 11:27:19 -0500 Subject: [PATCH 1/3] Helix fix dockers --- salt/top.sls | 1 + setup/so-functions | 81 +++++++++++++++++++++++++++------------------- setup/so-setup | 3 ++ 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/salt/top.sls b/salt/top.sls index b1ef40ae6..1a5c30e60 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -15,6 +15,7 @@ base: 'G@role:so-helix': - ca - ssl + - registry - common - firewall - idstools diff --git a/setup/so-functions b/setup/so-functions index 3693ee1e7..83f323b0b 100644 --- a/setup/so-functions +++ b/setup/so-functions @@ -453,42 +453,55 @@ docker_registry() { echo "Docker Registry Setup - Complete" >> $SETUPLOG 2>&1 } - docker_seed_registry() { VERSION="HH1.1.4" - TRUSTED_CONTAINERS=( \ - "so-acng:$VERSION" \ - "so-auth-api:$VERSION" \ - "so-auth-ui:$VERSION" \ - "so-core:$VERSION" \ - "so-thehive-cortex:$VERSION" \ - "so-curator:$VERSION" \ - "so-domainstats:$VERSION" \ - "so-elastalert:$VERSION" \ - "so-elasticsearch:$VERSION" \ - "so-filebeat:$VERSION" \ - "so-fleet:$VERSION" \ - "so-fleet-launcher:$VERSION" \ - "so-freqserver:$VERSION" \ - "so-grafana:$VERSION" \ - "so-idstools:$VERSION" \ - "so-influxdb:$VERSION" \ - "so-kibana:$VERSION" \ - "so-logstash:$VERSION" \ - "so-mysql:$VERSION" \ - "so-navigator:$VERSION" \ - "so-playbook:$VERSION" \ - "so-redis:$VERSION" \ - "so-sensoroni:$VERSION" \ - "so-soctopus:$VERSION" \ - "so-steno:$VERSION" \ - #"so-strelka:$VERSION" \ - "so-suricata:$VERSION" \ - "so-telegraf:$VERSION" \ - "so-thehive:$VERSION" \ - "so-thehive-es:$VERSION" \ - "so-wazuh:$VERSION" \ - "so-zeek:$VERSION" ) + if [ $INSTALLTYPE != 'HELIXSENSOR' ]; then + TRUSTED_CONTAINERS=( \ + "so-acng:$VERSION" \ + "so-auth-api:$VERSION" \ + "so-auth-ui:$VERSION" \ + "so-core:$VERSION" \ + "so-thehive-cortex:$VERSION" \ + "so-curator:$VERSION" \ + "so-domainstats:$VERSION" \ + "so-elastalert:$VERSION" \ + "so-elasticsearch:$VERSION" \ + "so-filebeat:$VERSION" \ + "so-fleet:$VERSION" \ + "so-fleet-launcher:$VERSION" \ + "so-freqserver:$VERSION" \ + "so-grafana:$VERSION" \ + "so-idstools:$VERSION" \ + "so-influxdb:$VERSION" \ + "so-kibana:$VERSION" \ + "so-logstash:$VERSION" \ + "so-mysql:$VERSION" \ + "so-navigator:$VERSION" \ + "so-playbook:$VERSION" \ + "so-redis:$VERSION" \ + "so-sensoroni:$VERSION" \ + "so-soctopus:$VERSION" \ + "so-steno:$VERSION" \ + #"so-strelka:$VERSION" \ + "so-suricata:$VERSION" \ + "so-telegraf:$VERSION" \ + "so-thehive:$VERSION" \ + "so-thehive-es:$VERSION" \ + "so-wazuh:$VERSION" \ + "so-zeek:$VERSION" ) + else + TRUSTED_CONTAINERS=( \ + "so-core:$VERSION" \ + "so-filebeat:$VERSION" \ + "so-idstools:$VERSION" \ + "so-logstash:$VERSION" \ + "so-redis:$VERSION" \ + "so-sensoroni:$VERSION" \ + "so-steno:$VERSION" \ + "so-suricata:$VERSION" \ + "so-telegraf:$VERSION" \ + "so-zeek:$VERSION" ) + fi if [ ! -f /nsm/docker-registry/docker/so-dockers-$VERSION.tar ]; then # Download the container from the interwebs diff --git a/setup/so-setup b/setup/so-setup index 755e21c62..addfbdd3c 100644 --- a/setup/so-setup +++ b/setup/so-setup @@ -213,6 +213,9 @@ if (whiptail_you_sure) ; then salt_checkin >> $SETUPLOG 2>&1 salt-call state.apply ca >> $SETUPLOG 2>&1 salt-call state.apply ssl >> $SETUPLOG 2>&1 + echo -e "XXX\n42\nDownloading Containers from the Internet... \nXXX" + salt-call state.apply registry >> $SETUPLOG 2>&1 + docker_seed_registry >> $SETUPLOG 2>&1 echo -e "XXX\n43\nInstalling Common Components... \nXXX" salt-call state.apply common >> $SETUPLOG 2>&1 echo -e "XXX\n45\nApplying firewall rules... \nXXX" From 799cf322654671e610825b978cff30b4715ec55b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 Feb 2020 14:45:07 -0500 Subject: [PATCH 2/3] fix logstash for so-helix role --- .../conf/pipelines/helix/{ => templates}/9997_output_helix.conf | 0 salt/logstash/etc/logstash.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename salt/logstash/conf/pipelines/helix/{ => templates}/9997_output_helix.conf (100%) diff --git a/salt/logstash/conf/pipelines/helix/9997_output_helix.conf b/salt/logstash/conf/pipelines/helix/templates/9997_output_helix.conf similarity index 100% rename from salt/logstash/conf/pipelines/helix/9997_output_helix.conf rename to salt/logstash/conf/pipelines/helix/templates/9997_output_helix.conf diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index 47b487ebe..d7a51df81 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -63,7 +63,7 @@ # # path.config: # /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image -{%- if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' and grains.role != 'so-master' and grains.role != 'so-eval' %} +{%- if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' and grains.role != 'so-master' and grains.role != 'so-eval' and grains.role != 'so-helix' %} path.config: /usr/share/logstash/pipeline.enabled/*.conf {%- else %} #path.config: /usr/share/logstash/pipeline.enabled/*.conf From ec11b9ea257f9939ab8c9b68ead3061c525cf5cf Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Wed, 5 Feb 2020 20:34:32 +0000 Subject: [PATCH 3/3] Remove Redis config from eval PL --- .../eval/templates/9999_output_redis.conf | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9999_output_redis.conf diff --git a/salt/logstash/conf/pipelines/eval/templates/9999_output_redis.conf b/salt/logstash/conf/pipelines/eval/templates/9999_output_redis.conf deleted file mode 100644 index f176e0b94..000000000 --- a/salt/logstash/conf/pipelines/eval/templates/9999_output_redis.conf +++ /dev/null @@ -1,26 +0,0 @@ -{%- if salt['grains.get']('role') == 'so-master' %} -{% set master = salt['pillar.get']('static:masterip', '') %} -{%- set nodetype = 'master' %} -{% elif grains.role == 'so-heavynode' %} -{% set master = salt['pillar.get']('node:mainip', '') %} -{%- set nodetype = salt['pillar.get']('node:node_type', 'search') %} -{%- else %} -{%- set nodetype = salt['pillar.get']('node:node_type', 'storage') %} -{% set master = salt['pillar.get']('static:masterip', '') %} -{%- endif %} - - -output { - redis { - host => '{{ master }}' - data_type => 'list' - {%- if nodetype == 'parser' %} - key => 'logstash:parsed' - {%- else %} - key => 'logstash:unparsed' - {%- endif %} - congestion_interval => 1 - congestion_threshold => 50000000 - # batch_events => 500 - } -}