From e171c1d176dd8714e69712f2441e5b4dbee51913 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 10:43:38 -0400 Subject: [PATCH 01/58] Hive Module - Fix heap size --- salt/hive/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 958ecf540..fa3d6e581 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -26,6 +26,7 @@ so-thehive-es: - thread_pool.index.queue_size=100000 - thread_pool.search.queue_size=100000 - thread_pool.bulk.queue_size=100000 + - ES_JAVA_OPTS=-Xms2g -Xmx2g # Install Cortex From f494ffcebe3be9dcb26fefd7ebae3cfda0a794b2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 11:16:24 -0400 Subject: [PATCH 02/58] Hive Module - Sync ES config --- salt/hive/init.sls | 30 +++++++++++++--------- salt/hive/thehive/etc/es/elasticsearch.yml | 19 ++++++++++++++ 2 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 salt/hive/thehive/etc/es/elasticsearch.yml diff --git a/salt/hive/init.sls b/salt/hive/init.sls index fa3d6e581..7772008d8 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -1,3 +1,19 @@ +hiveconfdir: + file.directory: + - name: /opt/so/conf/hive/etc + - makedirs: True + +hivelogdir: + file.directory: + - name: /opt/so/log/hive + - makedirs: True + +hiveconf: + file.recurse: + - name: /opt/so/conf/hive/etc + - source: salt://hive/thehive/etc + - template: jinja + # Install Elasticsearch # Made directory for ES data to live in @@ -15,6 +31,8 @@ so-thehive-es: - tty: True - binds: - /nsm/hive/esdata:/usr/share/elasticsearch/data:rw + - /opt/so/conf/hive/etc/es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + - /opt/so/log/hive:/var/log/elasticsearch:rw - environment: - http.host=0.0.0.0 - http.port=9400 @@ -36,18 +54,6 @@ so-cortex: - hostname: so-cortex - name: so-cortex -# Install Hive -hiveconfdir: - file.directory: - - name: /opt/so/conf/hive/etc - - makedirs: True - -hiveconf: - file.managed: - - name: /opt/so/conf/hive/etc/application.conf - - source: salt://hive/thehive/etc/application.conf - - template: jinja - so-thehive: docker_container.running: - image: thehiveproject/thehive:latest diff --git a/salt/hive/thehive/etc/es/elasticsearch.yml b/salt/hive/thehive/etc/es/elasticsearch.yml new file mode 100644 index 000000000..197c780a3 --- /dev/null +++ b/salt/hive/thehive/etc/es/elasticsearch.yml @@ -0,0 +1,19 @@ +cluster.name: "hive" +network.host: 0.0.0.0 +discovery.zen.minimum_master_nodes: 1 +# This is a test -- if this is here, then the volume is mounted correctly. +path.logs: /var/log/elasticsearch +action.destructive_requires_name: true +path.logs: /var/log/elasticsearch +transport.bind_host: 0.0.0.0 +transport.publish_host: 0.0.0.0 +transport.publish_port: 9500 +http.host: 0.0.0.0 +http.port: 9400 +transport.tcp.port: 9500 +transport.host: 0.0.0.0 +xpack.security.enabled: false +script.inline: true +thread_pool.index.queue_size: 100000 +thread_pool.search.queue_size: 100000 +thread_pool.bulk.queue_size: 100000 From 2b1f0719f1cbd4033b2b1cfac03c1be047be2f8f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 11:19:17 -0400 Subject: [PATCH 03/58] Hive Module - Fix ES memory usage --- salt/hive/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 7772008d8..d7f8757b3 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -44,7 +44,7 @@ so-thehive-es: - thread_pool.index.queue_size=100000 - thread_pool.search.queue_size=100000 - thread_pool.bulk.queue_size=100000 - - ES_JAVA_OPTS=-Xms2g -Xmx2g + - ES_JAVA_OPTS=-Xms512m -Xmx512m # Install Cortex From 2caf331e8edcad4c76afc06ffab16462e73f9ae4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 11:26:10 -0400 Subject: [PATCH 04/58] Hive Module - Fix ES logging --- salt/hive/thehive/etc/es/log4j2.properties | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 salt/hive/thehive/etc/es/log4j2.properties diff --git a/salt/hive/thehive/etc/es/log4j2.properties b/salt/hive/thehive/etc/es/log4j2.properties new file mode 100644 index 000000000..85cf5d8fb --- /dev/null +++ b/salt/hive/thehive/etc/es/log4j2.properties @@ -0,0 +1,20 @@ +status = error +#appender.console.type = Console +#appender.console.name = console +#appender.console.layout.type = PatternLayout +#appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n +#rootLogger.level = info +#rootLogger.appenderRef.console.ref = console +# This is a test -- if this here, then the volume is mounted correctly. +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +rootLogger.level = info +rootLogger.appenderRef.rolling.ref = rolling From 5a41d3e2ab7c8df2a85216d1973fa4267f4af5b9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 11:30:54 -0400 Subject: [PATCH 05/58] Hive Module - Fix ES logging --- salt/hive/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index d7f8757b3..e2b0bcfdb 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -32,6 +32,7 @@ so-thehive-es: - binds: - /nsm/hive/esdata:/usr/share/elasticsearch/data:rw - /opt/so/conf/hive/etc/es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + - /opt/so/conf/hive/etc/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro - /opt/so/log/hive:/var/log/elasticsearch:rw - environment: - http.host=0.0.0.0 From 7c278f95787f485ade424d09348cd55210deb484 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 11:34:21 -0400 Subject: [PATCH 06/58] Hive Module - Fix ES logging --- salt/hive/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index e2b0bcfdb..c6a8577fb 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -32,7 +32,7 @@ so-thehive-es: - binds: - /nsm/hive/esdata:/usr/share/elasticsearch/data:rw - /opt/so/conf/hive/etc/es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro - - /opt/so/conf/hive/etc/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro + - /opt/so/conf/hive/etc/es/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties:ro - /opt/so/log/hive:/var/log/elasticsearch:rw - environment: - http.host=0.0.0.0 From ccee0803d019b1d1ad17c1becd9a160df95f3744 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 11:43:52 -0400 Subject: [PATCH 07/58] Hive Module - Fix ES logging --- salt/hive/thehive/etc/es/elasticsearch.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/hive/thehive/etc/es/elasticsearch.yml b/salt/hive/thehive/etc/es/elasticsearch.yml index 197c780a3..2ce07e9dc 100644 --- a/salt/hive/thehive/etc/es/elasticsearch.yml +++ b/salt/hive/thehive/etc/es/elasticsearch.yml @@ -4,7 +4,6 @@ discovery.zen.minimum_master_nodes: 1 # This is a test -- if this is here, then the volume is mounted correctly. path.logs: /var/log/elasticsearch action.destructive_requires_name: true -path.logs: /var/log/elasticsearch transport.bind_host: 0.0.0.0 transport.publish_host: 0.0.0.0 transport.publish_port: 9500 From 9dd863e1abaad118f9ac7e07fe6ff61fd703a10f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 11:49:46 -0400 Subject: [PATCH 08/58] Hive Module - Add publish ports --- salt/hive/init.sls | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index c6a8577fb..a6aee3c9d 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -46,6 +46,9 @@ so-thehive-es: - thread_pool.search.queue_size=100000 - thread_pool.bulk.queue_size=100000 - ES_JAVA_OPTS=-Xms512m -Xmx512m + - port_bindings: + - 0.0.0.0:9400:9400 + - 0.0.0.0:9500:9500 # Install Cortex From e1b444b2bf9af399b5615eccce87dff5ec46281c Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 12:01:59 -0400 Subject: [PATCH 09/58] Hive Module - Add more publish ports --- salt/hive/init.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index a6aee3c9d..615447630 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -57,9 +57,13 @@ so-cortex: - image: thehiveproject/cortex:latest - hostname: so-cortex - name: so-cortex + - port_bindings: + - 0.0.0.0:9001:9001 so-thehive: docker_container.running: - image: thehiveproject/thehive:latest - hostname: so-thehive - name: so-thehive + - port_bindings: + - 0.0.0.0:9000:9000 From 33513f18106283cc95727a81a061072ca8805d8d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 12:05:14 -0400 Subject: [PATCH 10/58] Hive Module - Add application.conf --- salt/hive/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 615447630..fe38d4ea0 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -65,5 +65,7 @@ so-thehive: - image: thehiveproject/thehive:latest - hostname: so-thehive - name: so-thehive + - binds: + - /opt/so/conf/hive/application.conf:/etc/hive/application.conf - port_bindings: - 0.0.0.0:9000:9000 From c454850e19608171d90863302f8fa7214d21c166 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 12:10:23 -0400 Subject: [PATCH 11/58] Hive Module - Add application.conf --- salt/hive/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index fe38d4ea0..1f1a96529 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -66,6 +66,6 @@ so-thehive: - hostname: so-thehive - name: so-thehive - binds: - - /opt/so/conf/hive/application.conf:/etc/hive/application.conf + - /opt/so/conf/hive/application.conf:/etc/thehive/application.conf - port_bindings: - 0.0.0.0:9000:9000 From 57c92e724bfc401fcff13a73516fceca60b772c0 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 12:19:18 -0400 Subject: [PATCH 12/58] Hive Module - Add application.conf --- salt/hive/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 1f1a96529..3547cbf08 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -66,6 +66,6 @@ so-thehive: - hostname: so-thehive - name: so-thehive - binds: - - /opt/so/conf/hive/application.conf:/etc/thehive/application.conf + - /opt/so/conf/hive/application.conf:/etc/thehive/application.conf:ro - port_bindings: - 0.0.0.0:9000:9000 From 92818bebca75ea3e139a2dea4ac2920ae2ca6033 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 11 Mar 2019 12:20:03 -0400 Subject: [PATCH 13/58] Hive Module - Add application.conf --- salt/hive/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 3547cbf08..38907e98b 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -66,6 +66,6 @@ so-thehive: - hostname: so-thehive - name: so-thehive - binds: - - /opt/so/conf/hive/application.conf:/etc/thehive/application.conf:ro + - /opt/so/conf/hive/etc/application.conf:/etc/thehive/application.conf:ro - port_bindings: - 0.0.0.0:9000:9000 From 201668eed92d974375f7c5489d8a8939b0e5ff56 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 12 Mar 2019 12:35:22 -0400 Subject: [PATCH 14/58] TheHive Module - Add our new dockers --- salt/hive/init.sls | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 38907e98b..9ba83ad69 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -2,11 +2,15 @@ hiveconfdir: file.directory: - name: /opt/so/conf/hive/etc - makedirs: True + - user: 939 + - group: 939 hivelogdir: file.directory: - name: /opt/so/log/hive - makedirs: True + - user: 939 + - group: 939 hiveconf: file.recurse: @@ -21,12 +25,15 @@ hiveesdata: file.directory: - name: /nsm/hive/esdata - makedirs: True + - user: 939 + - group: 939 so-thehive-es: docker_container.running: - - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.0 + - image: soshybridhunter/so-thehive-es:HH1.0.7 - hostname: so-thehive-es - name: so-thehive-es + - user: 939 - interactive: True - tty: True - binds: @@ -62,9 +69,10 @@ so-cortex: so-thehive: docker_container.running: - - image: thehiveproject/thehive:latest + - image: soshybridhunter/so-thehive:HH1.0.7 - hostname: so-thehive - name: so-thehive + - user: 939 - binds: - /opt/so/conf/hive/etc/application.conf:/etc/thehive/application.conf:ro - port_bindings: From 1858bc83fb79a9f6e56ee8ca2fa9ed091663904a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 12 Mar 2019 13:09:28 -0400 Subject: [PATCH 15/58] TheHive Module - Change permissions --- salt/hive/init.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 9ba83ad69..7317bde59 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -16,6 +16,8 @@ hiveconf: file.recurse: - name: /opt/so/conf/hive/etc - source: salt://hive/thehive/etc + - user: 939 + - group: 939 - template: jinja # Install Elasticsearch From 02f9d2b9c5724b6ee9c27dab5b8a4c1c49108052 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 12 Mar 2019 15:17:12 -0400 Subject: [PATCH 16/58] TheHive Module - Fix ES --- salt/hive/thehive/etc/es/elasticsearch.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/hive/thehive/etc/es/elasticsearch.yml b/salt/hive/thehive/etc/es/elasticsearch.yml index 2ce07e9dc..d600830b6 100644 --- a/salt/hive/thehive/etc/es/elasticsearch.yml +++ b/salt/hive/thehive/etc/es/elasticsearch.yml @@ -11,7 +11,6 @@ http.host: 0.0.0.0 http.port: 9400 transport.tcp.port: 9500 transport.host: 0.0.0.0 -xpack.security.enabled: false script.inline: true thread_pool.index.queue_size: 100000 thread_pool.search.queue_size: 100000 From 6a5ab47870dbaed642e1d6893a51ee9abc654676 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 12 Mar 2019 15:37:07 -0400 Subject: [PATCH 17/58] TheHive Module - Fix ES --- salt/hive/thehive/etc/es/elasticsearch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/hive/thehive/etc/es/elasticsearch.yml b/salt/hive/thehive/etc/es/elasticsearch.yml index d600830b6..2ce07e9dc 100644 --- a/salt/hive/thehive/etc/es/elasticsearch.yml +++ b/salt/hive/thehive/etc/es/elasticsearch.yml @@ -11,6 +11,7 @@ http.host: 0.0.0.0 http.port: 9400 transport.tcp.port: 9500 transport.host: 0.0.0.0 +xpack.security.enabled: false script.inline: true thread_pool.index.queue_size: 100000 thread_pool.search.queue_size: 100000 From e2d3caa2f700eaa8b5fd4f3c6d2c9985f3fe45d9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 10:23:10 -0400 Subject: [PATCH 18/58] TheHive ES Docker - Change yml --- salt/hive/thehive/etc/es/elasticsearch.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/hive/thehive/etc/es/elasticsearch.yml b/salt/hive/thehive/etc/es/elasticsearch.yml index 2ce07e9dc..d600830b6 100644 --- a/salt/hive/thehive/etc/es/elasticsearch.yml +++ b/salt/hive/thehive/etc/es/elasticsearch.yml @@ -11,7 +11,6 @@ http.host: 0.0.0.0 http.port: 9400 transport.tcp.port: 9500 transport.host: 0.0.0.0 -xpack.security.enabled: false script.inline: true thread_pool.index.queue_size: 100000 thread_pool.search.queue_size: 100000 From e23c7c1a3c504ec9eefb36dbccd2492752a38c9b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 10:56:10 -0400 Subject: [PATCH 19/58] TheHive Module - Fix env variables --- salt/hive/init.sls | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index 7317bde59..c204f0d26 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -48,7 +48,6 @@ so-thehive-es: - http.port=9400 - transport.tcp.port=9500 - transport.host=0.0.0.0 - - xpack.security.enabled=false - cluster.name=hive - script.inline=true - thread_pool.index.queue_size=100000 From 4d567fd431ccf7f57b4289f2c978c88f5437732d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 11:04:29 -0400 Subject: [PATCH 20/58] TheHive Module - Fix application.conf --- salt/hive/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index c204f0d26..d0af62fc3 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -75,6 +75,6 @@ so-thehive: - name: so-thehive - user: 939 - binds: - - /opt/so/conf/hive/etc/application.conf:/etc/thehive/application.conf:ro + - /opt/so/conf/hive/etc/application.conf:/opt/thehive/conf/application.conf:ro - port_bindings: - 0.0.0.0:9000:9000 From e193ec797be1528520e1ac22fb0b4a2a304a29b8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 14:04:34 -0400 Subject: [PATCH 21/58] Setup Script - Make sure a management nic is selected --- so-setup-network.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index e0a2899de..b7b106898 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1171,9 +1171,11 @@ whiptail_log_size_limit() { whiptail_management_nic() { MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC" 20 78 12 ${NICS[@]} 3>&1 1>&2 2>&3 ) - - local exitstatus=$? - whiptail_check_exitstatus $exitstatus + if [[ ! $MNIC ]]; then + MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC" 20 78 12 ${NICS[@]} 3>&1 1>&2 2>&3 ) + else + local exitstatus=$? + whiptail_check_exitstatus $exitstatus } From bc09ee87b45b59145a0473b3b9232603ebd158f9 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 14:08:38 -0400 Subject: [PATCH 22/58] Setup Script - Make sure a management nic is selected --- so-setup-network.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index b7b106898..1a7816035 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1171,11 +1171,13 @@ whiptail_log_size_limit() { whiptail_management_nic() { MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC" 20 78 12 ${NICS[@]} 3>&1 1>&2 2>&3 ) - if [[ ! $MNIC ]]; then + + if [ -z "$MNIC" ]; then MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC" 20 78 12 ${NICS[@]} 3>&1 1>&2 2>&3 ) else local exitstatus=$? whiptail_check_exitstatus $exitstatus + fi } From 20441df78e66e0907ebddf821b60a98d7138c8d2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 14:15:32 -0400 Subject: [PATCH 23/58] Setup Script - Make sure a management nic is selected --- so-setup-network.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 1a7816035..2cc947a88 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1172,12 +1172,13 @@ whiptail_management_nic() { MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC" 20 78 12 ${NICS[@]} 3>&1 1>&2 2>&3 ) - if [ -z "$MNIC" ]; then + while [ -z "$MNIC" ] + do MNIC=$(whiptail --title "NIC Setup" --radiolist "Please select your management NIC" 20 78 12 ${NICS[@]} 3>&1 1>&2 2>&3 ) - else - local exitstatus=$? - whiptail_check_exitstatus $exitstatus - fi + done + + local exitstatus=$? + whiptail_check_exitstatus $exitstatus } From 924b061b502ef78d102c34eca5c4939290e36c3d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 14:21:27 -0400 Subject: [PATCH 24/58] Setup Script - Make sure you have at least 1 mon nic --- so-setup-network.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index 2cc947a88..ec2f97233 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1066,6 +1066,11 @@ whiptail_bond_nics() { BNICS=$(whiptail --title "NIC Setup" --checklist "Please add NICs to the Monitor Interface" 20 78 12 ${FNICS[@]} 3>&1 1>&2 2>&3 ) + while [ -z "$BNICS" ] + do + BNICS=$(whiptail --title "NIC Setup" --checklist "Please add NICs to the Monitor Interface" 20 78 12 ${FNICS[@]} 3>&1 1>&2 2>&3 ) + done + local exitstatus=$? whiptail_check_exitstatus $exitstatus From 7bc9085001a17a98439a0daecdec7e4dd635236f Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 14:36:12 -0400 Subject: [PATCH 25/58] Setup Script - Add entry to hosts file --- so-setup-network.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/so-setup-network.sh b/so-setup-network.sh index ec2f97233..a9020ff9e 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -410,6 +410,12 @@ es_heapsize() { } +eval_mode_hostsfile() { + + echo "127.0.0.1 $HOSTNAME" >> /etc/hosts + +} + filter_nics() { # Filter the NICs that we don't want to see in setup @@ -1711,6 +1717,7 @@ if (whiptail_you_sure); then BROVERSION=ZEEK CURCLOSEDAYS=30 whiptail_make_changes + eval_mode_hostsfile generate_passwords auth_pillar clear_master From 27d53a3a75039540da5a466b0517f1f3b4a9bac1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 14:40:21 -0400 Subject: [PATCH 26/58] Setup Script - Add entry to hosts file --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index a9020ff9e..8152718de 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -412,7 +412,7 @@ es_heapsize() { eval_mode_hostsfile() { - echo "127.0.0.1 $HOSTNAME" >> /etc/hosts + echo "127.0.0.1 $HOSTNAME" >> /etc/hosts } From d589a7ac732bcee148ba09b39760c5de6233fea8 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 13 Mar 2019 15:25:39 -0400 Subject: [PATCH 27/58] Setup Script - Remove hosts entry --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 8152718de..826e0e650 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1717,7 +1717,7 @@ if (whiptail_you_sure); then BROVERSION=ZEEK CURCLOSEDAYS=30 whiptail_make_changes - eval_mode_hostsfile + #eval_mode_hostsfile generate_passwords auth_pillar clear_master From 6e29ad90c7d9c38d5bceb14682c5f498303e3ea1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 14 Mar 2019 09:35:45 -0400 Subject: [PATCH 28/58] Nginx Module - add hive support --- salt/common/nginx/nginx.conf.so-eval | 12 ++++++++++++ salt/common/nginx/nginx.conf.so-master | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval index 50f48497d..f137ced15 100644 --- a/salt/common/nginx/nginx.conf.so-eval +++ b/salt/common/nginx/nginx.conf.so-eval @@ -136,6 +136,18 @@ http { } + location /thehive/ { + rewrite /thehive/(.*) /$1 break; + proxy_pass http://{{ masterip }}:9000/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + error_page 404 /404.html; location = /40x.html { } diff --git a/salt/common/nginx/nginx.conf.so-master b/salt/common/nginx/nginx.conf.so-master index 50f48497d..f73c9620a 100644 --- a/salt/common/nginx/nginx.conf.so-master +++ b/salt/common/nginx/nginx.conf.so-master @@ -136,6 +136,18 @@ http { } + location /thehive/ { + rewrite /thehive/(.*) /$1 break; + proxy_pass http://{{ masterip }}:9000/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + error_page 404 /404.html; location = /40x.html { } From 28c0cda16bc1a92bf073a7f9464118f34b83767e Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Fri, 15 Mar 2019 02:40:02 +0000 Subject: [PATCH 29/58] SOCtopus: Initial Commit --- salt/firewall/init.sls | 11 +++++++++++ salt/soctopus/files/SOCtopus.conf | 12 ++++++++++++ salt/soctopus/init.sls | 24 ++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 salt/soctopus/files/SOCtopus.conf create mode 100644 salt/soctopus/init.sls diff --git a/salt/firewall/init.sls b/salt/firewall/init.sls index 0d5fd174d..7ff009495 100644 --- a/salt/firewall/init.sls +++ b/salt/firewall/init.sls @@ -389,6 +389,17 @@ enable_standard_analyst_3000_{{ip}}: - position: 1 - save: True +enable_standard_analyst_7000_{{ip}}: + iptables.insert: + - table: filter + - chain: DOCKER-USER + - jump: ACCEPT + - proto: tcp + - source: {{ ip }} + - dport: 7000 + - position: 1 + - save: True + enable_standard_analyst_9000_{{ip}}: iptables.insert: - table: filter diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf new file mode 100644 index 000000000..eb766755f --- /dev/null +++ b/salt/soctopus/files/SOCtopus.conf @@ -0,0 +1,12 @@ +{%- set ip = salt['pillar.get']('static:masterip', '') %} + +[es] +es_url = http://{{ ip }}:9200 + +[hive] +hive_url = http://{{ ip }}:9000 +hive_key = YOURHIVEAPIKEYHERE -- TO LATER BE REPLACED BY JINJA + +[log] +logfile = /tmp/soctopus.log + diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls new file mode 100644 index 000000000..e811b587b --- /dev/null +++ b/salt/soctopus/init.sls @@ -0,0 +1,24 @@ +soctopusdir: + file.directory: + - name: /opt/so/conf/soctopus + - user: 939 + - group: 939 + - makedirs: True + +soctopussync: + file.recurse: + - name: /opt/so/conf/soctopus + - source: salt://soctopus/files + - user: 939 + - group: 939 + - template: jinja + +so-soctopus: + docker_container.running: + - image: wlambert/soctopus + - hostname: soctopus + - binds: + - /opt/so/conf/soctopus/SOCtopus.conf:/SOCtopus/SOCtopus.conf:ro + - port_bindings: + - 0.0.0.0:7000:7000 + From 681966a83d0750ef80f5d2fd12e19764013e5d6a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 15 Mar 2019 07:52:27 -0400 Subject: [PATCH 30/58] Setup Script - Change Eval memory settings --- salt/hive/thehive/etc/application.conf | 1 + so-setup-network.sh | 39 ++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/salt/hive/thehive/etc/application.conf b/salt/hive/thehive/etc/application.conf index 5a0a96b9b..1fd4b4816 100644 --- a/salt/hive/thehive/etc/application.conf +++ b/salt/hive/thehive/etc/application.conf @@ -4,6 +4,7 @@ # The secret key is used to secure cryptographic functions. # WARNING: If you deploy your application on several servers, make sure to use the same key. play.http.secret.key="letsdewdis" +play.http.context=/thehive/ # Elasticsearch search { diff --git a/so-setup-network.sh b/so-setup-network.sh index 826e0e650..9d9d3618b 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1104,7 +1104,7 @@ whiptail_cancel() { whiptail_check_exitstatus() { if [ $1 == '1' ]; then - echo " They hit cancel" + echo "They hit cancel" whiptail_cancel fi @@ -1120,6 +1120,30 @@ whiptail_cur_close_days() { } +whiptail_eval_adv() { + MASTERADV=$(whiptail --title "Security Onion Setup" --radiolist \ + "Choose your eval install:" 20 78 4 \ + "BASIC" "Install basic components for evaluation" ON \ + "ADVANCED" "Choose additional components to be installed" OFF 3>&1 1>&2 2>&3 ) +} + +whiptail_eval_adv_service_grafana() { + echo "blah" +} + +whiptail_eval_adv_service_osquery() { + echo "blah" + +} + +whiptail_eval_adv_service_wazuh() { + echo "blah" +} + +whiptail_eval_adv_warning() { + whiptail --title "Security Onion Setup" --msgbox "Please keep in mind the more services that you enable the more RAM that is required." 8 78 +} + whiptail_homenet_master() { # Ask for the HOME_NET on the master @@ -1702,11 +1726,20 @@ if (whiptail_you_sure); then # Snag the HOME_NET whiptail_homenet_master + # Ask about advanced mode + whiptail_eval_adv + if [ MASTERADV == 'ADVANCED' ]; then + whiptail_eval_adv_warning + whiptail_eval_adv_service_grafana + whiptail_eval_adv_service_osquery + whiptail_eval_adv_service_wazuh + fi + # Set a bunch of stuff since this is eval es_heapsize ls_heapsize - NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE - NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE + NODE_ES_HEAP_SIZE="600m" + NODE_LS_HEAP_SIZE="2000m" LSPIPELINEWORKERS=1 LSPIPELINEBATCH=125 LSINPUTTHREADS=1 From a35e90ad387542c5f95d6f5bd0e6031f60402067 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 15 Mar 2019 08:02:08 -0400 Subject: [PATCH 31/58] Setup Script - Eval Advanced Settings --- so-setup-network.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 9d9d3618b..7514cc478 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1121,7 +1121,7 @@ whiptail_cur_close_days() { } whiptail_eval_adv() { - MASTERADV=$(whiptail --title "Security Onion Setup" --radiolist \ + EVALADV=$(whiptail --title "Security Onion Setup" --radiolist \ "Choose your eval install:" 20 78 4 \ "BASIC" "Install basic components for evaluation" ON \ "ADVANCED" "Choose additional components to be installed" OFF 3>&1 1>&2 2>&3 ) @@ -1728,7 +1728,7 @@ if (whiptail_you_sure); then # Ask about advanced mode whiptail_eval_adv - if [ MASTERADV == 'ADVANCED' ]; then + if [ $EVALADVANCED == 'ADVANCED' ]; then whiptail_eval_adv_warning whiptail_eval_adv_service_grafana whiptail_eval_adv_service_osquery From 9a2b98aa17b46f99b29cf631f6d8dfbd4d430784 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 15 Mar 2019 08:12:40 -0400 Subject: [PATCH 32/58] Setup Script - Eval Advanced Settings --- so-setup-network.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 7514cc478..2542d3c9d 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1121,7 +1121,7 @@ whiptail_cur_close_days() { } whiptail_eval_adv() { - EVALADV=$(whiptail --title "Security Onion Setup" --radiolist \ + EVALADVANCED=$(whiptail --title "Security Onion Setup" --radiolist \ "Choose your eval install:" 20 78 4 \ "BASIC" "Install basic components for evaluation" ON \ "ADVANCED" "Choose additional components to be installed" OFF 3>&1 1>&2 2>&3 ) From c8102fe7b530e0292dcbc73f594f78f5a292fade Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 18 Mar 2019 10:17:33 -0400 Subject: [PATCH 33/58] Logstash Module - Disable stuff that isn't used in eval --- salt/logstash/conf/conf.enabled.txt.so-eval | 70 ++++++++++----------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval b/salt/logstash/conf/conf.enabled.txt.so-eval index 5c85ab8ac..bab35babc 100644 --- a/salt/logstash/conf/conf.enabled.txt.so-eval +++ b/salt/logstash/conf/conf.enabled.txt.so-eval @@ -7,25 +7,25 @@ # /usr/share/logstash/pipeline.custom/1234_input_custom.conf ## # All of the defaults are loaded. -/usr/share/logstash/pipeline.so/0000_input_syslogng.conf -/usr/share/logstash/pipeline.so/0001_input_json.conf -/usr/share/logstash/pipeline.so/0002_input_windows_json.conf -/usr/share/logstash/pipeline.so/0003_input_syslog.conf -/usr/share/logstash/pipeline.so/0005_input_suricata.conf +#/usr/share/logstash/pipeline.so/0000_input_syslogng.conf +#/usr/share/logstash/pipeline.so/0001_input_json.conf +#/usr/share/logstash/pipeline.so/0002_input_windows_json.conf +#/usr/share/logstash/pipeline.so/0003_input_syslog.conf +#/usr/share/logstash/pipeline.so/0005_input_suricata.conf /usr/share/logstash/pipeline.dynamic/0006_input_beats.conf /usr/share/logstash/pipeline.so/0007_input_import.conf -/usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf -/usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf -/usr/share/logstash/pipeline.so/1002_preprocess_json.conf +#/usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf +#/usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf +#/usr/share/logstash/pipeline.so/1002_preprocess_json.conf /usr/share/logstash/pipeline.so/1003_preprocess_bro.conf -/usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf +#/usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf /usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf -/usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf -/usr/share/logstash/pipeline.so/1030_preprocess_greensql.conf -/usr/share/logstash/pipeline.so/1031_preprocess_iis.conf -/usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf +#/usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf +#/usr/share/logstash/pipeline.so/1030_preprocess_greensql.conf +#/usr/share/logstash/pipeline.so/1031_preprocess_iis.conf +#/usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf /usr/share/logstash/pipeline.so/1033_preprocess_snort.conf -/usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf +#/usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf /usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf /usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf /usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf @@ -60,16 +60,16 @@ /usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf /usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf /usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf -/usr/share/logstash/pipeline.so/1998_test_data.conf -/usr/share/logstash/pipeline.so/2000_network_flow.conf +#/usr/share/logstash/pipeline.so/1998_test_data.conf +#/usr/share/logstash/pipeline.so/2000_network_flow.conf /usr/share/logstash/pipeline.so/6000_bro.conf /usr/share/logstash/pipeline.so/6001_bro_import.conf -/usr/share/logstash/pipeline.so/6002_syslog.conf -/usr/share/logstash/pipeline.so/6101_switch_brocade.conf -/usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf -/usr/share/logstash/pipeline.so/6201_firewall_pfsense.conf -/usr/share/logstash/pipeline.so/6300_windows.conf -/usr/share/logstash/pipeline.so/6301_dns_windows.conf +#/usr/share/logstash/pipeline.so/6002_syslog.conf +#/usr/share/logstash/pipeline.so/6101_switch_brocade.conf +#/usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf +#/usr/share/logstash/pipeline.so/6201_firewall_pfsense.conf +#/usr/share/logstash/pipeline.so/6300_windows.conf +#/usr/share/logstash/pipeline.so/6301_dns_windows.conf /usr/share/logstash/pipeline.so/6400_suricata.conf /usr/share/logstash/pipeline.so/6500_ossec.conf /usr/share/logstash/pipeline.so/6501_ossec_sysmon.conf @@ -87,25 +87,25 @@ #/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf #/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf #/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf -/usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf +#/usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf /usr/share/logstash/pipeline.so/8999_postprocess_rename_type.conf /usr/share/logstash/pipeline.dynamic/9000_output_bro.conf -/usr/share/logstash/pipeline.dynamic/9001_output_switch.conf +#/usr/share/logstash/pipeline.dynamic/9001_output_switch.conf /usr/share/logstash/pipeline.dynamic/9002_output_import.conf -/usr/share/logstash/pipeline.dynamic/9004_output_flow.conf -/usr/share/logstash/pipeline.dynamic/9026_output_dhcp.conf -/usr/share/logstash/pipeline.dynamic/9029_output_esxi.conf -/usr/share/logstash/pipeline.dynamic/9030_output_greensql.conf -/usr/share/logstash/pipeline.dynamic/9031_output_iis.conf -/usr/share/logstash/pipeline.dynamic/9032_output_mcafee.conf +#/usr/share/logstash/pipeline.dynamic/9004_output_flow.conf +#/usr/share/logstash/pipeline.dynamic/9026_output_dhcp.conf +#/usr/share/logstash/pipeline.dynamic/9029_output_esxi.conf +#/usr/share/logstash/pipeline.dynamic/9030_output_greensql.conf +#/usr/share/logstash/pipeline.dynamic/9031_output_iis.conf +#/usr/share/logstash/pipeline.dynamic/9032_output_mcafee.conf /usr/share/logstash/pipeline.dynamic/9033_output_snort.conf -/usr/share/logstash/pipeline.dynamic/9034_output_syslog.conf -/usr/share/logstash/pipeline.dynamic/9200_output_firewall.conf -/usr/share/logstash/pipeline.dynamic/9300_output_windows.conf -/usr/share/logstash/pipeline.dynamic/9301_output_dns_windows.conf +#/usr/share/logstash/pipeline.dynamic/9034_output_syslog.conf +#/usr/share/logstash/pipeline.dynamic/9200_output_firewall.conf +#/usr/share/logstash/pipeline.dynamic/9300_output_windows.conf +#/usr/share/logstash/pipeline.dynamic/9301_output_dns_windows.conf /usr/share/logstash/pipeline.dynamic/9400_output_suricata.conf /usr/share/logstash/pipeline.dynamic/9500_output_beats.conf /usr/share/logstash/pipeline.dynamic/9600_output_ossec.conf -/usr/share/logstash/pipeline.dynamic/9998_output_test_data.conf +#/usr/share/logstash/pipeline.dynamic/9998_output_test_data.conf /usr/share/logstash/pipeline.dynamic/7100_osquery_wel.conf /usr/share/logstash/pipeline.dynamic/9100_output_osquery.conf From 8663da0330dd2228821deb365634e7b2b2a46562 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 18 Mar 2019 10:23:43 -0400 Subject: [PATCH 34/58] Logstash Module - Disable stuff that isn't used in eval --- salt/logstash/conf/conf.enabled.txt.so-eval | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/conf/conf.enabled.txt.so-eval b/salt/logstash/conf/conf.enabled.txt.so-eval index bab35babc..d8eae00e6 100644 --- a/salt/logstash/conf/conf.enabled.txt.so-eval +++ b/salt/logstash/conf/conf.enabled.txt.so-eval @@ -70,7 +70,7 @@ #/usr/share/logstash/pipeline.so/6201_firewall_pfsense.conf #/usr/share/logstash/pipeline.so/6300_windows.conf #/usr/share/logstash/pipeline.so/6301_dns_windows.conf -/usr/share/logstash/pipeline.so/6400_suricata.conf +#/usr/share/logstash/pipeline.so/6400_suricata.conf /usr/share/logstash/pipeline.so/6500_ossec.conf /usr/share/logstash/pipeline.so/6501_ossec_sysmon.conf /usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf From bb608541a75c4ced75b97b6f4c2d08f3f0a83f19 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 18 Mar 2019 14:09:10 -0400 Subject: [PATCH 35/58] Setup Script - Eval mode memory savings --- so-setup-network.sh | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 2542d3c9d..339e572c3 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -534,6 +534,28 @@ master_pillar() { echo " ls_input_threads: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " ls_batch_count: 125" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " mtu: 1500" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + if [ $EVALADVANCED == 'ADVANCED' ]; then + if [ $EVALGRAFANA == '0' ]; then + echo " grafana: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + else + echo " grafana: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + fi + if [ $EVALOSQUERY == '0' ]; then + echo " osquery: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + else + echo " osquery: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + fi + if [ $EVALWAZUH == '0' ]; then + echo " wazuh: 1" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + else + echo " wazuh: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + fi + else + echo " grafana: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " osquery: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + echo " wazuh: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls + fi + else echo " freq: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " domainstats: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls @@ -1128,16 +1150,16 @@ whiptail_eval_adv() { } whiptail_eval_adv_service_grafana() { - echo "blah" + EVALGRAFANA=$(whiptail --title "Eval Advanced Setup" --yesno "Would you like to enable Grafana for detailed monitoring?" 8 78) } whiptail_eval_adv_service_osquery() { - echo "blah" + EVALOSQUERY=$(whiptail --title "Eval Advanced Setup" --yesno "Would you like to enable OSquery for client monitoring?" 8 78) } whiptail_eval_adv_service_wazuh() { - echo "blah" + EVALWAZUH=$(whiptail --title "Eval Advanced Setup" --yesno "Would you like to enable Wazuh for client monitoring?" 8 78) } whiptail_eval_adv_warning() { From 661f71b04e621f21d9794e4acfa68c276b3de533 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 18 Mar 2019 14:25:04 -0400 Subject: [PATCH 36/58] Setup Script - Eval mode memory savings --- so-setup-network.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 339e572c3..092f1b199 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -555,7 +555,7 @@ master_pillar() { echo " osquery: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " wazuh: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls fi - + else echo " freq: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls echo " domainstats: 0" >> /opt/so/saltstack/pillar/masters/$HOSTNAME.sls @@ -1150,7 +1150,9 @@ whiptail_eval_adv() { } whiptail_eval_adv_service_grafana() { - EVALGRAFANA=$(whiptail --title "Eval Advanced Setup" --yesno "Would you like to enable Grafana for detailed monitoring?" 8 78) + whiptail --title "Eval Advanced Setup" --yesno "Would you like to enable Grafana for detailed monitoring?" 8 78 + local exitstatus=$? + EVALGRAFANA=$exitstatus } whiptail_eval_adv_service_osquery() { From 2c13b0c663b634c1ec3d913177b079658f8931b5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 18 Mar 2019 14:43:33 -0400 Subject: [PATCH 37/58] Setup Script - Eval mode memory savings --- so-setup-network.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/so-setup-network.sh b/so-setup-network.sh index 092f1b199..1643d3e42 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -1156,12 +1156,15 @@ whiptail_eval_adv_service_grafana() { } whiptail_eval_adv_service_osquery() { - EVALOSQUERY=$(whiptail --title "Eval Advanced Setup" --yesno "Would you like to enable OSquery for client monitoring?" 8 78) - + whiptail --title "Eval Advanced Setup" --yesno "Would you like to enable OSquery for client monitoring?" 8 78 + local exitstatus=$? + EVALOSQUERY=$exitstatus } whiptail_eval_adv_service_wazuh() { - EVALWAZUH=$(whiptail --title "Eval Advanced Setup" --yesno "Would you like to enable Wazuh for client monitoring?" 8 78) + whiptail --title "Eval Advanced Setup" --yesno "Would you like to enable Wazuh for client monitoring?" 8 78 + local exitstatus=$? + EVALWAZUH=$exitstatus } whiptail_eval_adv_warning() { From dfd25567e14bc38a743fcdff03234127a0e2eb3b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 18 Mar 2019 15:17:31 -0400 Subject: [PATCH 38/58] Setup Script - Turn off wazuh and osq by default in eval mode --- salt/top.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/top.sls b/salt/top.sls index fcd68f1eb..0d6476779 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -1,4 +1,6 @@ {%- set BROVER = salt['pillar.get']('static:broversion', 'COMMUNITY') %} +{%- set OSQUERY = salt['pillar.get']('master:osquery', '0') %} +{%- set WAZUH = salt['pillar.get']('master:wazuh', '0') %} base: 'G@role:so-sensor': - ca @@ -31,8 +33,12 @@ base: - curator - elastalert - redis + {%- if OSQUERY != '0' %} - fleet + {%- endif %} + {%- if WAZUH != '0' %} - wazuh + {%- endif %} - filebeat - utility - schedule From 55e577a7a2076f1a6a87c5e17e59f7ffb935d5fd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 18 Mar 2019 15:46:47 -0400 Subject: [PATCH 39/58] MEGARAMS Savings - Add advanced logic for services disabled --- salt/common/init.sls | 3 ++- salt/top.sls | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 00d7f35d0..74735a185 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -1,3 +1,4 @@ +{%- set GRAFANA = salt['pillar.get']('master:grafana', '0') %} # Add socore Group socoregroup: group.present: @@ -187,7 +188,7 @@ so-telegraf: - /opt/so/conf/telegraf/scripts # If its a master or eval lets install the back end for now -{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %} +{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' and GRAFANA == 1 %} # Influx DB influxconfdir: diff --git a/salt/top.sls b/salt/top.sls index 0d6476779..b36f3b4c9 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -33,10 +33,10 @@ base: - curator - elastalert - redis - {%- if OSQUERY != '0' %} + {%- if OSQUERY != 0 %} - fleet {%- endif %} - {%- if WAZUH != '0' %} + {%- if WAZUH != 0 %} - wazuh {%- endif %} - filebeat From f72f70330a620b079d82ab044e75458a8f250814 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 18 Mar 2019 16:23:07 -0400 Subject: [PATCH 40/58] MEGARAMS Savings - Remove redis from eval --- salt/top.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/top.sls b/salt/top.sls index b36f3b4c9..2a3a2175f 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -23,7 +23,9 @@ base: - firewall - master - idstools + {%- if OSQUERY != 0 %} - mysql + {%- endif %} - elasticsearch - logstash - kibana @@ -32,7 +34,6 @@ base: - bro - curator - elastalert - - redis {%- if OSQUERY != 0 %} - fleet {%- endif %} From c126afe3f7eb556d06dc45de76b6415ef7260ae7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 18 Mar 2019 16:23:56 -0400 Subject: [PATCH 41/58] MEGARAMS Savings - Remove redis from eval --- salt/top.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/top.sls b/salt/top.sls index 2a3a2175f..bd917428e 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -36,6 +36,7 @@ base: - elastalert {%- if OSQUERY != 0 %} - fleet + - redis {%- endif %} {%- if WAZUH != 0 %} - wazuh From e7964f3a40723413c68d191469b3163b3f091114 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 20 Mar 2019 14:37:07 -0400 Subject: [PATCH 42/58] The Hive - Reverse Proxy config --- salt/common/nginx/nginx.conf.so-eval | 13 +++++++++++-- salt/common/nginx/nginx.conf.so-master | 14 ++++++++++++-- salt/hive/thehive/etc/application.conf | 2 +- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval index f137ced15..77c5e3d48 100644 --- a/salt/common/nginx/nginx.conf.so-eval +++ b/salt/common/nginx/nginx.conf.so-eval @@ -137,8 +137,7 @@ http { } location /thehive/ { - rewrite /thehive/(.*) /$1 break; - proxy_pass http://{{ masterip }}:9000/; + proxy_pass http://{{ masterip }}:9000/thehive/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; @@ -148,6 +147,16 @@ http { } + location /soctopus/ { + proxy_pass http://{{ masterip }}:7000/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } error_page 404 /404.html; location = /40x.html { } diff --git a/salt/common/nginx/nginx.conf.so-master b/salt/common/nginx/nginx.conf.so-master index f73c9620a..f0338070f 100644 --- a/salt/common/nginx/nginx.conf.so-master +++ b/salt/common/nginx/nginx.conf.so-master @@ -137,8 +137,18 @@ http { } location /thehive/ { - rewrite /thehive/(.*) /$1 break; - proxy_pass http://{{ masterip }}:9000/; + proxy_pass http://{{ masterip }}:9000/thehive/; + proxy_read_timeout 90; + proxy_connect_timeout 90; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Proxy ""; + + } + + location /soctopus/ { + proxy_pass http://{{ masterip }}:7000/; proxy_read_timeout 90; proxy_connect_timeout 90; proxy_set_header Host $host; diff --git a/salt/hive/thehive/etc/application.conf b/salt/hive/thehive/etc/application.conf index 1fd4b4816..1f27ac70d 100644 --- a/salt/hive/thehive/etc/application.conf +++ b/salt/hive/thehive/etc/application.conf @@ -4,7 +4,7 @@ # The secret key is used to secure cryptographic functions. # WARNING: If you deploy your application on several servers, make sure to use the same key. play.http.secret.key="letsdewdis" -play.http.context=/thehive/ +play.http.context=\/thehive\/ # Elasticsearch search { From 636692fbb3a36e7f6a6a83af106f040f8e0967be Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Wed, 20 Mar 2019 16:52:06 -0400 Subject: [PATCH 43/58] The Hive - Reverse Proxy config --- salt/hive/thehive/etc/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/thehive/etc/application.conf b/salt/hive/thehive/etc/application.conf index 1f27ac70d..1fd4b4816 100644 --- a/salt/hive/thehive/etc/application.conf +++ b/salt/hive/thehive/etc/application.conf @@ -4,7 +4,7 @@ # The secret key is used to secure cryptographic functions. # WARNING: If you deploy your application on several servers, make sure to use the same key. play.http.secret.key="letsdewdis" -play.http.context=\/thehive\/ +play.http.context=/thehive/ # Elasticsearch search { From dcc8db312ccd6344881f1200e33badacb6aed15b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 22 Mar 2019 13:44:36 -0400 Subject: [PATCH 44/58] The Hive - Reverse Proxy config --- salt/hive/thehive/etc/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/thehive/etc/application.conf b/salt/hive/thehive/etc/application.conf index 1fd4b4816..8be7ea8d6 100644 --- a/salt/hive/thehive/etc/application.conf +++ b/salt/hive/thehive/etc/application.conf @@ -4,7 +4,7 @@ # The secret key is used to secure cryptographic functions. # WARNING: If you deploy your application on several servers, make sure to use the same key. play.http.secret.key="letsdewdis" -play.http.context=/thehive/ +play.http.contex=/thehive/ # Elasticsearch search { From 392cde313f722603249352ce360e33221e302bdd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 22 Mar 2019 13:44:55 -0400 Subject: [PATCH 45/58] The Hive - Reverse Proxy config --- salt/hive/thehive/etc/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/thehive/etc/application.conf b/salt/hive/thehive/etc/application.conf index 8be7ea8d6..1fd4b4816 100644 --- a/salt/hive/thehive/etc/application.conf +++ b/salt/hive/thehive/etc/application.conf @@ -4,7 +4,7 @@ # The secret key is used to secure cryptographic functions. # WARNING: If you deploy your application on several servers, make sure to use the same key. play.http.secret.key="letsdewdis" -play.http.contex=/thehive/ +play.http.context=/thehive/ # Elasticsearch search { From b32558b2ac35a8b8dfa7db8a450722ae6655ab98 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 22 Mar 2019 14:20:09 -0400 Subject: [PATCH 46/58] The Hive - Reverse Proxy config --- salt/common/nginx/nginx.conf.so-eval | 1 + salt/common/nginx/nginx.conf.so-master | 1 + 2 files changed, 2 insertions(+) diff --git a/salt/common/nginx/nginx.conf.so-eval b/salt/common/nginx/nginx.conf.so-eval index 77c5e3d48..3b0a0d4a1 100644 --- a/salt/common/nginx/nginx.conf.so-eval +++ b/salt/common/nginx/nginx.conf.so-eval @@ -140,6 +140,7 @@ http { proxy_pass http://{{ masterip }}:9000/thehive/; proxy_read_timeout 90; proxy_connect_timeout 90; + proxy_http_version 1.1; # this is essential for chunked responses to work proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/salt/common/nginx/nginx.conf.so-master b/salt/common/nginx/nginx.conf.so-master index f0338070f..c0eada231 100644 --- a/salt/common/nginx/nginx.conf.so-master +++ b/salt/common/nginx/nginx.conf.so-master @@ -140,6 +140,7 @@ http { proxy_pass http://{{ masterip }}:9000/thehive/; proxy_read_timeout 90; proxy_connect_timeout 90; + proxy_http_version 1.1; # this is essential for chunked responses to work proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; From 03e42a009a60078d3e25abfb127d87a9de29c76f Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 25 Mar 2019 16:12:24 +0000 Subject: [PATCH 47/58] SOCtopus: update config --- salt/soctopus/files/SOCtopus.conf | 43 ++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index eb766755f..c1d580f3c 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -1,12 +1,47 @@ {%- set ip = salt['pillar.get']('static:masterip', '') %} [es] -es_url = http://{{ ip }}:9200 +es_url = http://{{ip}}:9200 + +[fir] +fir_url = YOURFIRURL +fir_token = YOURFIRTOKEN +fir_actor = 3 +fir_category = 3 +fir_confidentiality = 1 +fir_detection = 2 +fir_plan = 8 +fir_severity = 4 + +[grr] +grr_url = YOURGRRURL +grr_user = YOURGRRUSER +grr_pass = YOURGRRPASS [hive] -hive_url = http://{{ ip }}:9000 -hive_key = YOURHIVEAPIKEYHERE -- TO LATER BE REPLACED BY JINJA +hive_url = https://{{ip}}/thehive +hive_key = YOURHIVEKEY +tlp = 3 + +[misp] +misp_url = YOURMISPURL +misp_key = YOURMISPKEY +misp_verifycert = False +distrib = 0 +threat = 4 +analysis = 0 + +[rtir] +rtir_url = YOURRTIRURL +rtir_api = REST/1.0/ +rtir_user = YOURRTIRUSER +rtir_pass = YOURRTIRPASS +rtir_queue = Incidents +rtir_creator = root + +[slack] +slack_url = YOURSLACKWORKSPACE +slack_webhook = YOURSLACKWEBHOOK [log] logfile = /tmp/soctopus.log - From 5b8e00b8bc77919f97de92e10b748b7e1f2a11b4 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 25 Mar 2019 16:28:20 -0400 Subject: [PATCH 48/58] SOCtopus Module - Fixed Configs --- salt/soctopus/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index e811b587b..3902124e9 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -15,10 +15,10 @@ soctopussync: so-soctopus: docker_container.running: - - image: wlambert/soctopus + - image: soshybridhunter/so-soctopus:HH1.0.7 - hostname: soctopus + - name: so-soctopus - binds: - /opt/so/conf/soctopus/SOCtopus.conf:/SOCtopus/SOCtopus.conf:ro - port_bindings: - 0.0.0.0:7000:7000 - From 1d01192b9ee5c94ae8e15cadf7393a96065d4261 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 25 Mar 2019 21:21:21 +0000 Subject: [PATCH 49/58] SOCtopus: update config --- salt/soctopus/files/SOCtopus.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index c1d580f3c..f235ed612 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -19,9 +19,10 @@ grr_user = YOURGRRUSER grr_pass = YOURGRRPASS [hive] -hive_url = https://{{ip}}/thehive +hive_url = https://{{ip}}/thehive/ hive_key = YOURHIVEKEY -tlp = 3 +hive_tlp = 3 +hive_verify_ssl = False [misp] misp_url = YOURMISPURL From 8b8556213b1e29807e458c4da8879e33a5b3e0da Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Mon, 25 Mar 2019 21:26:53 +0000 Subject: [PATCH 50/58] SOCtopus: more config updates --- salt/soctopus/files/SOCtopus.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index f235ed612..3ce772082 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -22,7 +22,7 @@ grr_pass = YOURGRRPASS hive_url = https://{{ip}}/thehive/ hive_key = YOURHIVEKEY hive_tlp = 3 -hive_verify_ssl = False +hive_verifycert = False [misp] misp_url = YOURMISPURL From 4f15e14cc2ba15ebfe2a3ae9266f43321ec08bd1 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 28 Mar 2019 14:24:17 +0000 Subject: [PATCH 51/58] TheHive: Add initial user --- salt/hive/thehive/files/hive_init.sh | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 salt/hive/thehive/files/hive_init.sh diff --git a/salt/hive/thehive/files/hive_init.sh b/salt/hive/thehive/files/hive_init.sh new file mode 100755 index 000000000..d1893e200 --- /dev/null +++ b/salt/hive/thehive/files/hive_init.sh @@ -0,0 +1,37 @@ +#!/bin/bash +{%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} +{%- set HIVEUSER = salt['pillar.get']('static:hiveuser', '') %} +{%- set HIVEPASSWORD = salt['pillar.get']('static:hivepassword', '') %} +{%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') %} + +hive_init(){ + + HIVE_IP="{{MASTERIP}}" + HIVE_USER="{{HIVEUSER}}" + HIVE_PASSWORD="{{HIVEPASSWORD}}" + SOCTOPUS_CONFIG="/opt/so/saltstack/salt/soctopus/files/SOCtopus.conf" + + # Migrate DB + curl -v -k -XPOST "https://$HIVE_IP:/thehive/api/maintenance/migrate" + + # Generate unique ID for apikey + HIVE_KEY="{{HIVEKEY}}" + + # Create intial TheHive user + curl -v -k "https://$HIVE_IP/thehive/api/user" -H "Content-Type: application/json" -d "{\"login\" : \"$HIVE_$USER\",\"name\" : \"$HIVE_USER\",\"roles\" : [\"read\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$HIVE_PASSWORD\", \"key\": \"$HIVE_KEY\"}" + + # Update SOCtopus config with apikey value + sed -i "s/hive_key = .*/hive_key = $HIVE_KEY/" $SOCTOPUS_CONFIG + + # Check for correct authentication + #curl -v -k -H "Authorization: Bearer $HIVE_KEY" "https://$HIVE_IP/thehive/api/user/$USER" + + touch /opt/so/state/thehive.txt + +} + +if [ -f /opt/so/state/thehive.txt ]; then + exit 0 +else + hive_init +fi From fce80236de55ee4577eebc5a0747c0c73f54bfe1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Mar 2019 10:40:29 -0400 Subject: [PATCH 52/58] The Hive - Wes Mods --- salt/hive/thehive/{files => scripts}/hive_init.sh | 8 ++++---- salt/soctopus/files/SOCtopus.conf | 5 +++-- so-setup-network.sh | 4 ++++ 3 files changed, 11 insertions(+), 6 deletions(-) rename salt/hive/thehive/{files => scripts}/hive_init.sh (93%) diff --git a/salt/hive/thehive/files/hive_init.sh b/salt/hive/thehive/scripts/hive_init.sh similarity index 93% rename from salt/hive/thehive/files/hive_init.sh rename to salt/hive/thehive/scripts/hive_init.sh index d1893e200..255bf0502 100755 --- a/salt/hive/thehive/files/hive_init.sh +++ b/salt/hive/thehive/scripts/hive_init.sh @@ -5,7 +5,7 @@ {%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') %} hive_init(){ - + sleep 60 HIVE_IP="{{MASTERIP}}" HIVE_USER="{{HIVEUSER}}" HIVE_PASSWORD="{{HIVEPASSWORD}}" @@ -16,16 +16,16 @@ hive_init(){ # Generate unique ID for apikey HIVE_KEY="{{HIVEKEY}}" - + # Create intial TheHive user curl -v -k "https://$HIVE_IP/thehive/api/user" -H "Content-Type: application/json" -d "{\"login\" : \"$HIVE_$USER\",\"name\" : \"$HIVE_USER\",\"roles\" : [\"read\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$HIVE_PASSWORD\", \"key\": \"$HIVE_KEY\"}" # Update SOCtopus config with apikey value - sed -i "s/hive_key = .*/hive_key = $HIVE_KEY/" $SOCTOPUS_CONFIG + #sed -i "s/hive_key = .*/hive_key = $HIVE_KEY/" $SOCTOPUS_CONFIG # Check for correct authentication #curl -v -k -H "Authorization: Bearer $HIVE_KEY" "https://$HIVE_IP/thehive/api/user/$USER" - + touch /opt/so/state/thehive.txt } diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index 3ce772082..1a48ad92f 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -1,4 +1,5 @@ {%- set ip = salt['pillar.get']('static:masterip', '') %} +{%- set HIVEKEY = salt['pillar.get']('static:hivekey', '') %} [es] es_url = http://{{ip}}:9200 @@ -20,14 +21,14 @@ grr_pass = YOURGRRPASS [hive] hive_url = https://{{ip}}/thehive/ -hive_key = YOURHIVEKEY +hive_key = {{ HIVEKEY }} hive_tlp = 3 hive_verifycert = False [misp] misp_url = YOURMISPURL misp_key = YOURMISPKEY -misp_verifycert = False +misp_verifycert = False distrib = 0 threat = 4 analysis = 0 diff --git a/so-setup-network.sh b/so-setup-network.sh index 1643d3e42..dde654eb3 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -427,6 +427,7 @@ generate_passwords(){ # Generate Random Passwords for Things MYSQLPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) FLEETPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) + HIVEKEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) } get_filesystem_nsm(){ @@ -588,6 +589,9 @@ master_static() { echo " broversion: $BROVERSION" >> /opt/so/saltstack/pillar/static.sls echo " ids: $NIDS" >> /opt/so/saltstack/pillar/static.sls echo " masterip: $MAINIP" >> /opt/so/saltstack/pillar/static.sls + echo " hiveuser: hiveadmin" >> /opt/so/saltstack/pillar/static.sls + echo " hivepassword: hivechangeme" >> /opt/so/saltstack/pillar/static.sls + echo " hivekey: $HIVEKEY" >> /opt/so/saltstack/pillar/static.sls if [[ $MASTERUPDATES == 'MASTER' ]]; then echo " masterupdate: 1" >> /opt/so/saltstack/pillar/static.sls else From fd027cb95465aebd97c90e301b647f1de7b192bd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Mar 2019 10:55:32 -0400 Subject: [PATCH 53/58] The Hive - Fix the user creation script --- salt/hive/thehive/scripts/hive_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hive/thehive/scripts/hive_init.sh b/salt/hive/thehive/scripts/hive_init.sh index 255bf0502..54c658474 100755 --- a/salt/hive/thehive/scripts/hive_init.sh +++ b/salt/hive/thehive/scripts/hive_init.sh @@ -18,7 +18,7 @@ hive_init(){ HIVE_KEY="{{HIVEKEY}}" # Create intial TheHive user - curl -v -k "https://$HIVE_IP/thehive/api/user" -H "Content-Type: application/json" -d "{\"login\" : \"$HIVE_$USER\",\"name\" : \"$HIVE_USER\",\"roles\" : [\"read\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$HIVE_PASSWORD\", \"key\": \"$HIVE_KEY\"}" + curl -v -k "https://$HIVE_IP/thehive/api/user" -H "Content-Type: application/json" -d "{\"login\" : \"$HIVE_USER\",\"name\" : \"$HIVE_USER\",\"roles\" : [\"read\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$HIVE_PASSWORD\", \"key\": \"$HIVE_KEY\"}" # Update SOCtopus config with apikey value #sed -i "s/hive_key = .*/hive_key = $HIVE_KEY/" $SOCTOPUS_CONFIG From 77c90ce752bdbe4168c2304f8d1213216f2d0d84 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Mar 2019 11:23:06 -0400 Subject: [PATCH 54/58] The Hive - Just scripted filed left --- salt/hive/init.sls | 5 +++++ salt/hive/thehive/scripts/hive_init.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/salt/hive/init.sls b/salt/hive/init.sls index d0af62fc3..371e790de 100644 --- a/salt/hive/init.sls +++ b/salt/hive/init.sls @@ -78,3 +78,8 @@ so-thehive: - /opt/so/conf/hive/etc/application.conf:/opt/thehive/conf/application.conf:ro - port_bindings: - 0.0.0.0:9000:9000 + +hivescript: + cmd.script: + - source: salt://hive/thehive/scripts/hive_init.sh + - template: jinja diff --git a/salt/hive/thehive/scripts/hive_init.sh b/salt/hive/thehive/scripts/hive_init.sh index 54c658474..cb901e36b 100755 --- a/salt/hive/thehive/scripts/hive_init.sh +++ b/salt/hive/thehive/scripts/hive_init.sh @@ -18,7 +18,7 @@ hive_init(){ HIVE_KEY="{{HIVEKEY}}" # Create intial TheHive user - curl -v -k "https://$HIVE_IP/thehive/api/user" -H "Content-Type: application/json" -d "{\"login\" : \"$HIVE_USER\",\"name\" : \"$HIVE_USER\",\"roles\" : [\"read\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$HIVE_PASSWORD\", \"key\": \"$HIVE_KEY\"}" + curl -v -k "https://$HIVE_IP/thehive/api/user" -H "Content-Type: application/json" -d "{\"login\" : \"$HIVE_USER\",\"name\" : \"$HIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$HIVE_PASSWORD\", \"key\": \"$HIVE_KEY\"}" # Update SOCtopus config with apikey value #sed -i "s/hive_key = .*/hive_key = $HIVE_KEY/" $SOCTOPUS_CONFIG From 87fde50eb148bdbc557dad411cd5ca22cbf80d47 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 28 Mar 2019 14:46:20 -0400 Subject: [PATCH 55/58] Top.sls - Add SOCtopus as default docker to get loaded --- salt/top.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/top.sls b/salt/top.sls index bd917428e..2a34c7548 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -44,6 +44,7 @@ base: - filebeat - utility - schedule + - soctopus 'G@role:so-master': @@ -64,6 +65,7 @@ base: - utility - schedule - fleet + - soctopus # Storage node logic From 2dd6558826897b526d5aa1e6cf0411539c774146 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 1 Apr 2019 11:10:44 -0400 Subject: [PATCH 56/58] Suricata Module - Suricata 4.1.3 --- salt/suricata/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index ea29c69a0..48106a83a 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -72,7 +72,7 @@ suriconfigsync: so-suricata: docker_container.running: - - image: soshybridhunter/so-suricata:HH1.0.6 + - image: soshybridhunter/so-suricata:HH1.0.7 - privileged: True - environment: - INTERFACE={{ interface }} From 139f0cd281f941b87e5edad810d3b024751b67de Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 1 Apr 2019 17:22:22 -0400 Subject: [PATCH 57/58] 1.0.7 Upgrade --- salt/common/init.sls | 8 ++++---- salt/kibana/init.sls | 2 +- salt/master/init.sls | 2 +- salt/mysql/init.sls | 2 +- salt/redis/init.sls | 2 +- salt/wazuh/init.sls | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/salt/common/init.sls b/salt/common/init.sls index 74735a185..eadf4f142 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -103,7 +103,7 @@ nginxtmp: # Start the core docker so-core: docker_container.running: - - image: soshybridhunter/so-core:HH1.0.5 + - image: soshybridhunter/so-core:HH1.0.7 - hostname: so-core - user: socore - binds: @@ -156,7 +156,7 @@ tgrafconf: so-telegraf: docker_container.running: - - image: soshybridhunter/so-telegraf:HH1.0.4 + - image: soshybridhunter/so-telegraf:HH1.0.7 - environment: - HOST_PROC=/host/proc - HOST_ETC=/host/etc @@ -211,7 +211,7 @@ influxdbconf: so-influxdb: docker_container.running: - - image: soshybridhunter/so-influxdb:HH1.0.4 + - image: soshybridhunter/so-influxdb:HH1.0.7 - hostname: influxdb - environment: - INFLUXDB_HTTP_LOG_ENABLED=false @@ -368,7 +368,7 @@ dashboard-{{ SN }}: # Install the docker. This needs to be behind nginx at some point so-grafana: docker_container.running: - - image: soshybridhunter/so-grafana:HH1.0.4 + - image: soshybridhunter/so-grafana:HH1.0.7 - hostname: grafana - user: socore - binds: diff --git a/salt/kibana/init.sls b/salt/kibana/init.sls index 3b5037336..050582c82 100644 --- a/salt/kibana/init.sls +++ b/salt/kibana/init.sls @@ -57,7 +57,7 @@ synckibanacustom: # Start the kibana docker so-kibana: docker_container.running: - - image: soshybridhunter/so-kibana:HH1.0.6 + - image: soshybridhunter/so-kibana:HH1.0.7 - hostname: kibana - user: kibana - environment: diff --git a/salt/master/init.sls b/salt/master/init.sls index 35f6c5254..8f20ef69f 100644 --- a/salt/master/init.sls +++ b/salt/master/init.sls @@ -49,7 +49,7 @@ acngcopyconf: # Install the apt-cacher-ng container so-aptcacherng: docker_container.running: - - image: soshybridhunter/so-acng:HH1.0.5 + - image: soshybridhunter/so-acng:HH1.0.7 - hostname: so-acng - port_bindings: - 0.0.0.0:3142:3142 diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index af80030ee..b1e875578 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -50,7 +50,7 @@ mysqldatadir: so-mysql: docker_container.running: - - image: soshybridhunter/so-mysql:HH1.0.5 + - image: soshybridhunter/so-mysql:HH1.0.7 - hostname: so-mysql - user: socore - port_bindings: diff --git a/salt/redis/init.sls b/salt/redis/init.sls index cd982a137..6dfbb473d 100644 --- a/salt/redis/init.sls +++ b/salt/redis/init.sls @@ -49,7 +49,7 @@ toosmooth/so-redis:test2: so-redis: docker_container.running: - - image: soshybridhunter/so-redis:HH1.0.5 + - image: soshybridhunter/so-redis:HH1.0.7 - hostname: so-redis - user: socore - port_bindings: diff --git a/salt/wazuh/init.sls b/salt/wazuh/init.sls index ac05f1984..4e5c136b5 100644 --- a/salt/wazuh/init.sls +++ b/salt/wazuh/init.sls @@ -58,7 +58,7 @@ wazuhagentregister: so-wazuh: docker_container.running: - - image: soshybridhunter/so-wazuh:HH1.0.5 + - image: soshybridhunter/so-wazuh:HH1.0.7 - hostname: {{HOSTNAME}}-wazuh-manager - name: so-wazuh - detach: True From 2bbd31c9549101b4aebf02c0b3c774ccd94a343a Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Apr 2019 11:21:57 -0400 Subject: [PATCH 58/58] Core Module - Update packages mapping --- salt/common/init.sls | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/init.sls b/salt/common/init.sls index eadf4f142..22e36d1d2 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -114,6 +114,7 @@ so-core: - /opt/so/tmp/nginx/:/run:rw - /etc/pki/masterssl.crt:/etc/pki/nginx/server.crt:ro - /etc/pki/masterssl.key:/etc/pki/nginx/server.key:ro + - /opt/so/conf/fleet/packages:/opt/so/html/packages - cap_add: NET_BIND_SERVICE - port_bindings: - 80:80