Merge pull request #120 from defensivedepth/cortex-initial

Cortex - initial commit

salt/common/nginx/nginx.conf.so-eval

@@ -177,6 +177,18 @@ http {
 
         }
 
+        location /cortex/ {
+          proxy_pass http://{{ masterip }}:9001/cortex/;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_http_version 1.1; # this is essential for chunked responses to work
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+
+        }
+	
         location /soctopus/ {
           proxy_pass http://{{ masterip }}:7000/;
           proxy_read_timeout    90;

salt/common/nginx/nginx.conf.so-master

@@ -176,6 +176,18 @@ http {
 
         }
 
+        location /cortex/ {
+          proxy_pass http://{{ masterip }}:9001/cortex/;
+          proxy_read_timeout    90;
+          proxy_connect_timeout 90;
+          proxy_http_version 1.1; # this is essential for chunked responses to work
+          proxy_set_header      Host $host;
+          proxy_set_header      X-Real-IP $remote_addr;
+          proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header      Proxy "";
+
+        }
+	
         location /soctopus/ {
           proxy_pass http://{{ masterip }}:7000/;
           proxy_read_timeout    90;

salt/firewall/init.sls

@@ -265,6 +265,17 @@ enable_master_navigator_4200_{{ip}}:
     - position: 1
     - save: True
  
+enable_master_cortex_9001_{{ip}}:
+  iptables.insert:
+    - table: filter
+    - chain: DOCKER-USER
+    - jump: ACCEPT
+    - proto: tcp
+    - source: {{ ip }}
+    - dport: 9001
+    - position: 1
+    - save: True 
+
 {% endfor %}
 
 # Make it so all the minions can talk to salt and update etc.

salt/hive/init.sls

@@ -21,6 +21,28 @@ hiveconf:
     - group: 939
     - template: jinja
     
+cortexconfdir:
+  file.directory:
+    - name: /opt/so/conf/cortex
+    - makedirs: True
+    - user: 939
+    - group: 939
+
+cortexlogdir:
+  file.directory:
+    - name: /opt/so/log/cortex
+    - makedirs: True
+    - user: 939
+    - group: 939
+
+cortexconf:
+  file.recurse:
+    - name: /opt/so/conf/cortex
+    - source: salt://hive/thehive/etc
+    - user: 939
+    - group: 939
+    - template: jinja
+
 # Install Elasticsearch
 
 # Made directory for ES data to live in
@@ -66,17 +88,28 @@ so-thehive-es:
 
 # Install Cortex
 
-#so-corteximage:
-# cmd.run:
-#   - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-cortex:HH1.0.3
+so-corteximage:
+ cmd.run:
+   - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-cortex:HH1.1.1
 
-#so-cortex:
-#  docker_container.running:
-#    - image: thehiveproject/cortex:latest
-#    - hostname: so-cortex
-#    - name: so-cortex
-#    - port_bindings:
-#      - 0.0.0.0:9001:9001
+so-cortex:
+  docker_container.running:
+    - require:
+      - so-corteximage
+    - image: docker.io/soshybridhunter/so-cortex:HH1.1.1
+    - hostname: so-cortex
+    - name: so-cortex
+    - user: 939
+    - binds:
+      - /opt/so/conf/hive/etc/cortex-application.conf:/opt/cortex/conf/application.conf:ro
+    - port_bindings:
+      - 0.0.0.0:9001:9001
+    
+cortexscript:
+  cmd.script:
+    - source: salt://hive/thehive/scripts/cortex_init.sh
+    - cwd: /opt/so
+    - template: jinja
 
 so-thehiveimage:
  cmd.run:

salt/hive/thehive/etc/application.conf

@@ -1,4 +1,5 @@
 {%- set MASTERIP = salt['pillar.get']('static:masterip', '') %}
+{%- set CORTEXKEY = salt['pillar.get']('static:cortexkey', '') %}
 
 # Secret Key
 # The secret key is used to secure cryptographic functions.
@@ -130,15 +131,15 @@ play.http.parser.maxDiskBuffer = 1G
 #
 # In order to use Cortex, first you need to enable the Cortex module by uncommenting the next line
 
-#play.modules.enabled += connectors.cortex.CortexConnector
+play.modules.enabled += connectors.cortex.CortexConnector
 
 cortex {
-  #"CORTEX-SERVER-ID" {
-  #  url = ""
-  #  key = ""
+  "CORTEX-SERVER-ID" {
+    url = "http://{{ MASTERIP }}:9001/cortex/"
+    key = "{{ CORTEXKEY }}"
   #  # HTTP client configuration (SSL and proxy)
   #  ws {}
-  #}
+  }
 }
 
 # MISP

salt/hive/thehive/etc/cortex-application.conf

@@ -0,0 +1,130 @@
+{%- set MASTERIP = salt['pillar.get']('static:masterip', '') %}
+
+# Secret Key
+# The secret key is used to secure cryptographic functions.
+# WARNING: If you deploy your application on several servers, make sure to use the same key.
+play.http.secret.key="letsdewdis"
+play.http.context=/cortex/
+search.uri = "http://{{ MASTERIP }}:9400"
+
+# Elasticsearch
+search {
+ # Name of the index
+ index = cortex
+ # Name of the Elasticsearch cluster
+ cluster = hive
+ # Address of the Elasticsearch instance
+ host = ["{{ MASTERIP }}:9500"]
+ # Scroll keepalive
+ keepalive = 1m
+ # Size of the page for scroll
+ pagesize = 50
+ # Number of shards
+ nbshards = 5
+ # Number of replicas
+ nbreplicas = 1
+ # Arbitrary settings
+ settings {
+   # Maximum number of nested fields
+   mapping.nested_fields.limit = 100
+ }
+
+  ## Authentication configuration
+  #search.username = ""
+  #search.password = ""
+
+  ## SSL configuration
+  #search.keyStore {
+  #  path = "/path/to/keystore"
+  #  type = "JKS" # or PKCS12
+  #  password = "keystore-password"
+  #}
+  #search.trustStore {
+  #  path = "/path/to/trustStore"
+  #  type = "JKS" # or PKCS12
+  #  password = "trustStore-password"
+  #}
+}
+
+## Cache
+#
+# If an analyzer is executed against the same observable, the previous report can be returned without re-executing the
+# analyzer. The cache is used only if the second job occurs within cache.job (the default is 10 minutes).
+cache.job = 10 minutes
+
+## Authentication
+auth {
+	# "provider" parameter contains the authentication provider(s). It can be multi-valued, which is useful
+	# for migration.
+	# The available auth types are:
+	# - services.LocalAuthSrv : passwords are stored in the user entity within ElasticSearch). No
+	#   configuration are required.
+	# - ad : use ActiveDirectory to authenticate users. The associated configuration shall be done in
+	#   the "ad" section below.
+	# - ldap : use LDAP to authenticate users. The associated configuration shall be done in the
+	#   "ldap" section below.
+	provider = [local]
+
+	ad {
+		# The Windows domain name in DNS format. This parameter is required if you do not use
+		# 'serverNames' below.
+		#domainFQDN = "mydomain.local"
+
+		# Optionally you can specify the host names of the domain controllers instead of using 'domainFQDN
+		# above. If this parameter is not set, TheHive uses 'domainFQDN'.
+		#serverNames = [ad1.mydomain.local, ad2.mydomain.local]
+
+		# The Windows domain name using short format. This parameter is required.
+		#domainName = "MYDOMAIN"
+
+		# If 'true', use SSL to connect to the domain controller.
+		#useSSL = true
+	}
+
+	ldap {
+		# The LDAP server name or address. The port can be specified using the 'host:port'
+		# syntax. This parameter is required if you don't use 'serverNames' below.
+		#serverName = "ldap.mydomain.local:389"
+
+		# If you have multiple LDAP servers, use the multi-valued setting 'serverNames' instead.
+		#serverNames = [ldap1.mydomain.local, ldap2.mydomain.local]
+
+		# Account to use to bind to the LDAP server. This parameter is required.
+		#bindDN = "cn=thehive,ou=services,dc=mydomain,dc=local"
+
+		# Password of the binding account. This parameter is required.
+		#bindPW = "***secret*password***"
+
+		# Base DN to search users. This parameter is required.
+		#baseDN = "ou=users,dc=mydomain,dc=local"
+
+		# Filter to search user in the directory server. Please note that {0} is replaced
+		# by the actual user name. This parameter is required.
+		#filter = "(cn={0})"
+
+		# If 'true', use SSL to connect to the LDAP directory server.
+		#useSSL = true
+	}
+}
+
+## ANALYZERS
+#
+analyzer {
+  # Absolute path where you have pulled the Cortex-Analyzers repository.
+  path = ["/Cortex-Analyzers/analyzers"]
+
+  # Sane defaults. Do not change unless you know what you are doing.
+  fork-join-executor {
+
+    # Min number of threads available for analysis.
+    parallelism-min = 2
+
+    # Parallelism (threads) ... ceil(available processors * factor).
+    parallelism-factor = 2.0
+
+    # Max number of threads available for analysis.
+    parallelism-max = 4
+  }
+}
+
+# It's the end my friend. Happy hunting!

salt/hive/thehive/scripts/cortex_init.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+{%- set MASTERIP = salt['pillar.get']('static:masterip', '') %}
+{%- set CORTEXUSER = salt['pillar.get']('static:cortexuser', '') %}
+{%- set CORTEXPASSWORD = salt['pillar.get']('static:cortexpassword', '') %}
+{%- set CORTEXKEY = salt['pillar.get']('static:cortexkey', '') %}
+
+cortex_init(){
+    sleep 60
+    CORTEX_IP="{{MASTERIP}}"
+    CORTEX_USER="{{CORTEXUSER}}"
+    CORTEX_PASSWORD="{{CORTEXPASSWORD}}"
+    CORTEX_KEY="{{CORTEXKEY}}"    
+    SOCTOPUS_CONFIG="/opt/so/saltstack/salt/soctopus/files/SOCtopus.conf"
+
+    # Migrate DB
+    curl -v -k -XPOST "https://$CORTEX_IP:/cortex/api/maintenance/migrate"
+
+    # Create intial Cortex user
+    curl -v -k "https://$CORTEX_IP/cortex/api/user" -H "Content-Type: application/json" -d "{\"login\" : \"$CORTEX_USER\",\"name\" : \"$CORTEX_USER\",\"roles\" : [\"read\",\"analyze\",\"orgadmin\"],\"preferences\" : \"{}\",\"password\" : \"$CORTEX_PASSWORD\", \"key\": \"$CORTEX_KEY\"}"
+
+    # Enable URLScan.io Analyzer
+    curl -v -k -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" "https://$CORTEX_IP/cortex/api/organization/analyzer/Urlscan_io_Search_0_1_0" -d '{"name":"Urlscan_io_Search_0_1_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2}}'
+    
+    # Update SOCtopus config with apikey value
+    #sed -i "s/cortex_key = .*/cortex_key = $CORTEX_KEY/" $SOCTOPUS_CONFIG
+
+    touch /opt/so/state/cortex.txt
+
+}
+
+if [ -f /opt/so/state/cortex.txt ]; then
+    exit 0
+else
+    rm -f garbage_file
+    while ! wget -O garbage_file {{MASTERIP}}:9500 2>/dev/null
+    do
+      echo "Waiting for Elasticsearch..."
+      rm -f garbage_file
+      sleep 1
+    done
+    rm -f garbage_file
+    sleep 5
+    cortex_init
+fi

so-setup-network.sh

@@ -427,6 +427,7 @@ generate_passwords(){
   MYSQLPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1)
   FLEETPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1)
   HIVEKEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1)
+  CORTEXKEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1)
   SENSORONIKEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1)
 }
 
@@ -575,6 +576,9 @@ master_static() {
   echo "  hiveuser: hiveadmin" >> /opt/so/saltstack/pillar/static.sls
   echo "  hivepassword: hivechangeme" >> /opt/so/saltstack/pillar/static.sls
   echo "  hivekey: $HIVEKEY" >> /opt/so/saltstack/pillar/static.sls
+  echo "  cortexuser: cortexadmin" >> /opt/so/saltstack/pillar/static.sls
+  echo "  cortexpassword: cortexchangeme" >> /opt/so/saltstack/pillar/static.sls
+  echo "  cortexkey: $CORTEXKEY" >> /opt/so/saltstack/pillar/static.sls  
   echo "  fleetsetup: 0" >> /opt/so/saltstack/pillar/static.sls
   echo "  sensoronikey: $SENSORONIKEY" >> /opt/so/saltstack/pillar/static.sls
   if [[ $MASTERUPDATES == 'MASTER' ]]; then