Switch to Zeek and download registry

2025-12-06 17:22:49 +01:00 · 2020-01-13 15:51:38 -05:00
parent 686282da62
commit 4d00f26c35
16 changed files with 3504 additions and 5 deletions
--- a/salt/common/tools/sbin/so-elastic-download
+++ b/salt/common/tools/sbin/so-elastic-download
@@ -1,21 +1,24 @@
 #!/bin/bash
-MASTER={{ MASTER }}
+MASTER=MASTER
 VERSION="HH1.1.4"
 TRUSTED_CONTAINERS=( \
 "so-auth-api:$VERSION" \
 "so-auth-ui:$VERSION" \
-"so-bro:$VERSION" \
 "so-core:$VERSION" \
-"so-cortex:$VERSION" \
+"so-thehive-cortex:$VERSION" \
 "so-curator:$VERSION" \
+"so-domainstats:$VERSION" \
 "so-elastalert:$VERSION" \
 "so-elasticsearch:$VERSION" \
 "so-filebeat:$VERSION" \
 "so-fleet:$VERSION" \
+"so-fleet-launcher:$VERSION" \
+"so-freqserver:$VERSION" \
 "so-grafana:$VERSION" \
 "so-idstools:$VERSION" \
 "so-influxdb:$VERSION" \
 "so-kibana:$VERSION" \
+"so-logstash:$VERSION" \
 "so-mysql:$VERSION" \
 "so-navigator:$VERSION" \
 "so-playbook:$VERSION" \
@@ -23,17 +26,21 @@ TRUSTED_CONTAINERS=( \
 "so-sensoroni:$VERSION" \
 "so-soctopus:$VERSION" \
 "so-steno:$VERSION" \
+#"so-strelka:$VERSION" \
 "so-suricata:$VERSION" \
 "so-telegraf:$VERSION" \
 "so-thehive:$VERSION" \
 "so-thehive-es:$VERSION" \
-"so-wazuh:$VERSION" )
+"so-wazuh:$VERSION" \
+"so-zeek:$VERSION" )

 for i in "${TRUSTED_CONTAINERS[@]}"
 do
  # Pull down the trusted docker image
+  echo "Downloading $i"
  docker pull --disable-content-trust=false docker.io/soshybridhunter/$i
  # Tag it with the new registry destination
  docker tag soshybridhunter/$i $MASTER:5000/soshybridhunter/$i
  docker push $MASTER:5000/soshybridhunter/$i
+  docker rmi soshybridhunter/$i
 done
--- a/salt/zeek/cron/packetloss.sh
+++ b/salt/zeek/cron/packetloss.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/bin/docker exec so-zeek /opt/zeek/bin/zeekctl netstats | awk '{print $(NF-2),$(NF-1),$NF}' | awk -F '[ =]' '{RCVD += $2;DRP += $4;TTL += $6} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/zeek/logs/packetloss.log
--- a/salt/zeek/files/local.zeek
+++ b/salt/zeek/files/local.zeek
@@ -0,0 +1,120 @@
+##! Local site policy. Customize as appropriate.
+##!
+##! This file will not be overwritten when upgrading or reinstalling!
+
+# This script logs which scripts were loaded during each run.
+@load misc/loaded-scripts
+
+# Apply the default tuning scripts for common tuning settings.
+@load tuning/defaults
+
+# Estimate and log capture loss.
+@load misc/capture-loss
+
+# Enable logging of memory, packet and lag statistics.
+@load misc/stats
+
+# Load the scan detection script.  It's disabled by default because
+# it often causes performance issues.
+#@load misc/scan
+
+# Detect traceroute being run on the network. This could possibly cause
+# performance trouble when there are a lot of traceroutes on your network.
+# Enable cautiously.
+#@load misc/detect-traceroute
+
+# Generate notices when vulnerable versions of software are discovered.
+# The default is to only monitor software found in the address space defined
+# as "local".  Refer to the software framework's documentation for more
+# information.
+@load frameworks/software/vulnerable
+
+# Detect software changing (e.g. attacker installing hacked SSHD).
+@load frameworks/software/version-changes
+
+# This adds signatures to detect cleartext forward and reverse windows shells.
+@load-sigs frameworks/signatures/detect-windows-shells
+
+# Load all of the scripts that detect software in various protocols.
+@load protocols/ftp/software
+@load protocols/smtp/software
+@load protocols/ssh/software
+@load protocols/http/software
+# The detect-webapps script could possibly cause performance trouble when
+# running on live traffic.  Enable it cautiously.
+#@load protocols/http/detect-webapps
+
+# This script detects DNS results pointing toward your Site::local_nets
+# where the name is not part of your local DNS zone and is being hosted
+# externally.  Requires that the Site::local_zones variable is defined.
+@load protocols/dns/detect-external-names
+
+# Script to detect various activity in FTP sessions.
+@load protocols/ftp/detect
+
+# Scripts that do asset tracking.
+@load protocols/conn/known-hosts
+@load protocols/conn/known-services
+@load protocols/ssl/known-certs
+
+# This script enables SSL/TLS certificate validation.
+@load protocols/ssl/validate-certs
+
+# This script prevents the logging of SSL CA certificates in x509.log
+@load protocols/ssl/log-hostcerts-only
+
+# Uncomment the following line to check each SSL certificate hash against the ICSI
+# certificate notary service; see http://notary.icsi.berkeley.edu .
+# @load protocols/ssl/notary
+
+# If you have GeoIP support built in, do some geographic detections and
+# logging for SSH traffic.
+@load protocols/ssh/geo-data
+# Detect hosts doing SSH bruteforce attacks.
+@load protocols/ssh/detect-bruteforcing
+# Detect logins using "interesting" hostnames.
+@load protocols/ssh/interesting-hostnames
+
+# Detect SQL injection attacks.
+@load protocols/http/detect-sqli
+
+#### Network File Handling ####
+
+# Enable MD5 and SHA1 hashing for all files.
+@load frameworks/files/hash-all-files
+
+# Detect SHA1 sums in Team Cymru's Malware Hash Registry.
+@load frameworks/files/detect-MHR
+
+# Extend email alerting to include hostnames
+@load policy/frameworks/notice/extend-email/hostnames
+
+# Uncomment the following line to enable detection of the heartbleed attack. Enabling
+# this might impact performance a bit.
+# @load policy/protocols/ssl/heartbleed
+
+# Uncomment the following line to enable logging of connection VLANs. Enabling
+# this adds two VLAN fields to the conn.log file.
+# @load policy/protocols/conn/vlan-logging
+
+# Uncomment the following line to enable logging of link-layer addresses. Enabling
+# this adds the link-layer address for each connection endpoint to the conn.log file.
+# @load policy/protocols/conn/mac-logging
+
+# JA3 - SSL Detection Goodness
+@load policy/ja3
+
+# HASSH
+@load policy/hassh
+
+# You can load your own intel into:
+# /opt/so/saltstack/bro/policy/intel/ on the master
+@load intel
+
+# Load a custom Bro policy
+# /opt/so/saltstack/bro/policy/custom/ on the master
+#@load custom/somebropolicy.bro
+
+# Write logs in JSON
+redef LogAscii::use_json = T;
+redef LogAscii::json_timestamps = JSON::TS_ISO8601;
--- a/salt/zeek/files/node.cfg
+++ b/salt/zeek/files/node.cfg
@@ -0,0 +1,47 @@
+{%- set interface = salt['pillar.get']('sensor:interface', 'bond0') %}
+
+{%- if salt['pillar.get']('sensor:bro_pins') or salt['pillar.get']('sensor:bro_lbprocs') %}
+{%- if salt['pillar.get']('sensor:bro_proxies') %}
+  {%- set proxies =  salt['pillar.get']('sensor:bro_proxies', '1') %}
+{%- else %}
+  {%- if salt['pillar.get']('sensor:bro_pins') %}
+    {%- set proxies = (salt['pillar.get']('sensor:bro_pins')|length/10)|round(0, 'ceil')|int %}
+  {%- else %}
+    {%- set proxies = (salt['pillar.get']('sensor:bro_lbprocs')/10)|round(0, 'ceil')|int %}
+  {%- endif %}
+{%- endif %}
+[manager]
+type=manager
+host=localhost
+
+[logger]
+type=logger
+host=localhost
+
+[proxy]
+type=proxy
+host=localhost
+
+[worker-1]
+type=worker
+host=localhost
+interface=af_packet::{{ interface }}
+lb_method=custom
+
+{%- if salt['pillar.get']('sensor:bro_lbprocs') %}
+lb_procs={{ salt['pillar.get']('sensor:bro_lbprocs', '1') }}
+{%- else %}
+lb_procs={{ salt['pillar.get']('sensor:bro_pins')|length }}
+{%- endif %}
+{%- if salt['pillar.get']('sensor:bro_pins') %}
+pin_cpus={{ salt['pillar.get']('sensor:bro_pins')|join(", ") }}
+{%- endif %}
+af_packet_fanout_id=23
+af_packet_fanout_mode=AF_Packet::FANOUT_HASH
+af_packet_buffer_size=128*1024*1024
+{%- else %}
+[brosa]
+type=standalone
+host=localhost
+interface={{ interface }}
+{%- endif %}
--- a/salt/zeek/init.sls
+++ b/salt/zeek/init.sls
@@ -0,0 +1,118 @@
+{%- set VERSION = salt['pillar.get']('static:soversion', '1.1.4') -%}
+{%- set MASTER = salt['grains.get']('master') -%}
+# Zeek Salt State
+# Add Zeek group
+zeekgroup:
+  group.present:
+    - name: zeek
+    - gid: 937
+
+# Add Zeek User
+zeek:
+  user.present:
+    - uid: 937
+    - gid: 937
+    - home: /home/zeek
+
+# Create some directories
+zeekpolicydir:
+  file.directory:
+    - name: /opt/so/conf/zeek/policy
+    - user: 937
+    - group: 939
+    - makedirs: True
+
+# Zeek Log Directory
+zeeklogdir:
+  file.directory:
+    - name: /nsm/zeek/logs
+    - user: 937
+    - group: 939
+    - makedirs: True
+
+# Zeek Spool Directory
+zeekspooldir:
+  file.directory:
+    - name: /nsm/zeek/spool/manager
+    - user: 937
+    - makedirs: true
+
+# Zeek extracted
+zeekextractdir:
+  file.directory:
+    - name: /nsm/zeek/extracted
+    - user: 937
+    - group: 939
+    - makedirs: True
+
+zeeksfafincompletedir:
+  file.directory:
+    - name: /nsm/faf/files/incomplete
+    - user: 937
+    - makedirs: true
+
+zeeksfafcompletedir:
+  file.directory:
+    - name: /nsm/faf/files/complete
+    - user: 937
+    - makedirs: true
+
+# Sync the policies
+zeekpolicysync:
+  file.recurse:
+    - name: /opt/so/conf/zeek/policy
+    - source: salt://zeek/policy
+    - user: 937
+    - group: 939
+    - template: jinja
+
+# Sync node.cfg
+nodecfgsync:
+  file.managed:
+    - name: /opt/so/conf/zeek/node.cfg
+    - source: salt://zeek/files/node.cfg
+    - user: 937
+    - group: 939
+    - template: jinja
+
+plcronscript:
+  file.managed:
+    - name: /usr/local/bin/packetloss.sh
+    - source: salt://zeek/cron/packetloss.sh
+    - mode: 755
+
+/usr/local/bin/packetloss.sh:
+  cron.present:
+    - user: root
+    - minute: '*/10'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+localzeeksync:
+  file.managed:
+    - name: /opt/so/conf/zeek/local.zeek
+    - source: salt://zeek/files/local.zeek
+    - user: 937
+    - group: 939
+    - template: jinja
+
+so-zeek:
+  docker_container.running:
+    - image: {{ MASTER }}:5000/soshybridhunter/so-zeek:HH{{ VERSION }}
+    - privileged: True
+    - binds:
+      - /nsm/zeek/logs:/nsm/zeek/logs:rw
+      - /nsm/zeek/spool:/nsm/zeek/spool:rw
+      - /nsm/zeek/extracted:/nsm/zeek/extracted:rw
+      - /opt/so/conf/zeek/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro
+      - /opt/so/conf/zeek/node.cfg:/opt/zeek/etc/node.cfg:ro
+      - /opt/so/conf/zeek/policy/securityonion:/opt/zeek/share/zeek/policy/securityonion:ro
+      - /opt/so/conf/zeek/policy/custom:/opt/zeek/share/zeek/policy/custom:ro
+      - /opt/so/conf/zeek/policy/intel:/opt/zeek/share/zeek/policy/intel:rw
+    - network_mode: host
+    - watch:
+      - file: /opt/so/conf/zeek/local.zeek
+      - file: /opt/so/conf/zeek/node.cfg
+      - file: /opt/so/conf/zeek/policy
--- a/salt/zeek/policy/intel/load.zeek
+++ b/salt/zeek/policy/intel/load.zeek
@@ -0,0 +1 @@
+#Intel
--- a/salt/zeek/policy/securityonion/add-interface-to-logs.bro
+++ b/salt/zeek/policy/securityonion/add-interface-to-logs.bro
@@ -0,0 +1,20 @@
+{%- set interface = salt['pillar.get']('sensor:interface', '0') %}
+global interface = "{{ interface }}";
+
+event bro_init()
+	{
+	if ( ! reading_live_traffic() )
+		return;
+
+	Log::remove_default_filter(HTTP::LOG);
+	Log::add_filter(HTTP::LOG, [$name = "http-interfaces",
+	                            $path_func(id: Log::ID, path: string, rec: HTTP::Info) =
+	                            	{
+	                            	local peer = get_event_peer()$descr;
+	                            	if ( peer in Cluster::nodes && Cluster::nodes[peer]?$interface )
+	                            		return cat("http_", Cluster::nodes[peer]$interface);
+	                            	else
+	                            		return "http";
+	                            	}
+	                            ]);
+	}
--- a/salt/zeek/policy/securityonion/apt1/load.zeek
+++ b/salt/zeek/policy/securityonion/apt1/load.zeek
@@ -0,0 +1,9 @@
+@load frameworks/intel/seen
+@load frameworks/intel/do_notice
+@load frameworks/files/hash-all-files
+
+redef Intel::read_files += {
+  fmt("%s/apt1-fqdn.dat", @DIR),
+  fmt("%s/apt1-md5.dat", @DIR),
+  fmt("%s/apt1-certs.dat", @DIR)
+};
--- a/salt/zeek/policy/securityonion/apt1/apt1-certs.dat
+++ b/salt/zeek/policy/securityonion/apt1/apt1-certs.dat
@@ -0,0 +1,26 @@
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.do_notice
+b054e26ef827fbbf5829f84a9bdbb697a5b042fc	Intel::CERT_HASH	Mandiant APT1 Report	ALPHA	T
+7bc0cc2cf7c3a996c32dbe7e938993f7087105b4	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+7855c132af1390413d4e4ff4ead321f8802d8243	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+f3e3c590d7126bd227733e9d8313d2575c421243	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+d4d4e896ce7d73b573f0a0006080a246aec61fe7	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+bcdf4809c1886ac95478bbafde246d0603934298	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+6b4855df8afc8d57a671fe5ed628f6d88852a922	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+d50fdc82c328319ac60f256d3119b8708cd5717b	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+70b48d5177eebe9c762e9a37ecabebfd10e1b7e9	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+3a6a299b764500ce1b6e58a32a257139d61a3543	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+bf4f90e0029b2263af1141963ddf2a0c71a6b5fb	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+b21139583dec0dae344cca530690ec1f344acc79	Intel::CERT_HASH	Mandiant APT1 Report	AOL	T
+21971ffef58baf6f638df2f7e2cceb4c58b173c8	Intel::CERT_HASH	Mandiant APT1 Report	EMAIL	T
+04ecff66973c92a1c348666d5a4738557cce0cfc	Intel::CERT_HASH	Mandiant APT1 Report	IBM	T
+f97d1a703aec44d0f53a3a294e33acda43a49de1	Intel::CERT_HASH	Mandiant APT1 Report	IBM	T
+c0d32301a7c96ecb0bc8e381ec19e6b4eaf5d2fe	Intel::CERT_HASH	Mandiant APT1 Report	IBM	T
+1b27a897cda019da2c3a6dc838761871e8bf5b5d	Intel::CERT_HASH	Mandiant APT1 Report	LAME	T
+d515996e8696612dc78fc6db39006466fc6550df	Intel::CERT_HASH	Mandiant APT1 Report	MOON-NIGHT	T
+8f79315659e59c79f1301ef4aee67b18ae2d9f1c	Intel::CERT_HASH	Mandiant APT1 Report	NONAME	T
+a57a84975e31e376e3512da7b05ad06ef6441f53	Intel::CERT_HASH	Mandiant APT1 Report	NS	T
+b3db37a0edde97b3c3c15da5f2d81d27af82f583	Intel::CERT_HASH	Mandiant APT1 Report	SERVER (PEM)	T
+6d8f1454f6392361fb2464b744d4fc09eee5fcfd	Intel::CERT_HASH	Mandiant APT1 Report	SUR	T
+b66e230f404b2cc1c033ccacda5d0a14b74a2752	Intel::CERT_HASH	Mandiant APT1 Report	VIRTUALLYTHERE	T
+4acbadb86a91834493dde276736cdf8f7ef5d497	Intel::CERT_HASH	Mandiant APT1 Report	WEBMAIL	T
+86a48093d9b577955c4c9bd19e30536aae5543d4	Intel::CERT_HASH	Mandiant APT1 Report	YAHOO	T
--- a/salt/zeek/policy/securityonion/apt1/apt1-fqdn.dat
+++ b/salt/zeek/policy/securityonion/apt1/apt1-fqdn.dat
--- a/salt/zeek/policy/securityonion/apt1/apt1-md5.dat
+++ b/salt/zeek/policy/securityonion/apt1/apt1-md5.dat
--- a/salt/zeek/policy/securityonion/conn-add-sensorname.bro
+++ b/salt/zeek/policy/securityonion/conn-add-sensorname.bro
@@ -0,0 +1,10 @@
+global sensorname = "{{ grains.host }}";
+
+redef record Conn::Info += {
+	sensorname: string &log &optional;
+};
+
+event connection_state_remove(c: connection)
+	{
+	c$conn$sensorname = sensorname;
+	}
--- a/salt/zeek/policy/securityonion/file-extraction/load.zeek
+++ b/salt/zeek/policy/securityonion/file-extraction/load.zeek
@@ -0,0 +1 @@
+@load ./extract
--- a/salt/zeek/policy/securityonion/file-extraction/extract.zeek
+++ b/salt/zeek/policy/securityonion/file-extraction/extract.zeek
@@ -0,0 +1,21 @@
+global ext_map: table[string] of string = {
+    ["application/x-dosexec"] = "exe",
+    ["text/plain"] = "txt",
+    ["image/jpeg"] = "jpg",
+    ["image/png"] = "png",
+    ["text/html"] = "html",
+} &default ="";
+
+event file_sniff(f: fa_file, meta: fa_metadata)
+    {
+    if ( ! meta?$mime_type || meta$mime_type != "application/x-dosexec" )
+        return;
+
+    local ext = "";
+
+    if ( meta?$mime_type )
+        ext = ext_map[meta$mime_type];
+
+    local fname = fmt("/nsm/bro/extracted/%s-%s.%s", f$source, f$id, ext);
+    Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+    }
--- a/salt/zeek/policy/securityonion/json-logs/load.bro
+++ b/salt/zeek/policy/securityonion/json-logs/load.bro
@@ -0,0 +1,3 @@
+@load tuning/json-logs
+redef LogAscii::json_timestamps = JSON::TS_ISO8601;
+redef LogAscii::use_json = T;
--- a/setup/functions.sh
+++ b/setup/functions.sh
@@ -424,7 +424,9 @@ docker_install() {
 #      if [ $INSTALLTYPE != 'EVALMODE'  ]; then
 #        docker_registry >> $SETUPLOG 2>&1
 #      fi
-      docker_registry
+      docker_registry >> $SETUPLOG 2>&1
+      echo "Seeding the registry. This can take a bit" >> $SETUPLOG 2>&1
+      docker_seed_registry >> $SETUPLOG 2>&1
      echo "Restarting Docker" >> $SETUPLOG 2>&1
      systemctl restart docker >> $SETUPLOG 2>&1
    else
@@ -433,6 +435,8 @@ docker_install() {
      apt-get update >> $SETUPLOG 2>&1
      apt-get -y install docker-ce python3-docker >> $SETUPLOG 2>&1
      docker_registry >> $SETUPLOG 2>&1
+      echo "Seeding the registry. This can take a bit" >> $SETUPLOG 2>&1
+      docker_seed_registry >> $SETUPLOG 2>&1
      echo "Restarting Docker" >> $SETUPLOG 2>&1
      systemctl restart docker >> $SETUPLOG 2>&1
    fi
@@ -452,6 +456,54 @@ docker_registry() {

 }

+docker_seed_registry() {
+  VERSION="HH1.1.4"
+  TRUSTED_CONTAINERS=( \
+  "so-auth-api:$VERSION" \
+  "so-auth-ui:$VERSION" \
+  "so-core:$VERSION" \
+  "so-thehive-cortex:$VERSION" \
+  "so-curator:$VERSION" \
+  "so-domainstats:$VERSION" \
+  "so-elastalert:$VERSION" \
+  "so-elasticsearch:$VERSION" \
+  "so-filebeat:$VERSION" \
+  "so-fleet:$VERSION" \
+  "so-fleet-launcher:$VERSION" \
+  "so-freqserver:$VERSION" \
+  "so-grafana:$VERSION" \
+  "so-idstools:$VERSION" \
+  "so-influxdb:$VERSION" \
+  "so-kibana:$VERSION" \
+  "so-logstash:$VERSION" \
+  "so-mysql:$VERSION" \
+  "so-navigator:$VERSION" \
+  "so-playbook:$VERSION" \
+  "so-redis:$VERSION" \
+  "so-sensoroni:$VERSION" \
+  "so-soctopus:$VERSION" \
+  "so-steno:$VERSION" \
+  #"so-strelka:$VERSION" \
+  "so-suricata:$VERSION" \
+  "so-telegraf:$VERSION" \
+  "so-thehive:$VERSION" \
+  "so-thehive-es:$VERSION" \
+  "so-wazuh:$VERSION" \
+  "so-zeek:$VERSION" )
+
+  for i in "${TRUSTED_CONTAINERS[@]}"
+  do
+    # Pull down the trusted docker image
+    echo "Downloading $i"
+    docker pull --disable-content-trust=false docker.io/soshybridhunter/$i
+    # Tag it with the new registry destination
+    docker tag soshybridhunter/$i $MSRV:5000/soshybridhunter/$i
+    docker push $MSRV:5000/soshybridhunter/$i
+    echo "Removing $i locally"
+    docker rmi soshybridhunter/$i
+  done
+}
+
 es_heapsize() {

  # Determine ES Heap Size
@@ -646,6 +698,7 @@ master_static() {
  touch /opt/so/saltstack/pillar/static.sls

  echo "static:" > /opt/so/saltstack/pillar/static.sls
+  echo "  soversion: 1.1.4" >> /opt/so/saltstack/pillar/static.sls
  echo "  hnmaster: $HNMASTER" >> /opt/so/saltstack/pillar/static.sls
  echo "  ntpserver: $NTPSERVER" >> /opt/so/saltstack/pillar/static.sls
  echo "  proxy: $PROXY" >> /opt/so/saltstack/pillar/static.sls