From 7fe0182ede5df2378cc07eada6b93d4c956ae790 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 28 Dec 2020 11:26:56 -0500 Subject: [PATCH 1/2] Refactor so-test and so-tcpreplay to be compatible with SO 2.3.20+; Change hive_init and cortex_init to initialize the cortex and fleet services directly on the manager IP instead of attempting to use the public URL --- salt/common/tools/sbin/so-tcpreplay | 46 +++++++++++++++++++++++------ salt/common/tools/sbin/so-test | 29 +----------------- salt/tcpreplay/init.sls | 15 ++-------- salt/thehive/scripts/cortex_init | 19 ++++++------ salt/thehive/scripts/hive_init | 11 ++++--- 5 files changed, 55 insertions(+), 65 deletions(-) diff --git a/salt/common/tools/sbin/so-tcpreplay b/salt/common/tools/sbin/so-tcpreplay index 99ed5665c..4cd473c0c 100755 --- a/salt/common/tools/sbin/so-tcpreplay +++ b/salt/common/tools/sbin/so-tcpreplay @@ -17,14 +17,42 @@ # Usage: so-tcpreplay "/opt/samples/*" -REPLAY_ENABLED=$(docker images | grep so-tcpreplay) -REPLAY_RUNNING=$(docker ps | grep so-tcpreplay) +. /usr/sbin/so-common +. /usr/sbin/so-image-common -if [ "$REPLAY_ENABLED" != "" ] && [ "$REPLAY_RUNNING" != "" ]; then - docker cp so-tcpreplay:/opt/samples /opt/samples - docker exec -it so-tcpreplay /usr/local/bin/tcpreplay -i bond0 -M10 $1 -else - echo "Replay functionality not enabled! To enable, run `so-tcpreplay-start`" - echo - echo "Note that you will need internet access to download the appropriate components" +REPLAYIFACE=${REPLAYIFACE:-bond0} +REPLAYSPEED=${REPLAYSPEED:-10} + +if [[ $# -lt 1 ]]; then + echo "Replays one or more PCAP sample files to the Security Onion monitoring interface." + echo + echo "Usage: $0 " + echo + echo "All PCAPs must be placed in the /opt/so/samples directory unless replaying" + echo "a sample pcap that is included in the so-tcpreplay image. Those PCAP sampes" + echo "are located in the /opt/samples directory inside of the image." + echo + echo "Customer provided PCAP example:" + echo " $0 /opt/so/samples/some_event.pcap" + echo + echo "Security Onion-provided PCAP example:" + echo " $0 /opt/samples/4in6.pcap" + + exit 1 fi + +if ! docker ps | grep -q so-tcpreplay; then + echo "Replay functionality not enabled; attempting to enable now (may require Internet access)..." + echo + + TRUSTED_CONTAINERS=("so-tcpreplay") + update_docker_containers "tcpreplay" + so-tcpreplay-start || fail "Unable to initialize tcpreplay" + mkdir -p /opt/so/samples + docker cp so-tcpreplay:/opt/samples/* /opt/so/samples +fi + +echo "Replaying PCAP(s) at ${REPLAYSPEED} Mbps on interface ${REPLAYIFACE}..." +docker exec -it so-tcpreplay /usr/bin/bash -c "/usr/local/bin/tcpreplay -i ${REPLAYIFACE} -M${REPLAYSPEED} $@" + +echo "Replay completed. Warnings shown above are typically expected." diff --git a/salt/common/tools/sbin/so-test b/salt/common/tools/sbin/so-test index 4270c9e76..8e981aecf 100755 --- a/salt/common/tools/sbin/so-test +++ b/salt/common/tools/sbin/so-test @@ -15,31 +15,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Usage: so-test - -. /usr/sbin/so-common - -REPLAY_ENABLED=$(docker images | grep so-tcpreplay) -REPLAY_RUNNING=$(docker ps | grep so-tcpreplay) - -if [ "$REPLAY_ENABLED" != "" ] && [ "$REPLAY_RUNNING" != "" ]; then - echo - echo "Preparing to replay PCAPs..." - docker cp so-tcpreplay:/opt/samples /opt/samples - docker exec -it so-tcpreplay /usr/local/bin/tcpreplay -i bond0 -M10 /opt/samples/* - echo - echo "PCAP's have been replayed - it is normal to see some warnings." - echo -else - echo "Replay functionality not enabled! Enabling Now...." - echo - echo "Note that you will need internet access to download the appropriate components" - /usr/sbin/so-start tcpreplay - echo "Replay functionality enabled. Replaying PCAPs Now...." - docker cp so-tcpreplay:/opt/samples /opt/samples - docker exec -it so-tcpreplay /usr/local/bin/tcpreplay -i bond0 -M10 /opt/samples/* - echo - echo "PCAP's have been replayed - it is normal to see some warnings." - echo -fi - +so-tcpreplay /opt/samples/* diff --git a/salt/tcpreplay/init.sls b/salt/tcpreplay/init.sls index d9050ffbb..a684d2fef 100644 --- a/salt/tcpreplay/init.sls +++ b/salt/tcpreplay/init.sls @@ -3,18 +3,6 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% set MANAGER = salt['grains.get']('master') %} -so-tcpreplayimage: - cmd.run: - - name: docker pull --disable-content-trust=false docker.io/{{ IMAGEREPO }}/so-tcpreplay:{{ VERSION }} - -so-tcpreplaytag: - cmd.run: - - name: docker tag {{ IMAGEREPO }}/so-tcpreplay:{{ VERSION }} {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-tcpreplay:{{ VERSION }} - -so-tcpreplaypush: - cmd.run: - - name: docker push {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-tcpreplay:{{ VERSION }} - so-tcpreplay: docker_container.running: - network_mode: "host" @@ -23,6 +11,9 @@ so-tcpreplay: - user: root - interactive: True - tty: True + - binds: + - /opt/so/samples:/opt/so/samples:ro + {% else %} diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index 084957885..64a228f16 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -1,6 +1,5 @@ #!/bin/bash # {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} -# {%- set URLBASE = salt['pillar.get']('global:url_base', '') %} # {%- set CORTEXUSER = salt['pillar.get']('global:cortexuser', 'cortexadmin') %} # {%- set CORTEXPASSWORD = salt['pillar.get']('global:cortexpassword', 'cortexchangeme') %} # {%- set CORTEXKEY = salt['pillar.get']('global:cortexkey', '') %} @@ -19,7 +18,7 @@ cortex_clean(){ } cortex_init(){ - CORTEX_URL="{{URLBASE}}/cortex" + CORTEX_URL="http://{{MANAGERIP}}:9001" CORTEX_API_URL="$CORTEX_URL/api" CORTEX_USER="{{CORTEXUSER}}" CORTEX_PASSWORD="{{CORTEXPASSWORD}}" @@ -30,27 +29,27 @@ cortex_init(){ CORTEX_ORG_USER_KEY="{{CORTEXORGUSERKEY}}" SOCTOPUS_CONFIG="$default_salt_dir/salt/soctopus/files/SOCtopus.conf" - if wait_for_web_response https://$CORTEX_URL "Cortex"; then + if wait_for_web_response $CORTEX_URL "Cortex"; then # Migrate DB - curl -sk -XPOST -L "https://$CORTEX_API_URL/maintenance/migrate" + curl -sk -XPOST -L "$CORTEX_API_URL/maintenance/migrate" # Create intial Cortex superadmin - curl -sk -L "https://$CORTEX_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$CORTEX_USER\",\"name\" : \"$CORTEX_USER\",\"roles\" : [\"superadmin\"],\"preferences\" : \"{}\",\"password\" : \"$CORTEX_PASSWORD\", \"key\": \"$CORTEX_KEY\"}" + curl -sk -L "$CORTEX_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$CORTEX_USER\",\"name\" : \"$CORTEX_USER\",\"roles\" : [\"superadmin\"],\"preferences\" : \"{}\",\"password\" : \"$CORTEX_PASSWORD\", \"key\": \"$CORTEX_KEY\"}" # Create user-supplied org - curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization" -d "{ \"name\": \"$CORTEX_ORG_NAME\",\"description\": \"$CORTEX_ORG_DESC\",\"status\": \"Active\"}" + curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "$CORTEX_API_URL/organization" -d "{ \"name\": \"$CORTEX_ORG_NAME\",\"description\": \"$CORTEX_ORG_DESC\",\"status\": \"Active\"}" # Create user-supplied org user - curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/user" -d "{\"name\": \"$CORTEX_ORG_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_ORG_USER\",\"key\": \"$CORTEX_ORG_USER_KEY\" }" + curl -sk -XPOST -H "Authorization: Bearer $CORTEX_KEY" -H "Content-Type: application/json" -L "$CORTEX_API_URL/user" -d "{\"name\": \"$CORTEX_ORG_USER\",\"roles\": [\"read\",\"analyze\",\"orgadmin\"],\"organization\": \"$CORTEX_ORG_NAME\",\"login\": \"$CORTEX_ORG_USER\",\"key\": \"$CORTEX_ORG_USER_KEY\" }" # Enable URLScan.io Analyzer - curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization/analyzer/Urlscan_io_Search_0_1_0" -d '{"name":"Urlscan_io_Search_0_1_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2}}' + curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "$CORTEX_API_URL/organization/analyzer/Urlscan_io_Search_0_1_0" -d '{"name":"Urlscan_io_Search_0_1_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2}}' # Enable Cert PassiveDNS Analyzer - curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "https://$CORTEX_API_URL/organization/analyzer/CERTatPassiveDNS_2_0" -d '{"name":"CERTatPassiveDNS_2_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2, "limit": 100}}' + curl -sv -k -XPOST -H "Authorization: Bearer $CORTEX_ORG_USER_KEY" -H "Content-Type: application/json" -L "$CORTEX_API_URL/organization/analyzer/CERTatPassiveDNS_2_0" -d '{"name":"CERTatPassiveDNS_2_0","configuration":{"auto_extract_artifacts":false,"check_tlp":true,"max_tlp":2, "limit": 100}}' # Revoke $CORTEX_USER key - curl -sk -XDELETE -H "Authorization: Bearer $CORTEX_KEY" -L "https://$CORTEX_API_URL/user/$CORTEX_USER/key" + curl -sk -XDELETE -H "Authorization: Bearer $CORTEX_KEY" -L "$CORTEX_API_URL/user/$CORTEX_USER/key" # Update SOCtopus config with apikey value #sed -i "s/cortex_key = .*/cortex_key = $CORTEX_KEY/" $SOCTOPUS_CONFIG diff --git a/salt/thehive/scripts/hive_init b/salt/thehive/scripts/hive_init index f0f1a5f52..55a408f60 100755 --- a/salt/thehive/scripts/hive_init +++ b/salt/thehive/scripts/hive_init @@ -1,6 +1,5 @@ #!/bin/bash # {%- set MANAGERIP = salt['pillar.get']('global:managerip', '') %} -# {%- set URLBASE = salt['pillar.get']('global:url_base', '') %} # {%- set THEHIVEUSER = salt['pillar.get']('global:hiveuser', 'hiveadmin') %} # {%- set THEHIVEPASSWORD = salt['pillar.get']('global:hivepassword', 'hivechangeme') %} # {%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %} @@ -13,7 +12,7 @@ thehive_clean(){ } thehive_init(){ - THEHIVE_URL="{{URLBASE}}/thehive" + THEHIVE_URL="http://{{MANAGERIP}}:9000/thehive" THEHIVE_API_URL="$THEHIVE_URL/api" THEHIVE_USER="{{THEHIVEUSER}}" THEHIVE_PASSWORD="{{THEHIVEPASSWORD}}" @@ -21,17 +20,17 @@ thehive_init(){ SOCTOPUS_CONFIG="/opt/so/saltstack/salt/soctopus/files/SOCtopus.conf" echo -n "Waiting for TheHive..." - if wait_for_web_response https://$THEHIVE_URL "TheHive"; then + if wait_for_web_response $THEHIVE_URL "TheHive"; then # Migrate DB - curl -sk -XPOST -L "https://$THEHIVE_API_URL/maintenance/migrate" + curl -sk -XPOST -L "$THEHIVE_API_URL/maintenance/migrate" # Create intial TheHive user - curl -sk -L "https://$THEHIVE_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASSWORD\", \"key\": \"$THEHIVE_KEY\"}" + curl -sk -L "$THEHIVE_API_URL/user" -H "Content-Type: application/json" -d "{\"login\" : \"$THEHIVE_USER\",\"name\" : \"$THEHIVE_USER\",\"roles\" : [\"read\",\"alert\",\"write\",\"admin\"],\"preferences\" : \"{}\",\"password\" : \"$THEHIVE_PASSWORD\", \"key\": \"$THEHIVE_KEY\"}" # Pre-load custom fields # # reputation - curl -sk -L "https://$THEHIVE_API_URL/list/custom_fields" -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -d "{\"value\":{\"name\": \"reputation\", \"reference\": \"reputation\", \"description\": \"This field provides an overall reputation status for an address/domain.\", \"type\": \"string\", \"options\": []}}" + curl -sk -L "$THEHIVE_API_URL/list/custom_fields" -H "Authorization: Bearer $THEHIVE_KEY" -H "Content-Type: application/json" -d "{\"value\":{\"name\": \"reputation\", \"reference\": \"reputation\", \"description\": \"This field provides an overall reputation status for an address/domain.\", \"type\": \"string\", \"options\": []}}" touch /opt/so/state/thehive.txt else From deb38844bac91ab8ae7f5e2ac09168f96564be7b Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 28 Dec 2020 16:20:33 -0500 Subject: [PATCH 2/2] Correct hive init urls --- salt/thehive/scripts/cortex_init | 4 ++-- salt/thehive/scripts/hive_init | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/thehive/scripts/cortex_init b/salt/thehive/scripts/cortex_init index 64a228f16..c2f00390d 100644 --- a/salt/thehive/scripts/cortex_init +++ b/salt/thehive/scripts/cortex_init @@ -18,8 +18,8 @@ cortex_clean(){ } cortex_init(){ - CORTEX_URL="http://{{MANAGERIP}}:9001" - CORTEX_API_URL="$CORTEX_URL/api" + CORTEX_URL="http://{{MANAGERIP}}:9001/cortex/" + CORTEX_API_URL="${CORTEX_URL}api" CORTEX_USER="{{CORTEXUSER}}" CORTEX_PASSWORD="{{CORTEXPASSWORD}}" CORTEX_KEY="{{CORTEXKEY}}" diff --git a/salt/thehive/scripts/hive_init b/salt/thehive/scripts/hive_init index 55a408f60..ca6f7aa2b 100755 --- a/salt/thehive/scripts/hive_init +++ b/salt/thehive/scripts/hive_init @@ -12,8 +12,8 @@ thehive_clean(){ } thehive_init(){ - THEHIVE_URL="http://{{MANAGERIP}}:9000/thehive" - THEHIVE_API_URL="$THEHIVE_URL/api" + THEHIVE_URL="http://{{MANAGERIP}}:9000/thehive/" + THEHIVE_API_URL="${THEHIVE_URL}api" THEHIVE_USER="{{THEHIVEUSER}}" THEHIVE_PASSWORD="{{THEHIVEPASSWORD}}" THEHIVE_KEY="{{THEHIVEKEY}}"