merge

2025-12-09 10:42:54 +01:00 · 2020-04-08 16:22:49 -04:00
parent cd1caa5568 19c5ca5643
commit 638bf7e50b
9 changed files with 83 additions and 64 deletions
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -65,3 +65,10 @@ base:
    - data.*
    - secrets
    - minions.{{ grains.id }}
+
+  '*_searchnode':
+    - static
+    - firewall.*
+    - logstash
+    - logstash.search
+    - minions.{{ grains.id }}
--- a/salt/common/tools/sbin/so-kibana-config-export
+++ b/salt/common/tools/sbin/so-kibana-config-export
@@ -0,0 +1,6 @@
+
+#!/bin/bash
+KIBANA_HOST=10.66.166.141
+KSO_PORT=5601
+OUTFILE="saved_objects.json"
+curl -s -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST $KIBANA_HOST:$KSO_PORT/api/saved_objects/_export -d '{ "type": "index-pattern", "type": "config", "type": "dashboard", "type": "query", "type": "search", "type": "url", "type": "visualization" }' -o $OUTFILE
--- a/salt/kibana/bin/so-kibana-config-load
+++ b/salt/kibana/bin/so-kibana-config-load
@@ -1,7 +1,7 @@
 {%- set MASTER = salt['pillar.get']('static:masterip', '') %}
 #!/bin/bash
 KIBANA_VERSION="7.6.1"
-MAX_WAIT=60
+MAX_WAIT=120

 # Check to see if Kibana is available
 until curl "{{ MASTER }}:5601/nonexistenturl" 2>&1 |grep -q "Not Found" ; do
@@ -18,20 +18,20 @@ done
 sleep 30s

 # Load config
-curl -X PUT "localhost:5601/api/saved_objects/config/$KIBANA_VERSION" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'
-{ "attributes":
-        {
-          "defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29",
-          "defaultRoute":"/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645",
-          "discover:sampleSize":"100",
-          "dashboard:defaultDarkTheme":true,
-          "theme:darkMode":true,
-          "timepicker:timeDefaults":"{\n  \"from\": \"now-24h\",\n  \"to\": \"now\",\n  \"mode\": \"quick\"\n}"
-        }
-}'
+#curl -X PUT "localhost:5601/api/saved_objects/config/$KIBANA_VERSION" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'
+#{ "attributes":
+#        {
+#          "defaultIndex": "2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29",
+#          "defaultRoute":"/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645",
+#          "discover:sampleSize":"100",
+#          "dashboard:defaultDarkTheme":true,
+#          "theme:darkMode":true,
+#          "timepicker:timeDefaults":"{\n  \"from\": \"now-24h\",\n  \"to\": \"now\",\n  \"mode\": \"quick\"\n}"
+#        }
+#}'

 # Sub our IP for placholders
-for i in PLACEHOLDER FLEETPLACEHOLDER PCAPPLACEHOLDER SOCTOPUSPLACEHOLDER; do
+for i in FLEETPLACEHOLDER PCAPPLACEHOLDER SOCTOPUSPLACEHOLDER PLACEHOLDER; do
        sed -i "s/$i/{{ MASTER }}/g" /opt/so/saltstack/salt/kibana/saved_objects.ndjson
 done

--- a/salt/kibana/init.sls
+++ b/salt/kibana/init.sls
@@ -83,6 +83,7 @@ so-kibana-config-load:
  cmd.script:
    - shell: /bin/bash
    - runas: socore
+    - cwd: /opt/so
    - source: salt://kibana/bin/so-kibana-config-load
    - template: jinja

--- a/salt/kibana/saved_objects.ndjson
+++ b/salt/kibana/saved_objects.ndjson
--- a/salt/logstash/etc/logstash.yml
+++ b/salt/logstash/etc/logstash.yml
@@ -63,12 +63,6 @@
 #
 # path.config:
 # /etc/logstash/conf.d is mapped to /usr/share/logstash/pipeline in the Docker image
-{%- if grains.role != 'so-mastersearch' and grains.role != 'so-heavynode' and grains.role != 'so-master' and grains.role != 'so-eval' and grains.role != 'so-helix' %}
-path.config: /usr/share/logstash/pipeline.enabled/*.conf
-{%- else %} 
-#path.config: /usr/share/logstash/pipeline.enabled/*.conf
-{%- endif %}
-
 # Special Docker path
 # path.config: /usr/share/logstash/pipeline

--- a/salt/top.sls
+++ b/salt/top.sls
@@ -175,8 +175,7 @@ base:
    {%- endif %}
    - schedule

-  '*_node and I@node:node_type:search':
-    - match: compound
+  '*_searchnode':
    - ca
    - ssl
    - common
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -230,14 +230,16 @@ check_network_manager_conf() {
 	local nmconf="/etc/NetworkManager/NetworkManager.conf"
 	local preupdir="/etc/NetworkManager/dispatcher.d/pre-up.d"

-	if ! test -f "${gmdconf}.bak"; then
-		{
-			mv "$gmdconf" "${gmdconf}.bak"
-			touch "$gmdconf"
-			systemctl restart NetworkManager
-		} >> $SETUPLOG 2>&1
+	if test -f "$gmdconf"; then
+		if ! test -f "${gmdconf}.bak"; then
+			{
+				mv "$gmdconf" "${gmdconf}.bak"
+				touch "$gmdconf"
+				systemctl restart NetworkManager
+			} >> "$SETUPLOG" 2>&1
+		fi
 	fi
-
+	
 	if test -f "$nmconf"; then
 		sed -i 's/managed=false/managed=true/g' "$nmconf" >> $SETUPLOG 2>&1
 	fi
@@ -388,20 +390,23 @@ create_sensor_bond() {
 		MTU=1500
 	fi

-	# Create the bond interface
-	nmcli con add ifname bond0 con-name "bond0" type bond mode 0 -- \
-		ipv4.method disabled \
-		ipv6.method ignore \
-		ethernet.mtu $MTU \
-		connection.autoconnect "yes" >> $SETUPLOG 2>&1
+	# Create the bond interface only if it doesn't already exist
+	if ! [[ $(nmcli -f name,uuid -p con | sed -n 's/bond0 //p' | tr -d ' ') ]]; then
+		nmcli con add ifname bond0 con-name "bond0" type bond mode 0 -- \
+			ipv4.method disabled \
+			ipv6.method ignore \
+			ethernet.mtu $MTU \
+			connection.autoconnect "yes" >> "$SETUPLOG" 2>&1
+	fi
+

 	for BNIC in ${BNICS[@]}; do
 		BONDNIC="$(echo -e "${BNIC}" | tr -d '"')" # Strip the quotes from the NIC names

 		# Check if specific offload features are able to be disabled
 		for string in "generic-segmentation-offload" "generic-receive-offload" "tcp-segmentation-offload"; do
-			if ethtool -k $BONDNIC | egrep $string | egrep -q "on [fixed]"; then
-				echo "The hardware or driver for interface ${BONDNIC} is not supported, packet capture may not work as expected." >> $SETUPLOG 2>&1
+			if ethtool -k "$BONDNIC" | grep $string | grep -q "on [fixed]"; then
+				echo "The hardware or driver for interface ${BONDNIC} is not supported, packet capture may not work as expected." >> "$SETUPLOG" 2>&1
 				nic_error=1
 				break
 			fi
@@ -411,13 +416,17 @@ create_sensor_bond() {
 		for i in rx tx sg tso ufo gso gro lro; do
 			ethtool -K $BONDNIC $i off >> $SETUPLOG 2>&1
 		done
-
-		# Create the slave interface and assign it to the bond
-		nmcli con add type ethernet ifname $BONDNIC con-name "bond0-slave-$BONDNIC" master bond0 -- \
-			ethernet.mtu $MTU \
-			connection.autoconnect "yes" >> $SETUPLOG 2>&1
 		
-		nmcli con up bond0-slave-$BONDNIC >> $SETUPLOG 2>&1 # Bring the slave interface up
+		# Check if the bond slave connection has already been created
+		if ! [[ $(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BONDNIC //p" | tr -d ' ') ]]; then
+			# Create the slave interface and assign it to the bond
+			
+			nmcli con add type ethernet ifname "$BONDNIC" con-name "bond0-slave-$BONDNIC" master bond0 -- \
+				ethernet.mtu $MTU \
+				connection.autoconnect "yes" >> "$SETUPLOG" 2>&1
+		fi
+
+		nmcli con up "bond0-slave-$BONDNIC" >> "$SETUPLOG" 2>&1 # Bring the slave interface up
 	done

 	if [ $nic_error != 0 ]; then
@@ -494,15 +503,14 @@ disable_misc_network_features() {
 			connection.autoconnect "no" >> $SETUPLOG 2>&1

 		# Flush any existing IPs
-		ip addr flush "$UNUSED_NIC" >> $SETUPLOG 2>&1
-
-		# Disable IPv6
-		{ 
-			echo "net.ipv6.conf.all.disable_ipv6 = 1"
-			echo "net.ipv6.conf.default.disable_ipv6 = 1"
-			echo "net.ipv6.conf.lo.disable_ipv6 = 1" 
-		} >> /etc/sysctl.conf
+		ip addr flush "$UNUSED_NIC" >> "$SETUPLOG" 2>&1
 	done
+    # Disable IPv6
+    { 
+        echo "net.ipv6.conf.all.disable_ipv6 = 1"
+        echo "net.ipv6.conf.default.disable_ipv6 = 1"
+        echo "net.ipv6.conf.lo.disable_ipv6 = 1" 
+    } >> /etc/sysctl.conf
 }

 docker_install() {
@@ -924,22 +932,26 @@ minio_generate_keys() {
 }

 network_setup() {
-	echo "Finishing up network setup" >> $SETUPLOG 2>&1
+    {
+        echo "Finishing up network setup";

-	echo "... Verifying all network devices are managed by Network Manager" >> $SETUPLOG 2>&1
-	check_network_manager_conf >> $SETUPLOG 2>&1
+        echo "... Verifying all network devices are managed by Network Manager";
+        check_network_manager_conf;

-	echo "... Disabling unused NICs" >> $SETUPLOG 2>&1
-	disable_misc_network_features >> $SETUPLOG 2>&1
+        echo "... Disabling unused NICs";
+        disable_misc_network_features;

-	echo "... Setting ONBOOT for management interface" >> $SETUPLOG 2>&1
-	nmcli con mod $MAININT connection.autoconnect "yes" >> $SETUPLOG 2>&1
+        echo "... Setting ONBOOT for management interface";
+        if ! netplan > /dev/null 2>&1; then
+            nmcli con mod "$MAININT" connection.autoconnect "yes";
+        fi

-	echo "... Copying 99-so-checksum-offload-disable" >> $SETUPLOG 2>&1
-	cp $SCRIPTDIR/install_scripts/99-so-checksum-offload-disable /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable  >> $SETUPLOG 2>&1
+        echo "... Copying 99-so-checksum-offload-disable";
+        cp "$SCRIPTDIR/install_scripts/99-so-checksum-offload-disable" /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable ;

-	echo "... Modifying 99-so-checksum-offload-disable" >> $SETUPLOG 2>&1
-	sed -i "s/\$MAININT/${MAININT}/g" /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable >> $SETUPLOG 2>&1
+        echo "... Modifying 99-so-checksum-offload-disable";
+        sed -i "s/\$MAININT/${MAININT}/g" /etc/NetworkManager/dispatcher.d/pre-up.d/99-so-checksum-offload-disable;
+    } >> "$SETUPLOG" 2>&1
 }

 node_pillar() {
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -36,8 +36,8 @@ export PATH=$PATH:../salt/common/tools/sbin
 HOSTNAME=$(cat /etc/hostname)
 TOTAL_MEM=`grep MemTotal /proc/meminfo | awk '{print $2}' | sed -r 's/.{3}$//'`
 NICS=$(ip link | awk -F: '$0 !~ "lo|vir|veth|br|docker|wl|^[^0-9]"{print $2 " \"" "Interface" "\"" " OFF"}')
-CPUCORES=$(cat /proc/cpuinfo | grep processor | wc -l)
-LISTCORES=$(cat /proc/cpuinfo | grep processor | awk '{print $3 " \"" "core" "\""}')
+CPUCORES=$(cat /proc/cpuinfo | grep processor | grep -v KVM | wc -l)
+LISTCORES=$(cat /proc/cpuinfo | grep processor | grep -v KVM | awk '{print $3 " \"" "core" "\""}')
 RANDOMUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1)
 NODE_ES_PORT="9200"
 SETUPLOG="/root/sosetup.log"