From 040b43527820b5ecb6b64bb5f969a9d2d2709df5 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 30 Nov 2020 11:09:06 -0500 Subject: [PATCH 1/5] [refactor] Fail mysql_conn if the mainint has > 1 ip address --- salt/_modules/so.py | 61 ++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/salt/_modules/so.py b/salt/_modules/so.py index ff3cf27b2..8657722e2 100644 --- a/salt/_modules/so.py +++ b/salt/_modules/so.py @@ -18,34 +18,43 @@ def mysql_conn(retry): return False mainint = __salt__['pillar.get']('host:mainint') - mainip = __salt__['grains.get']('ip_interfaces').get(mainint)[0] + ip_arr = __salt__['grains.get']('ip_interfaces').get(mainint) mysql_up = False - for i in range(0, retry): - log.debug(f'Connection attempt {i+1}') - try: - db = _mysql.connect( - host=mainip, - user='root', - passwd=__salt__['pillar.get']('secrets:mysql') - ) - log.debug(f'Connected to MySQL server on {mainip} after {i} attempts.') - - db.query("""SELECT 1;""") - log.debug(f'Successfully completed query against MySQL server on {mainip}') - - db.close() - mysql_up = True - break - except _mysql.OperationalError as e: - log.debug(e) - except Exception as e: - log.error('Unexpected error occured.') - log.error(e) - break - sleep(1) - if not mysql_up: - log.error(f'Could not connect to MySQL server on {mainip} after {retry} attempts.') + if len(ip_arr) == 1: + mainip = ip_arr[0] + + for i in range(0, retry): + log.debug(f'Connection attempt {i+1}') + try: + db = _mysql.connect( + host=mainip, + user='root', + passwd=__salt__['pillar.get']('secrets:mysql') + ) + log.debug(f'Connected to MySQL server on {mainip} after {i} attempts.') + + db.query("""SELECT 1;""") + log.debug(f'Successfully completed query against MySQL server on {mainip}') + + db.close() + mysql_up = True + break + except _mysql.OperationalError as e: + log.debug(e) + except Exception as e: + log.error('Unexpected error occured.') + log.error(e) + break + sleep(1) + + if not mysql_up: + log.error(f'Could not connect to MySQL server on {mainip} after {retry} attempts.') + else: + log.error(f'Main interface {mainint} has more than one IP address assigned to it, which is not supported.') + log.debug(f'{mainint}:') + for addr in ip_arr: + log.debug(f' - {addr}') return mysql_up \ No newline at end of file From ec81e8565fd1dc487577b4db1eb18b3b3e0df5d1 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 30 Nov 2020 11:32:28 -0500 Subject: [PATCH 2/5] [fix] Add safety logic to retry var in mysql_conn --- salt/_modules/so.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/_modules/so.py b/salt/_modules/so.py index 8657722e2..fb61f3460 100644 --- a/salt/_modules/so.py +++ b/salt/_modules/so.py @@ -25,6 +25,10 @@ def mysql_conn(retry): if len(ip_arr) == 1: mainip = ip_arr[0] + if not(retry >= 1): + log.debug('`retry` set to value below 1, resetting it to 1 to prevent errors.') + retry = 1 + for i in range(0, retry): log.debug(f'Connection attempt {i+1}') try: From 8964444eebecf65c1a35a0c607fbb3024866a218 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 30 Nov 2020 11:32:43 -0500 Subject: [PATCH 3/5] [fix] Correct count print in mysql_conn --- salt/_modules/so.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/_modules/so.py b/salt/_modules/so.py index fb61f3460..bbbbe4ea8 100644 --- a/salt/_modules/so.py +++ b/salt/_modules/so.py @@ -37,7 +37,7 @@ def mysql_conn(retry): user='root', passwd=__salt__['pillar.get']('secrets:mysql') ) - log.debug(f'Connected to MySQL server on {mainip} after {i} attempts.') + log.debug(f'Connected to MySQL server on {mainip} after {i+1} attempts.') db.query("""SELECT 1;""") log.debug(f'Successfully completed query against MySQL server on {mainip}') From 5d2acf40117554eafa58d6d416745568de730a15 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 30 Nov 2020 12:06:02 -0500 Subject: [PATCH 4/5] Fix Fleet setup errors --- salt/fleet/event_enable-fleet.sls | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/salt/fleet/event_enable-fleet.sls b/salt/fleet/event_enable-fleet.sls index 34b031685..d09749a55 100644 --- a/salt/fleet/event_enable-fleet.sls +++ b/salt/fleet/event_enable-fleet.sls @@ -1,10 +1,4 @@ -{% set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) %} -{% set FLEETNODE = salt['pillar.get']('global:fleet_node', False) %} -{% if FLEETNODE or FLEETMANAGER %} - {% set ENROLLSECRET = salt['cmd.run']('docker exec so-fleet fleetctl get enroll-secret default') %} -{% else %} - {% set ENROLLSECRET = '' %} -{% endif %} +{% set ENROLLSECRET = salt['cmd.run']('docker exec so-fleet fleetctl get enroll-secret default') %} {% set MAININT = salt['pillar.get']('host:mainint') %} {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} From e7a927188b081e1c3b7ee6faaae5de48171d4e09 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 30 Nov 2020 17:28:11 -0500 Subject: [PATCH 5/5] Fleet Fixes - mysql race condition --- salt/common/tools/sbin/so-fleet-setup | 7 +++---- salt/fleet/event_enable-fleet.sls | 4 +--- salt/fleet/event_update-enroll-secret.sls | 7 +++++++ salt/reactor/fleet.sls | 19 ++++++++++++------- setup/so-setup | 3 +++ 5 files changed, 26 insertions(+), 14 deletions(-) create mode 100644 salt/fleet/event_update-enroll-secret.sls diff --git a/salt/common/tools/sbin/so-fleet-setup b/salt/common/tools/sbin/so-fleet-setup index b481ceb59..3e9fb1d74 100755 --- a/salt/common/tools/sbin/so-fleet-setup +++ b/salt/common/tools/sbin/so-fleet-setup @@ -26,10 +26,9 @@ docker exec so-fleet /bin/sh -c 'for pack in /packs/palantir/Fleet/Endpoints/pac docker exec so-fleet fleetctl apply -f /packs/osquery-config.conf -# Enable Fleet -echo "Enabling Fleet..." -sleep 5 -salt-call state.apply fleet.event_enable-fleet queue=True >> /root/fleet-setup.log +# Update the Enroll Secret +echo "Updating the Enroll Secret..." +salt-call state.apply fleet.event_update-enroll-secret queue=True >> /root/fleet-setup.log salt-call state.apply nginx queue=True >> /root/fleet-setup.log # Generate osquery install packages diff --git a/salt/fleet/event_enable-fleet.sls b/salt/fleet/event_enable-fleet.sls index d09749a55..52a15269c 100644 --- a/salt/fleet/event_enable-fleet.sls +++ b/salt/fleet/event_enable-fleet.sls @@ -1,4 +1,3 @@ -{% set ENROLLSECRET = salt['cmd.run']('docker exec so-fleet fleetctl get enroll-secret default') %} {% set MAININT = salt['pillar.get']('host:mainint') %} {% set MAINIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %} @@ -8,5 +7,4 @@ so/fleet: action: 'enablefleet' hostname: {{ grains.host }} mainip: {{ MAINIP }} - role: {{ grains.role }} - enroll-secret: {{ ENROLLSECRET }} \ No newline at end of file + role: {{ grains.role }} \ No newline at end of file diff --git a/salt/fleet/event_update-enroll-secret.sls b/salt/fleet/event_update-enroll-secret.sls new file mode 100644 index 000000000..609020247 --- /dev/null +++ b/salt/fleet/event_update-enroll-secret.sls @@ -0,0 +1,7 @@ +{% set ENROLLSECRET = salt['cmd.run']('docker exec so-fleet fleetctl get enroll-secret default') %} + +so/fleet: + event.send: + - data: + action: 'update-enrollsecret' + enroll-secret: {{ ENROLLSECRET }} \ No newline at end of file diff --git a/salt/reactor/fleet.sls b/salt/reactor/fleet.sls index a4226b027..bc2131427 100644 --- a/salt/reactor/fleet.sls +++ b/salt/reactor/fleet.sls @@ -17,7 +17,6 @@ def run(): if ACTION == 'enablefleet': logging.info('so/fleet enablefleet reactor') - ESECRET = data['data']['enroll-secret'] MAINIP = data['data']['mainip'] ROLE = data['data']['role'] HOSTNAME = data['data']['hostname'] @@ -30,12 +29,6 @@ def run(): line = re.sub(r'fleet_manager: \S*', f"fleet_manager: True", line.rstrip()) print(line) - # Update the enroll secret in the secrets pillar - if ESECRET != "": - for line in fileinput.input(SECRETSFILE, inplace=True): - line = re.sub(r'fleet_enroll-secret: \S*', f"fleet_enroll-secret: {ESECRET}", line.rstrip()) - print(line) - # Update the Fleet host in the static pillar for line in fileinput.input(STATICFILE, inplace=True): line = re.sub(r'fleet_hostname: \S*', f"fleet_hostname: '{HOSTNAME}'", line.rstrip()) @@ -46,6 +39,18 @@ def run(): line = re.sub(r'fleet_ip: \S*', f"fleet_ip: '{MAINIP}'", line.rstrip()) print(line) + if ACTION == 'update-enrollsecret': + logging.info('so/fleet update-enrollsecret reactor') + + ESECRET = data['data']['enroll-secret'] + + # Update the enroll secret in the secrets pillar + if ESECRET != "": + for line in fileinput.input(SECRETSFILE, inplace=True): + line = re.sub(r'fleet_enroll-secret: \S*', f"fleet_enroll-secret: {ESECRET}", line.rstrip()) + print(line) + + if ACTION == 'genpackages': logging.info('so/fleet genpackages reactor') diff --git a/setup/so-setup b/setup/so-setup index d83411b58..77c579cfc 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -691,6 +691,9 @@ fi if [[ "$OSQUERY" = 1 ]]; then + set_progress_str 75 "$(print_salt_state_apply 'fleet.event_enable-fleet')" + salt-call state.apply -l info fleet.event_enable-fleet >> $setup_log 2>&1 + set_progress_str 75 "$(print_salt_state_apply 'fleet')" salt-call state.apply -l info fleet >> $setup_log 2>&1