From 9960cf05921c8f121521f80857a0f24d472938d4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 9 Nov 2020 12:05:37 -0500 Subject: [PATCH 1/9] [feat] Add salt module to check if mysql is accepting db connections --- salt/_modules/mysql.py | 35 +++++++++++++++++++++++++++++++++++ salt/mysql/init.sls | 5 +++++ 2 files changed, 40 insertions(+) create mode 100644 salt/_modules/mysql.py diff --git a/salt/_modules/mysql.py b/salt/_modules/mysql.py new file mode 100644 index 000000000..f4e35ae76 --- /dev/null +++ b/salt/_modules/mysql.py @@ -0,0 +1,35 @@ +#!py + +from MySQLdb import _mysql +import logging +import time + +log = logging.getLogger(__name__) + + +def status(retry): + mainint = __salt__['pillar.get']('sensor:mainint', __salt__['pillar.get']('manager:mainint')) + mainip = __salt__['grains.get']('ip_interfaces').get(mainint)[0] + + mysql_up = False + for i in range(0, retry): + log.debug(f'Connection attempt {i+1}') + try: + _mysql.connect( + host=mainip, + user="root", + passwd=__salt__['pillar.get']('secrets:mysql') + ) + mysql_up = True + break + except _mysql.OperationalError as e: + log.debug(e) + except Exception as e: + log.error(e) + break + time.sleep(1) + + if not mysql_up: + log.error(f'Could not connect to MySQL server on {mainip} after {retry} attempts.') + + return mysql_up diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 818b5c303..e1f37f29c 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -97,6 +97,11 @@ so-mysql: - timeout: 900 - onchanges: - docker_container: so-mysql + module.run: + - mysql.status: + - retry: 900 + - onchanges: + - cmd: so-mysql {% endif %} {% else %} From 394fa727cbb87346411c46a1db2e0662b45968aa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 9 Nov 2020 13:05:29 -0500 Subject: [PATCH 2/9] [fix] Don't overwrite mysql module --- salt/_modules/mysql.py | 35 ----------------------------------- salt/_modules/so.py | 36 +++++++++++++++++++++++++++++++++++- salt/mysql/init.sls | 2 +- 3 files changed, 36 insertions(+), 37 deletions(-) delete mode 100644 salt/_modules/mysql.py diff --git a/salt/_modules/mysql.py b/salt/_modules/mysql.py deleted file mode 100644 index f4e35ae76..000000000 --- a/salt/_modules/mysql.py +++ /dev/null @@ -1,35 +0,0 @@ -#!py - -from MySQLdb import _mysql -import logging -import time - -log = logging.getLogger(__name__) - - -def status(retry): - mainint = __salt__['pillar.get']('sensor:mainint', __salt__['pillar.get']('manager:mainint')) - mainip = __salt__['grains.get']('ip_interfaces').get(mainint)[0] - - mysql_up = False - for i in range(0, retry): - log.debug(f'Connection attempt {i+1}') - try: - _mysql.connect( - host=mainip, - user="root", - passwd=__salt__['pillar.get']('secrets:mysql') - ) - mysql_up = True - break - except _mysql.OperationalError as e: - log.debug(e) - except Exception as e: - log.error(e) - break - time.sleep(1) - - if not mysql_up: - log.error(f'Could not connect to MySQL server on {mainip} after {retry} attempts.') - - return mysql_up diff --git a/salt/_modules/so.py b/salt/_modules/so.py index 50c29902f..43ffac250 100644 --- a/salt/_modules/so.py +++ b/salt/_modules/so.py @@ -1,4 +1,38 @@ #!py +import logging + def status(): - return __salt__['cmd.run']('/usr/sbin/so-status') \ No newline at end of file + return __salt__['cmd.run']('/usr/sbin/so-status') + + +def mysql_conn(retry): + from MySQLdb import _mysql + import time + + log = logging.getLogger(__name__) + mainint = __salt__['pillar.get']('sensor:mainint', __salt__['pillar.get']('manager:mainint')) + mainip = __salt__['grains.get']('ip_interfaces').get(mainint)[0] + + mysql_up = False + for i in range(0, retry): + log.debug(f'Connection attempt {i+1}') + try: + _mysql.connect( + host=mainip, + user="root", + passwd=__salt__['pillar.get']('secrets:mysql') + ) + mysql_up = True + break + except _mysql.OperationalError as e: + log.debug(e) + except Exception as e: + log.error(e) + break + time.sleep(1) + + if not mysql_up: + log.error(f'Could not connect to MySQL server on {mainip} after {retry} attempts.') + + return mysql_up \ No newline at end of file diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index e1f37f29c..121e689f8 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -98,7 +98,7 @@ so-mysql: - onchanges: - docker_container: so-mysql module.run: - - mysql.status: + - so.mysql_conn: - retry: 900 - onchanges: - cmd: so-mysql From ff4d7a6cb60654c68cde17cc896462bcb73f80a2 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 9 Nov 2020 14:01:19 -0500 Subject: [PATCH 3/9] [fix] Sync modules so states can use our modules during setup --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index c19490e73..51a9b01c0 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1729,6 +1729,7 @@ salt_checkin() { { salt-call state.apply ca; salt-call state.apply ssl; + salt-call saltutil.sync_modules; } >> "$setup_log" 2>&1 } From dba30fb0edb1d354dfbf62dbfa22d175f8595c4a Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 10 Nov 2020 09:48:20 -0500 Subject: [PATCH 4/9] [refactor] Split 15 min mysql startup between two wait states --- salt/mysql/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/mysql/init.sls b/salt/mysql/init.sls index 121e689f8..c8683b1a1 100644 --- a/salt/mysql/init.sls +++ b/salt/mysql/init.sls @@ -94,12 +94,12 @@ so-mysql: - /opt/so/conf/mysql/etc cmd.run: - name: until nc -z {{ MAINIP }} 3306; do sleep 1; done - - timeout: 900 + - timeout: 600 - onchanges: - docker_container: so-mysql module.run: - so.mysql_conn: - - retry: 900 + - retry: 300 - onchanges: - cmd: so-mysql {% endif %} From 22b7de819cd4a603eb44b78bce7f54c84eeb127b Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 10 Nov 2020 10:00:21 -0500 Subject: [PATCH 5/9] [fix] Put mysql import in try,catch in case it hasn't been installed --- salt/_modules/so.py | 57 ++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/salt/_modules/so.py b/salt/_modules/so.py index 43ffac250..de337c43f 100644 --- a/salt/_modules/so.py +++ b/salt/_modules/so.py @@ -3,36 +3,41 @@ import logging def status(): - return __salt__['cmd.run']('/usr/sbin/so-status') + return __salt__['cmd.run']('/usr/sbin/so-status') def mysql_conn(retry): - from MySQLdb import _mysql - import time + log = logging.getLogger(__name__) - log = logging.getLogger(__name__) - mainint = __salt__['pillar.get']('sensor:mainint', __salt__['pillar.get']('manager:mainint')) - mainip = __salt__['grains.get']('ip_interfaces').get(mainint)[0] + try: + from MySQLdb import _mysql + except ImportError as e: + log.error(e) + return False + from time import sleep - mysql_up = False - for i in range(0, retry): - log.debug(f'Connection attempt {i+1}') - try: - _mysql.connect( - host=mainip, - user="root", - passwd=__salt__['pillar.get']('secrets:mysql') - ) - mysql_up = True - break - except _mysql.OperationalError as e: - log.debug(e) - except Exception as e: - log.error(e) - break - time.sleep(1) + mainint = __salt__['pillar.get']('sensor:mainint', __salt__['pillar.get']('manager:mainint')) + mainip = __salt__['grains.get']('ip_interfaces').get(mainint)[0] - if not mysql_up: - log.error(f'Could not connect to MySQL server on {mainip} after {retry} attempts.') + mysql_up = False + for i in range(0, retry): + log.debug(f'Connection attempt {i+1}') + try: + _mysql.connect( + host=mainip, + user="root", + passwd=__salt__['pillar.get']('secrets:mysql') + ) + mysql_up = True + break + except _mysql.OperationalError as e: + log.debug(e) + except Exception as e: + log.error(e) + break + sleep(1) - return mysql_up \ No newline at end of file + if not mysql_up: + log.error(f'Could not connect to MySQL server on {mainip} after {retry} attempts.') + + return mysql_up \ No newline at end of file From 54d732a0602e12170a59ba464eb37adbc76e90aa Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 10 Nov 2020 10:01:10 -0500 Subject: [PATCH 6/9] [refactor] Code cleanup --- salt/_modules/so.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/_modules/so.py b/salt/_modules/so.py index de337c43f..9a3706c78 100644 --- a/salt/_modules/so.py +++ b/salt/_modules/so.py @@ -9,12 +9,13 @@ def status(): def mysql_conn(retry): log = logging.getLogger(__name__) + from time import sleep + try: from MySQLdb import _mysql except ImportError as e: log.error(e) return False - from time import sleep mainint = __salt__['pillar.get']('sensor:mainint', __salt__['pillar.get']('manager:mainint')) mainip = __salt__['grains.get']('ip_interfaces').get(mainint)[0] From b3c527e7a91ae2f266001b992d2e9fc257ba64e4 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 10 Nov 2020 10:05:06 -0500 Subject: [PATCH 7/9] [refactor] Code cleanup pt. 2 --- salt/_modules/so.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/_modules/so.py b/salt/_modules/so.py index 9a3706c78..a15e7ee66 100644 --- a/salt/_modules/so.py +++ b/salt/_modules/so.py @@ -25,15 +25,17 @@ def mysql_conn(retry): log.debug(f'Connection attempt {i+1}') try: _mysql.connect( - host=mainip, - user="root", - passwd=__salt__['pillar.get']('secrets:mysql') + host=mainip, + user='root', + passwd=__salt__['pillar.get']('secrets:mysql') ) + log.debug(f'Connected to MySQL server on {mainip} after {retry} attempts.') mysql_up = True break except _mysql.OperationalError as e: log.debug(e) except Exception as e: + log.error('Unexpected error occured.') log.error(e) break sleep(1) From 7f218e52973a96a5805fffbc652e8187bc61115d Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 10 Nov 2020 11:02:34 -0500 Subject: [PATCH 8/9] [feat] Also run query against mysql to ensure queries can complete --- salt/_modules/so.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/_modules/so.py b/salt/_modules/so.py index a15e7ee66..2356f68da 100644 --- a/salt/_modules/so.py +++ b/salt/_modules/so.py @@ -24,12 +24,14 @@ def mysql_conn(retry): for i in range(0, retry): log.debug(f'Connection attempt {i+1}') try: - _mysql.connect( + db = _mysql.connect( host=mainip, user='root', passwd=__salt__['pillar.get']('secrets:mysql') ) - log.debug(f'Connected to MySQL server on {mainip} after {retry} attempts.') + log.debug(f'Connected to MySQL server on {mainip} after {i} attempts.') + db.query("""SELECT 1;""") + log.debug(f'Successfully completed query against MySQL server on {mainip}') mysql_up = True break except _mysql.OperationalError as e: From d3227bbcb189e0a15a065e136f7e48d81a18ebbd Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 10 Nov 2020 11:03:43 -0500 Subject: [PATCH 9/9] [refactor] Code cleanup pt. 3 --- salt/_modules/so.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/_modules/so.py b/salt/_modules/so.py index 2356f68da..b9fd3c693 100644 --- a/salt/_modules/so.py +++ b/salt/_modules/so.py @@ -30,8 +30,10 @@ def mysql_conn(retry): passwd=__salt__['pillar.get']('secrets:mysql') ) log.debug(f'Connected to MySQL server on {mainip} after {i} attempts.') + db.query("""SELECT 1;""") log.debug(f'Successfully completed query against MySQL server on {mainip}') + mysql_up = True break except _mysql.OperationalError as e: