Merge pull request #118 from openvar/develop_v3

Develop v3 - Ready for release 1.0.3. Tested under stress in the API. See if Travis passes

VariantValidator/modules/utils.py

@@ -16,16 +16,7 @@ def handleCursor(func):
     """
     @functools.wraps(func)
     def wrapper(self, *args, **kwargs):
-        try:
-            self.conn.ping(reconnect=True, attempts=3, delay=5)
-        except mysql.connector.Error:
-            logger.warning("MySQL connection lost. Reconnecting.")
-            self.init_db()
-
-        self.cursor = self.conn.cursor(buffered=True)
         out = func(self, *args, **kwargs)
-        if self.cursor:
-            self.cursor.close()
         return out
     return wrapper
 

VariantValidator/modules/vvDBGet.py

@@ -12,20 +12,36 @@ class Mixin(vvDBInit.Mixin):
 
     @handleCursor
     def execute(self, query):
-        self.cursor.execute(query)
-        row = self.cursor.fetchone()
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
+        cursor.execute(query)
+        row = cursor.fetchone()
         if row is None:
             logger.debug("No data returned from query " + str(query))
             row = ['none', 'No data']
+
+        # Close conn
+        cursor.close()
+        conn.close()
         return row
 
     @handleCursor
     def execute_all(self, query):
-        self.cursor.execute(query)
-        rows = self.cursor.fetchall()
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
+        cursor.execute(query)
+        rows = cursor.fetchall()
         if not rows:
             logger.debug("No data returned from query " + str(query))
             rows = ['none', 'No data']
+
+        # Close conn
+        cursor.close()
+        conn.close()
         return rows
 
     # from dbfetchone

VariantValidator/modules/vvDBInit.py

@@ -7,28 +7,33 @@ class Mixin:
     A mixin containing the database initialisation routines.
     """
     def __init__(self, db_config):
-        self.conn = None
         self.pool = None
-        # self.cursor will be none UNLESS you're wrapping a function in @handleCursor, which automatically opens and
-        # closes connections for you.
-        self.cursor = None
         self.dbConfig = db_config
-
         self.init_db()
 
     def __del__(self):
-        if self.conn.is_connected():
-            try:
-                self.conn.close()
-            except mysql.connector.errors.NotSupportedError:
-                pass
-            self.conn = None
         if self.pool:
             self.pool = None
 
     def init_db(self):
         self.pool = mysql.connector.pooling.MySQLConnectionPool(pool_size=10, connect_timeout=1209600, **self.dbConfig)
-        self.conn = self.pool.get_connection()
+
+    def get_conn(self):
+        try:
+            conn = self.pool.get_connection()
+        except mysql.connector.Error:
+            self.init_db()
+            conn = self.pool.get_connection()
+        return conn
+
+    def get_cursor(self, conn):
+        try:
+            cursor = conn.cursor(buffered=True)
+        except mysql.connector.Error:
+            self.init_db()
+            self.get_conn()
+            cursor = conn.cursor(buffered=True)
+        return cursor
 
 # <LICENSE>
 # Copyright (C) 2019 VariantValidator Contributors

VariantValidator/modules/vvDBInsert.py

@@ -9,6 +9,10 @@ class Mixin(vvDBGet.Mixin):
 
     @handleCursor
     def insert(self, entry, data, table):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
         # MySQL queries
         if table == 'transcript_info':
             accession = entry
@@ -19,79 +23,111 @@ def insert(self, entry, data, table):
             uta_symbol = data[5]
             query = "INSERT INTO transcript_info(refSeqID, description, transcriptVariant, currentVersion, " \
                     "hgncSymbol, utaSymbol, updated) VALUES(%s,%s, %s, %s, %s, %s, NOW())"
-            self.cursor.execute(query, (accession, description, variant, version, hgnc_symbol, uta_symbol))
+            cursor.execute(query, (accession, description, variant, version, hgnc_symbol, uta_symbol))
         # Query report
-        if self.cursor.lastrowid:
+        if cursor.lastrowid:
             success = 'true'
         else:
             success = 'Unknown error'
 
-        # Commit and close connection (?close?)
-        self.conn.commit()
+        # Commit and close connection
+        conn.commit()
+        cursor.close()
+        conn.close()
         return success
 
     @handleCursor
     def insert_refseq_gene_data(self, rsg_data):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
         query = "INSERT INTO refSeqGene_loci(refSeqGeneID, refSeqChromosomeID, genomeBuild, startPos, endPos, " \
                 "orientation, totalLength, chrPos, rsgPos, entrezID, hgncSymbol, updated) " \
                 "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())"
-        self.cursor.execute(query, (rsg_data[0], rsg_data[1], rsg_data[2], rsg_data[3], rsg_data[4], rsg_data[5],
+        cursor.execute(query, (rsg_data[0], rsg_data[1], rsg_data[2], rsg_data[3], rsg_data[4], rsg_data[5],
                                     rsg_data[6], rsg_data[7], rsg_data[8], rsg_data[9], rsg_data[10]))
         # Query report
-        if self.cursor.lastrowid:
+        if cursor.lastrowid:
             success = 'true'
         else:
             success = 'Unknown error'
+
         # Commit and close connection
-        self.conn.commit()
+        conn.commit()
+        cursor.close()
+        conn.close()
         return success
 
     @handleCursor
     def insert_refseq_gene_id_from_lrg_id(self, lrg_rs_lookup):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
         query = "INSERT INTO LRG_RSG_lookup(lrgID, hgncSymbol, RefSeqGeneID, status) VALUES (%s,%s,%s,%s)"
-        self.cursor.execute(query, (lrg_rs_lookup[0], lrg_rs_lookup[1], lrg_rs_lookup[2], lrg_rs_lookup[3]))
+        cursor.execute(query, (lrg_rs_lookup[0], lrg_rs_lookup[1], lrg_rs_lookup[2], lrg_rs_lookup[3]))
         # Query report
-        if self.cursor.lastrowid:
+        if cursor.lastrowid:
             success = 'true'
         else:
             success = 'Unknown error'
+
         # Commit and close connection
-        self.conn.commit()
+        conn.commit()
+        cursor.close()
+        conn.close()
         return success
 
     @handleCursor
     def insert_lrg_transcript_data(self, lrgtx_to_rst_id):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
         query = "INSERT INTO LRG_transcripts(LRGtranscriptID, RefSeqTranscriptID) VALUES (%s,%s)"
-        self.cursor.execute(query, (lrgtx_to_rst_id[0], lrgtx_to_rst_id[1]))
+        cursor.execute(query, (lrgtx_to_rst_id[0], lrgtx_to_rst_id[1]))
         # Query report
-        if self.cursor.lastrowid:
+        if cursor.lastrowid:
             success = 'true'
         else:
             success = 'Unknown error'
 
         # Commit and close connection
-        self.conn.commit()
+        conn.commit()
+        cursor.close()
+        conn.close()
         return success
 
     @handleCursor
     def insert_lrg_protein_data(self, lrg_p, rs_p):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
         query = "INSERT INTO LRG_proteins(LRGproteinID, RefSeqProteinID) VALUES (%s,%s)"
-        self.cursor.execute(query, (lrg_p, rs_p))
+        cursor.execute(query, (lrg_p, rs_p))
         # Query report
-        if self.cursor.lastrowid:
+        if cursor.lastrowid:
             success = 'true'
         else:
             success = 'Unknown error'
 
         # Commit and close connection
-        self.conn.commit()
+        conn.commit()
+        cursor.close()
+        conn.close()
         return success
 
     @handleCursor
     def insert_gene_stable_ids(self, data):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
         query = "INSERT INTO stableGeneIds(hgnc_id, hgnc_symbol, entrez_id, ensembl_gene_id, omim_id, ucsc_id, " \
                 "vega_id, ccds_ids) VALUES (%s,%s,%s,%s,%s,%s,%s,%s)"
-        self.cursor.execute(query, (
+        cursor.execute(query, (
             data['hgnc_id'],
             data['hgnc_symbol'],
             data['entrez_id'],
@@ -102,16 +138,23 @@ def insert_gene_stable_ids(self, data):
             data['ccds_id']
         ))
 
-        if self.cursor.lastrowid:
+        if cursor.lastrowid:
             success = 'true'
         else:
             success = 'unknown error'
 
-        self.conn.commit()
+        # Commit and close connection
+        conn.commit()
+        cursor.close()
+        conn.close()
         return success
 
     @handleCursor
     def update(self, entry, data):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
         accession = entry
         description = data[1]
         variant = data[2]
@@ -120,27 +163,42 @@ def update(self, entry, data):
         uta_symbol = data[5]
         query = "UPDATE transcript_info SET description=%s, transcriptVariant=%s, currentVersion=%s, hgncSymbol=%s, " \
                 "utaSymbol=%s, updated=NOW() WHERE refSeqID = %s"
-        self.cursor.execute(query, (description, variant, version, hgnc_symbol, uta_symbol, accession))
+        cursor.execute(query, (description, variant, version, hgnc_symbol, uta_symbol, accession))
         success = 'true'
-        self.conn.commit()
+
+        # Commit and close connection
+        conn.commit()
+        cursor.close()
+        conn.close()
         return success
 
     @handleCursor
     def update_refseq_gene_data(self, rsg_data):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
         query = "UPDATE refSeqGene_loci SET hgncSymbol=%s, updated=NOW() WHERE refSeqGeneID=%s"
-        self.cursor.execute(query, (rsg_data[10], rsg_data[0]))
+        cursor.execute(query, (rsg_data[10], rsg_data[0]))
         success = 'true'
-        self.conn.commit()
+
+        # Commit and close connection
+        conn.commit()
+        cursor.close()
+        conn.close()
         return success
 
     @handleCursor
     def update_gene_stable_ids(self, gene_stable_ids):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
 
         # Insert or update combined statement
         query = "UPDATE stableGeneIds SET hgnc_symbol=%s, entrez_id=%s, ensembl_gene_id=%s, omim_id=%s, ucsc_id=%s, " \
                 "vega_id=%s, ccds_ids=%s WHERE hgnc_id=%s"
 
-        self.cursor.execute(query, (
+        cursor.execute(query, (
             gene_stable_ids["hgnc_symbol"],
             gene_stable_ids["entrez_id"],
             gene_stable_ids["ensembl_gene_id"],
@@ -151,7 +209,11 @@ def update_gene_stable_ids(self, gene_stable_ids):
             gene_stable_ids["hgnc_id"]
         ))
         success = 'true'
-        self.conn.commit()
+
+        # Commit and close connection
+        conn.commit()
+        cursor.close()
+        conn.close()
         return success
 
 # <LICENSE>

VariantValidator/modules/vvDatabase.py

@@ -25,14 +25,22 @@ class Database(vvDBInsert.Mixin):
     # from dbquery
     @handleCursor
     def query_with_fetchone(self, entry):
+        # Connect and create cursor
+        conn = self.get_conn()
+        cursor = self.get_cursor(conn)
+
         query = "SELECT refSeqID, description, transcriptVariant, currentVersion, hgncSymbol, utaSymbol, updated, " \
                 "IF(updated < NOW() - INTERVAL 3 MONTH , 'true', 'false') FROM transcript_info WHERE " \
                 "refSeqID = '%s'" % entry
-        self.cursor.execute(query)
-        row = self.cursor.fetchone()
+        cursor.execute(query)
+        row = cursor.fetchone()
         if row is None:
             row = ['none', 'No data']
             logger.debug("No data returned from query " + str(query))
+
+        # close conn
+        cursor.close()
+        conn.close()
         return row
 
     # From data

tests/test_aa_db_update.py

@@ -20,7 +20,8 @@ def test_connection(self):
         db_conn = update_vv_db.connect()
 
         self.assertIsInstance(db_conn, Database)
-        self.assertTrue(db_conn.conn.is_connected())
+        conn = db_conn.get_conn()
+        self.assertTrue(conn.is_connected())
 
     def test_deletion(self):
         db = update_vv_db.connect()