Python3: fixed tests and implementation (#3)

* Fixed `auto_increment` handling in MySQL driver
* Fixed auto_commit in SQLite driver
* Fixes initAlias/delAlias issue with testing
  multiple drivers at once
* Fixed GH action to run unit-tests as PyDO likes
* bumped version to 2.0.5

Signed-off-by: Yurii Shestakov <[email protected]>

.github/workflows/python-package.yml

@@ -26,6 +26,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
+        # sudo apt-get -yq install libmysqlclient-dev python3-mysqldb
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
@@ -37,4 +38,5 @@ jobs:
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
       run: |
-        pytest || true
+        # pytest || true
+        PYTHONPATH=src/ python tests/runtests.py -c tests/pydotest.rc -d "sqlite"

requirements.txt

@@ -0,0 +1 @@
+mysqlclient

src/pydo/__init__.py

@@ -29,4 +29,4 @@
 del getall
 
 
-__version__='2.0.4'
+__version__='2.0.5'

src/pydo/dbi.py

@@ -289,7 +289,8 @@ def init(conn):
             # get rid of connection for the sake of comparison
             old.pop('connection', None)
             if data!=old:
-                raise ValueError("already initialized: %s" % alias)
+                raise ValueError("already initialized: %s: %r != %r" %
+                                 (alias, data, old))
         else:
             _aliases[alias]=data
     finally:

src/pydo/drivers/mysqlconn.py

@@ -80,6 +80,13 @@ def execute(sql):
         nullableFields=[]
         for row in res:
             name, tipe, nullable, key, default, extra=row
+            if nullable == 'NO':
+                nullable = False
+            elif nullable == 'YES':
+                nullable = True
+            else:
+                raise RuntimeError("%s: unknown `nullable` for column '%s'" %
+                                   (nullable, name))
             if nullable:
                 nullableFields.append(name)
             if (not nullable) and extra=='auto_increment':

src/pydo/drivers/sqliteconn.py

@@ -96,6 +96,10 @@ class SqliteDBI(DBIBase):
    def __init__(self, connectArgs, pool=None, verbose=False, initFunc=None):
       if pool and not hasattr(pool, 'connect'):
          pool=ConnectionPool()
+      if sys.version_info[0] == 3:
+          # sqlite3.Connection object has no `autocommit` attribute
+          # https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection
+          connectArgs['isolation_level'] = None
       super(SqliteDBI, self).__init__(connectArgs,
                                       sqlite.connect,
                                       sqlite,
@@ -123,8 +127,6 @@ def listTables(self, schema=None):
          return sorted(x[0] for x in res)
       return ()
 
-
-
    def describeTable(self, table, schema=None):
       if schema is not None:
          raise ValueError("db schemas not supported by sqlite driver")
@@ -165,7 +167,8 @@ def execute(sql):
       execute(sql)
       res=c.fetchall()
       for row in res:
-         seq, name, uneek=row
+         #if self.verbose: print(".row: %s" % repr(row))
+         seq, name, uneek = row[0:3]
          if uneek:
             sql="pragma index_info('%s')" % name
             execute(sql)
@@ -177,9 +180,15 @@ def execute(sql):
       return fields, unique
 
    def autocommit():
+      # sqlite3.Connection object has no `autocommit` attribute
+      # https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection
       def fget(self):
+         if sys.version_info[0] == 3:
+             return False
          return self.conn.autocommit
       def fset(self, val):
+         if sys.version_info[0] == 3:
+            raise RuntimeError("sqliteconn.autocommit isn't implemented")
          self.conn.autocommit=val
       return fget, fset, None, None
    autocommit=property(*autocommit())

tests/config.py

@@ -15,15 +15,13 @@
 
 # https://stackoverflow.com/questions/436198/what-is-an-alternative-to-execfile-in-python-3
 def _execfile(cmd, g, l):
-    print("_execfile(%r, %r, %r)" % (cmd, g, l))
     exec(open(cmd).read(), g, l)
 if sys.version_info[0] == 3:
     execfile=_execfile
 
-
-DEFAULT_CONFIG='~/.pydotestrc'
-
-ALLDRIVERS=_driverConfig.keys()
+DEFAULT_CONFIG = '~/.pydotestrc'
+ALLDRIVERS = _driverConfig.keys()
+DRIVER = None
 
 def _readConfigFile(fname=DEFAULT_CONFIG):
     fname=os.path.expanduser(fname)
@@ -73,7 +71,7 @@ def readCmdLine(args, usage=None):
                       default=False)
     opts, args=parser.parse_args(args)
     try:
-        c=_readConfigFile(opts.config)
+        c = _readConfigFile(opts.config)
     except:
         traceback.print_exc()
         parser.error("error reading config file!")
@@ -93,18 +91,17 @@ def readCmdLine(args, usage=None):
     retdrivers={}
     # init all the aliases
     for d in drivers:
-        try:
-            connectArgs=c[d]
-        except:
+        if not d in c:
             print("no config for driver: %s" % d, file=sys.stderr)
         else:
+            connectArgs=c[d]
             connectArgs['driver']=d
             if opts.verbose is not None:
                 connectArgs['verbose']=opts.verbose
             # the connection alias will always be "pydotest"
-            connectArgs['alias']='pydotest'
             if not isinstance(connectArgs, dict):
                 parser.error("configuration error: must be a dict, got a %s" % type(connectArgs))
+            connectArgs['alias']='pydotest'
             retdrivers[d]=connectArgs
 
     if opts.pattern:
@@ -117,4 +114,3 @@ def readCmdLine(args, usage=None):
 
     return retdrivers, tags, pat, opts.unittest
 
-

tests/fixture.py

@@ -1,12 +1,18 @@
 from pydo.dbi import getConnection
 import pydo as P
 import config
+import sys
 
-def get_sequence_sql():
-    return dict(sqlite='INTEGER PRIMARY KEY NOT NULL',
+
+_SEQ_COL_SQL = dict(sqlite='INTEGER PRIMARY KEY NOT NULL',
                 sqlite2='INTEGER PRIMARY KEY NOT NULL',
                 psycopg='SERIAL PRIMARY KEY',
-                mysql='INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY')[config.DRIVER]
+                mysql='INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY')
+
+def get_sequence_sql():
+    if config.DRIVER not in _SEQ_COL_SQL:
+        raise KeyError("%r: incorect value of config.DRIVER" % config.DRIVER)
+    return _SEQ_COL_SQL[config.DRIVER]
 
 
 class Fixture(object):
@@ -29,7 +35,7 @@ def fget(self):
     def __call__(self):
         self.setup()
         try:
-           self.run ()
+           self.run ()
         finally:
            self.cleanup()
 
@@ -89,17 +95,17 @@ class base_fixture(Fixture):
     guess=False
 
     def setup(self):
-        c=self.db.cursor()
-        d=dict(seqsql=get_sequence_sql())
-        for table, cr in self.tables.items():
+        c=self.db.cursor()
+        d=dict(seqsql=get_sequence_sql())
+        for table, cr in self.tables.items():
             if self.usetables and table not in self.usetables:
                 continue
             cr %= d
-            c.execute(cr)
+            c.execute(cr)
             if self.useObjs:
-                self.setupObj(table, self.guess)
-        c.close()
-        self.pre()
+                self.setupObj(table, self.guess)
+        c.close()
+        self.pre()
 
 
     def pre(self):
@@ -195,11 +201,11 @@ class F(P.PyDO):
                             'c_id')
             self.F=F
 
-    def cleanup(self):
-        #
-        # sqlite2 transactions won't rollback table creation; only data
-        #
-        if self.db.autocommit or config.DRIVER == "sqlite2":
+    def cleanup(self):
+        #
+        # sqlite2 transactions won't rollback table creation; only data
+        #
+        if self.db.autocommit or config.DRIVER == "sqlite2" or sys.version_info[0] == 3:
             c=self.db.cursor()
             for table in self.tables:
                 if self.usetables and table not in self.usetables:

tests/pydotest.rc

@@ -0,0 +1,6 @@
+# -*-python-*-
+
+#psycopg=dict(connectArgs=dict(dsn='user=pydotest dbname=pydotest host=localhost'))
+sqlite=dict(connectArgs=dict(database='/tmp/pydotest_sqlite.db'))
+sqlite2=dict(connectArgs=dict(database=':memory:'))
+mysql=dict(connectArgs=dict(db='pydotest', user='pydotest'))

tests/runtests.py

@@ -3,12 +3,14 @@
 import sys
 import re
 import logging
+import os
 
 # munge sys.path
 
-sys.path.insert(0, '../src')
+_d = os.path.dirname(__file__)
+sys.path.insert(0, os.path.join(_d, '../src'))
 
-import config
+import config as _config
 from testingtesting import runNamespace, info, runModule, _defaultNamePat, _testsForModule
 from pydo import initAlias, delAlias, setLogLevel
 
@@ -24,20 +26,27 @@
 from test_operators import *
 
 if __name__=='__main__':
-    drivers, tags, pat, use_unit=config.readCmdLine(sys.argv[1:])
+    drivers, tags, pat, use_unit = _config.readCmdLine(sys.argv[1:])
     res=0
     import runtests
     for d, connectArgs in drivers.items():
+        _config.DRIVER = d
         initAlias(**connectArgs)
         curtags=list(tags)+[d]
+        # clean up existing sqlite db
+        if d.startswith('sqlite'):
+            _ca = connectArgs['connectArgs']
+            # print("d=%r _ca=%r" % (d, _ca))
+            db = _ca['database']
+            if db != ':memory:' and os.path.exists(db):
+                os.remove(db)
         if tags:
             info("testing with driver: %s, tags: %s", d, ", ".join(tags))
         else:
             info("testing with driver: %s", d)
-        config.DRIVER=d
         try:
             res |= runModule(runtests, curtags, pat, use_unit)
         finally:
             delAlias('pydotest')
-            del config.DRIVER
+            del _config.DRIVER
     sys.exit(res)

tests/test_base.py

@@ -349,11 +349,11 @@ class testclass(P.PyDO):
             connectionAlias='pydotest'
 
         cols=testclass.getColumns()
-
         uniq=testclass.getUniquenessConstraints()
         seq=testclass.getSequences()
         assert set(cols)==set(('id', 'x', 'y', 'z', 'r1', 'r2'))
-        assert list(seq.keys())==['id']
+        _sk = list(seq.keys())
+        assert _sk == ['id'], "seq.keys is %r" % _sk
         assert uniq==frozenset((frozenset(('r1', 'r2')),
                                 frozenset(('x',)),
                                 frozenset(('id',))))

tests/test_dbi.py

@@ -99,13 +99,13 @@ def test_swapConnection1():
 @tag(*dbitags)
 def test_pool1():
     stuff=D._aliases['pydotest'].copy()
-    D.initAlias('pydotestpool',
+    try:
+        D.initAlias('pydotestpool',
                 stuff['driver'],
                 stuff['connectArgs'],
                 D.ConnectionPool(max_poolsize=4, keep_poolsize=4),
                 stuff['verbose'])
-    db=D.getConnection('pydotestpool')
-    try:
+        db=D.getConnection('pydotestpool')
         assert len(db.pool._busy)==0
         assert len(db.pool._free)==0
         conn=db.conn
@@ -161,6 +161,7 @@ def run(self):
     # assertion means: no connection has been handed out
     # to a plural number of simultaneously active threads
     assert len(connids)== len(set(connids))
+    D.delAlias('pydotestpool')
 
 
 @tag(*dbitags)