"""XXX in progress: resultdb plugin for database logging of test results. Saves test results to a datastore. XXX this needs to be merged with resultlog plugin Also mixes in some early ideas about an archive abstraction for test results. """ import py py.test.skip("XXX needs to be merged with resultlog") from pytest_resultlog import ResultLog def pytest_addoption(parser): group = parser.addgroup("resultdb", "resultdb plugin options") group.addoption('--resultdb', action="store", dest="resultdb", metavar="path", help="path to the file to store test results.") group.addoption('--resultdb_format', action="store", dest="resultdbformat", default='json', help="data format (json, sqlite)") def pytest_configure(config): # XXX using config.XYZ is not good if config.getvalue('resultdb'): if config.option.resultdb: # local import so missing module won't crash py.test try: import sqlite3 except ImportError: raise config.Error('Could not import sqlite3 module') try: import simplejson except ImportError: raise config.Error('Could not import simplejson module') if config.option.resultdbformat.lower() == 'json': resultdb = ResultDB(JSONResultArchive, config.option.resultdb) elif config.option.resultdbformat.lower() == 'sqlite': resultdb = ResultDB(SQLiteResultArchive, config.option.resultdb) else: raise config.Error('Unknown --resultdb_format: %s' % config.option.resultdbformat) config.pluginmanager.register(resultdb) class JSONResultArchive(object): def __init__(self, archive_path): self.archive_path = archive_path import simplejson self.simplejson = simplejson def init_db(self): if os.path.exists(self.archive_path): data_file = open(self.archive_path) archive = self.simplejson.load(data_file) self.archive = archive else: self.archive = [] self._flush() def append_data(self, data): runid = py.std.uuid.uuid4() for item in data: item = item.copy() item['runid'] = str(runid) self.archive.append(item) self._flush() def get_all_data(self): return self.archive def _flush(self): data_file = open(self.archive_path, 'w') self.simplejson.dump(self.archive, data_file) data_file.close() class SQLiteResultArchive(object): def __init__(self, archive_path): self.archive_path = archive_path import sqlite3 self.sqlite3 = sqlite3 def init_db(self): if not os.path.exists(self.archive_path): conn = self.sqlite3.connect(self.archive_path) cursor = conn.cursor() try: cursor.execute(SQL_CREATE_TABLES) conn.commit() finally: cursor.close() conn.close() def append_data(self, data): flat_data = [] runid = py.std.uuid.uuid4() for item in data: item = item.copy() item['runid'] = str(runid) flat_data.append(self.flatten(item)) conn = self.sqlite3.connect(self.archive_path) cursor = conn.cursor() cursor.executemany(SQL_INSERT_DATA, flat_data) conn.commit() cursor.close() conn.close() def get_all_data(self): conn = self.sqlite3.connect(self.archive_path) conn.row_factory = self.sqlite3.Row cursor = conn.cursor() cursor.execute(SQL_SELECT_DATA) data = cursor.fetchall() cursor.close() conn.close() data = [self.unflatten(item) for item in data] return data def flatten(self, item): return (item.get('runid', None), item.get('name', None), item.get('passed', False), item.get('skipped', False), item.get('failed', False), item.get('shortrepr', None), item.get('longrepr', None), item.get('fspath', None), item.get('itemname', None), ) def unflatten(self, item): names = ("runid name passed skipped failed shortrepr " "longrepr fspath itemname").split() d = {} for i, name in enumerate(names): d[name] = item[i] return d class ResultDB(ResultLog): def __init__(self, cls, db_path): self.archive = cls(db_path) self.archive.init_db() def write_log_entry(self, testpath, shortrepr, longrepr): data = {} event_excludes = ['colitem', 'longrepr'] for item in vars(event).keys(): if item not in event_excludes: data[item] = getattr(event, item) # use the locally calculated longrepr & shortrepr data['longrepr'] = longrepr data['shortrepr'] = shortrepr data['testpath'] = unicode(testpath) self.archive.append_data([data]) SQL_CREATE_TABLES = """ create table pytest_results ( runid varchar(36), name varchar, passed int, skipped int, failed int, shortrepr varchar, longrepr varchar, fspath varchar, itemname varchar ); """ SQL_INSERT_DATA = """ insert into pytest_results ( runid, name, passed, skipped, failed, shortrepr, longrepr, fspath, itemname) values (?, ?, ?, ?, ?, ?, ?, ?, ?); """ SQL_SELECT_DATA = """ select runid, name, passed, skipped, failed, shortrepr, longrepr, fspath, itemname from pytest_results; """ # =============================================================================== # # plugin tests # # =============================================================================== import os, StringIO class BaseResultArchiveTests(object): cls = None def setup_class(cls): # XXX refactor setup into a funcarg? cls.tempdb = "test_tempdb" def test_init_db(self, testdir): tempdb_path = unicode(testdir.tmpdir.join(self.tempdb)) archive = self.cls(tempdb_path) archive.init_db() assert os.path.exists(tempdb_path) def test_db_insert(self, testdir): tempdb_path = unicode(testdir.tmpdir.join(self.tempdb)) archive = self.cls(tempdb_path) archive.init_db() assert len(archive.get_all_data()) == 0 data = [{'name': 'tmppackage/test_whatever.py:test_hello', 'fspath': '/Users/brian/work/tmppackage/test_whatever.py', 'name': 'test_hello', 'longrepr': '', 'passed': True, 'shortrepr': '.' }] archive.append_data(data) result = archive.get_all_data() print result assert len(result) == 1 for key, value in data[0].items(): assert value == result[0][key] assert 'runid' in result[0] # make sure the data is persisted tempdb_path = unicode(testdir.tmpdir.join(self.tempdb)) archive = self.cls(tempdb_path) archive.init_db() assert len(archive.get_all_data()) == 1 class TestJSONResultArchive(BaseResultArchiveTests): cls = JSONResultArchive def setup_method(self, method): py.test.importorskip("simplejson") class TestSQLiteResultArchive(BaseResultArchiveTests): cls = SQLiteResultArchive def setup_method(self, method): py.test.importorskip("sqlite3") def test_init_db_sql(self, testdir): py.test.importorskip("sqlite3") tempdb_path = unicode(testdir.tmpdir.join(self.tempdb)) archive = self.cls(tempdb_path) archive.init_db() assert os.path.exists(tempdb_path) # is table in the database? import sqlite3 conn = sqlite3.connect(tempdb_path) cursor = conn.cursor() cursor.execute("""SELECT name FROM sqlite_master ORDER BY name;""") tables = cursor.fetchall() cursor.close() conn.close() assert len(tables) == 1 def verify_archive_item_shape(item): names = ("runid name passed skipped failed shortrepr " "longrepr fspath itemname").split() for name in names: assert name in item class TestWithFunctionIntegration: def getarchive(self, testdir, arg): py.test.importorskip("sqlite3") py.test.importorskip("simplejson") resultdb = testdir.tmpdir.join("resultdb") args = ["--resultdb=%s" % resultdb, "--resultdb_format=sqlite"] + [arg] testdir.runpytest(*args) assert resultdb.check(file=1) archive = SQLiteResultArchive(unicode(resultdb)) archive.init_db() return archive def test_collection_report(self, testdir): py.test.skip("Needs a rewrite for db version.") ok = testdir.makepyfile(test_collection_ok="") skip = testdir.makepyfile(test_collection_skip="import py ; py.test.skip('hello')") fail = testdir.makepyfile(test_collection_fail="XXX") lines = self.getresultdb(testdir, ok) assert not lines lines = self.getresultdb(testdir, skip) assert len(lines) == 2 assert lines[0].startswith("S ") assert lines[0].endswith("test_collection_skip.py") assert lines[1].startswith(" ") assert lines[1].endswith("test_collection_skip.py:1: Skipped: 'hello'") lines = self.getresultdb(testdir, fail) assert lines assert lines[0].startswith("F ") assert lines[0].endswith("test_collection_fail.py"), lines[0] for x in lines[1:]: assert x.startswith(" ") assert "XXX" in "".join(lines[1:]) def test_log_test_outcomes(self, testdir): mod = testdir.makepyfile(test_mod=""" import py def test_pass(): pass def test_skip(): py.test.skip("hello") def test_fail(): raise ValueError("val") """) archive = self.getarchive(testdir, mod) data = archive.get_all_data() for item in data: verify_archive_item_shape(item) assert len(data) == 3 assert len([item for item in data if item['passed'] == True]) == 1 assert len([item for item in data if item['skipped'] == True]) == 1 assert len([item for item in data if item['failed'] == True]) == 1 def test_internal_exception(self): py.test.skip("Needs a rewrite for db version.") # they are produced for example by a teardown failing # at the end of the run try: raise ValueError except ValueError: excinfo = py.code.ExceptionInfo() reslog = ResultDB(StringIO.StringIO()) reslog.pytest_internalerror(excinfo.getrepr) entry = reslog.logfile.getvalue() entry_lines = entry.splitlines() assert entry_lines[0].startswith('! ') assert os.path.basename(__file__)[:-1] in entry_lines[0] #.py/.pyc assert entry_lines[-1][0] == ' ' assert 'ValueError' in entry def test_generic(testdir): testdir.makepyfile(""" import py def test_pass(): pass def test_fail(): assert 0 def test_skip(): py.test.skip("") """) testdir.runpytest("--resultdb=result.sqlite") #testdir.tmpdir.join("result.sqlite")