From 352e3054314d18b63347039fc9435d36a197b017 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 10 Jan 2010 23:52:23 +0100 Subject: [PATCH] fix and test bug: dist-testing now works again without execnet/pylib installed remotely. fixes issue65. --HG-- branch : trunk --- CHANGELOG | 3 ++ ISSUES.txt | 9 ----- bin-for-dist/test_install.py | 49 ++++++++++++++++++++++++++++ py/impl/test/config.py | 4 +-- py/impl/test/dist/dsession.py | 6 ---- py/impl/test/dist/gwmanage.py | 17 +++++----- py/impl/test/dist/mypickle.py | 8 ++--- py/impl/test/dist/txnode.py | 2 ++ testing/pytest/dist/test_mypickle.py | 14 -------- 9 files changed, 67 insertions(+), 45 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0e875dccf..124c78413 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -62,6 +62,9 @@ Changes between 1.X and 1.1.1 - fix assert reinterpreation that sees a call containing "keyword=..." +- fix issue65: properly handle dist-testing if no + execnet/py lib installed remotely. + - skip some install-tests if no execnet is available - fix docs, fix internal bin/ script generation diff --git a/ISSUES.txt b/ISSUES.txt index a0e826384..6a81bf05e 100644 --- a/ISSUES.txt +++ b/ISSUES.txt @@ -42,15 +42,6 @@ test: testing/pytest/dist/test_dsession.py - test_terminate_on_hanging_node Call gateway group termination with a small timeout if available. Should make dist-testing less likely to leave lost processes. -fix dist-testing: execnet needs to be rsynced over automatically ------------------------------------------------------------------- - -tags: bug 1.2 -bb: http://bitbucket.org/hpk42/py-trunk/issue/65/ - -execnet is not rsynced so fails if run in an ssh-situation. -write test and fix. - dist-testing: fix session hook / setup calling ----------------------------------------------------- tags: bug 1.2 diff --git a/bin-for-dist/test_install.py b/bin-for-dist/test_install.py index 7cd13af76..5dcd16b52 100644 --- a/bin-for-dist/test_install.py +++ b/bin-for-dist/test_install.py @@ -174,3 +174,52 @@ def test_cmdline_entrypoints(monkeypatch): assert expected in points for script in unversioned_scripts: assert script in points + +def test_slave_popen_needs_no_pylib(testdir, venv): + venv.ensure() + #xxx execnet optimizes popen + #ch = venv.makegateway().remote_exec("import execnet") + #py.test.raises(ch.RemoteError, ch.waitclose) + python = venv._cmd("python") + p = testdir.makepyfile(""" + import py + def test_func(): + pass + """) + result = testdir.runpytest(p, '--rsyncdir=%s' % str(p), + '--dist=each', '--tx=popen//python=%s' % python) + result.stdout.fnmatch_lines([ + "*1 passed*" + ]) + +def test_slave_needs_no_execnet(testdir, specssh): + gw = execnet.makegateway(specssh) + ch = gw.remote_exec(""" + import os, subprocess + subprocess.call(["virtualenv", "--no-site-packages", "subdir"]) + channel.send(os.path.join(os.path.abspath("subdir"), 'bin', 'python')) + channel.send(os.path.join(os.path.abspath("subdir"))) + """) + try: + path = ch.receive() + chdir = ch.receive() + except ch.RemoteError: + e = sys.exc_info()[1] + py.test.skip("could not prepare ssh slave:%s" % str(e)) + gw.exit() + newspec = "%s//python=%s//chdir=%s" % (specssh, path, chdir) + gw = execnet.makegateway(newspec) + ch = gw.remote_exec("import execnet") + py.test.raises(ch.RemoteError, ch.waitclose) + gw.exit() + + p = testdir.makepyfile(""" + import py + def test_func(): + pass + """) + result = testdir.runpytest(p, '--rsyncdir=%s' % str(p), + '--dist=each', '--tx=%s' % newspec) + result.stdout.fnmatch_lines([ + "*1 passed*" + ]) diff --git a/py/impl/test/config.py b/py/impl/test/config.py index ef70a1550..aa924fe97 100644 --- a/py/impl/test/config.py +++ b/py/impl/test/config.py @@ -166,8 +166,8 @@ class Config(object): raise self.Error("path %r is not relative to %r" % (str(path), str(topdir))) # assumtion: pytest's fs-collector tree follows the filesystem tree - names = filter(None, path.relto(topdir).split(path.sep)) - names.extend(parts) + names = list(filter(None, path.relto(topdir).split(path.sep))) + names += parts try: return self._rootcol.getbynames(names) except ValueError: diff --git a/py/impl/test/dist/dsession.py b/py/impl/test/dist/dsession.py index 7f6b01d3c..1ce0b52e4 100644 --- a/py/impl/test/dist/dsession.py +++ b/py/impl/test/dist/dsession.py @@ -1,9 +1,3 @@ -""" - - EXPERIMENTAL dsession session (for dist/non-dist unification) - -""" - import py from py.impl.test.session import Session from py.impl.test import outcome diff --git a/py/impl/test/dist/gwmanage.py b/py/impl/test/dist/gwmanage.py index 343a3de9b..b7822d08e 100644 --- a/py/impl/test/dist/gwmanage.py +++ b/py/impl/test/dist/gwmanage.py @@ -35,14 +35,15 @@ class GatewayManager: gateways = [] for gateway in self.group: spec = gateway.spec - if not spec._samefilesystem(): - if spec not in seen: - def finished(): - if notify: - notify("rsyncrootready", spec, source) - rsync.add_target_host(gateway, finished=finished) - seen.add(spec) - gateways.append(gateway) + if spec.popen and not spec.chdir and not spec.python: + continue + if spec not in seen: + def finished(): + if notify: + notify("rsyncrootready", spec, source) + rsync.add_target_host(gateway, finished=finished) + seen.add(spec) + gateways.append(gateway) if seen: self.hook.pytest_gwmanage_rsyncstart( source=source, diff --git a/py/impl/test/dist/mypickle.py b/py/impl/test/dist/mypickle.py index fb2bdbd9f..c87005e1b 100644 --- a/py/impl/test/dist/mypickle.py +++ b/py/impl/test/dist/mypickle.py @@ -138,12 +138,8 @@ class PickleChannel(object): self.RemoteError = channel.RemoteError def send(self, obj): - from execnet.gateway_base import Channel - if not isinstance(obj, Channel): - pickled_obj = self._ipickle.dumps(obj) - self._channel.send(pickled_obj) - else: - self._channel.send(obj) + pickled_obj = self._ipickle.dumps(obj) + self._channel.send(pickled_obj) def receive(self): pickled_obj = self._channel.receive() diff --git a/py/impl/test/dist/txnode.py b/py/impl/test/dist/txnode.py index 429f8dc5c..ea76363bd 100644 --- a/py/impl/test/dist/txnode.py +++ b/py/impl/test/dist/txnode.py @@ -86,10 +86,12 @@ def install_slave(gateway, config): sys.path.insert(0, os.getcwd()) from py.impl.test.dist.mypickle import PickleChannel from py.impl.test.dist.txnode import SlaveNode + channel.send("basicimport") channel = PickleChannel(channel) slavenode = SlaveNode(channel) slavenode.run() """) + channel.receive() channel = PickleChannel(channel) basetemp = None if gateway.spec.popen: diff --git a/testing/pytest/dist/test_mypickle.py b/testing/pytest/dist/test_mypickle.py index cff9b7087..c5ca55575 100644 --- a/testing/pytest/dist/test_mypickle.py +++ b/testing/pytest/dist/test_mypickle.py @@ -239,20 +239,6 @@ class TestPickleChannelFunctional: error = channel._getremoteerror() assert isinstance(error, UnpickleError) - def test_popen_with_newchannel(self): - channel = self.gw.remote_exec(""" - from py.impl.test.dist.mypickle import PickleChannel - channel = PickleChannel(channel) - newchannel = channel.receive() - newchannel.send(42) - """) - channel = PickleChannel(channel) - newchannel = self.gw.newchannel() - channel.send(newchannel) - channel.waitclose() - res = newchannel.receive() - assert res == 42 - def test_popen_with_various_methods(self): channel = self.gw.remote_exec(""" from py.impl.test.dist.mypickle import PickleChannel