diff --git a/MANIFEST.in b/MANIFEST.in index 84715a5fa..42c013e59 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,16 +1,14 @@ include CHANGELOG include README.txt include setup.py +include distribute_setup.py include LICENSE -include py/LICENSE -include py/path/testing/repotest.dump -include py/rest/rest.sty.template graft doc graft contrib graft example -graft py/bin -graft py/rest/testing/data +graft bin exclude *.orig exclude *.rej +exclude .hginore prune .svn prune .hg diff --git a/bin-for-dist/test_install.py b/bin-for-dist/test_install.py index 14ada3c48..d30c90079 100644 --- a/bin-for-dist/test_install.py +++ b/bin-for-dist/test_install.py @@ -1,6 +1,7 @@ import py import subprocess import os +import execnet # diff --git a/distribute_setup.py b/distribute_setup.py new file mode 100644 index 000000000..728533560 --- /dev/null +++ b/distribute_setup.py @@ -0,0 +1,456 @@ +#!python +"""Bootstrap distribute installation + +If you want to use setuptools in your package's setup.py, just include this +file in the same directory with it, and add this to the top of your setup.py:: + + from distribute_setup import use_setuptools + use_setuptools() + +If you want to require a specific version of setuptools, set a download +mirror, or use an alternate download directory, you can do so by supplying +the appropriate options to ``use_setuptools()``. + +This file can also be run as a script to install or upgrade setuptools. +""" +import os +import sys +import time +import fnmatch +import tempfile +import tarfile +from distutils import log + +try: + from site import USER_SITE +except ImportError: + USER_SITE = None + +try: + import subprocess + + def _python_cmd(*args): + args = (sys.executable,) + args + return subprocess.call(args) == 0 + +except ImportError: + # will be used for python 2.3 + def _python_cmd(*args): + args = (sys.executable,) + args + # quoting arguments if windows + if sys.platform == 'win32': + def quote(arg): + if ' ' in arg: + return '"%s"' % arg + return arg + args = [quote(arg) for arg in args] + return os.spawnl(os.P_WAIT, sys.executable, *args) == 0 + +DEFAULT_VERSION = "0.6.6" +DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/" +SETUPTOOLS_PKG_INFO = """\ +Metadata-Version: 1.0 +Name: setuptools +Version: 0.6c9 +Summary: xxxx +Home-page: xxx +Author: xxx +Author-email: xxx +License: xxx +Description: xxx +""" + + +def _install(tarball): + # extracting the tarball + tmpdir = tempfile.mkdtemp() + log.warn('Extracting in %s', tmpdir) + old_wd = os.getcwd() + try: + os.chdir(tmpdir) + tar = tarfile.open(tarball) + _extractall(tar) + tar.close() + + # going in the directory + subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) + os.chdir(subdir) + log.warn('Now working in %s', subdir) + + # installing + log.warn('Installing Distribute') + assert _python_cmd('setup.py', 'install') + finally: + os.chdir(old_wd) + + +def _build_egg(tarball, to_dir): + # extracting the tarball + tmpdir = tempfile.mkdtemp() + log.warn('Extracting in %s', tmpdir) + old_wd = os.getcwd() + try: + os.chdir(tmpdir) + tar = tarfile.open(tarball) + _extractall(tar) + tar.close() + + # going in the directory + subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) + os.chdir(subdir) + log.warn('Now working in %s', subdir) + + # building an egg + log.warn('Building a Distribute egg in %s', to_dir) + _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) + + # returning the result + for file in os.listdir(to_dir): + if fnmatch.fnmatch(file, 'distribute-%s*.egg' % DEFAULT_VERSION): + return os.path.join(to_dir, file) + + raise IOError('Could not build the egg.') + finally: + os.chdir(old_wd) + + +def _do_download(version, download_base, to_dir, download_delay): + tarball = download_setuptools(version, download_base, + to_dir, download_delay) + egg = _build_egg(tarball, to_dir) + sys.path.insert(0, egg) + import setuptools + setuptools.bootstrap_install_from = egg + + +def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=os.curdir, download_delay=15, no_fake=False): + # making sure we use the absolute path + to_dir = os.path.abspath(to_dir) + was_imported = 'pkg_resources' in sys.modules or \ + 'setuptools' in sys.modules + try: + try: + import pkg_resources + if not hasattr(pkg_resources, '_distribute'): + if not no_fake: + fake_setuptools() + raise ImportError + except ImportError: + return _do_download(version, download_base, to_dir, download_delay) + try: + pkg_resources.require("distribute>="+version) + return + except pkg_resources.VersionConflict: + e = sys.exc_info()[1] + if was_imported: + sys.stderr.write( + "The required version of distribute (>=%s) is not available,\n" + "and can't be installed while this script is running. Please\n" + "install a more recent version first, using\n" + "'easy_install -U distribute'." + "\n\n(Currently using %r)\n" % (version, e.args[0])) + sys.exit(2) + else: + del pkg_resources, sys.modules['pkg_resources'] # reload ok + return _do_download(version, download_base, to_dir, + download_delay) + except pkg_resources.DistributionNotFound: + return _do_download(version, download_base, to_dir, + download_delay) + finally: + _create_fake_setuptools_pkg_info(to_dir) + +def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, + to_dir=os.curdir, delay=15): + """Download distribute from a specified location and return its filename + + `version` should be a valid distribute version number that is available + as an egg for download under the `download_base` URL (which should end + with a '/'). `to_dir` is the directory where the egg will be downloaded. + `delay` is the number of seconds to pause before an actual download + attempt. + """ + # making sure we use the absolute path + to_dir = os.path.abspath(to_dir) + try: + from urllib.request import urlopen + except ImportError: + from urllib2 import urlopen + tgz_name = "distribute-%s.tar.gz" % version + url = download_base + tgz_name + saveto = os.path.join(to_dir, tgz_name) + src = dst = None + if not os.path.exists(saveto): # Avoid repeated downloads + try: + log.warn("Downloading %s", url) + src = urlopen(url) + # Read/write all in one block, so we don't create a corrupt file + # if the download is interrupted. + data = src.read() + dst = open(saveto, "wb") + dst.write(data) + finally: + if src: + src.close() + if dst: + dst.close() + return os.path.realpath(saveto) + + +def _patch_file(path, content): + """Will backup the file then patch it""" + existing_content = open(path).read() + if existing_content == content: + # already patched + log.warn('Already patched.') + return False + log.warn('Patching...') + _rename_path(path) + f = open(path, 'w') + try: + f.write(content) + finally: + f.close() + return True + + +def _same_content(path, content): + return open(path).read() == content + + +def _rename_path(path): + new_name = path + '.OLD.%s' % time.time() + log.warn('Renaming %s into %s', path, new_name) + try: + from setuptools.sandbox import DirectorySandbox + def _violation(*args): + pass + DirectorySandbox._violation = _violation + except ImportError: + pass + + os.rename(path, new_name) + return new_name + + +def _remove_flat_installation(placeholder): + if not os.path.isdir(placeholder): + log.warn('Unkown installation at %s', placeholder) + return False + found = False + for file in os.listdir(placeholder): + if fnmatch.fnmatch(file, 'setuptools*.egg-info'): + found = True + break + if not found: + log.warn('Could not locate setuptools*.egg-info') + return + + log.warn('Removing elements out of the way...') + pkg_info = os.path.join(placeholder, file) + if os.path.isdir(pkg_info): + patched = _patch_egg_dir(pkg_info) + else: + patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO) + + if not patched: + log.warn('%s already patched.', pkg_info) + return False + # now let's move the files out of the way + for element in ('setuptools', 'pkg_resources.py', 'site.py'): + element = os.path.join(placeholder, element) + if os.path.exists(element): + _rename_path(element) + else: + log.warn('Could not find the %s element of the ' + 'Setuptools distribution', element) + return True + + +def _after_install(dist): + log.warn('After install bootstrap.') + placeholder = dist.get_command_obj('install').install_purelib + _create_fake_setuptools_pkg_info(placeholder) + +def _create_fake_setuptools_pkg_info(placeholder): + if not placeholder or not os.path.exists(placeholder): + log.warn('Could not find the install location') + return + pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1]) + setuptools_file = 'setuptools-0.6c9-py%s.egg-info' % pyver + pkg_info = os.path.join(placeholder, setuptools_file) + if os.path.exists(pkg_info): + log.warn('%s already exists', pkg_info) + return + log.warn('Creating %s', pkg_info) + f = open(pkg_info, 'w') + try: + f.write(SETUPTOOLS_PKG_INFO) + finally: + f.close() + pth_file = os.path.join(placeholder, 'setuptools.pth') + log.warn('Creating %s', pth_file) + f = open(pth_file, 'w') + try: + f.write(os.path.join(os.curdir, setuptools_file)) + finally: + f.close() + + +def _patch_egg_dir(path): + # let's check if it's already patched + pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') + if os.path.exists(pkg_info): + if _same_content(pkg_info, SETUPTOOLS_PKG_INFO): + log.warn('%s already patched.', pkg_info) + return False + _rename_path(path) + os.mkdir(path) + os.mkdir(os.path.join(path, 'EGG-INFO')) + pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') + f = open(pkg_info, 'w') + try: + f.write(SETUPTOOLS_PKG_INFO) + finally: + f.close() + return True + + +def _before_install(): + log.warn('Before install bootstrap.') + fake_setuptools() + + +def _under_prefix(location): + if 'install' not in sys.argv: + return True + args = sys.argv[sys.argv.index('install')+1:] + for index, arg in enumerate(args): + for option in ('--root', '--prefix'): + if arg.startswith('%s=' % option): + top_dir = arg.split('root=')[-1] + return location.startswith(top_dir) + elif arg == option: + if len(args) > index: + top_dir = args[index+1] + return location.startswith(top_dir) + elif option == '--user' and USER_SITE is not None: + return location.startswith(USER_SITE) + return True + + +def fake_setuptools(): + log.warn('Scanning installed packages') + try: + import pkg_resources + except ImportError: + # we're cool + log.warn('Setuptools or Distribute does not seem to be installed.') + return + ws = pkg_resources.working_set + try: + setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools', + replacement=False)) + except TypeError: + # old distribute API + setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools')) + + if setuptools_dist is None: + log.warn('No setuptools distribution found') + return + # detecting if it was already faked + setuptools_location = setuptools_dist.location + log.warn('Setuptools installation detected at %s', setuptools_location) + + # if --root or --preix was provided, and if + # setuptools is not located in them, we don't patch it + if not _under_prefix(setuptools_location): + log.warn('Not patching, --root or --prefix is installing Distribute' + ' in another location') + return + + # let's see if its an egg + if not setuptools_location.endswith('.egg'): + log.warn('Non-egg installation') + res = _remove_flat_installation(setuptools_location) + if not res: + return + else: + log.warn('Egg installation') + pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO') + if (os.path.exists(pkg_info) and + _same_content(pkg_info, SETUPTOOLS_PKG_INFO)): + log.warn('Already patched.') + return + log.warn('Patching...') + # let's create a fake egg replacing setuptools one + res = _patch_egg_dir(setuptools_location) + if not res: + return + log.warn('Patched done.') + _relaunch() + + +def _relaunch(): + log.warn('Relaunching...') + # we have to relaunch the process + args = [sys.executable] + sys.argv + sys.exit(subprocess.call(args)) + + +def _extractall(self, path=".", members=None): + """Extract all members from the archive to the current working + directory and set owner, modification time and permissions on + directories afterwards. `path' specifies a different directory + to extract to. `members' is optional and must be a subset of the + list returned by getmembers(). + """ + import copy + import operator + from tarfile import ExtractError + directories = [] + + if members is None: + members = self + + for tarinfo in members: + if tarinfo.isdir(): + # Extract directories with a safe mode. + directories.append(tarinfo) + tarinfo = copy.copy(tarinfo) + tarinfo.mode = 448 # decimal for oct 0700 + self.extract(tarinfo, path) + + # Reverse sort directories. + if sys.version_info < (2, 4): + def sorter(dir1, dir2): + return cmp(dir1.name, dir2.name) + directories.sort(sorter) + directories.reverse() + else: + directories.sort(key=operator.attrgetter('name'), reverse=True) + + # Set correct owner, mtime and filemode on directories. + for tarinfo in directories: + dirpath = os.path.join(path, tarinfo.name) + try: + self.chown(tarinfo, dirpath) + self.utime(tarinfo, dirpath) + self.chmod(tarinfo, dirpath) + except ExtractError: + e = sys.exc_info()[1] + if self.errorlevel > 1: + raise + else: + self._dbg(1, "tarfile: %s" % e) + + +def main(argv, version=DEFAULT_VERSION): + """Install or upgrade setuptools and EasyInstall""" + tarball = download_setuptools() + _install(tarball) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/example/execnet/popen_read_multiple.py b/example/execnet/popen_read_multiple.py deleted file mode 100644 index 34ffbb65d..000000000 --- a/example/execnet/popen_read_multiple.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -example - -reading results from possibly blocking code running in sub processes. -""" -import py - -NUM_PROCESSES = 5 - -channels = [] -for i in range(NUM_PROCESSES): - gw = execnet.PopenGateway() # or use SSH or socket gateways - channel = gw.remote_exec(""" - import time - secs = channel.receive() - time.sleep(secs) - channel.send("waited %d secs" % secs) - """) - channels.append(channel) - print "*** instantiated subprocess", gw - -mc = execnet.MultiChannel(channels) -queue = mc.make_receive_queue() - -print "***", "verifying that timeout on receiving results from blocked subprocesses works" -try: - queue.get(timeout=1.0) -except Exception: - pass - -print "*** sending subprocesses some data to have them unblock" -mc.send_each(1) - -print "*** receiving results asynchronously" -for i in range(NUM_PROCESSES): - channel, result = queue.get(timeout=2.0) - print "result", channel.gateway, result diff --git a/example/execnet/redirect_remote_output.py b/example/execnet/redirect_remote_output.py deleted file mode 100644 index e36d41462..000000000 --- a/example/execnet/redirect_remote_output.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -redirect output from remote to a local function -showcasing features of the channel object: - -- sending a channel over a channel -- adapting a channel to a file object -- setting a callback for receiving channel data - -""" - -import py - -gw = execnet.PopenGateway() - -outchan = gw.remote_exec(""" - import sys - outchan = channel.gateway.newchannel() - sys.stdout = outchan.makefile("w") - channel.send(outchan) -""").receive() - -# note: callbacks execute in receiver thread! -def write(data): - print "received:", repr(data) -outchan.setcallback(write) - -gw.remote_exec(""" - print 'hello world' - print 'remote execution ends' -""").waitclose() - diff --git a/example/execnet/svn-sync-repo.py b/example/execnet/svn-sync-repo.py deleted file mode 100644 index 5e677a5f6..000000000 --- a/example/execnet/svn-sync-repo.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python - -import py -import sys, os - -def usage(): - arg0 = sys.argv[0] - print """%s [user@]remote-host:/repo/location localrepo [identity keyfile]""" % (arg0,) - - -def main(args): - remote = args[0] - localrepo = py.path.local(args[1]) - if not localrepo.check(dir=1): - raise SystemExit("localrepo %s does not exist" %(localrepo,)) - if len(args) == 3: - keyfile = py.path.local(args[2]) - else: - keyfile = None - remote_host, path = remote.split(':', 1) - print "ssh-connecting to", remote_host - gw = getgateway(remote_host, keyfile) - - local_rev = get_svn_youngest(localrepo) - - # local protocol - # 1. client sends rev/repo -> server - # 2. server checks for newer revisions and sends dumps - # 3. client receives dumps, updates local repo - # 4. client goes back to step 1 - c = gw.remote_exec(""" - import py - import os - remote_rev, repopath = channel.receive() - while 1: - rev = py.process.cmdexec('svnlook youngest "%s"' % repopath) - rev = int(rev) - if rev > remote_rev: - revrange = (remote_rev+1, rev) - dumpchannel = channel.gateway.newchannel() - channel.send(revrange) - channel.send(dumpchannel) - - f = os.popen( - "svnadmin dump -q --incremental -r %s:%s %s" - % (revrange[0], revrange[1], repopath), 'r') - try: - while 1: - s = f.read(8192) - if not s: - raise EOFError - dumpchannel.send(s) - except EOFError: - dumpchannel.close() - remote_rev = rev - else: - # using svn-hook instead would be nice here - py.std.time.sleep(30) - """) - - c.send((local_rev, path)) - print "checking revisions from %d in %s" %(local_rev, remote) - while 1: - revstart, revend = c.receive() - dumpchannel = c.receive() - - print "receiving revisions", revstart, "-", revend, "replaying..." - svn_load(localrepo, dumpchannel) - print "current revision", revend - -def svn_load(repo, dumpchannel): - f = os.popen("svnadmin load -q %s" %(repo, ), "w") - for x in dumpchannel: - sys.stdout.write(".") - sys.stdout.flush() - f.write(x) - print >>sys.stdout - f.close() - -def get_svn_youngest(repo): - rev = py.process.cmdexec('svnlook youngest "%s"' % repo) - return int(rev) - -def getgateway(host, keyfile=None): - return execnet.SshGateway(host, identity=keyfile) - -if __name__ == '__main__': - if len(sys.argv) < 3: - usage() - raise SystemExit(1) - - main(sys.argv[1:]) - diff --git a/example/execnet/sysinfo.py b/example/execnet/sysinfo.py deleted file mode 100644 index a4c44dd98..000000000 --- a/example/execnet/sysinfo.py +++ /dev/null @@ -1,139 +0,0 @@ -""" -sysinfo.py [host1] [host2] [options] - -obtain system info from remote machine. -""" - -import py -import sys - - -parser = py.std.optparse.OptionParser(usage=__doc__) -parser.add_option("-f", "--sshconfig", action="store", dest="ssh_config", default=None, - help="use given ssh config file, and add info all contained hosts for getting info") -parser.add_option("-i", "--ignore", action="store", dest="ignores", default=None, - help="ignore hosts (useful if the list of hostnames come from a file list)") - -def parsehosts(path): - path = py.path.local(path) - l = [] - rex = py.std.re.compile(r'Host\s*(\S+)') - for line in path.readlines(): - m = rex.match(line) - if m is not None: - sshname, = m.groups() - l.append(sshname) - return l - -class RemoteInfo: - def __init__(self, gateway): - self.gw = gateway - self._cache = {} - - def exreceive(self, execstring): - if execstring not in self._cache: - channel = self.gw.remote_exec(execstring) - self._cache[execstring] = channel.receive() - return self._cache[execstring] - - def getmodattr(self, modpath): - module = modpath.split(".")[0] - return self.exreceive(""" - import %s - channel.send(%s) - """ %(module, modpath)) - - def islinux(self): - return self.getmodattr('sys.platform').find("linux") != -1 - - def getfqdn(self): - return self.exreceive(""" - import socket - channel.send(socket.getfqdn()) - """) - - def getmemswap(self): - if self.islinux(): - return self.exreceive(""" - import commands, re - out = commands.getoutput("free") - mem = re.search(r"Mem:\s+(\S*)", out).group(1) - swap = re.search(r"Swap:\s+(\S*)", out).group(1) - channel.send((mem, swap)) - """) - - def getcpuinfo(self): - if self.islinux(): - return self.exreceive(""" - # a hyperthreaded cpu core only counts as 1, although it - # is present as 2 in /proc/cpuinfo. Counting it as 2 is - # misleading because it is *by far* not as efficient as - # two independent cores. - cpus = {} - cpuinfo = {} - f = open("/proc/cpuinfo") - lines = f.readlines() - f.close() - for line in lines + ['']: - if line.strip(): - key, value = line.split(":", 1) - cpuinfo[key.strip()] = value.strip() - else: - corekey = (cpuinfo.get("physical id"), - cpuinfo.get("core id")) - cpus[corekey] = 1 - numcpus = len(cpus) - model = cpuinfo.get("model name") - channel.send((numcpus, model)) - """) - -def debug(*args): - print >>sys.stderr, " ".join(map(str, args)) -def error(*args): - debug("ERROR", args[0] + ":", *args[1:]) - -def getinfo(sshname, ssh_config=None, loginfo=sys.stdout): - debug("connecting to", sshname) - try: - gw = execnet.SshGateway(sshname, ssh_config=ssh_config) - except IOError: - error("could not get sshagteway", sshname) - else: - ri = RemoteInfo(gw) - #print "%s info:" % sshname - prefix = sshname.upper() + " " - print >>loginfo, prefix, "fqdn:", ri.getfqdn() - for attr in ( - "sys.platform", - "sys.version_info", - ): - loginfo.write("%s %s: " %(prefix, attr,)) - loginfo.flush() - value = ri.getmodattr(attr) - loginfo.write(str(value)) - loginfo.write("\n") - loginfo.flush() - memswap = ri.getmemswap() - if memswap: - mem,swap = memswap - print >>loginfo, prefix, "Memory:", mem, "Swap:", swap - cpuinfo = ri.getcpuinfo() - if cpuinfo: - numcpu, model = cpuinfo - print >>loginfo, prefix, "number of cpus:", numcpu - print >>loginfo, prefix, "cpu model", model - return ri - -if __name__ == '__main__': - options, args = parser.parse_args() - hosts = list(args) - ssh_config = options.ssh_config - if ssh_config: - hosts.extend(parsehosts(ssh_config)) - ignores = options.ignores or () - if ignores: - ignores = ignores.split(",") - for host in hosts: - if host not in ignores: - getinfo(host, ssh_config=ssh_config) - diff --git a/setup.py b/setup.py index 57c3c2150..8961bfafa 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,19 @@ """py lib / py.test setup.py file""" import os, sys +if sys.version_info >= (3,0): + from distribute_setup import use_setuptools + use_setuptools() from setuptools import setup -long_description = """ +long_description = """ py.test and pylib: rapid testing and development utils - `py.test`_: cross-project testing tool with many advanced features - `py.path`_: path abstractions over local and subversion files - `py.code`_: dynamic code compile and traceback printing support -Compatibility: Linux, Win32, OSX, Python versions 2.4 through to 3.1. +Platforms: Linux, Win32, OSX +Interpreters: Python versions 2.4 through to 3.1, Jython 2.5.1. For questions please check out http://pylib.org/contact.html .. _`py.test`: http://pylib.org/test.html @@ -17,7 +21,6 @@ For questions please check out http://pylib.org/contact.html .. _`py.code`: http://pylib.org/code.html (c) Holger Krekel and others, 2009 - """ trunk = None def main(): @@ -58,30 +61,12 @@ def main(): '_py.io', '_py.log', '_py.path', - '_py.path.gateway', '_py.process', '_py.test', '_py.test.dist', '_py.test.looponfail', '_py.test.plugin',], - package_data={'py': ['bin/_findpy.py', - 'bin/env.cmd', - 'bin/env.py', - 'bin/py.cleanup', - 'bin/py.convert_unittest', - 'bin/py.countloc', - 'bin/py.lookup', - 'bin/py.svnwcrevert', - 'bin/py.test', - 'bin/py.which', - 'bin/win32/py.cleanup.cmd', - 'bin/win32/py.convert_unittest.cmd', - 'bin/win32/py.countloc.cmd', - 'bin/win32/py.lookup.cmd', - 'bin/win32/py.svnwcrevert.cmd', - 'bin/win32/py.test.cmd', - 'bin/win32/py.which.cmd',], - zip_safe=True, + zip_safe=False, ) if __name__ == '__main__':