argcomplete: FastFileCompleter that doesn't call bash in subprocess, strip prefix dir

```
timeit result for 10000 iterations of expanding '/d' (lowered the count in the code afterwards)
#                      2.7.5     3.3.2
# FilesCompleter       75.1109   69.2116
# FastFilesCompleter    0.7383    1.0760
```
- does not display prefix dir (like bash, not like compgen), py.test /usr/<TAB> does not show /usr/bin/ but bin/

_pytest/_argcomplete.py

@@ -22,7 +22,19 @@ doing the add_argument calls as they need to be specified as .completer
 attributes as well. (If argcomplete is not installed, the function the
 attribute points to will not be used).
 
----
+SPEEDUP
+=======
+The generic argcomplete script for bash-completion
+(/etc/bash_completion.d/python-argcomplete.sh )
+uses a python program to determine startup script generated by pip.
+You can speed up completion somewhat by changing this script to include
+  # PYTHON_ARGCOMPLETE_OK
+so the the python-argcomplete-check-easy-install-script does not
+need to be called to find the entry point of the code and see if that is
+marked  with PYTHON_ARGCOMPLETE_OK
+
+INSTALL/DEBUGGING
+=================
 To include this support in another application that has setup.py generated
 scripts:
 - add the line:
@@ -44,11 +56,32 @@ If things do not work right away:
     _ARGCOMPLETE=1 _ARC_DEBUG=1 appname
   which should throw a KeyError: 'COMPLINE' (which is properly set by the
   global argcomplete script).
-   
 """
 
 import sys
 import os
+from glob import glob
+
+class FastFilesCompleter:
+    'Fast file completer class'
+    def __init__(self, directories=True):
+        self.directories = directories
+
+    def __call__(self, prefix, **kwargs):
+        """only called on non option completions"""
+        if os.path.sep in prefix[1:]: #
+            prefix_dir = len(os.path.dirname(prefix) + os.path.sep)
+        else:
+            prefix_dir = 0
+        completion = []
+        if '*' not in prefix and '?' not in prefix:
+            prefix += '*'
+        for x in sorted(glob(prefix)):
+            if os.path.isdir(x):
+                x += '/'
+            # append stripping the prefix (like bash, not like compgen)
+            completion.append(x[prefix_dir:])
+        return completion
 
 if os.environ.get('_ARGCOMPLETE'):
     # argcomplete 0.5.6 is not compatible with python 2.5.6: print/with/format
@@ -58,7 +91,7 @@ if os.environ.get('_ARGCOMPLETE'):
         import argcomplete.completers
     except ImportError:
         sys.exit(-1)
-    filescompleter = argcomplete.completers.FilesCompleter()
+    filescompleter = FastFilesCompleter()
 
     def try_argcomplete(parser):
         argcomplete.autocomplete(parser)

bench/bench_argcomplete.py

@@ -0,0 +1,19 @@
+
+
+# 10000 iterations, just for relative comparison
+#                      2.7.5     3.3.2
+# FilesCompleter       75.1109   69.2116
+# FastFilesCompleter    0.7383    1.0760
+
+
+if __name__ == '__main__':
+    import sys
+    import timeit
+    from argcomplete.completers import FilesCompleter
+    from _pytest._argcomplete import FastFilesCompleter
+    count = 1000 # only a few seconds
+    setup = 'from __main__ import FastFilesCompleter\nfc = FastFilesCompleter()'
+    run = 'fc("/d")'
+    sys.stdout.write('%s\n' % (timeit.timeit(run,
+                                setup=setup.replace('Fast', ''), number=count)))
+    sys.stdout.write('%s\n' % (timeit.timeit(run, setup=setup, number=count)))

testing/test_argcomplete.py

@@ -0,0 +1,92 @@
+from __future__ import with_statement
+import py, pytest
+
+# test for _argcomplete but not specific for any application
+
+def equal_with_bash(prefix, ffc, fc, out=None):
+    res = ffc(prefix)
+    res_bash = set(fc(prefix))
+    retval = set(res) == res_bash
+    if out:
+        out.write('equal_with_bash %s %s\n' % (retval, res))
+        if not retval:
+            out.write(' python - bash: %s\n' % (set(res) - res_bash))
+            out.write(' bash - python: %s\n' % (res_bash - set(res)))
+    return retval
+
+# copied from argcomplete.completers as import from there
+# also pulls in argcomplete.__init__ which opens filedescriptor 9
+# this gives an IOError at the end of testrun
+def _wrapcall(*args, **kargs):
+    try:
+        if py.std.sys.version_info > (2,7):
+            return py.std.subprocess.check_output(*args,**kargs).decode().splitlines()
+        if 'stdout' in kargs:
+            raise ValueError('stdout argument not allowed, it will be overridden.')
+        process = py.std.subprocess.Popen(
+            stdout=py.std.subprocess.PIPE, *args, **kargs)
+        output, unused_err = process.communicate()
+        retcode = process.poll()
+        if retcode:
+            cmd = kargs.get("args")
+            if cmd is None:
+                cmd = args[0]
+            raise py.std.subprocess.CalledProcessError(retcode, cmd)
+        return output.decode().splitlines()
+    except py.std.subprocess.CalledProcessError:
+        return []
+
+class FilesCompleter(object):
+    'File completer class, optionally takes a list of allowed extensions'
+    def __init__(self,allowednames=(),directories=True):
+        # Fix if someone passes in a string instead of a list
+        if type(allowednames) is str:
+            allowednames = [allowednames]
+
+        self.allowednames = [x.lstrip('*').lstrip('.') for x in allowednames]
+        self.directories = directories
+
+    def __call__(self, prefix, **kwargs):
+        completion = []
+        if self.allowednames:
+            if self.directories:
+                files = _wrapcall(['bash','-c',
+                    "compgen -A directory -- '{p}'".format(p=prefix)])
+                completion += [ f + '/' for f in files]
+            for x in self.allowednames:
+                completion += _wrapcall(['bash', '-c',
+                    "compgen -A file -X '!*.{0}' -- '{p}'".format(x,p=prefix)])
+        else:
+            completion += _wrapcall(['bash', '-c',
+                "compgen -A file -- '{p}'".format(p=prefix)])
+
+            anticomp = _wrapcall(['bash', '-c',
+                "compgen -A directory -- '{p}'".format(p=prefix)])
+
+            completion = list( set(completion) - set(anticomp))
+
+            if self.directories:
+                completion += [f + '/' for f in anticomp]
+        return completion
+
+# the following barfs with a syntax error on py2.5
+# @pytest.mark.skipif("sys.version_info < (2,6)")
+class TestArgComplete:
+    @pytest.mark.skipif("sys.version_info < (2,6)")
+    def test_compare_with_compgen(self):
+        from _pytest._argcomplete import FastFilesCompleter
+        ffc = FastFilesCompleter()
+        fc = FilesCompleter()
+        for x in '/ /d /data qqq'.split():
+            assert equal_with_bash(x, ffc, fc, out=py.std.sys.stdout)
+
+    @pytest.mark.skipif("sys.version_info < (2,6)")
+    def test_remove_dir_prefix(self):
+        """this is not compatible with compgen but it is with bash itself:
+        ls /usr/<TAB>
+        """
+        from _pytest._argcomplete import FastFilesCompleter
+        ffc = FastFilesCompleter()
+        fc = FilesCompleter()
+        for x in '/usr/'.split():
+            assert not equal_with_bash(x, ffc, fc, out=py.std.sys.stdout)