Use atomicrewrites only on Windows
Fixes https://github.com/pytest-dev/pytest/issues/6147
This commit is contained in:
parent
ab101658f0
commit
45c4a8fb3d
|
@ -0,0 +1 @@
|
||||||
|
``python-atomicwrites`` is only used on Windows, fixing a performance regression with assertion rewriting on Unix.
|
2
setup.py
2
setup.py
|
@ -7,7 +7,7 @@ INSTALL_REQUIRES = [
|
||||||
"packaging",
|
"packaging",
|
||||||
"attrs>=17.4.0", # should match oldattrs tox env.
|
"attrs>=17.4.0", # should match oldattrs tox env.
|
||||||
"more-itertools>=4.0.0",
|
"more-itertools>=4.0.0",
|
||||||
"atomicwrites>=1.0",
|
'atomicwrites>=1.0;sys_platform=="win32"',
|
||||||
'pathlib2>=2.2.0;python_version<"3.6"',
|
'pathlib2>=2.2.0;python_version<"3.6"',
|
||||||
'colorama;sys_platform=="win32"',
|
'colorama;sys_platform=="win32"',
|
||||||
"pluggy>=0.12,<1.0",
|
"pluggy>=0.12,<1.0",
|
||||||
|
|
|
@ -20,8 +20,6 @@ from typing import Optional
|
||||||
from typing import Set
|
from typing import Set
|
||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
|
|
||||||
import atomicwrites
|
|
||||||
|
|
||||||
from _pytest._io.saferepr import saferepr
|
from _pytest._io.saferepr import saferepr
|
||||||
from _pytest._version import version
|
from _pytest._version import version
|
||||||
from _pytest.assertion import util
|
from _pytest.assertion import util
|
||||||
|
@ -255,12 +253,10 @@ class AssertionRewritingHook(importlib.abc.MetaPathFinder):
|
||||||
return f.read()
|
return f.read()
|
||||||
|
|
||||||
|
|
||||||
def _write_pyc(state, co, source_stat, pyc):
|
def _write_pyc_fp(fp, source_stat, co):
|
||||||
# Technically, we don't have to have the same pyc format as
|
# Technically, we don't have to have the same pyc format as
|
||||||
# (C)Python, since these "pycs" should never be seen by builtin
|
# (C)Python, since these "pycs" should never be seen by builtin
|
||||||
# import. However, there's little reason deviate.
|
# import. However, there's little reason deviate.
|
||||||
try:
|
|
||||||
with atomicwrites.atomic_write(fspath(pyc), mode="wb", overwrite=True) as fp:
|
|
||||||
fp.write(importlib.util.MAGIC_NUMBER)
|
fp.write(importlib.util.MAGIC_NUMBER)
|
||||||
# as of now, bytecode header expects 32-bit numbers for size and mtime (#4903)
|
# as of now, bytecode header expects 32-bit numbers for size and mtime (#4903)
|
||||||
mtime = int(source_stat.st_mtime) & 0xFFFFFFFF
|
mtime = int(source_stat.st_mtime) & 0xFFFFFFFF
|
||||||
|
@ -268,6 +264,15 @@ def _write_pyc(state, co, source_stat, pyc):
|
||||||
# "<LL" stands for 2 unsigned longs, little-ending
|
# "<LL" stands for 2 unsigned longs, little-ending
|
||||||
fp.write(struct.pack("<LL", mtime, size))
|
fp.write(struct.pack("<LL", mtime, size))
|
||||||
fp.write(marshal.dumps(co))
|
fp.write(marshal.dumps(co))
|
||||||
|
|
||||||
|
|
||||||
|
if sys.platform == "win32":
|
||||||
|
from atomicwrites import atomic_write
|
||||||
|
|
||||||
|
def _write_pyc(state, co, source_stat, pyc):
|
||||||
|
try:
|
||||||
|
with atomic_write(fspath(pyc), mode="wb", overwrite=True) as fp:
|
||||||
|
_write_pyc_fp(fp, source_stat, co)
|
||||||
except EnvironmentError as e:
|
except EnvironmentError as e:
|
||||||
state.trace("error writing pyc file at {}: errno={}".format(pyc, e.errno))
|
state.trace("error writing pyc file at {}: errno={}".format(pyc, e.errno))
|
||||||
# we ignore any failure to write the cache file
|
# we ignore any failure to write the cache file
|
||||||
|
@ -277,6 +282,32 @@ def _write_pyc(state, co, source_stat, pyc):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
def _write_pyc(state, co, source_stat, pyc):
|
||||||
|
proc_pyc = "{}.{}".format(pyc, os.getpid())
|
||||||
|
try:
|
||||||
|
fp = open(proc_pyc, "wb")
|
||||||
|
except EnvironmentError as e:
|
||||||
|
state.trace(
|
||||||
|
"error writing pyc file at {}: errno={}".format(proc_pyc, e.errno)
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
_write_pyc_fp(fp, source_stat, co)
|
||||||
|
os.rename(proc_pyc, fspath(pyc))
|
||||||
|
except BaseException as e:
|
||||||
|
state.trace("error writing pyc file at {}: errno={}".format(pyc, e.errno))
|
||||||
|
# we ignore any failure to write the cache file
|
||||||
|
# there are many reasons, permission-denied, pycache dir being a
|
||||||
|
# file etc.
|
||||||
|
return False
|
||||||
|
finally:
|
||||||
|
fp.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def _rewrite_test(fn, config):
|
def _rewrite_test(fn, config):
|
||||||
"""read and rewrite *fn* and return the code object."""
|
"""read and rewrite *fn* and return the code object."""
|
||||||
fn = fspath(fn)
|
fn = fspath(fn)
|
||||||
|
|
|
@ -959,14 +959,15 @@ class TestAssertionRewriteHookDetails:
|
||||||
def test_write_pyc(self, testdir, tmpdir, monkeypatch):
|
def test_write_pyc(self, testdir, tmpdir, monkeypatch):
|
||||||
from _pytest.assertion.rewrite import _write_pyc
|
from _pytest.assertion.rewrite import _write_pyc
|
||||||
from _pytest.assertion import AssertionState
|
from _pytest.assertion import AssertionState
|
||||||
import atomicwrites
|
|
||||||
from contextlib import contextmanager
|
|
||||||
|
|
||||||
config = testdir.parseconfig([])
|
config = testdir.parseconfig([])
|
||||||
state = AssertionState(config, "rewrite")
|
state = AssertionState(config, "rewrite")
|
||||||
source_path = tmpdir.ensure("source.py")
|
source_path = str(tmpdir.ensure("source.py"))
|
||||||
pycpath = tmpdir.join("pyc").strpath
|
pycpath = tmpdir.join("pyc").strpath
|
||||||
assert _write_pyc(state, [1], os.stat(source_path.strpath), pycpath)
|
assert _write_pyc(state, [1], os.stat(source_path), pycpath)
|
||||||
|
|
||||||
|
if sys.platform == "win32":
|
||||||
|
from contextlib import contextmanager
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def atomic_write_failed(fn, mode="r", overwrite=False):
|
def atomic_write_failed(fn, mode="r", overwrite=False):
|
||||||
|
@ -975,8 +976,17 @@ class TestAssertionRewriteHookDetails:
|
||||||
raise e
|
raise e
|
||||||
yield
|
yield
|
||||||
|
|
||||||
monkeypatch.setattr(atomicwrites, "atomic_write", atomic_write_failed)
|
monkeypatch.setattr(
|
||||||
assert not _write_pyc(state, [1], source_path.stat(), pycpath)
|
_pytest.assertion.rewrite, "atomic_write", atomic_write_failed
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
|
||||||
|
def raise_ioerror(*args):
|
||||||
|
raise IOError()
|
||||||
|
|
||||||
|
monkeypatch.setattr("os.rename", raise_ioerror)
|
||||||
|
|
||||||
|
assert not _write_pyc(state, [1], os.stat(source_path), pycpath)
|
||||||
|
|
||||||
def test_resources_provider_for_loader(self, testdir):
|
def test_resources_provider_for_loader(self, testdir):
|
||||||
"""
|
"""
|
||||||
|
|
Loading…
Reference in New Issue