consolidate py/log into fewer files, remove one old approach, sketch simplified API

--HG--
branch : trunk
This commit is contained in:
holger krekel 2009-08-22 09:42:12 +02:00
parent 2b8f489d60
commit 27c08ac235
9 changed files with 230 additions and 378 deletions

View File

@ -1,6 +1,11 @@
Changes between 1.0.x and 'trunk'
=====================================
* consolidate py.log implementation, remove old approach.
* introduce py.io.TextIO and py.io.BytesIO for distinguishing between
text/unicode and byte-streams (uses underlying standard lib io.*
if available)
* introduce py.io.TextIO and py.io.BytesIO for distinguishing between
text/unicode and byte-streams (uses underlying standard lib io.*
if available)

View File

@ -176,17 +176,15 @@ initpkg(__name__,
# logging API ('producers' and 'consumers' connected via keywords)
'log.__doc__' : ('./log/__init__.py', '__doc__'),
'log._apiwarn' : ('./log/warning.py', '_apiwarn'),
'log.Producer' : ('./log/producer.py', 'Producer'),
'log.default' : ('./log/producer.py', 'default'),
'log._getstate' : ('./log/producer.py', '_getstate'),
'log._setstate' : ('./log/producer.py', '_setstate'),
'log.setconsumer' : ('./log/consumer.py', 'setconsumer'),
'log.Path' : ('./log/consumer.py', 'Path'),
'log.STDOUT' : ('./log/consumer.py', 'STDOUT'),
'log.STDERR' : ('./log/consumer.py', 'STDERR'),
'log.Syslog' : ('./log/consumer.py', 'Syslog'),
'log.get' : ('./log/logger.py', 'get'),
'log._apiwarn' : ('./log/warning.py', '_apiwarn'),
'log.Producer' : ('./log/log.py', 'Producer'),
'log.setconsumer' : ('./log/log.py', 'setconsumer'),
'log._setstate' : ('./log/log.py', 'setstate'),
'log._getstate' : ('./log/log.py', 'getstate'),
'log.Path' : ('./log/log.py', 'Path'),
'log.STDOUT' : ('./log/log.py', 'STDOUT'),
'log.STDERR' : ('./log/log.py', 'STDERR'),
'log.Syslog' : ('./log/log.py', 'Syslog'),
# compatibility modules (taken from 2.4.4)
'compat.__doc__' : ('./compat/__init__.py', '__doc__'),

View File

@ -1,79 +0,0 @@
import py
import sys
class File(object):
""" log consumer wrapping a file(-like) object
"""
def __init__(self, f):
assert hasattr(f, 'write')
assert isinstance(f, file) or not hasattr(f, 'open')
self._file = f
def __call__(self, msg):
""" write a message to the log """
print >>self._file, str(msg)
class Path(object):
""" log consumer able to write log messages into
"""
def __init__(self, filename, append=False, delayed_create=False,
buffering=1):
self._append = append
self._filename = filename
self._buffering = buffering
if not delayed_create:
self._openfile()
def _openfile(self):
mode = self._append and 'a' or 'w'
f = open(str(self._filename), mode, buffering=self._buffering)
self._file = f
def __call__(self, msg):
""" write a message to the log """
if not hasattr(self, "_file"):
self._openfile()
print >> self._file, msg
def STDOUT(msg):
""" consumer that writes to sys.stdout """
print >>sys.stdout, str(msg)
def STDERR(msg):
""" consumer that writes to sys.stderr """
print >>sys.stderr, str(msg)
class Syslog:
""" consumer that writes to the syslog daemon """
for priority in "LOG_EMERG LOG_ALERT LOG_CRIT LOG_ERR LOG_WARNING LOG_NOTICE LOG_INFO LOG_DEBUG".split():
try:
exec("%s = py.std.syslog.%s" % (priority, priority))
except AttributeError:
pass
def __init__(self, priority = None):
self.priority = self.LOG_INFO
if priority is not None:
self.priority = priority
def __call__(self, msg):
""" write a message to the log """
py.std.syslog.syslog(self.priority, str(msg))
def setconsumer(keywords, consumer):
""" create a consumer for a set of keywords """
# normalize to tuples
if isinstance(keywords, str):
keywords = tuple(map(None, keywords.split()))
elif hasattr(keywords, 'keywords'):
keywords = keywords.keywords
elif not isinstance(keywords, tuple):
raise TypeError("key %r is not a string or tuple" % (keywords,))
if consumer is not None and not callable(consumer):
if not hasattr(consumer, 'write'):
raise TypeError("%r should be None, callable or file-like" % (consumer,))
consumer = File(consumer)
py.log.Producer(keywords).set_consumer(consumer)

183
py/log/log.py Normal file
View File

@ -0,0 +1,183 @@
"""
basic logging functionality based on a producer/consumer scheme.
XXX implement this API: (maybe put it into slogger.py?)
log = Logger(
info=py.log.STDOUT,
debug=py.log.STDOUT,
command=None)
log.info("hello", "world")
log.command("hello", "world")
log = Logger(info=Logger(something=...),
debug=py.log.STDOUT,
command=None)
"""
import py, sys
class Message(object):
def __init__(self, keywords, args):
self.keywords = keywords
self.args = args
def content(self):
return " ".join(map(str, self.args))
def prefix(self):
return "[%s] " % (":".join(self.keywords))
def __str__(self):
return self.prefix() + self.content()
class Producer(object):
""" (deprecated) Log producer API which sends messages to be logged
to a 'consumer' object, which then prints them to stdout,
stderr, files, etc. Used extensively by PyPy-1.1.
"""
Message = Message # to allow later customization
keywords2consumer = {}
def __init__(self, keywords, keywordmapper=None, **kw):
if hasattr(keywords, 'split'):
keywords = tuple(keywords.split())
self._keywords = keywords
if keywordmapper is None:
keywordmapper = default_keywordmapper
self._keywordmapper = keywordmapper
def __repr__(self):
return "<py.log.Producer %s>" % ":".join(self._keywords)
def __getattr__(self, name):
if '_' in name:
raise AttributeError(name)
producer = self.__class__(self._keywords + (name,))
setattr(self, name, producer)
return producer
def __call__(self, *args):
""" write a message to the appropriate consumer(s) """
func = self._keywordmapper.getconsumer(self._keywords)
if func is not None:
func(self.Message(self._keywords, args))
class KeywordMapper:
def __init__(self):
self.keywords2consumer = {}
def getstate(self):
return self.keywords2consumer.copy()
def setstate(self, state):
self.keywords2consumer.clear()
self.keywords2consumer.update(state)
def getconsumer(self, keywords):
""" return a consumer matching the given keywords.
tries to find the most suitable consumer by walking, starting from
the back, the list of keywords, the first consumer matching a
keyword is returned (falling back to py.log.default)
"""
for i in range(len(keywords), 0, -1):
try:
return self.keywords2consumer[keywords[:i]]
except KeyError:
continue
return self.keywords2consumer.get('default', default_consumer)
def setconsumer(self, keywords, consumer):
""" set a consumer for a set of keywords. """
# normalize to tuples
if isinstance(keywords, str):
keywords = tuple(map(None, keywords.split()))
elif hasattr(keywords, '_keywords'):
keywords = keywords._keywords
elif not isinstance(keywords, tuple):
raise TypeError("key %r is not a string or tuple" % (keywords,))
if consumer is not None and not callable(consumer):
if not hasattr(consumer, 'write'):
raise TypeError(
"%r should be None, callable or file-like" % (consumer,))
consumer = File(consumer)
self.keywords2consumer[keywords] = consumer
def default_consumer(msg):
""" the default consumer, prints the message to stdout (using 'print') """
sys.stderr.write(str(msg)+"\n")
default_keywordmapper = KeywordMapper()
def setconsumer(keywords, consumer):
default_keywordmapper.setconsumer(keywords, consumer)
def setstate(state):
default_keywordmapper.setstate(state)
def getstate():
return default_keywordmapper.getstate()
#
# Consumers
#
class File(object):
""" log consumer wrapping a file(-like) object
"""
def __init__(self, f):
assert hasattr(f, 'write')
assert isinstance(f, file) or not hasattr(f, 'open')
self._file = f
def __call__(self, msg):
""" write a message to the log """
self._file.write(str(msg) + "\n")
class Path(object):
""" log consumer able to write log messages into
"""
def __init__(self, filename, append=False, delayed_create=False,
buffering=1):
self._append = append
self._filename = filename
self._buffering = buffering
if not delayed_create:
self._openfile()
def _openfile(self):
mode = self._append and 'a' or 'w'
f = open(str(self._filename), mode, buffering=self._buffering)
self._file = f
def __call__(self, msg):
""" write a message to the log """
if not hasattr(self, "_file"):
self._openfile()
self._file.write(str(msg) + "\n")
def STDOUT(msg):
""" consumer that writes to sys.stdout """
sys.stdout.write(str(msg)+"\n")
def STDERR(msg):
""" consumer that writes to sys.stderr """
sys.stderr.write(str(msg)+"\n")
class Syslog:
""" consumer that writes to the syslog daemon """
for priority in "LOG_EMERG LOG_ALERT LOG_CRIT LOG_ERR LOG_WARNING LOG_NOTICE LOG_INFO LOG_DEBUG".split():
try:
exec("%s = py.std.syslog.%s" % (priority, priority))
except AttributeError:
pass
def __init__(self, priority = None):
if priority is None:
priority = self.LOG_INFO
self.priority = priority
def __call__(self, msg):
""" write a message to the log """
py.std.syslog.syslog(self.priority, str(msg))

View File

@ -1,71 +0,0 @@
class Message(object):
def __init__(self, processor, *args):
self.content = args
self.processor = processor
self.keywords = (processor.logger._ident,
processor.name)
def strcontent(self):
return " ".join(map(str, self.content))
def strprefix(self):
return '[%s] ' % ":".join(map(str, self.keywords))
def __str__(self):
return self.strprefix() + self.strcontent()
class Processor(object):
def __init__(self, logger, name, consume):
self.logger = logger
self.name = name
self.consume = consume
def __call__(self, *args):
try:
consume = self.logger._override
except AttributeError:
consume = self.consume
if consume is not None:
msg = Message(self, *args)
consume(msg)
class Logger(object):
_key2logger = {}
def __init__(self, ident):
self._ident = ident
self._key2logger[ident] = self
self._keywords = ()
def set_sub(self, **kwargs):
for name, value in kwargs.items():
self._setsub(name, value)
def ensure_sub(self, **kwargs):
for name, value in kwargs.items():
if not hasattr(self, name):
self._setsub(name, value)
def set_override(self, consumer):
self._override = lambda msg: consumer(msg)
def del_override(self):
try:
del self._override
except AttributeError:
pass
def _setsub(self, name, dest):
assert "_" not in name
setattr(self, name, Processor(self, name, dest))
def get(ident="global", **kwargs):
""" return the Logger with id 'ident', instantiating if appropriate """
try:
log = Logger._key2logger[ident]
except KeyError:
log = Logger(ident)
log.ensure_sub(**kwargs)
return log

View File

@ -1,88 +0,0 @@
"""
py lib's basic logging/tracing functionality
EXPERIMENTAL EXPERIMENTAL EXPERIMENTAL (especially the dispatching)
WARNING: this module is not allowed to contain any 'py' imports,
Instead, it is very self-contained and should not depend on
CPython/stdlib versions, either. One reason for these
restrictions is that this module should be sendable
via py.execnet across the network in an very early phase.
"""
class Message(object):
def __init__(self, keywords, args):
self.keywords = keywords
self.args = args
def content(self):
return " ".join(map(str, self.args))
def prefix(self):
return "[%s] " % (":".join(self.keywords))
def __str__(self):
return self.prefix() + self.content()
class Producer(object):
""" Log producer API which sends messages to be logged
to a 'consumer' object, which then prints them to stdout,
stderr, files, etc.
"""
Message = Message # to allow later customization
keywords2consumer = {}
def __init__(self, keywords):
if isinstance(keywords, str):
keywords = tuple(keywords.split())
self.keywords = keywords
def __repr__(self):
return "<py.log.Producer %s>" % ":".join(self.keywords)
def __getattr__(self, name):
if '_' in name:
raise AttributeError, name
producer = self.__class__(self.keywords + (name,))
setattr(self, name, producer)
return producer
def __call__(self, *args):
""" write a message to the appropriate consumer(s) """
func = self.get_consumer(self.keywords)
if func is not None:
func(self.Message(self.keywords, args))
def get_consumer(self, keywords):
""" return a consumer matching keywords
tries to find the most suitable consumer by walking, starting from
the back, the list of keywords, the first consumer matching a
keyword is returned (falling back to py.log.default)
"""
for i in range(len(self.keywords), 0, -1):
try:
return self.keywords2consumer[self.keywords[:i]]
except KeyError:
continue
return self.keywords2consumer.get('default', default_consumer)
def set_consumer(self, consumer):
""" register a consumer matching our own keywords """
self.keywords2consumer[self.keywords] = consumer
default = Producer('default')
def _getstate():
return Producer.keywords2consumer.copy()
def _setstate(state):
Producer.keywords2consumer.clear()
Producer.keywords2consumer.update(state)
def default_consumer(msg):
""" the default consumer, prints the message to stdout (using 'print') """
print str(msg)
Producer.keywords2consumer['default'] = default_consumer

View File

@ -1,24 +1,32 @@
import py
import sys
from py.__.log.log import default_keywordmapper
callcapture = py.io.StdCapture.call
def setup_module(mod):
mod.tempdir = py.test.ensuretemp("py.log-test")
mod.logstate = py.log._getstate()
mod._oldstate = default_keywordmapper.getstate()
def teardown_module(mod):
py.log._setstate(mod.logstate)
default_keywordmapper.setstate(mod._oldstate)
class TestLogProducer:
def setup_method(self, meth):
self.state = py.log._getstate()
default_keywordmapper.setstate(_oldstate)
def teardown_method(self, meth):
py.log._setstate(self.state)
def test_getstate_setstate(self):
state = py.log._getstate()
py.log.setconsumer("hello", [].append)
state2 = py.log._getstate()
assert state2 != state
py.log._setstate(state)
state3 = py.log._getstate()
assert state3 == state
def test_producer_repr(self):
d = py.log.default
d = py.log.Producer("default")
assert repr(d).find('default') != -1
def test_produce_one_keyword(self):
@ -34,7 +42,7 @@ class TestLogProducer:
def test_producer_class(self):
p = py.log.Producer('x1')
l = []
py.log.setconsumer(p.keywords, l.append)
py.log.setconsumer(p._keywords, l.append)
p("hello")
assert len(l) == 1
assert len(l[0].keywords) == 1
@ -47,10 +55,7 @@ class TestLogProducer:
class TestLogConsumer:
def setup_method(self, meth):
self.state = py.log._getstate()
def teardown_method(self, meth):
py.log._setstate(self.state)
default_keywordmapper.setstate(_oldstate)
def test_log_none(self):
log = py.log.Producer("XXX")
l = []
@ -62,9 +67,9 @@ class TestLogConsumer:
log("2")
assert not l
def test_log_default_stdout(self):
res, out, err = callcapture(py.log.default, "hello")
assert out.strip() == "[default] hello"
def test_log_default_stderr(self):
res, out, err = callcapture(py.log.Producer("default"), "hello")
assert err.strip() == "[default] hello"
def test_simple_consumer_match(self):
l = []
@ -77,7 +82,7 @@ class TestLogConsumer:
def test_simple_consumer_match_2(self):
l = []
p = py.log.Producer("x1 x2")
p.set_consumer(l.append)
py.log.setconsumer(p._keywords, l.append)
p("42")
assert l
assert l[0].content() == "42"
@ -106,19 +111,19 @@ class TestLogConsumer:
assert l[0].content() == "hello"
def test_log_stderr(self):
py.log.setconsumer("default", py.log.STDERR)
res, out, err = callcapture(py.log.default, "hello")
assert not out
assert err.strip() == '[default] hello'
py.log.setconsumer("xyz", py.log.STDOUT)
res, out, err = callcapture(py.log.Producer("xyz"), "hello")
assert not err
assert out.strip() == '[xyz] hello'
def test_log_file(self):
custom_log = tempdir.join('log.out')
py.log.setconsumer("default", open(str(custom_log), 'w', buffering=0))
py.log.default("hello world #1")
py.log.Producer("default")("hello world #1")
assert custom_log.readlines() == ['[default] hello world #1\n']
py.log.setconsumer("default", py.log.Path(custom_log, buffering=0))
py.log.default("hello world #2")
py.log.Producer("default")("hello world #2")
assert custom_log.readlines() == ['[default] hello world #2\n'] # no append by default!
def test_log_file_append_mode(self):
@ -128,12 +133,12 @@ class TestLogConsumer:
py.log.setconsumer("default", py.log.Path(logfilefn, append=True,
buffering=0))
assert logfilefn.check()
py.log.default("hello world #1")
py.log.Producer("default")("hello world #1")
lines = logfilefn.readlines()
assert lines == ['[default] hello world #1\n']
py.log.setconsumer("default", py.log.Path(logfilefn, append=True,
buffering=0))
py.log.default("hello world #1")
py.log.Producer("default")("hello world #1")
lines = logfilefn.readlines()
assert lines == ['[default] hello world #1\n',
'[default] hello world #1\n']
@ -144,7 +149,7 @@ class TestLogConsumer:
py.log.setconsumer("default", py.log.Path(logfilefn,
delayed_create=True, buffering=0))
assert not logfilefn.check()
py.log.default("hello world #1")
py.log.Producer("default")("hello world #1")
lines = logfilefn.readlines()
assert lines == ['[default] hello world #1\n']

View File

@ -1,101 +0,0 @@
import py
def test_logger_identity():
assert py.log.get() is py.log.get()
otherkey = object()
for key in "name1", object():
log = py.log.get(key)
assert py.log.get(key) is log
assert py.log.get(otherkey) is not log
def test_log_preset():
log = py.log.get(test_log_preset)
l2 = []
log.set_sub(x1=None, x2=l2.append)
l3 = []
log2 = py.log.get(test_log_preset,
x2=None,
x3=l3.append)
log2.x2("hello")
log2.x3("world")
assert l2[0].strcontent() == "hello"
assert l3[0].strcontent() == "world"
def test_log_override():
l2 = []
log = py.log.get(object(), x1=None, x2=l2.append)
l = []
log.set_override(l.append)
log.x1("hello")
log.x2("world")
log.ensure_sub(x3=None)
log.x3(42)
assert len(l) == 3
assert not l2
r = [x.strcontent() for x in l]
assert r == ["hello", "world", "42"]
l[:] = []
log.del_override()
log.del_override()
log.x2("hello")
assert l2[0].strcontent() == "hello"
def test_log_basic():
l1 = []
class SomeKey:
def __str__(self):
return "somekey"
for key in "name1", SomeKey():
log = py.log.get(key)
log.set_sub(x1=l1.append)
log.x1(42)
assert l1[-1].content == (42,)
assert l1[-1].strcontent() == "42"
assert l1[-1].keywords == (key, 'x1')
assert l1[-1].strprefix() == "[%s:x1] " %(key,)
#log.set_prefix("hello")
#assert l1[0].strprefix() == "hello"
#log("world")
#assert str(l1[-1]) == "hello world"
class TestLogger:
def setup_method(self, method):
self._x1 = []
self._x2 = []
self.log = py.log.get()
self.log.set_sub(x1=self._x1.append,
x2=self._x2.append)
#def teardown_method(self, method):
# self.log.close()
def test_simple(self):
self.log.x1("hello")
self.log.x2("world")
assert self._x1[0].strcontent() == 'hello'
assert self._x1[0].strprefix() == '[global:x1] '
assert self._x2[0].strcontent() == 'world'
assert self._x2[0].strprefix() == '[global:x2] '
py.test.raises(AttributeError, "self.log.x3")
def test_reconfig(self):
self.log.set_sub(x1=None)
self.log.x1("asdasd")
assert not self._x1
def test_reconfig_add(self):
l = []
self.log.set_sub(x2=None, x3=l.append)
self.log.x2("asdhello")
assert not self._x2
self.log.x3(123)
assert l[0].content == (123,)
def test_logger_del(self):
del self.log.x2
py.test.raises(AttributeError, "self.log.x2")

View File

@ -5,10 +5,10 @@
import py
import sys
log = py.log.get("dynpkg",
info=py.log.STDOUT,
debug=py.log.STDOUT,
command=None) # py.log.STDOUT)
log = py.log.Logger("dynpkg",
info=py.log.STDOUT,
debug=py.log.STDOUT,
command=None)
from distutils import util