diff --git a/newsfragments/742.fixed.1.rst b/newsfragments/742.fixed.1.rst new file mode 100644 index 00000000..8b2d9354 --- /dev/null +++ b/newsfragments/742.fixed.1.rst @@ -0,0 +1 @@ +:class:`~logbook.queues.RedisHandler` made logging calls wait while Redis was unreachable, and could stop pushing records after a push failed. diff --git a/newsfragments/742.fixed.rst b/newsfragments/742.fixed.rst new file mode 100644 index 00000000..ffb03189 --- /dev/null +++ b/newsfragments/742.fixed.rst @@ -0,0 +1 @@ +:meth:`~logbook.queues.RedisHandler.disable_buffering` could leave records buffered until the handler was closed. diff --git a/src/logbook/queues.py b/src/logbook/queues.py index 1c8e4f3c..59055481 100644 --- a/src/logbook/queues.py +++ b/src/logbook/queues.py @@ -9,6 +9,7 @@ """ import json +import os import platform import sys import threading @@ -38,6 +39,12 @@ class RedisHandler(Handler): handler = RedisHandler(password="your_redis_password") + If the flush thread cannot push, the records stay buffered and it tries + again later. The first of consecutive failures is reported through + :meth:`~logbook.Handler.handle_error` when the handler next emits a + record. Call :meth:`close` before the process exits, or the records still + buffered are lost. + More info about the default buffer size: wp.me/p3tYJu-3b """ @@ -75,22 +82,60 @@ def __init__( self.flush_threshold = flush_threshold self.queue = [] self.lock = Lock() + self._buffering = True self.push_method = push_method # Set up a thread that flushes the queue every specified seconds + self._flush_time = flush_time + self._start_flushing() + + def _start_flushing(self): + self._failing = False + self._error = None self._stop_event = threading.Event() - self._flushing_t = threading.Thread( - target=self._flush_task, args=(flush_time, self._stop_event) + self._flush_event = threading.Event() + thread = threading.Thread( + target=self._flush_task, args=(self._flush_time, self._stop_event) ) - self._flushing_t.daemon = True - self._flushing_t.start() + thread.daemon = True + thread.start() + self._flushing_t = thread + self._pid = os.getpid() def _flush_task(self, time, stop_event): - """Calls the method _flush_buffer every certain time.""" - while not self._stop_event.is_set(): + pid, flush_event = os.getpid(), self._flush_event + while not stop_event.is_set() and os.getpid() == pid: + failing = self._failing + try: + self._push() + except Exception as e: + if not failing: + with self.lock: + self._error = e + flush_event.wait(time) + flush_event.clear() + + def _stop_flushing(self): + self._stop_event.set() + self._flush_event.set() + if self._flushing_t is not threading.current_thread(): + self._flushing_t.join() + self._buffering = False + self._error = None + + def _push(self): + with self.lock: + batch, self.queue = self.queue, [] + if not batch: + return + try: + getattr(self.redis, self.push_method)(self.key, *batch) + except BaseException: with self.lock: - self._flush_buffer() - self._stop_event.wait(time) + self.queue[:0] = batch + self._failing = True + raise + self._failing = False def _flush_buffer(self): """Flushes the messaging queue into Redis. @@ -107,9 +152,13 @@ def disable_buffering(self): """Disables buffering. If called, every single message will be directly pushed to Redis. + Buffered records are pushed first. If that fails, the error is raised + and they stay buffered. """ - self._stop_event.set() + self._stop_flushing() self.flush_threshold = 1 + with self.lock: + self._flush_buffer() def emit(self, record): """Emits a pair (key, value) to redis. @@ -129,11 +178,33 @@ def emit(self, record): r.update(self.extra_fields) r.update(record.kwargs) self.queue.append(json.dumps(r)) - if len(self.queue) == self.flush_threshold: + if not self._buffering: self._flush_buffer() + elif self._pid != os.getpid(): + self._start_flushing() + elif len(self.queue) >= self.flush_threshold and not self._failing: + self._flush_event.set() + error, self._error = self._error, None + if error is not None: + raise error + + def emit_batch(self, records, reason): + """Handles each record as a logging call would, so a failure is + reported through :meth:`~logbook.Handler.handle_error` with that + record. + """ + for record in records: + self.handle(record) def close(self): - self._flush_buffer() + """Stop periodic flushing and push the buffered records. + + If that fails, the error is raised and the records stay buffered. + Records emitted after closing are pushed immediately. + """ + self._stop_flushing() + with self.lock: + self._flush_buffer() class MessageQueueHandler(Handler): diff --git a/tests/test_queues.py b/tests/test_queues.py index 89b894eb..cb46e79a 100644 --- a/tests/test_queues.py +++ b/tests/test_queues.py @@ -8,6 +8,7 @@ import logbook +from .fork_locks import fork from .utils import LETTERS, missing, require_module, unused_tcp_address REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") @@ -562,6 +563,151 @@ def test_redis_handler_rpush(): r.delete(KEY) +def redis_record(message): + record = logbook.LogRecord("redis", logbook.INFO, message) + record.heavy_init() + return record + + +def redis_messages(r, key): + return [json.loads(value)["message"] for value in r.lrange(key, 0, -1)] + + +def wait_for_redis_messages(r, key, messages): + deadline = time.monotonic() + 2 + while redis_messages(r, key) != messages and time.monotonic() < deadline: + time.sleep(0.01) + assert redis_messages(r, key) == messages + + +@require_module("redis") +def test_redis_handler_flushes_without_blocking_emit(monkeypatch): + import redis + + from logbook.queues import RedisHandler + + KEY = f"redis-{os.getpid()}-flush" + r = redis.Redis(REDIS_HOST, REDIS_PORT, decode_responses=True) + r.delete(KEY) + handler = RedisHandler(key=KEY, flush_threshold=1, flush_time=60) + entered = threading.Event() + release = threading.Event() + rpush = handler.redis.rpush + + def blocking_rpush(*args): + entered.set() + assert release.wait(2) + return rpush(*args) + + monkeypatch.setattr(handler.redis, "rpush", blocking_rpush) + handler.emit(redis_record("a")) + assert entered.wait(2) + handler.emit(redis_record("b")) + closer = threading.Thread(target=handler.close) + closer.start() + closer.join(0.1) + assert closer.is_alive() + release.set() + closer.join(2) + assert redis_messages(r, KEY) == ["a", "b"] + assert not handler._flushing_t.is_alive() + r.delete(KEY) + + +@require_module("redis") +def test_redis_handler_retries_failed_flush(monkeypatch, capsys): + import redis + + from logbook.queues import RedisHandler + + KEY = f"redis-{os.getpid()}-retry" + r = redis.Redis(REDIS_HOST, REDIS_PORT, decode_responses=True) + r.delete(KEY) + handler = RedisHandler(key=KEY, flush_time=0.01) + failures = 2 + entered = threading.Event() + release = threading.Event() + rpush = handler.redis.rpush + + def failing_rpush(*args): + nonlocal failures + if failures: + failures -= 1 + if not failures: + entered.set() + assert release.wait(2) + raise redis.ConnectionError("Redis is down") + return rpush(*args) + + monkeypatch.setattr(handler.redis, "rpush", failing_rpush) + handler.emit(redis_record("a")) + assert entered.wait(2) + assert capsys.readouterr().err == "" + handler.emit_batch([redis_record("b"), redis_record("c")], "group") + err = capsys.readouterr().err + assert err.count("ConnectionError: Redis is down") == 1 + assert f"Logged from file {os.path.abspath(__file__)}, line " in err + release.set() + wait_for_redis_messages(r, KEY, ["a", "b", "c"]) + handler.handle(redis_record("d")) + assert capsys.readouterr().err == "" + wait_for_redis_messages(r, KEY, ["a", "b", "c", "d"]) + failures = 1 + handler.emit(redis_record("e")) + wait_for_redis_messages(r, KEY, ["a", "b", "c", "d", "e"]) + with logbook.Flags(errors="raise"), pytest.raises(redis.ConnectionError): + handler.handle(redis_record("f")) + handler.close() + assert redis_messages(r, KEY) == ["a", "b", "c", "d", "e", "f"] + assert capsys.readouterr().err == "" + r.delete(KEY) + + +@require_module("redis") +def test_redis_handler_pushes_directly_when_not_buffering(): + import redis + + from logbook.queues import RedisHandler + + KEY = f"redis-{os.getpid()}-direct" + r = redis.Redis(REDIS_HOST, REDIS_PORT, decode_responses=True) + r.delete(KEY) + handler = RedisHandler(key=KEY, flush_time=60) + handler.emit(redis_record("a")) + handler.disable_buffering() + handler.emit(redis_record("b")) + assert redis_messages(r, KEY) == ["a", "b"] + handler.close() + + handler = RedisHandler(key=KEY, flush_time=60) + handler.close() + handler.emit(redis_record("c")) + assert redis_messages(r, KEY) == ["a", "b", "c"] + r.delete(KEY) + + +@require_module("redis") +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork") +@pytest.mark.filterwarnings("ignore:This process:DeprecationWarning") +def test_redis_handler_pushes_from_forked_child(): + import redis + + from logbook.queues import RedisHandler + + KEY = f"redis-{os.getpid()}-fork" + r = redis.Redis(REDIS_HOST, REDIS_PORT, decode_responses=True) + r.delete(KEY) + handler = RedisHandler(key=KEY, flush_threshold=1, flush_time=60) + + def child(): + handler.emit(redis_record("a")) + wait_for_redis_messages(r, KEY, ["a"]) + + fork(child) + handler.close() + r.delete(KEY) + + @pytest.fixture def handlers(handlers_subscriber): return handlers_subscriber[0]