Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions newsfragments/742.fixed.1.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
:class:`~logbook.queues.RedisHandler` made logging calls wait while Redis was unreachable, and could stop pushing records after a push failed.
1 change: 1 addition & 0 deletions newsfragments/742.fixed.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
:meth:`~logbook.queues.RedisHandler.disable_buffering` could leave records buffered until the handler was closed.
93 changes: 82 additions & 11 deletions src/logbook/queues.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
"""

import json
import os
import platform
import sys
import threading
Expand Down Expand Up @@ -38,6 +39,12 @@ class RedisHandler(Handler):

handler = RedisHandler(password="your_redis_password")

If the flush thread cannot push, the records stay buffered and it tries
again later. The first of consecutive failures is reported through
:meth:`~logbook.Handler.handle_error` when the handler next emits a
record. Call :meth:`close` before the process exits, or the records still
buffered are lost.

More info about the default buffer size: wp.me/p3tYJu-3b
"""

Expand Down Expand Up @@ -75,22 +82,60 @@ def __init__(
self.flush_threshold = flush_threshold
self.queue = []
self.lock = Lock()
self._buffering = True
self.push_method = push_method

# Set up a thread that flushes the queue every specified seconds
self._flush_time = flush_time
self._start_flushing()

def _start_flushing(self):
self._failing = False
self._error = None
self._stop_event = threading.Event()
self._flushing_t = threading.Thread(
target=self._flush_task, args=(flush_time, self._stop_event)
self._flush_event = threading.Event()
thread = threading.Thread(
target=self._flush_task, args=(self._flush_time, self._stop_event)
)
self._flushing_t.daemon = True
self._flushing_t.start()
thread.daemon = True
thread.start()
self._flushing_t = thread
self._pid = os.getpid()

def _flush_task(self, time, stop_event):
"""Calls the method _flush_buffer every certain time."""
while not self._stop_event.is_set():
pid, flush_event = os.getpid(), self._flush_event
while not stop_event.is_set() and os.getpid() == pid:
failing = self._failing
try:
self._push()
except Exception as e:
if not failing:
with self.lock:
self._error = e
flush_event.wait(time)
flush_event.clear()

def _stop_flushing(self):
self._stop_event.set()
self._flush_event.set()
if self._flushing_t is not threading.current_thread():
self._flushing_t.join()
self._buffering = False
self._error = None

def _push(self):
with self.lock:
batch, self.queue = self.queue, []
if not batch:
return
try:
getattr(self.redis, self.push_method)(self.key, *batch)
except BaseException:
with self.lock:
self._flush_buffer()
self._stop_event.wait(time)
self.queue[:0] = batch
self._failing = True
raise
self._failing = False

def _flush_buffer(self):
"""Flushes the messaging queue into Redis.
Expand All @@ -107,9 +152,13 @@ def disable_buffering(self):
"""Disables buffering.

If called, every single message will be directly pushed to Redis.
Buffered records are pushed first. If that fails, the error is raised
and they stay buffered.
"""
self._stop_event.set()
self._stop_flushing()
self.flush_threshold = 1
with self.lock:
self._flush_buffer()

def emit(self, record):
"""Emits a pair (key, value) to redis.
Expand All @@ -129,11 +178,33 @@ def emit(self, record):
r.update(self.extra_fields)
r.update(record.kwargs)
self.queue.append(json.dumps(r))
if len(self.queue) == self.flush_threshold:
if not self._buffering:
self._flush_buffer()
elif self._pid != os.getpid():
self._start_flushing()
elif len(self.queue) >= self.flush_threshold and not self._failing:
self._flush_event.set()
error, self._error = self._error, None
if error is not None:
raise error

def emit_batch(self, records, reason):
"""Handles each record as a logging call would, so a failure is
reported through :meth:`~logbook.Handler.handle_error` with that
record.
"""
for record in records:
self.handle(record)

def close(self):
self._flush_buffer()
"""Stop periodic flushing and push the buffered records.

If that fails, the error is raised and the records stay buffered.
Records emitted after closing are pushed immediately.
"""
self._stop_flushing()
with self.lock:
self._flush_buffer()


class MessageQueueHandler(Handler):
Expand Down
146 changes: 146 additions & 0 deletions tests/test_queues.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import logbook

from .fork_locks import fork
from .utils import LETTERS, missing, require_module, unused_tcp_address

REDIS_HOST = os.environ.get("REDIS_HOST", "localhost")
Expand Down Expand Up @@ -562,6 +563,151 @@ def test_redis_handler_rpush():
r.delete(KEY)


def redis_record(message):
record = logbook.LogRecord("redis", logbook.INFO, message)
record.heavy_init()
return record


def redis_messages(r, key):
return [json.loads(value)["message"] for value in r.lrange(key, 0, -1)]


def wait_for_redis_messages(r, key, messages):
deadline = time.monotonic() + 2
while redis_messages(r, key) != messages and time.monotonic() < deadline:
time.sleep(0.01)
assert redis_messages(r, key) == messages


@require_module("redis")
def test_redis_handler_flushes_without_blocking_emit(monkeypatch):
import redis

from logbook.queues import RedisHandler

KEY = f"redis-{os.getpid()}-flush"
r = redis.Redis(REDIS_HOST, REDIS_PORT, decode_responses=True)
r.delete(KEY)
handler = RedisHandler(key=KEY, flush_threshold=1, flush_time=60)
entered = threading.Event()
release = threading.Event()
rpush = handler.redis.rpush

def blocking_rpush(*args):
entered.set()
assert release.wait(2)
return rpush(*args)

monkeypatch.setattr(handler.redis, "rpush", blocking_rpush)
handler.emit(redis_record("a"))
assert entered.wait(2)
handler.emit(redis_record("b"))
closer = threading.Thread(target=handler.close)
closer.start()
closer.join(0.1)
assert closer.is_alive()
release.set()
closer.join(2)
assert redis_messages(r, KEY) == ["a", "b"]
assert not handler._flushing_t.is_alive()
r.delete(KEY)


@require_module("redis")
def test_redis_handler_retries_failed_flush(monkeypatch, capsys):
import redis

from logbook.queues import RedisHandler

KEY = f"redis-{os.getpid()}-retry"
r = redis.Redis(REDIS_HOST, REDIS_PORT, decode_responses=True)
r.delete(KEY)
handler = RedisHandler(key=KEY, flush_time=0.01)
failures = 2
entered = threading.Event()
release = threading.Event()
rpush = handler.redis.rpush

def failing_rpush(*args):
nonlocal failures
if failures:
failures -= 1
if not failures:
entered.set()
assert release.wait(2)
raise redis.ConnectionError("Redis is down")
return rpush(*args)

monkeypatch.setattr(handler.redis, "rpush", failing_rpush)
handler.emit(redis_record("a"))
assert entered.wait(2)
assert capsys.readouterr().err == ""
handler.emit_batch([redis_record("b"), redis_record("c")], "group")
err = capsys.readouterr().err
assert err.count("ConnectionError: Redis is down") == 1
assert f"Logged from file {os.path.abspath(__file__)}, line " in err
release.set()
wait_for_redis_messages(r, KEY, ["a", "b", "c"])
handler.handle(redis_record("d"))
assert capsys.readouterr().err == ""
wait_for_redis_messages(r, KEY, ["a", "b", "c", "d"])
failures = 1
handler.emit(redis_record("e"))
wait_for_redis_messages(r, KEY, ["a", "b", "c", "d", "e"])
with logbook.Flags(errors="raise"), pytest.raises(redis.ConnectionError):
handler.handle(redis_record("f"))
handler.close()
assert redis_messages(r, KEY) == ["a", "b", "c", "d", "e", "f"]
assert capsys.readouterr().err == ""
r.delete(KEY)


@require_module("redis")
def test_redis_handler_pushes_directly_when_not_buffering():
import redis

from logbook.queues import RedisHandler

KEY = f"redis-{os.getpid()}-direct"
r = redis.Redis(REDIS_HOST, REDIS_PORT, decode_responses=True)
r.delete(KEY)
handler = RedisHandler(key=KEY, flush_time=60)
handler.emit(redis_record("a"))
handler.disable_buffering()
handler.emit(redis_record("b"))
assert redis_messages(r, KEY) == ["a", "b"]
handler.close()

handler = RedisHandler(key=KEY, flush_time=60)
handler.close()
handler.emit(redis_record("c"))
assert redis_messages(r, KEY) == ["a", "b", "c"]
r.delete(KEY)


@require_module("redis")
@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork")
@pytest.mark.filterwarnings("ignore:This process:DeprecationWarning")
def test_redis_handler_pushes_from_forked_child():
import redis

from logbook.queues import RedisHandler

KEY = f"redis-{os.getpid()}-fork"
r = redis.Redis(REDIS_HOST, REDIS_PORT, decode_responses=True)
r.delete(KEY)
handler = RedisHandler(key=KEY, flush_threshold=1, flush_time=60)

def child():
handler.emit(redis_record("a"))
wait_for_redis_messages(r, KEY, ["a"])

fork(child)
handler.close()
r.delete(KEY)


@pytest.fixture
def handlers(handlers_subscriber):
return handlers_subscriber[0]
Expand Down
Loading