Repository navigation
Expand file tree
/
Copy pathtest_pool.py
More file actions
69 lines (52 loc) · 1.86 KB
/
Copy pathtest_pool.py
File metadata and controls
69 lines (52 loc) · 1.86 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
"""The shared decode pool."""
import threading
import time
import pytest
import micro_reader
from micro_reader import _pool
@pytest.fixture
def threads():
old = micro_reader.decode_threads()
yield micro_reader.set_decode_threads
micro_reader.set_decode_threads(old)
@pytest.mark.parametrize("n", [1, 3])
def test_sums_task_results(threads, n):
threads(n)
assert _pool.run(lambda k=k: k for k in range(100)) == sum(range(100))
def test_bounded_in_flight(threads):
"""The task generator is consumed lazily: at most two tasks per thread
are created ahead of the finished ones, so whatever tasks hold (bytes,
decoded segments) stays bounded; and at most one runs per thread."""
threads(2)
lock = threading.Lock()
created = finished = running = peak_running = peak_ahead = 0
def task():
nonlocal finished, running, peak_running
with lock:
running += 1
peak_running = max(peak_running, running)
time.sleep(0.005)
with lock:
running -= 1
finished += 1
return 1
def tasks():
nonlocal created, peak_ahead
for _ in range(40):
with lock:
created += 1
peak_ahead = max(peak_ahead, created - finished)
yield task
assert _pool.run(tasks()) == 40
assert peak_running <= 2
assert peak_ahead <= 2 * 2 + 1 # the window, plus the one being submitted
def test_errors_reach_the_caller(threads):
threads(3)
def boom():
raise ValueError("bad segment")
with pytest.raises(ValueError, match="bad segment"):
_pool.run([lambda: 1, boom, lambda: 1])
def test_environment_variable(monkeypatch, threads):
monkeypatch.setattr(_pool, "_threads", None)
monkeypatch.setenv("MICRO_READER_DECODE_THREADS", "5")
assert micro_reader.decode_threads() == 5