118 lines
4.3 KiB
Python
118 lines
4.3 KiB
Python
"""Catch lost/duplicated events across pagination, partial writes and rotation."""
|
|
from pathlib import Path
|
|
from threading import RLock
|
|
|
|
import pytest
|
|
|
|
from dx_productivity.trial_task_center.task_logs import TaskLogReader
|
|
from dx_productivity.trial_task_center.read_control import QueryCancelled, QueryControl
|
|
|
|
|
|
def test_pages_cover_rotated_history_without_duplicates(tmp_path):
|
|
path = tmp_path / 'events.jsonl'
|
|
path.with_name(path.name + '.1').write_text('旧一\n旧二\n', encoding='utf-8')
|
|
path.write_text('新一\n新二\n新三\n', encoding='utf-8')
|
|
reader = TaskLogReader(path, RLock(), backup_count=5)
|
|
latest = reader.latest(limit=2)
|
|
assert latest.lines == ('新二', '新三') and latest.has_older
|
|
previous = reader.older(latest.older_cursor, limit=2)
|
|
assert previous.lines == ('旧二', '新一') and previous.has_older
|
|
first = reader.older(previous.older_cursor, limit=2)
|
|
assert first.lines == ('旧一',) and not first.has_older
|
|
assert reader.poll(latest.cursor).lines == ()
|
|
|
|
|
|
def test_incremental_poll_handles_partial_utf8_and_rotation(tmp_path):
|
|
path = tmp_path / 'events.jsonl'
|
|
path.write_bytes('first\n'.encode())
|
|
reader = TaskLogReader(path, RLock(), backup_count=5)
|
|
cursor = reader.latest().cursor
|
|
payload = '位号 R1\n'.encode('utf-8')
|
|
with path.open('ab') as stream:
|
|
stream.write(payload[:2])
|
|
incomplete = reader.poll(cursor)
|
|
assert incomplete.lines == () and incomplete.cursor == cursor
|
|
with path.open('ab') as stream:
|
|
stream.write(payload[2:])
|
|
path.rename(path.with_name(path.name + '.1'))
|
|
path.write_text('next\n', encoding='utf-8')
|
|
page = reader.poll(cursor)
|
|
assert page.lines == ('位号 R1', 'next') and not page.reset
|
|
assert reader.poll(page.cursor).lines == ()
|
|
|
|
|
|
def test_truncated_log_resets_and_empty_log_can_later_be_created(tmp_path):
|
|
path = tmp_path / 'events.jsonl'
|
|
reader = TaskLogReader(path, RLock(), backup_count=5)
|
|
empty = reader.latest()
|
|
assert not empty.lines
|
|
path.write_text('abcdef\n', encoding='utf-8')
|
|
first = reader.poll(empty.cursor)
|
|
assert first.lines == ('abcdef',)
|
|
path.write_text('xy\n', encoding='utf-8')
|
|
reset = reader.poll(first.cursor)
|
|
assert reset.reset and reset.lines == ('xy',)
|
|
|
|
|
|
def test_poll_is_bounded_and_retains_unread_events(tmp_path):
|
|
path = tmp_path / 'events.jsonl'
|
|
path.touch()
|
|
reader = TaskLogReader(path, RLock(), backup_count=5)
|
|
cursor = reader.latest().cursor
|
|
path.write_text('1\n2\n3\n4\n5\n', encoding='utf-8')
|
|
a = reader.poll(cursor, limit=2)
|
|
b = reader.poll(a.cursor, limit=2)
|
|
c = reader.poll(b.cursor, limit=2)
|
|
assert (a.lines, b.lines, c.lines) == (('1', '2'), ('3', '4'), ('5',))
|
|
|
|
|
|
def test_cancelled_read_releases_while_another_thread_holds_log_lock(tmp_path):
|
|
from threading import Event, Thread
|
|
lock, entered, release, finished = RLock(), Event(), Event(), Event()
|
|
control = QueryControl()
|
|
failures = []
|
|
def holder():
|
|
with lock:
|
|
entered.set()
|
|
release.wait(5)
|
|
def read():
|
|
try:
|
|
TaskLogReader(tmp_path / 'events.jsonl', lock, read_control=control).latest()
|
|
except Exception as error:
|
|
failures.append(error)
|
|
finally:
|
|
finished.set()
|
|
writer = Thread(target=holder)
|
|
reader = Thread(target=read)
|
|
writer.start()
|
|
try:
|
|
assert entered.wait(2)
|
|
reader.start()
|
|
control.cancel()
|
|
assert finished.wait(1)
|
|
assert len(failures) == 1 and isinstance(failures[0], QueryCancelled)
|
|
assert not release.is_set()
|
|
finally:
|
|
release.set()
|
|
writer.join(2)
|
|
if reader.ident:
|
|
reader.join(2)
|
|
|
|
|
|
@pytest.mark.parametrize('mode', ['latest', 'poll'])
|
|
def test_large_single_log_line_can_be_cancelled_between_chunks(tmp_path, mode):
|
|
path = tmp_path / 'events.jsonl'
|
|
path.touch()
|
|
reader = TaskLogReader(path, RLock())
|
|
cursor = reader.latest().cursor
|
|
path.write_text('x' * (1024 * 1024) + '\n', encoding='utf-8')
|
|
class CancelAfterSegments:
|
|
calls = 0
|
|
def checkpoint(self):
|
|
self.calls += 1
|
|
if self.calls == 8:
|
|
raise QueryCancelled()
|
|
reader.read_control = CancelAfterSegments()
|
|
with pytest.raises(QueryCancelled):
|
|
reader.latest() if mode == 'latest' else reader.poll(cursor)
|