fix: stop batch when native session recovery is exhausted
This commit is contained in:
1 parent
a011e8eab6
commit
2c88be99ce
3 files changed
+54
-2
No files matched your search
@@ -166,9 +166,12 @@ class NativeInventoryReader:
|
||||
if record['state'] == 'uncertain':
|
||||
raise TaskCenterError('inventory_cleanup', f'原生采集清理状态不确定:{evidence}') from error
|
||||
_verify_sources(source_hashes)
|
||||
record.update(source_hashes_unchanged=True,
|
||||
source_hashes={str(path): digest for path, digest in source_hashes.items()})
|
||||
retry = False
|
||||
cleanup = getattr(desktop, 'cleanup_record', None)
|
||||
if (allow_retry and isinstance(error, _NativeInventoryProcessError)
|
||||
session_failure = runtime is not None and isinstance(error, _NativeInventoryProcessError)
|
||||
if (session_failure
|
||||
and clean and isinstance(cleanup, dict) and cleanup.get('workers_closed') is True):
|
||||
check_cancelled(stop_event)
|
||||
from .batch_resources import write_record
|
||||
@@ -179,7 +182,12 @@ class NativeInventoryReader:
|
||||
'binding': dict(desktop.identity),
|
||||
})
|
||||
_verify_sources(source_hashes)
|
||||
retry = runtime.recover_native_failure(project, evidence, str(error))
|
||||
if allow_retry:
|
||||
retry = runtime.recover_native_failure(project, evidence, str(error))
|
||||
if session_failure and not retry:
|
||||
record.update(state='session_failed', reason=str(error))
|
||||
raise TaskCenterError('inventory_session',
|
||||
f'原生读取会话异常,未能完成安全重试,批次已停止。请保留诊断记录后排查:{evidence};{error}') from error
|
||||
record.update(state='retrying' if retry else 'fallback', reason=str(error))
|
||||
if retry:
|
||||
result = _RETRY_NATIVE_READ
|
||||
|
||||
@@ -5,6 +5,7 @@ import re
|
||||
|
||||
ERROR_TEXT = {
|
||||
"batch_resources_pending": "已核验的替换结果已保留。后台资源尚未释放,请使用“重新清理资源”后再开始下一批。",
|
||||
"inventory_session": "原生读取会话异常,安全重试未完成,已停止本批处理。请查看诊断记录并排除原因后重试。",
|
||||
"task_required": "请先选择任务。",
|
||||
"directory_required": "请先选择工程目录。",
|
||||
"discovery_root_locked": "旧目录已有执行记录,不能在同一任务中更换目录;请新建任务。",
|
||||
|
||||
@@ -166,6 +166,49 @@ def test_only_transient_native_process_failure_requests_one_proven_rebuild(tmp_p
|
||||
assert result is None and len(scopes) == 1 and not recoveries
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['timeout', 'crash'])
|
||||
@pytest.mark.parametrize('recovery_allowed', [True, False])
|
||||
def test_native_session_failure_stops_batch_after_retry_or_denied_budget_without_cad_fallback(
|
||||
tmp_path, monkeypatch, failure, recovery_allowed):
|
||||
from dx_productivity.trial_task_center.quick_run import QuickRunPreparer
|
||||
project, reader, stop, calls, released = reader_fixture(tmp_path)
|
||||
reader.timeout = 0
|
||||
scopes, recoveries, cad_calls = [], [], []
|
||||
class BatchRuntime:
|
||||
def native_scope(self, evidence):
|
||||
scope = reader.desktop_factory('shared')
|
||||
scope.cleanup_record = {'workers_closed': True}
|
||||
scope.identity = {'session_id': 'batch', 'generation': 1,
|
||||
'attempt_id': f'native-{len(scopes) + 1}'}
|
||||
original = scope.start_process
|
||||
def start(command, **options):
|
||||
process = original(command, **options)
|
||||
process.poll = lambda: None if failure == 'timeout' else 0xC0000005
|
||||
process.close = lambda: None
|
||||
return process
|
||||
scope.start_process = start
|
||||
scopes.append(evidence)
|
||||
return scope
|
||||
def recover_native_failure(self, project, evidence, reason):
|
||||
recoveries.append(evidence)
|
||||
return recovery_allowed
|
||||
def run_worker(self, *args, **kwargs):
|
||||
cad_calls.append(args)
|
||||
raise AssertionError('native session failure must stop before CAD fallback')
|
||||
monkeypatch.setattr(runtime, 'NativeInventoryReader', lambda *args, **kwargs: reader)
|
||||
preparer = QuickRunPreparer(tmp_path / 'cad-data')
|
||||
with pytest.raises(TaskCenterError) as caught:
|
||||
preparer._discover_project_sheets(project, ('OLD',), collect_components=True, runtime=BatchRuntime())
|
||||
assert caught.value.code == 'inventory_session'
|
||||
assert len(scopes) == len(released) == (2 if recovery_allowed else 1)
|
||||
assert len(recoveries) == 1 and not cad_calls
|
||||
record = json.loads((scopes[-1] / 'native-inventory.json').read_text(encoding='utf-8'))
|
||||
assert record['state'] == 'session_failed' and record['source_hashes_unchanged']
|
||||
assert record['cleanup']['workers_closed']
|
||||
proof = json.loads((scopes[-1] / 'native-retry-proof.json').read_text(encoding='utf-8'))
|
||||
assert proof['workers_closed'] and proof['binding']['attempt_id'] == f'native-{len(scopes)}'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['malformed', 'timeout'])
|
||||
def test_safe_native_failure_allows_com_fallback_only_after_desktop_release(tmp_path, failure):
|
||||
project, reader, stop, calls, released = reader_fixture(
|
||||
|
||||
Reference in new issue
Block a user