Add owned desktop resource diagnostics and sanitized support export

This commit is contained in:
阿白叔 committed 2026-10-23 15:29:43 +08:00
1 parent 7d5ea814ec
commit 6d0b92c515
4 files changed
+407 -1

No files matched your search

@@ -0,0 +1,139 @@
"""Allowlisted, bounded support export; never packages project or CAD inputs."""
from __future__ import annotations
import json
import os
from pathlib import Path
import re
import stat
import zipfile
from .path_access import canonical_path, internal_path
MAX_FILE_BYTES = 4 * 1024 * 1024
MAX_BUNDLE_BYTES = 32 * 1024 * 1024
MAX_FILES = 512
_BATCH_NAMES = {'session.json', 'supervisor.json', 'request.json', 'result.json',
'business-result.json', 'supervisor.log', 'resource-release.log', 'cancel.json'}
_QUICK_NAMES = {'private-desktop.json', 'checkpoint.json', 'status.json',
'uncertain.flag', 'resolved-uncertain.flag', 'protection.json', 'cleanup.json',
'project-session.json', 'project-checkpoint.json', 'status.tsv'}
_SECRET_KEY = re.compile(r'(?:password|passwd|(?:client[_-]?)?secret|credentials?|authorization|api[_-]?key|access[_-]?token|refresh[_-]?token|token)', re.I)
_USER_KEY = re.compile(r'(?:user(?:[_-]?name)?|login[_-]?name)', re.I)
_SECRET_TEXT = re.compile(r'''(?ix)(\b(?:password|passwd|(?:client[_-]?)?secret|credentials?|authorization|api[_-]?key|access[_-]?token|refresh[_-]?token|token)\b["']?\s*[=:]\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,;}]+)''')
def _scrub_text(text):
# Scrub other users as well as the exporting user's home, including JSON
# escaped Windows paths. Preserve suffixes useful for fault diagnosis.
text = re.sub(r'(?i)(?:[A-Z]:[\\/]+Users[\\/]+|/Users/|/home/)[^\\/\s"\']+', '<USER_HOME>', text)
home = str(Path.home())
for prefix in {home, home.replace('\\', '/'), home.replace('\\', '\\\\')}:
text = re.sub(re.escape(prefix), '<USER_HOME>', text, flags=re.I)
username = os.environ.get('USERNAME') or os.environ.get('USER')
if username and len(username) > 2:
text = re.sub(r'(?<![\w])' + re.escape(username) + r'(?![\w])', '<USER>', text, flags=re.I)
text = re.sub(r'(?i)\bBearer\s+[A-Za-z0-9._~+/-]+=*', 'Bearer <REDACTED>', text)
text = _SECRET_TEXT.sub(r'\1<REDACTED>', text)
text = re.sub(r'''(?ix)(\b(?:user(?:[_-]?name)?|login[_-]?name)\b["']?\s*[=:]\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,;}]+)''', r'\1<USER>', text)
text = re.sub(r'(?i)(https?://)[^\s/@:]+:[^\s/@]+@', r'\1<REDACTED>@', text)
return text
def _scrub_json(value):
if isinstance(value, dict):
return {_scrub_text(str(key)): ('<REDACTED>' if _SECRET_KEY.fullmatch(str(key))
else '<USER>' if _USER_KEY.fullmatch(str(key)) else _scrub_json(item))
for key, item in value.items()}
if isinstance(value, list):
return [_scrub_json(item) for item in value]
return _scrub_text(value) if isinstance(value, str) else value
def _allowed(parts):
if len(parts) == 2 and parts[0] == 'logs':
return bool(re.fullmatch(r'[^/]+\.log(?:\.\d+)?', parts[-1]))
if len(parts) == 3 and parts[0] == 'batch-resources':
return parts[-1] in _BATCH_NAMES
if len(parts) >= 5 and parts[0] == 'batch-resources' and parts[2] == 'attempts':
return parts[-1].endswith('.json')
if len(parts) >= 2 and parts[0] == 'attempts':
return parts[-1].endswith('.json')
if len(parts) == 3 and parts[0] == 'tasks':
return bool(re.fullmatch(r'events\.jsonl(?:\.\d+)?', parts[-1]))
if len(parts) >= 2 and parts[0] in {'queue', 'queues', 'task-queues'}:
return parts[-1].endswith('.json')
if len(parts) >= 2 and any(value in parts for value in ('quick-run', 'quickrun', 'quick-run-history')):
return parts[-1] in _QUICK_NAMES
return False
def export_diagnostic_bundle(data_root, destination: Path) -> Path:
"""Write a new ZIP, excluding links, binaries, inputs and credential configs.
This is a point-in-time best-effort read: active writers may produce partial
log lines. Unsupported/unreadable items are documented, never dereferenced.
"""
root, destination = canonical_path(data_root), canonical_path(destination)
if destination.suffix.lower() != '.zip':
raise ValueError('diagnostic destination must have a .zip suffix')
manifest = {'version': 1, 'included': [], 'omissions': [], 'unknowns': [
'External desktop handle holders are unknown; empty window/thread lists do not prove release.',
'Active logs and metadata are point-in-time reads; no source CAD, libraries, BOM, database or environment configuration is included.'],
'limits': {'max_file_bytes': MAX_FILE_BYTES, 'max_bundle_bytes': MAX_BUNDLE_BYTES, 'max_files': MAX_FILES}}
total = 0
physical = internal_path(root)
# os.walk does not descend symlinks; explicitly prune Windows junctions too.
with zipfile.ZipFile(internal_path(destination), 'x', compression=zipfile.ZIP_DEFLATED) as archive:
for directory, folders, files in os.walk(physical, followlinks=False):
base = Path(directory)
safe_folders = []
for name in sorted(folders):
entry = base / name
relative = entry.relative_to(physical).as_posix()
info = entry.lstat()
if stat.S_ISLNK(info.st_mode) or getattr(info, 'st_file_attributes', 0) & 0x400:
manifest['omissions'].append({'path': _scrub_text(relative), 'reason': 'link_or_reparse_point'})
elif base == physical and name not in {'logs', 'batch-resources', 'attempts', 'tasks', 'queue', 'queues', 'task-queues', 'quick-run', 'quick-run-history'}:
manifest['omissions'].append({'path': _scrub_text(relative), 'reason': 'outside_allowlist'})
else:
safe_folders.append(name)
folders[:] = safe_folders
for name in sorted(files):
path = base / name
relative = path.relative_to(physical)
label = _scrub_text(relative.as_posix())
reason = None
try:
info = path.lstat()
if not _allowed(relative.parts):
reason = 'outside_allowlist'
elif stat.S_ISLNK(info.st_mode) or getattr(info, 'st_file_attributes', 0) & 0x400 or info.st_nlink != 1:
reason = 'link_or_reparse_point'
elif not stat.S_ISREG(info.st_mode):
reason = 'not_regular_file'
elif info.st_size > MAX_FILE_BYTES or total + info.st_size > MAX_BUNDLE_BYTES or len(manifest['included']) >= MAX_FILES:
reason = 'size_or_count_limit'
else:
with path.open('rb') as source:
payload = source.read(MAX_FILE_BYTES + 1)
if len(payload) > MAX_FILE_BYTES or b'\x00' in payload:
reason = 'binary_or_size_limit'
else:
text = payload.decode('utf-8-sig')
if relative.suffix == '.json':
text = json.dumps(_scrub_json(json.loads(text)), ensure_ascii=False, indent=2)
elif '.jsonl' in relative.name:
text = '\n'.join(json.dumps(_scrub_json(json.loads(line)), ensure_ascii=False)
for line in text.splitlines() if line.strip()) + '\n'
else:
text = _scrub_text(text)
archive.writestr(label, text)
total += len(payload)
manifest['included'].append(label)
except (OSError, UnicodeError, ValueError):
reason = 'unreadable_or_invalid_text'
if reason:
manifest['omissions'].append({'path': label, 'reason': reason})
archive.writestr('manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2))
return destination
@@ -8,6 +8,7 @@ from __future__ import annotations
from dataclasses import dataclass
from contextlib import ExitStack
from copy import deepcopy
import ctypes
from ctypes import wintypes
import math
@@ -24,6 +25,19 @@ from .errors import TaskCenterError
DESKTOP_RELEASE_TIMEOUT_SECONDS = 5.0
def _native_handle(handle):
try:
return int(handle) if handle is not None else None
except (TypeError, ValueError):
return None
def _resource_event(operation, handle, success, error=None):
return {'operation': operation, 'handle': _native_handle(handle), 'success': success,
'pid': os.getpid(), 'thread_id': get_native_id(), 'time_ns': time.time_ns(),
'error': str(error) if error is not None else None}
@dataclass(frozen=True, slots=True)
class StartupInfo:
lpDesktop: str
@@ -435,6 +449,8 @@ class PrivateDesktop:
self._processes: list[subprocess.Popen[str]] = []
self._api = api
self.cleanup_record = None
self._desktop_handle_history = []
self._process_history = []
def __enter__(self) -> "PrivateDesktop":
if not self.enabled:
@@ -451,7 +467,12 @@ class PrivateDesktop:
if os.name != "nt" and self._api is None:
return
self._api = self._api or _Win32Api()
self._handle = self._api.create_desktop(self.name)
try:
self._handle = self._api.create_desktop(self.name)
except Exception as error:
self._desktop_handle_history.append(_resource_event('create', None, False, error))
raise
self._desktop_handle_history.append(_resource_event('create', self._handle, self._handle is not None))
def start_process(
self,
@@ -471,8 +492,20 @@ class PrivateDesktop:
options = {} if stdout is None and stderr is None else {'stdout': stdout, 'stderr': stderr}
process = self._api.create_process(command, build_startup_info(self.name).lpDesktop, cwd, env, **options)
self._processes.append(process)
self._process_history.append({'pid': getattr(process, 'pid', None),
'owner_pid': os.getpid(), 'owner_thread_id': get_native_id(),
'started_time_ns': time.time_ns(), 'process_handle': _native_handle(getattr(process, 'handle', None)),
'job_handle': _native_handle(getattr(process, 'job', None)), 'forgotten': False})
return process
def _remember_process_cleanup(self, process, *, forgotten=False):
for record in reversed(self._process_history):
if record['pid'] == getattr(process, 'pid', None) and not record['forgotten']:
record['cleanup'] = deepcopy(getattr(process, 'cleanup_record', {}))
record['forgotten'] = forgotten
record['observed_time_ns'] = time.time_ns()
break
def forget_closed_process(self, process):
"""Drop a worker only after its owning wrapper confirmed all cleanup."""
record = getattr(process, 'cleanup_record', {})
@@ -482,6 +515,7 @@ class PrivateDesktop:
or record.get('errors') or process.poll() is None):
raise OSError('cannot forget a worker with unconfirmed owned resources')
self._processes = [item for item in self._processes if item is not process]
self._remember_process_cleanup(process, forgotten=True)
def close(self) -> None:
errors, remaining = [], []
@@ -497,6 +531,7 @@ class PrivateDesktop:
failed = True
self.cleanup_record['processes'].append(getattr(process, 'cleanup_record', {
'pid': getattr(process, 'pid', None), 'exit_code': getattr(process, 'returncode', None)}))
self._remember_process_cleanup(process)
if failed:
remaining.append(process)
self._processes = list(reversed(remaining))
@@ -508,11 +543,15 @@ class PrivateDesktop:
errors.append(error)
close_desktop = getattr(self._handle, "CloseDesktop", None)
if close_desktop:
event = _resource_event('close', self._handle, False)
try:
close_desktop()
event['success'] = True
self._handle = None
except Exception as error:
errors.append(error)
event['error'] = str(error)
self._desktop_handle_history.append(event)
confirm = getattr(self._api, 'confirm_desktop_released', None)
if confirm and self._handle is None:
try:
@@ -521,5 +560,50 @@ class PrivateDesktop:
errors.append(error)
self.cleanup_record.update(desktop_handle_closed=self._handle is None,
desktop_confirmation=getattr(self._api, 'desktop_release_check', None),
desktop_handle_history=deepcopy(self._desktop_handle_history),
process_history=deepcopy(self._process_history),
diagnostic_scope={'external_handle_holders': 'unknown',
'thread_desktops': 'unknown unless observed through residual windows; no exhaustive holder scan',
'empty_windows_prove_no_holders': False},
errors=[str(error) for error in errors])
if errors:
self.cleanup_record['owned_resource_diagnostics'] = self._owned_resource_diagnostics()
_raise_cleanup_errors(errors)
def _owned_resource_diagnostics(self):
"""Bounded queries of our handles only; never terminate/query foreign PIDs.
GetThreadDesktop can introduce borrowed USER references. Do not run a
global thread scan in this long-lived owner while retrying release.
Window observations identify only window-owning threads, not every
thread or an external OpenDesktop handle holder.
"""
resources = []
for process in self._processes:
item = {'pid': getattr(process, 'pid', None),
'process_handle': _native_handle(getattr(process, 'handle', None)),
'job_handle': _native_handle(getattr(process, 'job', None)),
'thread_handle': _native_handle(getattr(process, 'thread', None)),
'cleanup': deepcopy(getattr(process, 'cleanup_record', {}))}
try:
item['exit_code'] = process.poll()
except Exception as error:
item['poll_error'] = str(error)
job = getattr(process, 'job', None)
job_api = getattr(process, '_job_api', None)
if job is not None and job_api is not None:
try:
item['job_accounting'] = job_api.QueryInformationJobObject(
job, job_api.JobObjectBasicAccountingInformation)
except Exception as error:
item['job_query_error'] = str(error)
resources.append(item)
confirmation = getattr(self._api, 'desktop_release_check', None) or {}
return {'owned_processes': resources,
'pending_launch_cleanup_count': len(getattr(self._api, '_pending_cleanup', [])),
'residual_windows': deepcopy(confirmation.get('remaining_windows')),
'window_query_error': confirmation.get('window_query_error'),
'external_handle_holders': 'unknown',
'thread_desktops': {'observations': deepcopy(confirmation.get('remaining_windows', [])),
'complete': False, 'unobserved_threads': 'unknown'},
'empty_windows_prove_no_holders': False}
+114
View File
@@ -0,0 +1,114 @@
import json
from pathlib import Path
import zipfile
import pytest
from dx_productivity.trial_task_center import diagnostics_bundle as bundle
def test_bundle_allowlist_redacts_secrets_and_home_and_omits_inputs(tmp_path):
root = tmp_path / 'data'
session = root / 'batch-resources' / 'batch1'
session.mkdir(parents=True)
(root / 'logs').mkdir()
(root / 'logs/application.log.1').write_text('user=C:\\Users\\Alice token=super-secret password="a b"', encoding='utf-8')
(session / 'session.json').write_text(json.dumps({'pid': 123, 'api_key': 'hidden', 'input': 'C:\\Users\\Alice\\work\\bom.xlsx'}))
(session / 'source.xlsx').write_bytes(b'private BOM')
(root / 'task-center.db').write_bytes(b'private DB')
target = tmp_path / 'support.zip'
assert bundle.export_diagnostic_bundle(root, target) == target
with zipfile.ZipFile(target) as archive:
text = '\n'.join(archive.read(name).decode('utf-8') for name in archive.namelist())
assert 'super-secret' not in text and 'hidden' not in text and 'a b' not in text
assert 'Alice' not in text and 'private BOM' not in text and 'private DB' not in text
assert json.loads(archive.read('batch-resources/batch1/session.json'))['pid'] == 123
manifest = json.loads(archive.read('manifest.json'))
assert manifest['omissions'] and manifest['unknowns']
with pytest.raises(FileExistsError):
bundle.export_diagnostic_bundle(root, target)
def test_bundle_rejects_non_zip_destination(tmp_path):
with pytest.raises(ValueError):
bundle.export_diagnostic_bundle(tmp_path, tmp_path / 'output.txt')
def test_bundle_does_not_follow_links_or_export_binary_logs(tmp_path):
root = tmp_path / 'data'
(root / 'logs').mkdir(parents=True)
(root / 'logs/binary.log').write_bytes(b'\x00\xff\x00')
external = tmp_path / 'secret.log'
external.write_text('outside data root')
try:
(root / 'logs/link.log').symlink_to(external)
except OSError:
pass
output = bundle.export_diagnostic_bundle(root, tmp_path / 'support.zip')
with zipfile.ZipFile(output) as archive:
assert 'logs/binary.log' not in archive.namelist()
assert 'logs/link.log' not in archive.namelist()
def test_bundle_redacts_nested_client_secret_and_preserves_support_ids(tmp_path):
root = tmp_path / 'data'
directory = root / 'attempts'
directory.mkdir(parents=True)
(directory / 'attempt.json').write_text(json.dumps({'client_secret': 'oauth-private',
'support_id': 'DXT-123', 'username': 'ExternalUser',
'nested': {'authorization': 'Bearer a.b.c'}}))
output = bundle.export_diagnostic_bundle(root, tmp_path / 'support.zip')
with zipfile.ZipFile(output) as archive:
text = archive.read('attempts/attempt.json').decode()
assert 'oauth-private' not in text and 'a.b.c' not in text
assert 'ExternalUser' not in text
assert 'DXT-123' in text
def test_bundle_excludes_hardlinks_and_redacts_jsonl_credentials(tmp_path):
import os
root = tmp_path / 'data'
logs = root / 'logs'
logs.mkdir(parents=True)
external = tmp_path / 'private.log'
external.write_text('private content')
os.link(external, logs / 'linked.log')
tasks = root / 'tasks' / 'task1'
tasks.mkdir(parents=True)
(tasks / 'events.jsonl').write_text(json.dumps({'token': 'private-token'}) + '\n')
output = bundle.export_diagnostic_bundle(root, tmp_path / 'support.zip')
with zipfile.ZipFile(output) as archive:
assert 'logs/linked.log' not in archive.namelist()
assert 'private-token' not in archive.read('tasks/task1/events.jsonl').decode()
assert json.loads(archive.read('tasks/task1/events.jsonl'))['token'] == '<REDACTED>'
def test_bundle_size_limit_omits_oversized_files(tmp_path, monkeypatch):
(tmp_path / 'logs').mkdir()
(tmp_path / 'logs/application.log').write_text('too large for configured limit')
monkeypatch.setattr(bundle, 'MAX_FILE_BYTES', 4)
output = bundle.export_diagnostic_bundle(tmp_path, tmp_path / 'support.zip')
with zipfile.ZipFile(output) as archive:
assert 'logs/application.log' not in archive.namelist()
assert any(row['reason'] == 'size_or_count_limit' for row in json.loads(archive.read('manifest.json'))['omissions'])
def test_bundle_includes_real_nested_attempt_queue_and_protection_locations(tmp_path):
names = ['batch-resources/batch1/attempts/g1/attempt1/job1.json',
'task-queues/queue1.json', 'tasks/task1/quick-run/project-checkpoint.json',
'quick-run-history/recovery1/uncertain.flag']
for name in names:
path = tmp_path / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text('{}')
output = bundle.export_diagnostic_bundle(tmp_path, tmp_path / 'support.zip')
with zipfile.ZipFile(output) as archive:
assert set(names).issubset(archive.namelist())
def test_plain_log_authorization_redacts_bearer_value(tmp_path):
(tmp_path / 'logs').mkdir()
(tmp_path / 'logs/application.log').write_text('Authorization=Bearer private-bearer-value\n')
output = bundle.export_diagnostic_bundle(tmp_path, tmp_path / 'support.zip')
with zipfile.ZipFile(output) as archive:
assert 'private-bearer-value' not in archive.read('logs/application.log').decode()
+69
View File
@@ -68,6 +68,75 @@ def test_startup_info_always_targets_private_desktop():
assert info.lpDesktop == 'winsta0\\dx-private-test'
def test_desktop_records_native_handle_before_close_and_failed_retry():
class Desktop:
fail = True
def __int__(self):
return 761
def CloseDesktop(self):
if self.fail:
self.fail = False
raise OSError('busy')
handle = Desktop()
desktop = PrivateDesktop('private', enabled=True,
api=SimpleNamespace(create_desktop=lambda _: handle))
desktop.__enter__()
with pytest.raises(TaskCenterError):
desktop.close()
desktop.close()
history = desktop.cleanup_record['desktop_handle_history']
assert [event['operation'] for event in history] == ['create', 'close', 'close']
assert [event['success'] for event in history] == [True, False, True]
assert all(event['handle'] == 761 and event['pid'] == os.getpid() and event['thread_id'] for event in history)
def test_desktop_preserves_forgotten_worker_history_and_unknown_holder_scope():
api, _, state, *_ = native_api()
api.confirm_desktop_released = lambda _: None
desktop = PrivateDesktop('private', enabled=True, api=api)
desktop._handle = SimpleNamespace(CloseDesktop=lambda: None)
process = desktop.start_process(('python.exe',))
process.close()
desktop.forget_closed_process(process)
desktop.close()
history = desktop.cleanup_record['process_history']
assert history[0]['pid'] == 42 and history[0]['cleanup']['process_handle_closed']
assert history[0]['forgotten'] is True
assert desktop.cleanup_record['diagnostic_scope']['external_handle_holders'] == 'unknown'
def test_failed_release_reports_windowless_holder_unknown_and_owned_job_failure():
api, _, _, *_ = native_api()
api._pending_cleanup = []
def confirm(_):
api.desktop_release_check = {'released': False, 'remaining_windows': []}
raise OSError('still exists')
api.confirm_desktop_released = confirm
desktop = PrivateDesktop('private', enabled=True, api=api)
desktop._handle = SimpleNamespace(CloseDesktop=lambda: None)
process = desktop.start_process(('python.exe',))
process.close = lambda: (_ for _ in ()).throw(OSError('owned handles retained'))
with pytest.raises(TaskCenterError):
desktop.close()
report = desktop.cleanup_record['owned_resource_diagnostics']
assert report['residual_windows'] == []
assert report['external_handle_holders'] == 'unknown'
assert report['thread_desktops']['complete'] is False
assert report['owned_processes'][0]['pid'] == 42
assert report['owned_processes'][0]['job_accounting']['ActiveProcesses'] == 0
@pytest.mark.skipif(os.name != 'nt', reason='native Windows resource history')
def test_real_helper_retains_completed_lifecycle_after_forget():
with PrivateDesktop('dx-test-' + uuid4().hex, enabled=True) as desktop:
process = desktop.start_process((sys.executable, '-c', 'pass'))
assert process.wait(timeout=5) == 0
process.close()
desktop.forget_closed_process(process)
assert desktop.cleanup_record['process_history'][0]['pid'] == process.pid
assert desktop.cleanup_record['desktop_handle_history'][0]['handle'] > 0
def test_private_desktop_never_falls_back_to_input_desktop(monkeypatch):
desktop = PrivateDesktop('dx-private-test', enabled=True)
monkeypatch.setattr(desktop, '_create', lambda: None)