diff --git a/src/dx_productivity/trial_task_center/diagnostics_bundle.py b/src/dx_productivity/trial_task_center/diagnostics_bundle.py new file mode 100644 index 0000000..12d60da --- /dev/null +++ b/src/dx_productivity/trial_task_center/diagnostics_bundle.py @@ -0,0 +1,139 @@ +"""Allowlisted, bounded support export; never packages project or CAD inputs.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import re +import stat +import zipfile + +from .path_access import canonical_path, internal_path + +MAX_FILE_BYTES = 4 * 1024 * 1024 +MAX_BUNDLE_BYTES = 32 * 1024 * 1024 +MAX_FILES = 512 +_BATCH_NAMES = {'session.json', 'supervisor.json', 'request.json', 'result.json', + 'business-result.json', 'supervisor.log', 'resource-release.log', 'cancel.json'} +_QUICK_NAMES = {'private-desktop.json', 'checkpoint.json', 'status.json', + 'uncertain.flag', 'resolved-uncertain.flag', 'protection.json', 'cleanup.json', + 'project-session.json', 'project-checkpoint.json', 'status.tsv'} +_SECRET_KEY = re.compile(r'(?:password|passwd|(?:client[_-]?)?secret|credentials?|authorization|api[_-]?key|access[_-]?token|refresh[_-]?token|token)', re.I) +_USER_KEY = re.compile(r'(?:user(?:[_-]?name)?|login[_-]?name)', re.I) +_SECRET_TEXT = re.compile(r'''(?ix)(\b(?:password|passwd|(?:client[_-]?)?secret|credentials?|authorization|api[_-]?key|access[_-]?token|refresh[_-]?token|token)\b["']?\s*[=:]\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,;}]+)''') + + +def _scrub_text(text): + # Scrub other users as well as the exporting user's home, including JSON + # escaped Windows paths. Preserve suffixes useful for fault diagnosis. + text = re.sub(r'(?i)(?:[A-Z]:[\\/]+Users[\\/]+|/Users/|/home/)[^\\/\s"\']+', '', text) + home = str(Path.home()) + for prefix in {home, home.replace('\\', '/'), home.replace('\\', '\\\\')}: + text = re.sub(re.escape(prefix), '', text, flags=re.I) + username = os.environ.get('USERNAME') or os.environ.get('USER') + if username and len(username) > 2: + text = re.sub(r'(?', text, flags=re.I) + text = re.sub(r'(?i)\bBearer\s+[A-Za-z0-9._~+/-]+=*', 'Bearer ', text) + text = _SECRET_TEXT.sub(r'\1', text) + text = re.sub(r'''(?ix)(\b(?:user(?:[_-]?name)?|login[_-]?name)\b["']?\s*[=:]\s*)(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,;}]+)''', r'\1', text) + text = re.sub(r'(?i)(https?://)[^\s/@:]+:[^\s/@]+@', r'\1@', text) + return text + + +def _scrub_json(value): + if isinstance(value, dict): + return {_scrub_text(str(key)): ('' if _SECRET_KEY.fullmatch(str(key)) + else '' if _USER_KEY.fullmatch(str(key)) else _scrub_json(item)) + for key, item in value.items()} + if isinstance(value, list): + return [_scrub_json(item) for item in value] + return _scrub_text(value) if isinstance(value, str) else value + + +def _allowed(parts): + if len(parts) == 2 and parts[0] == 'logs': + return bool(re.fullmatch(r'[^/]+\.log(?:\.\d+)?', parts[-1])) + if len(parts) == 3 and parts[0] == 'batch-resources': + return parts[-1] in _BATCH_NAMES + if len(parts) >= 5 and parts[0] == 'batch-resources' and parts[2] == 'attempts': + return parts[-1].endswith('.json') + if len(parts) >= 2 and parts[0] == 'attempts': + return parts[-1].endswith('.json') + if len(parts) == 3 and parts[0] == 'tasks': + return bool(re.fullmatch(r'events\.jsonl(?:\.\d+)?', parts[-1])) + if len(parts) >= 2 and parts[0] in {'queue', 'queues', 'task-queues'}: + return parts[-1].endswith('.json') + if len(parts) >= 2 and any(value in parts for value in ('quick-run', 'quickrun', 'quick-run-history')): + return parts[-1] in _QUICK_NAMES + return False + + +def export_diagnostic_bundle(data_root, destination: Path) -> Path: + """Write a new ZIP, excluding links, binaries, inputs and credential configs. + + This is a point-in-time best-effort read: active writers may produce partial + log lines. Unsupported/unreadable items are documented, never dereferenced. + """ + root, destination = canonical_path(data_root), canonical_path(destination) + if destination.suffix.lower() != '.zip': + raise ValueError('diagnostic destination must have a .zip suffix') + manifest = {'version': 1, 'included': [], 'omissions': [], 'unknowns': [ + 'External desktop handle holders are unknown; empty window/thread lists do not prove release.', + 'Active logs and metadata are point-in-time reads; no source CAD, libraries, BOM, database or environment configuration is included.'], + 'limits': {'max_file_bytes': MAX_FILE_BYTES, 'max_bundle_bytes': MAX_BUNDLE_BYTES, 'max_files': MAX_FILES}} + total = 0 + physical = internal_path(root) + # os.walk does not descend symlinks; explicitly prune Windows junctions too. + with zipfile.ZipFile(internal_path(destination), 'x', compression=zipfile.ZIP_DEFLATED) as archive: + for directory, folders, files in os.walk(physical, followlinks=False): + base = Path(directory) + safe_folders = [] + for name in sorted(folders): + entry = base / name + relative = entry.relative_to(physical).as_posix() + info = entry.lstat() + if stat.S_ISLNK(info.st_mode) or getattr(info, 'st_file_attributes', 0) & 0x400: + manifest['omissions'].append({'path': _scrub_text(relative), 'reason': 'link_or_reparse_point'}) + elif base == physical and name not in {'logs', 'batch-resources', 'attempts', 'tasks', 'queue', 'queues', 'task-queues', 'quick-run', 'quick-run-history'}: + manifest['omissions'].append({'path': _scrub_text(relative), 'reason': 'outside_allowlist'}) + else: + safe_folders.append(name) + folders[:] = safe_folders + for name in sorted(files): + path = base / name + relative = path.relative_to(physical) + label = _scrub_text(relative.as_posix()) + reason = None + try: + info = path.lstat() + if not _allowed(relative.parts): + reason = 'outside_allowlist' + elif stat.S_ISLNK(info.st_mode) or getattr(info, 'st_file_attributes', 0) & 0x400 or info.st_nlink != 1: + reason = 'link_or_reparse_point' + elif not stat.S_ISREG(info.st_mode): + reason = 'not_regular_file' + elif info.st_size > MAX_FILE_BYTES or total + info.st_size > MAX_BUNDLE_BYTES or len(manifest['included']) >= MAX_FILES: + reason = 'size_or_count_limit' + else: + with path.open('rb') as source: + payload = source.read(MAX_FILE_BYTES + 1) + if len(payload) > MAX_FILE_BYTES or b'\x00' in payload: + reason = 'binary_or_size_limit' + else: + text = payload.decode('utf-8-sig') + if relative.suffix == '.json': + text = json.dumps(_scrub_json(json.loads(text)), ensure_ascii=False, indent=2) + elif '.jsonl' in relative.name: + text = '\n'.join(json.dumps(_scrub_json(json.loads(line)), ensure_ascii=False) + for line in text.splitlines() if line.strip()) + '\n' + else: + text = _scrub_text(text) + archive.writestr(label, text) + total += len(payload) + manifest['included'].append(label) + except (OSError, UnicodeError, ValueError): + reason = 'unreadable_or_invalid_text' + if reason: + manifest['omissions'].append({'path': label, 'reason': reason}) + archive.writestr('manifest.json', json.dumps(manifest, ensure_ascii=False, indent=2)) + return destination diff --git a/src/dx_productivity/trial_task_center/isolated_desktop.py b/src/dx_productivity/trial_task_center/isolated_desktop.py index cef0146..c1561a2 100644 --- a/src/dx_productivity/trial_task_center/isolated_desktop.py +++ b/src/dx_productivity/trial_task_center/isolated_desktop.py @@ -8,6 +8,7 @@ from __future__ import annotations from dataclasses import dataclass from contextlib import ExitStack +from copy import deepcopy import ctypes from ctypes import wintypes import math @@ -24,6 +25,19 @@ from .errors import TaskCenterError DESKTOP_RELEASE_TIMEOUT_SECONDS = 5.0 +def _native_handle(handle): + try: + return int(handle) if handle is not None else None + except (TypeError, ValueError): + return None + + +def _resource_event(operation, handle, success, error=None): + return {'operation': operation, 'handle': _native_handle(handle), 'success': success, + 'pid': os.getpid(), 'thread_id': get_native_id(), 'time_ns': time.time_ns(), + 'error': str(error) if error is not None else None} + + @dataclass(frozen=True, slots=True) class StartupInfo: lpDesktop: str @@ -435,6 +449,8 @@ class PrivateDesktop: self._processes: list[subprocess.Popen[str]] = [] self._api = api self.cleanup_record = None + self._desktop_handle_history = [] + self._process_history = [] def __enter__(self) -> "PrivateDesktop": if not self.enabled: @@ -451,7 +467,12 @@ class PrivateDesktop: if os.name != "nt" and self._api is None: return self._api = self._api or _Win32Api() - self._handle = self._api.create_desktop(self.name) + try: + self._handle = self._api.create_desktop(self.name) + except Exception as error: + self._desktop_handle_history.append(_resource_event('create', None, False, error)) + raise + self._desktop_handle_history.append(_resource_event('create', self._handle, self._handle is not None)) def start_process( self, @@ -471,8 +492,20 @@ class PrivateDesktop: options = {} if stdout is None and stderr is None else {'stdout': stdout, 'stderr': stderr} process = self._api.create_process(command, build_startup_info(self.name).lpDesktop, cwd, env, **options) self._processes.append(process) + self._process_history.append({'pid': getattr(process, 'pid', None), + 'owner_pid': os.getpid(), 'owner_thread_id': get_native_id(), + 'started_time_ns': time.time_ns(), 'process_handle': _native_handle(getattr(process, 'handle', None)), + 'job_handle': _native_handle(getattr(process, 'job', None)), 'forgotten': False}) return process + def _remember_process_cleanup(self, process, *, forgotten=False): + for record in reversed(self._process_history): + if record['pid'] == getattr(process, 'pid', None) and not record['forgotten']: + record['cleanup'] = deepcopy(getattr(process, 'cleanup_record', {})) + record['forgotten'] = forgotten + record['observed_time_ns'] = time.time_ns() + break + def forget_closed_process(self, process): """Drop a worker only after its owning wrapper confirmed all cleanup.""" record = getattr(process, 'cleanup_record', {}) @@ -482,6 +515,7 @@ class PrivateDesktop: or record.get('errors') or process.poll() is None): raise OSError('cannot forget a worker with unconfirmed owned resources') self._processes = [item for item in self._processes if item is not process] + self._remember_process_cleanup(process, forgotten=True) def close(self) -> None: errors, remaining = [], [] @@ -497,6 +531,7 @@ class PrivateDesktop: failed = True self.cleanup_record['processes'].append(getattr(process, 'cleanup_record', { 'pid': getattr(process, 'pid', None), 'exit_code': getattr(process, 'returncode', None)})) + self._remember_process_cleanup(process) if failed: remaining.append(process) self._processes = list(reversed(remaining)) @@ -508,11 +543,15 @@ class PrivateDesktop: errors.append(error) close_desktop = getattr(self._handle, "CloseDesktop", None) if close_desktop: + event = _resource_event('close', self._handle, False) try: close_desktop() + event['success'] = True self._handle = None except Exception as error: errors.append(error) + event['error'] = str(error) + self._desktop_handle_history.append(event) confirm = getattr(self._api, 'confirm_desktop_released', None) if confirm and self._handle is None: try: @@ -521,5 +560,50 @@ class PrivateDesktop: errors.append(error) self.cleanup_record.update(desktop_handle_closed=self._handle is None, desktop_confirmation=getattr(self._api, 'desktop_release_check', None), + desktop_handle_history=deepcopy(self._desktop_handle_history), + process_history=deepcopy(self._process_history), + diagnostic_scope={'external_handle_holders': 'unknown', + 'thread_desktops': 'unknown unless observed through residual windows; no exhaustive holder scan', + 'empty_windows_prove_no_holders': False}, errors=[str(error) for error in errors]) + if errors: + self.cleanup_record['owned_resource_diagnostics'] = self._owned_resource_diagnostics() _raise_cleanup_errors(errors) + + def _owned_resource_diagnostics(self): + """Bounded queries of our handles only; never terminate/query foreign PIDs. + + GetThreadDesktop can introduce borrowed USER references. Do not run a + global thread scan in this long-lived owner while retrying release. + Window observations identify only window-owning threads, not every + thread or an external OpenDesktop handle holder. + """ + resources = [] + for process in self._processes: + item = {'pid': getattr(process, 'pid', None), + 'process_handle': _native_handle(getattr(process, 'handle', None)), + 'job_handle': _native_handle(getattr(process, 'job', None)), + 'thread_handle': _native_handle(getattr(process, 'thread', None)), + 'cleanup': deepcopy(getattr(process, 'cleanup_record', {}))} + try: + item['exit_code'] = process.poll() + except Exception as error: + item['poll_error'] = str(error) + job = getattr(process, 'job', None) + job_api = getattr(process, '_job_api', None) + if job is not None and job_api is not None: + try: + item['job_accounting'] = job_api.QueryInformationJobObject( + job, job_api.JobObjectBasicAccountingInformation) + except Exception as error: + item['job_query_error'] = str(error) + resources.append(item) + confirmation = getattr(self._api, 'desktop_release_check', None) or {} + return {'owned_processes': resources, + 'pending_launch_cleanup_count': len(getattr(self._api, '_pending_cleanup', [])), + 'residual_windows': deepcopy(confirmation.get('remaining_windows')), + 'window_query_error': confirmation.get('window_query_error'), + 'external_handle_holders': 'unknown', + 'thread_desktops': {'observations': deepcopy(confirmation.get('remaining_windows', [])), + 'complete': False, 'unobserved_threads': 'unknown'}, + 'empty_windows_prove_no_holders': False} diff --git a/tests/v1_1/test_diagnostics_bundle.py b/tests/v1_1/test_diagnostics_bundle.py new file mode 100644 index 0000000..5c92206 --- /dev/null +++ b/tests/v1_1/test_diagnostics_bundle.py @@ -0,0 +1,114 @@ +import json +from pathlib import Path +import zipfile + +import pytest + +from dx_productivity.trial_task_center import diagnostics_bundle as bundle + + +def test_bundle_allowlist_redacts_secrets_and_home_and_omits_inputs(tmp_path): + root = tmp_path / 'data' + session = root / 'batch-resources' / 'batch1' + session.mkdir(parents=True) + (root / 'logs').mkdir() + (root / 'logs/application.log.1').write_text('user=C:\\Users\\Alice token=super-secret password="a b"', encoding='utf-8') + (session / 'session.json').write_text(json.dumps({'pid': 123, 'api_key': 'hidden', 'input': 'C:\\Users\\Alice\\work\\bom.xlsx'})) + (session / 'source.xlsx').write_bytes(b'private BOM') + (root / 'task-center.db').write_bytes(b'private DB') + target = tmp_path / 'support.zip' + assert bundle.export_diagnostic_bundle(root, target) == target + with zipfile.ZipFile(target) as archive: + text = '\n'.join(archive.read(name).decode('utf-8') for name in archive.namelist()) + assert 'super-secret' not in text and 'hidden' not in text and 'a b' not in text + assert 'Alice' not in text and 'private BOM' not in text and 'private DB' not in text + assert json.loads(archive.read('batch-resources/batch1/session.json'))['pid'] == 123 + manifest = json.loads(archive.read('manifest.json')) + assert manifest['omissions'] and manifest['unknowns'] + with pytest.raises(FileExistsError): + bundle.export_diagnostic_bundle(root, target) + + +def test_bundle_rejects_non_zip_destination(tmp_path): + with pytest.raises(ValueError): + bundle.export_diagnostic_bundle(tmp_path, tmp_path / 'output.txt') + + +def test_bundle_does_not_follow_links_or_export_binary_logs(tmp_path): + root = tmp_path / 'data' + (root / 'logs').mkdir(parents=True) + (root / 'logs/binary.log').write_bytes(b'\x00\xff\x00') + external = tmp_path / 'secret.log' + external.write_text('outside data root') + try: + (root / 'logs/link.log').symlink_to(external) + except OSError: + pass + output = bundle.export_diagnostic_bundle(root, tmp_path / 'support.zip') + with zipfile.ZipFile(output) as archive: + assert 'logs/binary.log' not in archive.namelist() + assert 'logs/link.log' not in archive.namelist() + + +def test_bundle_redacts_nested_client_secret_and_preserves_support_ids(tmp_path): + root = tmp_path / 'data' + directory = root / 'attempts' + directory.mkdir(parents=True) + (directory / 'attempt.json').write_text(json.dumps({'client_secret': 'oauth-private', + 'support_id': 'DXT-123', 'username': 'ExternalUser', + 'nested': {'authorization': 'Bearer a.b.c'}})) + output = bundle.export_diagnostic_bundle(root, tmp_path / 'support.zip') + with zipfile.ZipFile(output) as archive: + text = archive.read('attempts/attempt.json').decode() + assert 'oauth-private' not in text and 'a.b.c' not in text + assert 'ExternalUser' not in text + assert 'DXT-123' in text + + +def test_bundle_excludes_hardlinks_and_redacts_jsonl_credentials(tmp_path): + import os + root = tmp_path / 'data' + logs = root / 'logs' + logs.mkdir(parents=True) + external = tmp_path / 'private.log' + external.write_text('private content') + os.link(external, logs / 'linked.log') + tasks = root / 'tasks' / 'task1' + tasks.mkdir(parents=True) + (tasks / 'events.jsonl').write_text(json.dumps({'token': 'private-token'}) + '\n') + output = bundle.export_diagnostic_bundle(root, tmp_path / 'support.zip') + with zipfile.ZipFile(output) as archive: + assert 'logs/linked.log' not in archive.namelist() + assert 'private-token' not in archive.read('tasks/task1/events.jsonl').decode() + assert json.loads(archive.read('tasks/task1/events.jsonl'))['token'] == '' + + +def test_bundle_size_limit_omits_oversized_files(tmp_path, monkeypatch): + (tmp_path / 'logs').mkdir() + (tmp_path / 'logs/application.log').write_text('too large for configured limit') + monkeypatch.setattr(bundle, 'MAX_FILE_BYTES', 4) + output = bundle.export_diagnostic_bundle(tmp_path, tmp_path / 'support.zip') + with zipfile.ZipFile(output) as archive: + assert 'logs/application.log' not in archive.namelist() + assert any(row['reason'] == 'size_or_count_limit' for row in json.loads(archive.read('manifest.json'))['omissions']) + + +def test_bundle_includes_real_nested_attempt_queue_and_protection_locations(tmp_path): + names = ['batch-resources/batch1/attempts/g1/attempt1/job1.json', + 'task-queues/queue1.json', 'tasks/task1/quick-run/project-checkpoint.json', + 'quick-run-history/recovery1/uncertain.flag'] + for name in names: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('{}') + output = bundle.export_diagnostic_bundle(tmp_path, tmp_path / 'support.zip') + with zipfile.ZipFile(output) as archive: + assert set(names).issubset(archive.namelist()) + + +def test_plain_log_authorization_redacts_bearer_value(tmp_path): + (tmp_path / 'logs').mkdir() + (tmp_path / 'logs/application.log').write_text('Authorization=Bearer private-bearer-value\n') + output = bundle.export_diagnostic_bundle(tmp_path, tmp_path / 'support.zip') + with zipfile.ZipFile(output) as archive: + assert 'private-bearer-value' not in archive.read('logs/application.log').decode() diff --git a/tests/v1_1/test_isolated_desktop.py b/tests/v1_1/test_isolated_desktop.py index b62092a..19b10ac 100644 --- a/tests/v1_1/test_isolated_desktop.py +++ b/tests/v1_1/test_isolated_desktop.py @@ -68,6 +68,75 @@ def test_startup_info_always_targets_private_desktop(): assert info.lpDesktop == 'winsta0\\dx-private-test' +def test_desktop_records_native_handle_before_close_and_failed_retry(): + class Desktop: + fail = True + def __int__(self): + return 761 + def CloseDesktop(self): + if self.fail: + self.fail = False + raise OSError('busy') + handle = Desktop() + desktop = PrivateDesktop('private', enabled=True, + api=SimpleNamespace(create_desktop=lambda _: handle)) + desktop.__enter__() + with pytest.raises(TaskCenterError): + desktop.close() + desktop.close() + history = desktop.cleanup_record['desktop_handle_history'] + assert [event['operation'] for event in history] == ['create', 'close', 'close'] + assert [event['success'] for event in history] == [True, False, True] + assert all(event['handle'] == 761 and event['pid'] == os.getpid() and event['thread_id'] for event in history) + + +def test_desktop_preserves_forgotten_worker_history_and_unknown_holder_scope(): + api, _, state, *_ = native_api() + api.confirm_desktop_released = lambda _: None + desktop = PrivateDesktop('private', enabled=True, api=api) + desktop._handle = SimpleNamespace(CloseDesktop=lambda: None) + process = desktop.start_process(('python.exe',)) + process.close() + desktop.forget_closed_process(process) + desktop.close() + history = desktop.cleanup_record['process_history'] + assert history[0]['pid'] == 42 and history[0]['cleanup']['process_handle_closed'] + assert history[0]['forgotten'] is True + assert desktop.cleanup_record['diagnostic_scope']['external_handle_holders'] == 'unknown' + + +def test_failed_release_reports_windowless_holder_unknown_and_owned_job_failure(): + api, _, _, *_ = native_api() + api._pending_cleanup = [] + def confirm(_): + api.desktop_release_check = {'released': False, 'remaining_windows': []} + raise OSError('still exists') + api.confirm_desktop_released = confirm + desktop = PrivateDesktop('private', enabled=True, api=api) + desktop._handle = SimpleNamespace(CloseDesktop=lambda: None) + process = desktop.start_process(('python.exe',)) + process.close = lambda: (_ for _ in ()).throw(OSError('owned handles retained')) + with pytest.raises(TaskCenterError): + desktop.close() + report = desktop.cleanup_record['owned_resource_diagnostics'] + assert report['residual_windows'] == [] + assert report['external_handle_holders'] == 'unknown' + assert report['thread_desktops']['complete'] is False + assert report['owned_processes'][0]['pid'] == 42 + assert report['owned_processes'][0]['job_accounting']['ActiveProcesses'] == 0 + + +@pytest.mark.skipif(os.name != 'nt', reason='native Windows resource history') +def test_real_helper_retains_completed_lifecycle_after_forget(): + with PrivateDesktop('dx-test-' + uuid4().hex, enabled=True) as desktop: + process = desktop.start_process((sys.executable, '-c', 'pass')) + assert process.wait(timeout=5) == 0 + process.close() + desktop.forget_closed_process(process) + assert desktop.cleanup_record['process_history'][0]['pid'] == process.pid + assert desktop.cleanup_record['desktop_handle_history'][0]['handle'] > 0 + + def test_private_desktop_never_falls_back_to_input_desktop(monkeypatch): desktop = PrivateDesktop('dx-private-test', enabled=True) monkeypatch.setattr(desktop, '_create', lambda: None)