feat(dsh): harden health decisions and issue transport

This commit is contained in:
阿白叔 committed 2026-10-31 09:50:35 +08:00
1 parent 736b582bc1
commit eee4e6bde9
18 files changed
+592 -11

No files matched your search

+85
View File
@@ -0,0 +1,85 @@
"""Append-only, hash-linked records for automatic DSH decisions."""
from __future__ import annotations
from datetime import datetime, timezone
from hashlib import sha256
import json
from pathlib import Path
import re
from .evidence import scrub
_ID = re.compile(r"^[0-9a-f]{24}$")
_MAX = 2000
def _bounded(value, limit=1000):
value = scrub(value)
if isinstance(value, str):
return value[:limit]
if isinstance(value, list):
return [_bounded(item, limit) for item in value[:20]]
if isinstance(value, dict):
return {str(key)[:80]: _bounded(item, limit) for key, item in list(value.items())[:40]}
return value
class DecisionLog:
def __init__(self, data_root):
self.path = Path(data_root) / "dsh" / "decisions.jsonl"
def append(self, record):
if not isinstance(record, dict) or not _ID.fullmatch(str(record.get("incident_id", ""))):
raise ValueError("incident_id")
rows = self.list(limit=_MAX)
previous = rows[-1]["record_hash"] if rows else ""
value = {"sequence": len(rows) + 1, "at_utc": datetime.now(timezone.utc).isoformat(),
"previous_hash": previous, "incident_id": record["incident_id"],
"trigger": _bounded(record.get("trigger", ""), 160),
"evidence_refs": _bounded(record.get("evidence_refs", []), 300),
"gates": _bounded(record.get("gates", {}), 300),
"candidate_action": _bounded(record.get("candidate_action", "none"), 160),
"authorized_scope": _bounded(record.get("authorized_scope", {}), 600),
"outcome": _bounded(record.get("outcome", "unknown"), 160),
"human_takeover": bool(record.get("human_takeover", False))}
payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
value["record_hash"] = sha256(payload.encode("utf8")).hexdigest()
self.path.parent.mkdir(parents=True, exist_ok=True)
with self.path.open("a", encoding="utf8") as stream:
stream.write(json.dumps(value, ensure_ascii=False, sort_keys=True) + "\n")
stream.flush()
return dict(value)
def list(self, limit=20):
if not isinstance(limit, int) or limit <= 0:
return []
rows = []
try:
with self.path.open(encoding="utf8") as stream:
for line in stream:
if len(rows) >= min(limit, _MAX):
break
try:
value = json.loads(line)
except (ValueError, TypeError):
continue
if isinstance(value, dict) and _ID.fullmatch(str(value.get("incident_id", ""))):
rows.append(_bounded(value, 2000))
except OSError:
return []
return rows
def verify(self, sequence=None):
rows = self.list(limit=_MAX)
previous = ""
checked = 0
for value in rows:
if sequence is not None and value.get("sequence") > sequence:
break
expected = value.get("record_hash")
body = {key: item for key, item in value.items() if key != "record_hash"}
payload = json.dumps(body, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
if value.get("previous_hash") != previous or expected != sha256(payload.encode("utf8")).hexdigest():
return {"valid": False, "checked": checked, "error": "hash_mismatch"}
previous, checked = expected, checked + 1
return {"valid": True, "checked": checked, "last_hash": previous}
+4
View File
@@ -13,6 +13,7 @@ from uuid import uuid4
from .evidence import scrub
from .issue_credentials import issue_target, load_bootstrap
from .issue_transport import IssueTransport
class NoRedirect(urllib.request.HTTPRedirectHandler):
@@ -34,6 +35,9 @@ class GitIssuePublisher:
owner, repo = config['repository'].split('/')
if not all(value and all(c.isalnum() or c in '-_.' for c in value) and value not in {'.', '..'} for value in (owner, repo)):
raise ValueError('invalid_issue_repository')
transport = IssueTransport(self.config_path).credentials(config)
if transport.get('headers'):
return config, transport['headers'], transport.get('source', 'issue_transport')
bootstrap = load_bootstrap()
if (bootstrap and bootstrap['server'].rstrip('/') == config['server'].rstrip('/')
and bootstrap['repository'] == config['repository']):
+70
View File
@@ -0,0 +1,70 @@
"""Read-only, bounded health checks for the DSH assistant."""
from __future__ import annotations
from datetime import datetime, timezone
import json
from pathlib import Path
from .evidence import scrub
from .issue_credentials import load_bootstrap
def _now():
return datetime.now(timezone.utc).isoformat(timespec="seconds")
class HealthStore:
def __init__(self, data_root):
self.data_root = Path(data_root)
def snapshot(self, service, *, now=None):
checked = now.isoformat(timespec="seconds") if now is not None else _now()
checks = [self._provider(), self._execution(service), self._stores(), self._tickets(), self._credentials()]
status = "ok" if all(row["status"] == "ok" for row in checks) else (
"blocked" if any(row["status"] == "blocked" for row in checks) else "degraded")
return scrub({"status": status, "checked_utc": checked, "checks": checks})
def _provider(self):
path = self.data_root / "dsh" / "provider.json"
try:
value = json.loads(path.read_text(encoding="utf8"))
configured = isinstance(value, dict) and bool(value.get("base_url")) and bool(value.get("default_model"))
except (OSError, ValueError, TypeError):
configured = False
return {"name": "provider", "status": "ok" if configured else "degraded",
"code": "ready" if configured else "provider_unconfigured",
"message": "模型提供方已配置" if configured else "尚未配置模型提供方"}
def _execution(self, service):
active = bool(getattr(service, "_workers", set()))
return {"name": "execution", "status": "degraded" if active else "ok",
"code": "worker_active" if active else "idle",
"message": "任务中心有后台操作" if active else "任务中心空闲"}
def _stores(self):
dsh = self.data_root / "dsh"
try:
if dsh.exists() and not dsh.is_dir():
raise OSError("state path is not a directory")
# Health is observational: do not create files or alter timestamps.
return {"name": "stores", "status": "ok", "code": "ready", "message": "本地状态目录可用"}
except OSError:
return {"name": "stores", "status": "blocked", "code": "store_unavailable", "message": "本地状态目录不可写"}
def _tickets(self):
cache = self.data_root / "dsh" / "ticket-cache"
if not cache.exists():
return {"name": "tickets", "status": "ok", "code": "no_cache", "message": "暂无工单同步缓存"}
try:
count = sum(1 for path in cache.iterdir() if path.is_file() and path.suffix == ".json")
except OSError:
return {"name": "tickets", "status": "degraded", "code": "ticket_cache_unreadable", "message": "工单缓存不可读"}
return {"name": "tickets", "status": "ok", "code": "cache_available", "message": f"已有 {min(count, 100)} 条工单缓存"}
def _credentials(self):
value = load_bootstrap()
if value:
return {"name": "issue_credentials", "status": "degraded", "code": "credential_rotation_required",
"message": "当前使用预置工单凭据,建议切换短期中转凭据"}
return {"name": "issue_credentials", "status": "ok", "code": "machine_credential_or_disabled",
"message": "未发现预置工单凭据"}
+29 -1
View File
@@ -9,6 +9,7 @@ import re
from threading import Event, Lock
from .evidence import IncidentEvidence, scrub, write_json
from .decision_log import DecisionLog
from .rules import RETRYABLE_CODES
from .snapshot import TaskSnapshot
@@ -35,6 +36,28 @@ class IncidentRunner:
def __init__(self, data_root, service, analyze, publish, notify=lambda row: None):
self.data_root, self.service = Path(data_root), service
self.analyze, self.publish, self.notify = analyze, publish, notify
self.decisions = DecisionLog(self.data_root)
def _record_decision(self, record, evidence, *, outcome, human_takeover=False):
refs = []
for path in sorted(evidence.directory.iterdir()):
if path.is_file() and path.name in {'snapshot.json', 'snapshot-after.json', 'gates.json',
'actions.json', 'verification.json', 'diagnose.json', 'reflect.json', 'evidence.zip'}:
refs.append(path.relative_to(self.data_root).as_posix())
try:
value = self.decisions.append({
'incident_id': record['incident_id'], 'trigger': record.get('issue_code', ''),
'evidence_refs': refs, 'gates': record.get('gates', {}),
'candidate_action': 'retry_failed' if record.get('attempts') else 'none',
'authorized_scope': {'task_id': record.get('task_id'), 'run_id': record.get('run_id')},
'outcome': outcome, 'human_takeover': human_takeover})
record['decision_sequence'] = value['sequence']
record['decision_hash'] = value['record_hash']
except Exception as error:
# Evidence and the incident state remain authoritative; make the
# missing append visible rather than claiming an auditable result.
record['decision_log_status'] = 'unavailable'
evidence.event('decision_log_unavailable', error=type(error).__name__)
def run(self, task_id, issue_code, message, *, run_id='observation', execution_authorized=False, stop_event=None):
with _RUN_LOCK:
@@ -136,6 +159,8 @@ class IncidentRunner:
if p.queue_complete and not p.failed_items and not current.interrupted and not outcome.paused:
update('resolved')
evidence.capture('diagnostics-after')
self._record_decision(record, evidence, outcome='resolved')
write_json(state_path, record)
evidence.pack(record)
return record
except Exception as error:
@@ -158,7 +183,10 @@ class IncidentRunner:
update('pending_issue')
self._draft(record, evidence)
archive = evidence.pack(record)
return self._publish(record, evidence, archive)
result = self._publish(record, evidence, archive)
self._record_decision(record, evidence, outcome=record.get('status', 'pending_issue'))
write_json(state_path, record)
return result
def _snapshot(self, task_id, issue_code, message):
try:
@@ -0,0 +1,60 @@
"""Fixed-target issue credentials with optional short-lived broker exchange."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import json
from pathlib import Path
import urllib.parse
import urllib.request
from .issue_credentials import issue_target, load_bootstrap
def _post_broker(url, payload, timeout=10):
request = urllib.request.Request(url, data=json.dumps(payload).encode("utf8"),
headers={"Content-Type": "application/json", "Accept": "application/json"})
with urllib.request.build_opener(_NoRedirect()).open(request, timeout=timeout) as response:
return json.loads(response.read(64 * 1024))
class _NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *args, **kwargs):
return None
class IssueTransport:
def __init__(self, config_path):
self.config_path = Path(config_path)
def credentials(self, target, *, now=None):
config = issue_target(self.config_path)
if not self._same_target(config, target):
return {"source": "invalid_target", "rotation_required": False}
broker = config.get("broker_url")
if broker:
try:
parsed = urllib.parse.urlsplit(str(broker))
if parsed.scheme != "https" or not parsed.hostname:
raise ValueError("invalid_broker_url")
response = _post_broker(broker, {"server": target["server"], "repository": target["repository"],
"instance_id": str(config.get("instance_id", ""))[:128]})
token = response.get("token")
expires = int(response.get("expires_in", 0))
if not isinstance(token, str) or not token or not 30 <= expires <= 3600:
raise ValueError("invalid_broker_response")
stamp = now or datetime.now(timezone.utc)
return {"headers": {"Authorization": "token " + token}, "source": "broker",
"expires_utc": (stamp + timedelta(seconds=expires)).isoformat(), "rotation_required": False}
except Exception as error:
return {"source": "broker_unavailable", "error": type(error).__name__, "rotation_required": True}
bootstrap = load_bootstrap()
if bootstrap and self._same_target(bootstrap, target):
return {"headers": {"Authorization": "token " + bootstrap["token"]},
"source": "prefilled_issue_token", "expires_utc": None, "rotation_required": True}
return {"source": "credential_unavailable", "rotation_required": True}
@staticmethod
def _same_target(left, right):
return (isinstance(left, dict) and isinstance(right, dict)
and str(left.get("server", "")).rstrip("/") == str(right.get("server", "")).rstrip("/")
and left.get("repository") == right.get("repository"))
@@ -17,6 +17,8 @@ from ...dsh.scheduling import ScheduleStore
from ...dsh.experience import ExperienceStore
from ...dsh.ticket_tracking import TicketTracker
from ...dsh.impact import execution_impact
from ...dsh.health import HealthStore
from ...dsh.decision_log import DecisionLog
from ..errors import TaskCenterError
from .dsh_task_lifecycle import DshLifecycleDialog
from .workers import ServiceWorker
@@ -66,11 +68,14 @@ class DshOperationsController(QObject):
self.schedules = ScheduleStore(self.root)
self.experiences = ExperienceStore(self.root)
self.tickets = TicketTracker(self.root)
self.health = HealthStore(self.root)
self.decisions = DecisionLog(self.root)
self.dialog = None
self.worker = None
self.request_id = None
self.results = {}
self.cache = {'runs': [], 'schedules': [], 'experiences': [], 'tickets': [], 'activity': {}}
self.cache = {'runs': [], 'schedules': [], 'experiences': [], 'tickets': [], 'decisions': [],
'health': {'status': 'degraded', 'checks': []}, 'activity': {}}
self.timer = QTimer(self)
self.timer.setInterval(15000)
self.timer.timeout.connect(self.tick)
@@ -161,7 +166,8 @@ class DshOperationsController(QObject):
self.tickets.refresh(candidates[0]['incident_id'])
tickets = self.tickets.list_tickets()
return {'runs': runs, 'schedules': self.schedules.evaluate(service),
'experiences': self.experiences.refresh()[:20], 'tickets': tickets[:20]}
'experiences': self.experiences.refresh()[:20], 'tickets': tickets[:20],
'health': self.health.snapshot(service), 'decisions': self.decisions.list(20)}
def refresh(self, *, synchronize_tickets=False):
if self.window._closed or self.window._workers or self.busy or self._other_dialog():
@@ -230,13 +236,13 @@ class DshOperationsController(QObject):
self.request_id = key
self._state('checking')
allowed = {
'refresh': set(), 'history': set(), 'impact': {'task_ids'}, 'experience': {'issue_code'}, 'tickets': set(),
'refresh': set(), 'history': set(), 'health': set(), 'decisions': set(), 'impact': {'task_ids'}, 'experience': {'issue_code'}, 'tickets': set(),
'refresh_ticket': {'incident_id'}, 'schedule_dialog': {'task_ids'}, 'cancel_schedule': {'schedule_id'},
'run_schedule': {'schedule_id'}, 'open_incident': {'incident_id'},
'schedule': {'task_ids', 'priority', 'not_before', 'due_at', 'dependencies', 'auto_start', 'idle_only', 'title'}}
if op not in allowed or set(spec) - ({'operation'} | allowed[op]):
self._failed(key, ValueError('unknown operation fields')); return
if op in {'refresh', 'history', 'tickets'}:
if op in {'refresh', 'history', 'health', 'decisions', 'tickets'}:
def read(value):
self.cache.update(value)
self._respond(key, status='read', **value)