feat(dsh): harden health decisions and issue transport
This commit is contained in:
1 parent
736b582bc1
commit
eee4e6bde9
18 files changed
+592
-11
No files matched your search
@@ -0,0 +1,85 @@
|
||||
"""Append-only, hash-linked records for automatic DSH decisions."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from .evidence import scrub
|
||||
|
||||
_ID = re.compile(r"^[0-9a-f]{24}$")
|
||||
_MAX = 2000
|
||||
|
||||
|
||||
def _bounded(value, limit=1000):
|
||||
value = scrub(value)
|
||||
if isinstance(value, str):
|
||||
return value[:limit]
|
||||
if isinstance(value, list):
|
||||
return [_bounded(item, limit) for item in value[:20]]
|
||||
if isinstance(value, dict):
|
||||
return {str(key)[:80]: _bounded(item, limit) for key, item in list(value.items())[:40]}
|
||||
return value
|
||||
|
||||
|
||||
class DecisionLog:
|
||||
def __init__(self, data_root):
|
||||
self.path = Path(data_root) / "dsh" / "decisions.jsonl"
|
||||
|
||||
def append(self, record):
|
||||
if not isinstance(record, dict) or not _ID.fullmatch(str(record.get("incident_id", ""))):
|
||||
raise ValueError("incident_id")
|
||||
rows = self.list(limit=_MAX)
|
||||
previous = rows[-1]["record_hash"] if rows else ""
|
||||
value = {"sequence": len(rows) + 1, "at_utc": datetime.now(timezone.utc).isoformat(),
|
||||
"previous_hash": previous, "incident_id": record["incident_id"],
|
||||
"trigger": _bounded(record.get("trigger", ""), 160),
|
||||
"evidence_refs": _bounded(record.get("evidence_refs", []), 300),
|
||||
"gates": _bounded(record.get("gates", {}), 300),
|
||||
"candidate_action": _bounded(record.get("candidate_action", "none"), 160),
|
||||
"authorized_scope": _bounded(record.get("authorized_scope", {}), 600),
|
||||
"outcome": _bounded(record.get("outcome", "unknown"), 160),
|
||||
"human_takeover": bool(record.get("human_takeover", False))}
|
||||
payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
value["record_hash"] = sha256(payload.encode("utf8")).hexdigest()
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.path.open("a", encoding="utf8") as stream:
|
||||
stream.write(json.dumps(value, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
stream.flush()
|
||||
return dict(value)
|
||||
|
||||
def list(self, limit=20):
|
||||
if not isinstance(limit, int) or limit <= 0:
|
||||
return []
|
||||
rows = []
|
||||
try:
|
||||
with self.path.open(encoding="utf8") as stream:
|
||||
for line in stream:
|
||||
if len(rows) >= min(limit, _MAX):
|
||||
break
|
||||
try:
|
||||
value = json.loads(line)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if isinstance(value, dict) and _ID.fullmatch(str(value.get("incident_id", ""))):
|
||||
rows.append(_bounded(value, 2000))
|
||||
except OSError:
|
||||
return []
|
||||
return rows
|
||||
|
||||
def verify(self, sequence=None):
|
||||
rows = self.list(limit=_MAX)
|
||||
previous = ""
|
||||
checked = 0
|
||||
for value in rows:
|
||||
if sequence is not None and value.get("sequence") > sequence:
|
||||
break
|
||||
expected = value.get("record_hash")
|
||||
body = {key: item for key, item in value.items() if key != "record_hash"}
|
||||
payload = json.dumps(body, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
if value.get("previous_hash") != previous or expected != sha256(payload.encode("utf8")).hexdigest():
|
||||
return {"valid": False, "checked": checked, "error": "hash_mismatch"}
|
||||
previous, checked = expected, checked + 1
|
||||
return {"valid": True, "checked": checked, "last_hash": previous}
|
||||
@@ -13,6 +13,7 @@ from uuid import uuid4
|
||||
|
||||
from .evidence import scrub
|
||||
from .issue_credentials import issue_target, load_bootstrap
|
||||
from .issue_transport import IssueTransport
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
@@ -34,6 +35,9 @@ class GitIssuePublisher:
|
||||
owner, repo = config['repository'].split('/')
|
||||
if not all(value and all(c.isalnum() or c in '-_.' for c in value) and value not in {'.', '..'} for value in (owner, repo)):
|
||||
raise ValueError('invalid_issue_repository')
|
||||
transport = IssueTransport(self.config_path).credentials(config)
|
||||
if transport.get('headers'):
|
||||
return config, transport['headers'], transport.get('source', 'issue_transport')
|
||||
bootstrap = load_bootstrap()
|
||||
if (bootstrap and bootstrap['server'].rstrip('/') == config['server'].rstrip('/')
|
||||
and bootstrap['repository'] == config['repository']):
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Read-only, bounded health checks for the DSH assistant."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .evidence import scrub
|
||||
from .issue_credentials import load_bootstrap
|
||||
|
||||
|
||||
def _now():
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class HealthStore:
|
||||
def __init__(self, data_root):
|
||||
self.data_root = Path(data_root)
|
||||
|
||||
def snapshot(self, service, *, now=None):
|
||||
checked = now.isoformat(timespec="seconds") if now is not None else _now()
|
||||
checks = [self._provider(), self._execution(service), self._stores(), self._tickets(), self._credentials()]
|
||||
status = "ok" if all(row["status"] == "ok" for row in checks) else (
|
||||
"blocked" if any(row["status"] == "blocked" for row in checks) else "degraded")
|
||||
return scrub({"status": status, "checked_utc": checked, "checks": checks})
|
||||
|
||||
def _provider(self):
|
||||
path = self.data_root / "dsh" / "provider.json"
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf8"))
|
||||
configured = isinstance(value, dict) and bool(value.get("base_url")) and bool(value.get("default_model"))
|
||||
except (OSError, ValueError, TypeError):
|
||||
configured = False
|
||||
return {"name": "provider", "status": "ok" if configured else "degraded",
|
||||
"code": "ready" if configured else "provider_unconfigured",
|
||||
"message": "模型提供方已配置" if configured else "尚未配置模型提供方"}
|
||||
|
||||
def _execution(self, service):
|
||||
active = bool(getattr(service, "_workers", set()))
|
||||
return {"name": "execution", "status": "degraded" if active else "ok",
|
||||
"code": "worker_active" if active else "idle",
|
||||
"message": "任务中心有后台操作" if active else "任务中心空闲"}
|
||||
|
||||
def _stores(self):
|
||||
dsh = self.data_root / "dsh"
|
||||
try:
|
||||
if dsh.exists() and not dsh.is_dir():
|
||||
raise OSError("state path is not a directory")
|
||||
# Health is observational: do not create files or alter timestamps.
|
||||
return {"name": "stores", "status": "ok", "code": "ready", "message": "本地状态目录可用"}
|
||||
except OSError:
|
||||
return {"name": "stores", "status": "blocked", "code": "store_unavailable", "message": "本地状态目录不可写"}
|
||||
|
||||
def _tickets(self):
|
||||
cache = self.data_root / "dsh" / "ticket-cache"
|
||||
if not cache.exists():
|
||||
return {"name": "tickets", "status": "ok", "code": "no_cache", "message": "暂无工单同步缓存"}
|
||||
try:
|
||||
count = sum(1 for path in cache.iterdir() if path.is_file() and path.suffix == ".json")
|
||||
except OSError:
|
||||
return {"name": "tickets", "status": "degraded", "code": "ticket_cache_unreadable", "message": "工单缓存不可读"}
|
||||
return {"name": "tickets", "status": "ok", "code": "cache_available", "message": f"已有 {min(count, 100)} 条工单缓存"}
|
||||
|
||||
def _credentials(self):
|
||||
value = load_bootstrap()
|
||||
if value:
|
||||
return {"name": "issue_credentials", "status": "degraded", "code": "credential_rotation_required",
|
||||
"message": "当前使用预置工单凭据,建议切换短期中转凭据"}
|
||||
return {"name": "issue_credentials", "status": "ok", "code": "machine_credential_or_disabled",
|
||||
"message": "未发现预置工单凭据"}
|
||||
@@ -9,6 +9,7 @@ import re
|
||||
from threading import Event, Lock
|
||||
|
||||
from .evidence import IncidentEvidence, scrub, write_json
|
||||
from .decision_log import DecisionLog
|
||||
from .rules import RETRYABLE_CODES
|
||||
from .snapshot import TaskSnapshot
|
||||
|
||||
@@ -35,6 +36,28 @@ class IncidentRunner:
|
||||
def __init__(self, data_root, service, analyze, publish, notify=lambda row: None):
|
||||
self.data_root, self.service = Path(data_root), service
|
||||
self.analyze, self.publish, self.notify = analyze, publish, notify
|
||||
self.decisions = DecisionLog(self.data_root)
|
||||
|
||||
def _record_decision(self, record, evidence, *, outcome, human_takeover=False):
|
||||
refs = []
|
||||
for path in sorted(evidence.directory.iterdir()):
|
||||
if path.is_file() and path.name in {'snapshot.json', 'snapshot-after.json', 'gates.json',
|
||||
'actions.json', 'verification.json', 'diagnose.json', 'reflect.json', 'evidence.zip'}:
|
||||
refs.append(path.relative_to(self.data_root).as_posix())
|
||||
try:
|
||||
value = self.decisions.append({
|
||||
'incident_id': record['incident_id'], 'trigger': record.get('issue_code', ''),
|
||||
'evidence_refs': refs, 'gates': record.get('gates', {}),
|
||||
'candidate_action': 'retry_failed' if record.get('attempts') else 'none',
|
||||
'authorized_scope': {'task_id': record.get('task_id'), 'run_id': record.get('run_id')},
|
||||
'outcome': outcome, 'human_takeover': human_takeover})
|
||||
record['decision_sequence'] = value['sequence']
|
||||
record['decision_hash'] = value['record_hash']
|
||||
except Exception as error:
|
||||
# Evidence and the incident state remain authoritative; make the
|
||||
# missing append visible rather than claiming an auditable result.
|
||||
record['decision_log_status'] = 'unavailable'
|
||||
evidence.event('decision_log_unavailable', error=type(error).__name__)
|
||||
|
||||
def run(self, task_id, issue_code, message, *, run_id='observation', execution_authorized=False, stop_event=None):
|
||||
with _RUN_LOCK:
|
||||
@@ -136,6 +159,8 @@ class IncidentRunner:
|
||||
if p.queue_complete and not p.failed_items and not current.interrupted and not outcome.paused:
|
||||
update('resolved')
|
||||
evidence.capture('diagnostics-after')
|
||||
self._record_decision(record, evidence, outcome='resolved')
|
||||
write_json(state_path, record)
|
||||
evidence.pack(record)
|
||||
return record
|
||||
except Exception as error:
|
||||
@@ -158,7 +183,10 @@ class IncidentRunner:
|
||||
update('pending_issue')
|
||||
self._draft(record, evidence)
|
||||
archive = evidence.pack(record)
|
||||
return self._publish(record, evidence, archive)
|
||||
result = self._publish(record, evidence, archive)
|
||||
self._record_decision(record, evidence, outcome=record.get('status', 'pending_issue'))
|
||||
write_json(state_path, record)
|
||||
return result
|
||||
|
||||
def _snapshot(self, task_id, issue_code, message):
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Fixed-target issue credentials with optional short-lived broker exchange."""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
from .issue_credentials import issue_target, load_bootstrap
|
||||
|
||||
|
||||
def _post_broker(url, payload, timeout=10):
|
||||
request = urllib.request.Request(url, data=json.dumps(payload).encode("utf8"),
|
||||
headers={"Content-Type": "application/json", "Accept": "application/json"})
|
||||
with urllib.request.build_opener(_NoRedirect()).open(request, timeout=timeout) as response:
|
||||
return json.loads(response.read(64 * 1024))
|
||||
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
class IssueTransport:
|
||||
def __init__(self, config_path):
|
||||
self.config_path = Path(config_path)
|
||||
|
||||
def credentials(self, target, *, now=None):
|
||||
config = issue_target(self.config_path)
|
||||
if not self._same_target(config, target):
|
||||
return {"source": "invalid_target", "rotation_required": False}
|
||||
broker = config.get("broker_url")
|
||||
if broker:
|
||||
try:
|
||||
parsed = urllib.parse.urlsplit(str(broker))
|
||||
if parsed.scheme != "https" or not parsed.hostname:
|
||||
raise ValueError("invalid_broker_url")
|
||||
response = _post_broker(broker, {"server": target["server"], "repository": target["repository"],
|
||||
"instance_id": str(config.get("instance_id", ""))[:128]})
|
||||
token = response.get("token")
|
||||
expires = int(response.get("expires_in", 0))
|
||||
if not isinstance(token, str) or not token or not 30 <= expires <= 3600:
|
||||
raise ValueError("invalid_broker_response")
|
||||
stamp = now or datetime.now(timezone.utc)
|
||||
return {"headers": {"Authorization": "token " + token}, "source": "broker",
|
||||
"expires_utc": (stamp + timedelta(seconds=expires)).isoformat(), "rotation_required": False}
|
||||
except Exception as error:
|
||||
return {"source": "broker_unavailable", "error": type(error).__name__, "rotation_required": True}
|
||||
bootstrap = load_bootstrap()
|
||||
if bootstrap and self._same_target(bootstrap, target):
|
||||
return {"headers": {"Authorization": "token " + bootstrap["token"]},
|
||||
"source": "prefilled_issue_token", "expires_utc": None, "rotation_required": True}
|
||||
return {"source": "credential_unavailable", "rotation_required": True}
|
||||
|
||||
@staticmethod
|
||||
def _same_target(left, right):
|
||||
return (isinstance(left, dict) and isinstance(right, dict)
|
||||
and str(left.get("server", "")).rstrip("/") == str(right.get("server", "")).rstrip("/")
|
||||
and left.get("repository") == right.get("repository"))
|
||||
@@ -17,6 +17,8 @@ from ...dsh.scheduling import ScheduleStore
|
||||
from ...dsh.experience import ExperienceStore
|
||||
from ...dsh.ticket_tracking import TicketTracker
|
||||
from ...dsh.impact import execution_impact
|
||||
from ...dsh.health import HealthStore
|
||||
from ...dsh.decision_log import DecisionLog
|
||||
from ..errors import TaskCenterError
|
||||
from .dsh_task_lifecycle import DshLifecycleDialog
|
||||
from .workers import ServiceWorker
|
||||
@@ -66,11 +68,14 @@ class DshOperationsController(QObject):
|
||||
self.schedules = ScheduleStore(self.root)
|
||||
self.experiences = ExperienceStore(self.root)
|
||||
self.tickets = TicketTracker(self.root)
|
||||
self.health = HealthStore(self.root)
|
||||
self.decisions = DecisionLog(self.root)
|
||||
self.dialog = None
|
||||
self.worker = None
|
||||
self.request_id = None
|
||||
self.results = {}
|
||||
self.cache = {'runs': [], 'schedules': [], 'experiences': [], 'tickets': [], 'activity': {}}
|
||||
self.cache = {'runs': [], 'schedules': [], 'experiences': [], 'tickets': [], 'decisions': [],
|
||||
'health': {'status': 'degraded', 'checks': []}, 'activity': {}}
|
||||
self.timer = QTimer(self)
|
||||
self.timer.setInterval(15000)
|
||||
self.timer.timeout.connect(self.tick)
|
||||
@@ -161,7 +166,8 @@ class DshOperationsController(QObject):
|
||||
self.tickets.refresh(candidates[0]['incident_id'])
|
||||
tickets = self.tickets.list_tickets()
|
||||
return {'runs': runs, 'schedules': self.schedules.evaluate(service),
|
||||
'experiences': self.experiences.refresh()[:20], 'tickets': tickets[:20]}
|
||||
'experiences': self.experiences.refresh()[:20], 'tickets': tickets[:20],
|
||||
'health': self.health.snapshot(service), 'decisions': self.decisions.list(20)}
|
||||
|
||||
def refresh(self, *, synchronize_tickets=False):
|
||||
if self.window._closed or self.window._workers or self.busy or self._other_dialog():
|
||||
@@ -230,13 +236,13 @@ class DshOperationsController(QObject):
|
||||
self.request_id = key
|
||||
self._state('checking')
|
||||
allowed = {
|
||||
'refresh': set(), 'history': set(), 'impact': {'task_ids'}, 'experience': {'issue_code'}, 'tickets': set(),
|
||||
'refresh': set(), 'history': set(), 'health': set(), 'decisions': set(), 'impact': {'task_ids'}, 'experience': {'issue_code'}, 'tickets': set(),
|
||||
'refresh_ticket': {'incident_id'}, 'schedule_dialog': {'task_ids'}, 'cancel_schedule': {'schedule_id'},
|
||||
'run_schedule': {'schedule_id'}, 'open_incident': {'incident_id'},
|
||||
'schedule': {'task_ids', 'priority', 'not_before', 'due_at', 'dependencies', 'auto_start', 'idle_only', 'title'}}
|
||||
if op not in allowed or set(spec) - ({'operation'} | allowed[op]):
|
||||
self._failed(key, ValueError('unknown operation fields')); return
|
||||
if op in {'refresh', 'history', 'tickets'}:
|
||||
if op in {'refresh', 'history', 'health', 'decisions', 'tickets'}:
|
||||
def read(value):
|
||||
self.cache.update(value)
|
||||
self._respond(key, status='read', **value)
|
||||
|
||||
Reference in new issue
Block a user