fix: reconcile BPM sources and retain safe storage diagnostics

This commit is contained in:
abaishu0730 committed 2026-10-21 09:33:57 +08:00
1 parent 7e42a6dcc6
commit 33e9e9c108
9 files changed
+179 -40

No files matched your search

+28 -15
View File
@@ -8,13 +8,20 @@ function todoArrival(row){
return {...normalizeArrival({taskId:row.taskId,startTime:row.receivedAt,serialNum:row.serialNum,processName:row.processName,activityName:row.nodeName,processInstanceSubject:row.title},'todo:'+row.portalSource),kind:'todo',instanceId:String(row.instanceId),revision:row.revision,processVersion:row.processVersion||'',recordKey:JSON.stringify([row.instanceId,row.taskId,row.revision])};
}
class BpmArrivalMonitor{
constructor({workspace,manager,scanner,provider,settings,hubClient=new BpmSignalRClient(),clock=()=>Date.now(),debounceMs=750,pollMs=5000,retryMs=[2000,5000,15000,60000]}){
Object.assign(this,{workspace,manager,scanner,provider,settings,hubClient,clock,debounceMs,pollMs,retryMs});this.entries=new Map();this.todoPending=new Map();this.stopped=false;
constructor({workspace,manager,scanner,provider,settings,onTodoReconciled,hubClient=new BpmSignalRClient(),clock=()=>Date.now(),debounceMs=750,pollMs=5000,retryMs=[2000,5000,15000,60000],sourceRetryMs=[2000,5000,15000,60000,300000]}){
Object.assign(this,{workspace,manager,scanner,provider,settings,onTodoReconciled,hubClient,clock,debounceMs,pollMs,retryMs,sourceRetryMs});this.entries=new Map();this.todoPending=new Map();this.stopped=false;
this.principal={user:{username:'系统监听',role:'admin'},sessionId:'monitor:'+crypto.randomUUID()};
}
allowed(){const config=this.scanner.store.config();return config.enabled&&dayPolicy(new Date(this.clock()),this.settings().schedule,this.workspace.snapshot().calendar.overrides).scan;}
active(entry){return !this.stopped&&this.entries.get(entry.id)===entry&&this.workspace.getConnection(entry.id).status!=='disabled';}
snapshot(id){const e=this.entries.get(id);return e?{status:e.status,lastSignalAt:e.lastSignalAt||null,lastSyncAt:e.lastSyncAt||null,lastFailureAt:e.lastFailureAt||null,nextRetryAt:Number.isFinite(e.nextRetry)?new Date(e.nextRetry).toISOString():null}: {status:this.workspace.getConnection(id).status==='disabled'||!this.scanner.store.config().enabled?'paused':'waiting',lastSignalAt:null,lastSyncAt:null};}
active(entry){try{return !this.stopped&&this.entries.get(entry.id)===entry&&this.workspace.getConnection(entry.id).status!=='disabled';}catch{return false;}}
source(entry,kind){return entry.sources[kind]||=( {status:'pending',failures:0,nextAttempt:0,lastSuccessAt:null,lastFailureAt:null} );}
sourceSuccess(entry,kind){if(!entry||!this.active(entry))return;const source=this.source(entry,kind);Object.assign(source,{status:'healthy',failures:0,nextAttempt:0,lastSuccessAt:new Date(this.clock()).toISOString(),message:null});if(['todo','inform'].every(k=>this.source(entry,k).status==='healthy'))entry.lastSyncAt=new Date(this.clock()).toISOString();}
sourceFailure(entry,kind,error){if(!entry||!this.active(entry))return;const source=this.source(entry,kind),delay=this.sourceRetryMs[Math.min(source.failures++,this.sourceRetryMs.length-1)];Object.assign(source,{status:'error',lastFailureAt:new Date(this.clock()).toISOString(),nextAttempt:this.clock()+delay,message:error?.status===401?'需要恢复BPM登录。':error?.status===403?'没有读取权限。':error?.status===409?'连接忙碌或已暂停。':'读取失败,等待重试。'});entry.lastFailureAt=source.lastFailureAt;entry.dirty.add(kind);}
snapshot(id){const e=this.entries.get(id);if(e)return {status:e.status,lastSignalAt:e.lastSignalAt||null,lastSyncAt:e.lastSyncAt||null,lastFailureAt:e.lastFailureAt||null,nextRetryAt:Number.isFinite(e.nextRetry)?new Date(e.nextRetry).toISOString():null,nextReconcileAt:new Date(e.nextReconcile).toISOString(),sources:Object.fromEntries(['todo','inform'].map(kind=>{const s=this.source(e,kind);return [kind,{status:s.status,lastSuccessAt:s.lastSuccessAt,lastFailureAt:s.lastFailureAt,retryAt:s.nextAttempt?new Date(s.nextAttempt).toISOString():null,message:s.message||null}];}))};
let status='removed';try{status=this.workspace.getConnection(id).status==='disabled'||!this.scanner.store.config().enabled?'paused':'waiting';}catch{}return {status,lastSignalAt:null,lastSyncAt:null,nextReconcileAt:null,sources:{}};
}
updateInterval(entry,config){if(entry.intervalMinutes===config.intervalMinutes)return;entry.intervalMinutes=config.intervalMinutes;entry.nextReconcile=(entry.lastReconcileAt??this.clock())+config.intervalMinutes*60000;}
async reconfigure(){await this.tick();}
start(){if(this.timer||this.stopped)return;this.timer=setInterval(()=>this.tick().catch(()=>{}),this.pollMs);this.timer.unref?.();this.tick().catch(()=>{});}
async tick(){
if(this.stopped)return;const config=this.scanner.store.config(),connections=this.workspace.snapshot().connections,ids=new Set(connections.map(c=>c.id));
@@ -22,10 +29,11 @@ class BpmArrivalMonitor{
if(!config.enabled)return;
for(const c of connections){
if(c.status==='disabled'||c.status==='disconnected'&&!c.lastConnectedAt)continue;
let entry=this.entries.get(c.id);if(!entry){entry={id:c.id,status:'connecting',nextRetry:0,nextReconcile:this.clock()+config.intervalMinutes*60000,failures:0,dirty:new Set(),controller:new AbortController(),attempt:0};this.entries.set(c.id,entry);}
let entry=this.entries.get(c.id);if(!entry){entry={id:c.id,status:'connecting',nextRetry:0,nextReconcile:this.clock()+config.intervalMinutes*60000,lastReconcileAt:this.clock(),intervalMinutes:config.intervalMinutes,sources:{},failures:0,dirty:new Set(),controller:new AbortController(),attempt:0};this.entries.set(c.id,entry);}
this.updateInterval(entry,config);
if(entry.handle&&entry.epoch!==(this.manager.epoch?.(c.id)||0)){await this.stopConnection(c.id);continue;}
if(!entry.handle&&!entry.connecting&&this.clock()>=entry.nextRetry)this.connect(entry,c);
if(this.clock()>=entry.nextReconcile){entry.nextReconcile=this.clock()+config.intervalMinutes*60000;this.queue(entry,['todo','inform']);}
if(this.clock()>=entry.nextReconcile){entry.nextReconcile=this.clock()+config.intervalMinutes*60000;for(const kind of ['todo','inform'])this.source(entry,kind).nextAttempt=0;this.queue(entry,['todo','inform']);}
else if(entry.dirty.size&&!entry.sync&&!entry.debounce)this.queue(entry,[]);
}
}
@@ -42,30 +50,35 @@ class BpmArrivalMonitor{
return this.hubClient.connect(saved?.storageState,{signal:entry.controller.signal,onEvent:name=>{if(!this.active(entry)||entry.attempt!==attempt||entry.epoch!==(this.manager.epoch?.(entry.id)||0))return;entry.lastSignalAt=new Date(this.clock()).toISOString();this.queue(entry,name==='worklistChanged'?['todo']:name==='informlistChanged'?['inform']:['todo','inform']);},onClose:()=>{closed=true;if(this.active(entry)&&entry.attempt===attempt){entry.handle=null;this.retry(entry);}}});
});
if(!handle)return;if(!this.active(entry)||closed){await handle.stop();return;}
entry.handle=handle;entry.connectedAt=this.clock();entry.status='listening';entry.nextRetry=Infinity;entry.nextReconcile=this.clock()+this.scanner.store.config().intervalMinutes*60000;this.queue(entry,['todo','inform']);
entry.handle=handle;entry.connectedAt=this.clock();entry.status='listening';entry.nextRetry=Infinity;entry.lastReconcileAt=this.clock();entry.nextReconcile=this.clock()+this.scanner.store.config().intervalMinutes*60000;this.queue(entry,['todo','inform']);
})().catch(error=>{if(!this.active(entry))return;if(error.status===401)this.manager.observe?.(entry.id,'portal',error);this.retry(entry,error.status===409?'busy':error.status===401?'login_required':'polling',error.status===409?5000:error.status===401?5*60000:undefined);if(![401,409].includes(error.status))this.queue(entry,['todo','inform']);}).finally(()=>{entry.connecting=null;});
}
queue(entry,kinds){if(!this.active(entry))return;for(const kind of kinds)entry.dirty.add(kind);if(entry.sync||entry.debounce||!entry.dirty.size||!this.allowed()||this.manager.sessions.has(entry.id))return;
entry.debounce=setTimeout(()=>{entry.debounce=null;if(!this.active(entry)||!this.allowed())return;const kinds=[...entry.dirty];entry.dirty.clear();entry.sync=this.synchronize(entry,kinds).finally(()=>{entry.sync=null;if(entry.dirty.size&&this.active(entry))this.queue(entry,[]);});},this.debounceMs);entry.debounce.unref?.();
dueKinds(entry){return [...entry.dirty].filter(kind=>this.clock()>=this.source(entry,kind).nextAttempt);}
queue(entry,kinds){if(!this.active(entry))return;for(const kind of kinds)entry.dirty.add(kind);if(entry.sync||entry.debounce||!this.dueKinds(entry).length||!this.allowed()||this.manager.sessions.has(entry.id))return;
entry.debounce=setTimeout(()=>{entry.debounce=null;if(!this.active(entry)||!this.allowed())return;const kinds=this.dueKinds(entry);for(const kind of kinds)entry.dirty.delete(kind);entry.sync=this.synchronize(entry,kinds).finally(()=>{entry.sync=null;if(entry.dirty.size&&this.active(entry))this.queue(entry,[]);});},this.debounceMs);entry.debounce.unref?.();
}
async synchronize(entry,kinds){let valid=true;
for(const kind of kinds){if(!this.active(entry))return;
try{
if(kind==='inform'){await this.scanner.scan(entry.id,this.principal);}
if(kind==='inform'){this.source(entry,kind).status='running';await this.scanner.scan(entry.id,this.principal);this.sourceSuccess(entry,kind);}
else await this.startTodoScan(entry.id,this.principal).promise;
}catch(error){valid=false;entry.lastFailureAt=new Date(this.clock()).toISOString();}
}catch(error){valid=false;if(kind==='inform')this.sourceFailure(entry,kind,error);else if(this.active(entry)){entry.dirty.add(kind);const source=this.source(entry,kind);if(!source.nextAttempt)source.nextAttempt=this.clock()+this.sourceRetryMs[0];}}
}
if(valid&&this.active(entry))entry.lastSyncAt=new Date(this.clock()).toISOString();
if(valid&&this.active(entry)&&kinds.includes('todo')&&kinds.includes('inform')){entry.lastReconcileAt=this.clock();entry.nextReconcile=this.clock()+entry.intervalMinutes*60000;}
}
startTodoScan(id,principal){
const fail=(message,status)=>{throw Object.assign(new Error(message),{status});};if(this.stopped)fail('监听服务正在停止。',503);
const connection=this.workspace.getConnection(id);if(!principal?.sessionId||!['admin','operator'].includes(principal.user?.role)||principal.user.role!=='admin'&&connection.owner!==principal.user.username)fail('仅连接负责人或管理员可以同步。',403);
if(connection.status==='disabled')fail('此连接已停用。',409);if(this.todoPending.has(id))fail('待办对账正在运行。',409);
const runId=this.scanner.store.begin(id,principal.user.username,'todo');
const runId=this.scanner.store.begin(id,principal.user.username,'todo'),startedAt=new Date(this.clock()).toISOString(),epoch=this.manager.epoch?.(id)||0,entry=this.entries.get(id);
if(entry)this.source(entry,'todo').status='running';
const promise=Promise.resolve().then(async()=>{
try{let rows;try{rows=await this.provider.scan(connection,principal);}catch(error){if(error.status!==401)throw error;const result=await this.manager.ensure(connection,principal);if(result.status!=='connected')throw error;rows=await this.provider.scan(connection,principal);}
const current=this.workspace.getConnection(id);if(this.stopped||current.status==='disabled')fail('连接已暂停,本次对账未提交。',409);return {...this.scanner.store.complete(runId,id,rows.map(todoArrival),current),runId};
}catch(error){this.scanner.store.fail(runId,error.status===401?'需要恢复BPM登录。':error.status===409?'连接忙碌或已暂停。':'待办对账失败,请稍后重试。');throw error;}
const current=this.workspace.getConnection(id);if(this.stopped||current.status==='disabled'||epoch!==(this.manager.epoch?.(id)||0))fail('连接已暂停或身份已变化,本次对账未提交。',409);
const result=this.scanner.store.complete(runId,id,rows.map(todoArrival),current),finishedAt=new Date(this.clock()).toISOString();
if(this.onTodoReconciled)await this.onTodoReconciled({connectionId:id,runId,rows:structuredClone(rows),startedAt,finishedAt});
this.sourceSuccess(entry,'todo');return {...result,runId};
}catch(error){this.scanner.store.fail(runId,error.status===401?'需要恢复BPM登录。':error.status===409?'连接忙碌或已暂停。':'待办对账失败,请稍后重试。');this.sourceFailure(entry,'todo',error);throw error;}
}).finally(()=>this.todoPending.delete(id));this.todoPending.set(id,promise);return {runId,promise};
}
async stopConnection(id){const entry=this.entries.get(id);if(!entry){await this.todoPending.get(id)?.catch(()=>{});return;}this.entries.delete(id);entry.controller.abort();clearTimeout(entry.debounce);await Promise.allSettled([entry.handle?.stop(),entry.connecting,entry.sync,this.todoPending.get(id)]);}
+1 -1
View File
@@ -16,7 +16,7 @@ async function bpmScanApi({scanner,workspace,bpmMonitor,orderRules,user,sessionI
if(route==='/api/bpm-scan'&&method==='GET'){reply(200,{...scanner.store.status(),connections:workspace.snapshot().connections.map(({id,name,status,owner})=>({id,name,status,owner,monitoring:bpmMonitor?.snapshot(id)}))});return true;}
if(route==='/api/bpm-scan/config'&&method==='POST'){
if(user.role!=='admin')fail('仅管理员可以配置自动扫描。',403);
const config=scanner.store.saveConfig(body);workspace.record(user,'保存自动扫描',`${config.enabled?'开启':'暂停'},间隔 ${config.intervalMinutes} 分钟`);reply(200,config);return true;
const config=scanner.store.saveConfig(body);await bpmMonitor?.reconfigure?.();workspace.record(user,'保存自动扫描',`${config.enabled?'开启':'暂停'},间隔 ${config.intervalMinutes} 分钟`);reply(200,config);return true;
}
if(route==='/api/bpm-scan/run'&&method==='POST'){
if(!body||Object.keys(body).some(k=>!['connectionId','kind'].includes(k))||typeof body.connectionId!=='string'||body.kind!==undefined&&!['todo','inform'].includes(body.kind))fail('请选择 BPM 连接及有效到达类型。');
+21 -8
View File
@@ -1,12 +1,13 @@
'use strict';
const fs=require('node:fs'),path=require('node:path'),{DatabaseSync}=require('node:sqlite'),{readSupervisor}=require('../../platform/service/control.cjs'),{redact}=require('../../shared/redact.cjs');
const {monitorEventLoopDelay}=require('node:perf_hooks');
function timestamp(value){if(typeof value!=='string')return null;const normalized=/^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d$/.test(value)?value.replace(' ','T')+'Z':value;const n=Date.parse(normalized);return Number.isFinite(n)?new Date(n).toISOString():null;}
const latest=(...values)=>values.map(timestamp).filter(Boolean).sort().at(-1)||null;
const groupStatus=items=>!items.length?'unconfigured':['error','unknown','running','pending','healthy','paused','disabled'].find(s=>items.some(i=>i.status===s))||'pending';
class RuntimeHealth {
constructor({stateDir,startedAt=new Date().toISOString(),instanceId='',...sources}){
constructor({stateDir,startedAt=new Date().toISOString(),instanceId='',clock=()=>Date.now(),...sources}){
try {
Object.assign(this,{stateDir,startedAt,instanceId,sources});const dir=path.join(stateDir,'health');fs.mkdirSync(dir,{recursive:true,mode:0o700});this.db=new DatabaseSync(path.join(dir,'health.sqlite'));
Object.assign(this,{stateDir,startedAt,instanceId,sources,clock});const dir=path.join(stateDir,'health');fs.mkdirSync(dir,{recursive:true,mode:0o700});this.db=new DatabaseSync(path.join(dir,'health.sqlite'));
this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=2000;
CREATE TABLE IF NOT EXISTS observations(kind TEXT NOT NULL,id TEXT NOT NULL,last_success TEXT,last_failure TEXT,PRIMARY KEY(kind,id));`);
} catch(error) {
@@ -29,9 +30,20 @@ class RuntimeHealth {
});});
safe('scan','BPM 扫描','#records',()=>{
const {config={},runs=[]}=this.sources.scanner?.store.status()||{};
for(const r of runs)this.remember('scan',r.connectionId,{successAt:r.status==='success'?r.finishedAt:null,failureAt:['error','interrupted'].includes(r.status)?r.finishedAt:null});
return connections.filter(c=>c.status!=='disabled').map(c=>{const run=runs.find(r=>r.connectionId===c.id),status=run?.status==='running'?'running':config.enabled===false?'paused':!run?'pending':run.status==='success'?'healthy':'error';return {id:c.id,name:redact(c.name||'BPM 连接'),owner:c.owner,status,checkedAt:timestamp(run?.finishedAt||run?.startedAt),detail:config.enabled===false?'定时扫描已暂停,保留最近运行记录。':run?.message?redact(run.message):run?.status==='success'?`最近扫描 ${run.seen||0} 条,新增 ${run.added||0} 条。`:'等待扫描结果。',...this.remember('scan',c.id)};});
for(const r of runs)this.remember('scan',r.connectionId+':'+(r.kind||'inform'),{successAt:r.status==='success'?r.finishedAt:null,failureAt:['error','interrupted'].includes(r.status)?r.finishedAt:null});
return connections.filter(c=>c.status!=='disabled').flatMap(c=>['todo','inform'].map(kind=>{
const id=c.id+':'+kind,run=runs.find(r=>r.connectionId===c.id&&(r.kind||'inform')===kind),history=this.remember('scan',id),checkedAt=timestamp(run?.finishedAt||run?.startedAt)||latest(history.lastSuccessAt,history.lastFailureAt);
const failed=history.lastFailureAt&&(!history.lastSuccessAt||Date.parse(history.lastFailureAt)>Date.parse(history.lastSuccessAt));
const stale=!!checkedAt&&this.clock()-Date.parse(checkedAt)>Math.max(60000,(config.intervalMinutes||5)*120000+30000);
const status=config.enabled===false?'paused':run?.status==='running'?'running':failed||run&&['error','interrupted'].includes(run.status)?'error':!history.lastSuccessAt||stale?'pending':'healthy';
const label=kind==='todo'?'待办':'待阅',detail=config.enabled===false?'自动同步已暂停,保留最近观测。':status==='error'?redact(run?.message||label+'上次读取失败,等待重试。'):stale?'最近成功已超过完整对账周期,等待核实。':history.lastSuccessAt?`最近读取 ${run?.seen||0} 条;结果仅代表最近对账。`:'等待首次完整对账。';
return {id,connectionId:c.id,kind,name:redact(c.name||'BPM 连接')+' · '+label,owner:c.owner,status,stale,checkedAt,detail,...history};
}));
});
if(this.sources.monitor)safe('listener','BPM 到达监听','#bpm',()=>connections.filter(c=>c.status!=='disabled').map(c=>{
const observed=this.sources.monitor.snapshot(c.id),status=observed.status==='listening'?'healthy':['login_required','error'].includes(observed.status)?'error':observed.status==='paused'?'paused':'pending';
return {id:c.id,owner:c.owner,name:redact(c.name||'BPM 连接'),status,checkedAt:timestamp(observed.lastSignalAt||observed.lastSyncAt),nextReconcileAt:timestamp(observed.nextReconcileAt),detail:status==='healthy'?'推送已连接;实际读取结果请查看待办/待阅对账。':status==='error'?'监听不可用,请核对连接;完整对账继续作为补偿。':'等待监听或通过完整对账发现到达。'};
}));
safe('storage','SMB 存储','#storage',()=> (this.sources.storage?.store.servers()||[]).map(s=>{
const checkedAt=timestamp(s.state?.checkedAt),status=!s.enabled?'disabled':!checkedAt?'pending':s.state.readable&&s.state.writable?'healthy':s.state.readable?'pending':'error';
return {id:s.id,name:redact(s.name||'存储服务器'),status,checkedAt,detail:redact(s.state?.message||'尚未检查连接。'),...this.remember('storage',s.id,{successAt:status==='healthy'?checkedAt:null,failureAt:status==='error'?checkedAt:null})};
@@ -50,14 +62,15 @@ class RuntimeHealth {
return groups;
}
snapshot(user){
const now=Date.now(),groups=this.collect().map(g=>{if(!['bpm','scan','execution'].includes(g.key)||user.role==='admin'||g.status==='unknown')return g;const items=g.items.filter(i=>i.global||i.owner===user.username);return {...g,items,status:groupStatus(items)};});
const now=this.clock(),groups=this.collect().map(g=>{if(!['bpm','scan','listener','execution'].includes(g.key)||user.role==='admin'||g.status==='unknown')return g;const items=g.items.filter(i=>i.global||i.owner===user.username);return {...g,items,status:groupStatus(items)};});
let saved;try{saved=readSupervisor(this.stateDir);}catch{saved={state:null,events:[],autostart:null};}
const s=saved.state,managed=Boolean(this.instanceId&&s?.childInstanceId===this.instanceId),fresh=managed&&now-Date.parse(s.updatedAt)<45000;
const guard=managed?{status:fresh?s.status:'stale',heartbeatAt:s.heartbeatAt,updatedAt:s.updatedAt,restartCount:s.restartCount,childStartedAt:s.childStartedAt,reason:s.reason,nextRestartAt:s.nextRestartAt,events:saved.events.map(e=>({at:e.at,kind:e.kind,code:e.detail.code,delayMs:e.detail.delayMs,reason:e.detail.reason}))}:{status:'unmanaged',restartCount:0,events:[]};
const autostart={enabled:!!(saved.autostart?.enabled&&saved.autostart?.linkPath&&fs.existsSync(saved.autostart.linkPath)),method:'current_user_logon',installedAt:saved.autostart?.installedAt||null};
return {at:new Date(now).toISOString(),runtime:{startedAt:this.startedAt,uptimeSeconds:Math.max(0,Math.floor((now-Date.parse(this.startedAt))/1000)),managed,guard,autostart},groups};
const eventLoop={observed:!!this.loopDelay,since:this.loopDelay?this.loopObservedAt:null,maxMs:this.loopDelay?Math.round(this.loopDelay.max/1e4)/100:0,meanMs:this.loopDelay&&Number.isFinite(this.loopDelay.mean)?Math.round(this.loopDelay.mean/1e4)/100:0};
return {at:new Date(now).toISOString(),runtime:{startedAt:this.startedAt,uptimeSeconds:Math.max(0,Math.floor((now-Date.parse(this.startedAt))/1000)),managed,guard,autostart,eventLoop},groups};
}
start(){if(this.timer)return;const observe=()=>{try{this.collect();}catch{}};observe();this.timer=setInterval(observe,15000);this.timer.unref?.();}
close(){clearInterval(this.timer);this.timer=null;this.db?.close();this.db=null;}
start(){if(this.timer)return;this.loopDelay=monitorEventLoopDelay({resolution:20});this.loopDelay.enable();this.loopObservedAt=new Date(this.clock()).toISOString();const observe=()=>{try{this.collect();}catch{}};observe();this.timer=setInterval(observe,15000);this.timer.unref?.();}
close(){clearInterval(this.timer);this.timer=null;this.loopDelay?.disable();this.loopDelay=null;this.db?.close();this.db=null;}
}
module.exports={RuntimeHealth,timestamp};