Files
order-notification-automation/tests/assistant/review-recovery.test.cjs
T
abaishu0730 5ceffb935a fix(assistant): close review gaps in delivery and action recovery
Address PR4 comments 66-69 with safe delivery retries, capacity stops, preserved effect evidence and stale proposal rejection. Prevent archived queue snapshots from resurrecting cancelled reminders.
2026-10-29 11:51:03 +08:00

620 lines
25 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
'use strict';
const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const crypto = require('node:crypto');
const { managementFixture } = require('./management-fixture.cjs');
const { OperationJournal } = require('../../src/modules/assistant/operation-journal.cjs');
const approval = require('../../src/modules/assistant/configuration-approval.cjs');
const configuration = require('../../src/modules/assistant/configuration.cjs');
const { NativeSchedules } = require('../../src/modules/assistant/native-schedules.cjs');
async function until(predicate) {
const end = Date.now() + 10000;
while (!(await predicate())) {
if (Date.now() > end) throw Error('Recovery did not settle');
await new Promise((resolve) => setTimeout(resolve, 20));
}
}
function enqueue(schedules, conversation, task, owner) {
schedules.flush({
id: conversation.id,
owner,
pending: {
id: crypto.randomUUID(),
source: { kind: 'schedule' },
content: [
{
type: 'text',
text:
'[SCHEDULE REMINDER]\nThis is a scheduled message from the user\nschedule_id_json: ' +
JSON.stringify(task.id) +
'\noccurrence_at: ' +
task.scheduledAt +
'\nreminder_prompt_json: ' +
JSON.stringify(task.prompt),
},
],
},
});
}
async function setup(t, browser = false) {
require('../fixtures/isolated-vault-key.cjs')(t);
const f = await managementFixture(t, { browser });
const journalFile = path.join(os.tmpdir(), 'review-recovery-' + crypto.randomUUID() + '.sqlite');
f.assistant.journal = new OperationJournal({ file: journalFile });
t.after(() => {
f.assistant.journal.close();
fs.rmSync(journalFile, { force: true });
});
f.store.saveSettings({
...f.store.settings(),
runtimePath: process.execPath,
apiKey: 'TEST_REVIEW_RECOVERY',
allowActions: true,
});
const user = f.users.users().find((u) => u.username === 'admin');
return { ...f, principal: { user, sessionId: 'review-session' } };
}
test('expired or version-invalid proposals can be rejected after actions are disabled, without applying them', async (t) => {
const f = await setup(t),
c = f.store.create(f.principal.user.id);
const proposal = configuration.propose(
{ targetId: 'assistant', changes: { dailyLimit: 80 } },
configuration.snapshot(f.assistant.deps, f.store, f.principal.user),
f.assistant.deps,
f.store,
f.principal.user,
);
const action = { ...proposal, id: crypto.randomUUID(), status: 'proposed' };
const turn = f.store.beginTurn(c.id, f.principal.user.id, 'proposal');
f.store.updateTurn(turn.id, 'success', { reply: 'review', actions: [action] });
const review = approval.review(f.assistant, c.id, action.id, f.principal);
f.store.action(c.id, f.principal.user.id, action.id, (a) => {
a.expiresAt = 0;
});
f.store.saveSettings({
version: f.store.settings().version,
dailyLimit: 60,
allowActions: false,
});
assert.deepEqual(approval.reject(f.assistant, c.id, action.id, f.principal), {
status: 'rejected',
});
assert.equal(f.store.settings().dailyLimit, 60);
assert.equal(
f.store.conversation(c.id, f.principal.user.id).turns[0].actions[0].status,
'rejected',
);
assert.equal(
f.store.db
.prepare('SELECT COUNT(*) n FROM assistant_configuration_approvals WHERE action_id=?')
.get(action.id).n,
0,
);
await assert.rejects(
approval.execute(f.assistant, c.id, action.id, f.principal, review.approvalToken),
);
assert.throws(
() => approval.reject(f.assistant, c.id, action.id, f.principal),
(e) => e.status === 409,
);
});
for (const fault of ['workspace', 'journal', 'result-save'])
test(
'a completed fixed operation retains evidence and becomes needs_review when ' +
fault +
' fails',
async (t) => {
const f = await setup(t),
c = f.store.create(f.principal.user.id),
actionId = crypto.randomUUID();
let effects = 0;
f.assistant.deps.storage = {
store: { servers: () => [{ id: 'server', name: 'fixture', enabled: true, state: {} }] },
check: async () => {
effects++;
return { state: { readable: true, writable: false } };
},
};
const turn = f.store.beginTurn(c.id, f.principal.user.id, 'fixed operation');
f.store.updateTurn(turn.id, 'success', {
reply: 'review',
actions: [
{
id: actionId,
kind: 'check_storage',
targetId: 'server',
label: '检查存储连接',
status: 'proposed',
},
],
});
if (fault === 'workspace')
f.assistant.deps.workspace = {
record() {
throw Error('audit write failed after effect');
},
};
if (fault === 'journal') {
const record = f.assistant.record.bind(f.assistant);
t.mock.method(f.assistant, 'record', (...args) => {
if (args[2] === 'execute' && args[3] === 'success')
throw Error('final journal unavailable');
return record(...args);
});
}
if (fault === 'result-save') {
const action = f.store.action.bind(f.store);
t.mock.method(f.store, 'action', (id, owner, aid, work) =>
action(id, owner, aid, (a) => {
const value = work(a);
if (a.status === 'success') throw Error('result persistence failed');
return value;
}),
);
}
await assert.rejects(f.assistant.executeAction(c.id, actionId, f.principal));
const saved = f.store.conversation(c.id, f.principal.user.id).turns[0].actions[0];
assert.equal(saved.status, 'needs_review');
assert.deepEqual(saved.result, { readable: true, writable: false });
assert.equal(effects, 1);
await assert.rejects(
f.assistant.executeAction(c.id, actionId, f.principal),
(e) => e.status === 409,
);
assert.equal(effects, 1);
},
);
test('an unstarted native delivery survives restart and supports one confirmed owner-scoped retry', async (t) => {
const f = await setup(t),
c = f.store.create(f.principal.user.id);
f.store.saveSettings({ version: f.store.settings().version, dailyLimit: 1 });
const previous = f.store.beginTurn(c.id, f.principal.user.id, 'old call');
f.store.updateTurn(previous.id, 'success', { reply: 'done' });
let calls = 0;
f.assistant.runtime.chat = async () => {
calls++;
return JSON.stringify({ reply: 'Recovered reminder', actions: [] });
};
let schedules = await NativeSchedules.create({ assistant: f.assistant });
const task = await schedules.createTask(c.id, f.principal, {
confirm: true,
request: { title: 'Quota reminder', prompt: 'Read only', after_seconds: 1 },
});
await until(async () =>
(await schedules.list(c.id, f.principal)).history.some(
(h) => h.taskId === task.id && h.status === 'not_started',
),
);
let h = (await schedules.list(c.id, f.principal)).history[0];
assert.equal(h.status, 'not_started');
assert.equal(h.retryable, true);
assert.equal(calls, 0);
await schedules.stop();
schedules = await NativeSchedules.create({ assistant: f.assistant });
h = (await schedules.list(c.id, f.principal)).history[0];
assert.equal(h.status, 'not_started');
await assert.rejects(
schedules.retryDelivery(c.id, h.id, f.principal, {}),
(e) => e.status === 403,
);
await assert.rejects(
schedules.retryDelivery(
c.id,
h.id,
{ user: f.users.users().find((u) => u.username === 'viewer'), sessionId: 'viewer' },
{ confirm: true },
),
(e) => [403, 404].includes(e.status),
);
f.store.saveSettings({ version: f.store.settings().version, dailyLimit: 5 });
const results = await Promise.allSettled(
[0, 1].map(() => schedules.retryDelivery(c.id, h.id, f.principal, { confirm: true })),
);
assert.equal(results.filter((r) => r.status === 'fulfilled').length, 1);
await until(
async () => (await schedules.list(c.id, f.principal)).history[0].status === 'success',
);
assert.equal(calls, 1);
assert.equal(f.store.conversation(c.id, f.principal.user.id).turns.length, 2);
await assert.rejects(
schedules.retryDelivery(c.id, h.id, f.principal, { confirm: true }),
(e) => e.status === 409,
);
});
test('a delivery with an existing turn or unknown model result is never admitted as a safe retry', async (t) => {
const f = await setup(t),
c = f.store.create(f.principal.user.id);
let calls = 0;
f.assistant.runtime.chat = async () => {
calls++;
return JSON.stringify({ reply: 'Unexpected replay', actions: [] });
};
let schedules = await NativeSchedules.create({ assistant: f.assistant });
const task = await schedules.createTask(c.id, f.principal, {
confirm: true,
request: { title: 'Interrupted reminder', prompt: 'Read only', after_seconds: 3600 },
});
enqueue(schedules, c, task, f.principal.user.id);
const row = f.store.db.prepare('SELECT * FROM assistant_schedule_inbox').get();
const text = '[定时任务:' + task.title + ']\n' + task.prompt;
const turn = f.store.beginTurn(c.id, f.principal.user.id, text, {
requestId: 'schedule-' + row.id,
requestPayload: { text, diagnostic: null, repair: null },
});
f.store.db
.prepare("UPDATE assistant_schedule_inbox SET status='running',turn_id=? WHERE id=?")
.run(turn.id, row.id);
await schedules.stop();
f.store.recover();
schedules = await NativeSchedules.create({ assistant: f.assistant });
await until(
async () => (await schedules.list(c.id, f.principal)).history[0].status === 'needs_review',
);
const h = (await schedules.list(c.id, f.principal)).history[0];
assert.equal(h.retryable, false);
assert.equal(calls, 0);
await assert.rejects(
schedules.retryDelivery(c.id, h.id, f.principal, { confirm: true }),
(e) => e.status === 409,
);
assert.equal(calls, 0);
});
test('a full conversation stops its recurring native task and its undelivered message can move once to an owned conversation', async (t) => {
const f = await setup(t),
c = f.store.create(f.principal.user.id);
f.store.saveSettings({ version: f.store.settings().version, dailyLimit: 1000 });
for (let i = 0; i < 90; i++) {
const turn = f.store.beginTurn(c.id, f.principal.user.id, 'history ' + i);
f.store.updateTurn(turn.id, 'success', { reply: 'done' });
}
const schedules = await NativeSchedules.create({ assistant: f.assistant });
const task = await schedules.createTask(c.id, f.principal, {
confirm: true,
request: { title: 'Recurring reminder', prompt: 'Read only', every_seconds: 60 },
});
assert.match((await schedules.list(c.id, f.principal)).tasks[0].capacityWarning, /10/);
for (let i = 90; i < 100; i++) {
const turn = f.store.beginTurn(c.id, f.principal.user.id, 'history ' + i);
f.store.updateTurn(turn.id, 'success', { reply: 'done' });
}
let calls = 0;
f.assistant.runtime.chat = async () => {
calls++;
return JSON.stringify({ reply: 'Continued safely', actions: [] });
};
enqueue(schedules, c, task, f.principal.user.id);
await until(
async () => (await schedules.list(c.id, f.principal)).history[0]?.status === 'waiting_capacity',
);
await until(
async () => (await schedules.list(c.id, f.principal)).tasks[0]?.status === 'capacity_stopped',
);
assert.equal((await schedules.native.list({ sessionId: c.id })).length, 0);
assert.equal(calls, 0);
const h = (await schedules.list(c.id, f.principal)).history[0];
assert.equal(h.retryable, true);
await assert.rejects(
schedules.retryDelivery(c.id, h.id, f.principal, { confirm: true }),
(e) => e.status === 409,
);
const other = f.store.create(f.users.users().find((u) => u.username === 'viewer').id);
await assert.rejects(
schedules.retryDelivery(c.id, h.id, f.principal, {
confirm: true,
targetConversationId: other.id,
}),
(e) => [403, 404].includes(e.status),
);
const target = f.store.create(f.principal.user.id);
await schedules.retryDelivery(c.id, h.id, f.principal, {
confirm: true,
targetConversationId: target.id,
});
await until(() => calls === 1);
await until(
async () => (await schedules.list(target.id, f.principal)).history[0]?.status === 'success',
);
assert.equal(f.store.conversation(c.id, f.principal.user.id).turns.length, 100);
assert.equal(
f.store.conversation(target.id, f.principal.user.id).turns[0].reply,
'Continued safely',
);
assert.equal((await schedules.list(c.id, f.principal)).tasks[0].status, 'capacity_stopped');
await schedules.deleteTask(c.id, task.id, f.principal, { confirm: true });
assert.equal((await schedules.list(c.id, f.principal)).tasks.length, 0);
});
test('the owner can reject an expired card directly from the operation panel', async (t) => {
const f = await setup(t, true),
c = f.store.create(f.principal.user.id);
const proposal = configuration.propose(
{ targetId: 'assistant', changes: { dailyLimit: 80 } },
configuration.snapshot(f.assistant.deps, f.store, f.principal.user),
f.assistant.deps,
f.store,
f.principal.user,
);
const action = { ...proposal, id: crypto.randomUUID(), status: 'proposed', expiresAt: 0 };
const turn = f.store.beginTurn(c.id, f.principal.user.id, 'Expired suggestion');
f.store.updateTurn(turn.id, 'success', { reply: 'Please review', actions: [action] });
f.store.saveSettings({ version: f.store.settings().version, allowActions: false });
let reviews = 0;
await f.page.route('**/actions/*/review', (route) => {
reviews++;
return route.abort();
});
await f.open();
await f.page.getByRole('button', { name: '查看操作(1)', exact: true }).click();
await f.page.getByRole('button', { name: '拒绝建议', exact: true }).click();
await f.page.getByText('已拒绝,未修改', { exact: true }).waitFor();
assert.equal(reviews, 0);
assert.equal(f.store.settings().dailyLimit, 100);
});
test('the task manager offers a confirmed retry only for an unstarted delivery', async (t) => {
const f = await setup(t, true),
c = f.store.create(f.principal.user.id);
f.store.saveSettings({ version: f.store.settings().version, dailyLimit: 1 });
const old = f.store.beginTurn(c.id, f.principal.user.id, 'Old call');
f.store.updateTurn(old.id, 'success', { reply: 'Done' });
let calls = 0;
f.assistant.runtime.chat = async () => {
calls++;
return JSON.stringify({ reply: 'Recovered through UI', actions: [] });
};
const schedules = await NativeSchedules.create({ assistant: f.assistant });
await schedules.createTask(c.id, f.principal, {
confirm: true,
request: { title: 'UI recovery reminder', prompt: 'Read only', after_seconds: 1 },
});
await until(
async () => (await schedules.list(c.id, f.principal)).history[0]?.status === 'not_started',
);
f.store.saveSettings({ version: f.store.settings().version, dailyLimit: 5 });
await f.open();
await f.page.getByRole('button', { name: '定时任务', exact: true }).click();
await f.page
.getByRole('button', { name: '重试定时消息 UI recovery reminder', exact: true })
.click();
const dialog = f.page.getByRole('dialog', { name: '重试定时消息', exact: true });
assert.equal(
await dialog.getByRole('button', { name: '确认重试', exact: true }).isDisabled(),
true,
);
await dialog.getByRole('checkbox', { name: '我确认仅重试这条未调用消息', exact: true }).check();
await dialog.getByRole('button', { name: '确认重试', exact: true }).click();
await until(() => calls === 1);
await until(
async () => (await schedules.list(c.id, f.principal)).history[0]?.status === 'success',
);
await f.page.getByRole('button', { name: '返回对话', exact: true }).click();
await f.page.getByText('Recovered through UI', { exact: true }).waitFor();
assert.equal(calls, 1);
});
for (const restore of [false, true])
test(
'a queued snapshot cannot resurrect a delivery cancelled by archive; restored=' + restore,
async (t) => {
const f = await setup(t),
a = f.store.create(f.principal.user.id),
b = f.store.create(f.principal.user.id);
const schedules = await NativeSchedules.create({ assistant: f.assistant });
const ta = await schedules.createTask(a.id, f.principal, {
confirm: true,
request: { title: 'First gate', prompt: 'Wait', after_seconds: 3600 },
});
const tb = await schedules.createTask(b.id, f.principal, {
confirm: true,
request: { title: 'Cancelled next', prompt: 'Do not call', after_seconds: 3600 },
});
let release,
calls = 0;
const gate = new Promise((r) => (release = r));
t.after(() => release());
f.assistant.runtime.chat = async () => {
calls++;
if (calls === 1) await gate;
return JSON.stringify({ reply: 'Done', actions: [] });
};
enqueue(schedules, a, ta, f.principal.user.id);
enqueue(schedules, b, tb, f.principal.user.id);
f.store.db
.prepare('UPDATE assistant_schedule_inbox SET created_at=? WHERE task_id=?')
.run(new Date(Date.now() + 1000).toISOString(), tb.id);
await until(() => calls === 1);
const archived = f.store.archive(b.id, f.principal.user.id, {
version: f.store.conversation(b.id, f.principal.user.id).version,
});
await schedules.stopSession(b.id);
if (restore) f.store.restore(b.id, f.principal.user.id, { version: archived.version });
release();
await schedules.draining;
const cancelled = f.store.db
.prepare('SELECT * FROM assistant_schedule_inbox WHERE task_id=?')
.get(tb.id);
assert.equal(cancelled.status, 'cancelled');
assert.equal(calls, 1);
assert.equal(f.store.conversation(b.id, f.principal.user.id).turns.length, 0);
},
);
test('a send failure after durable turn admission cannot be retried as unstarted', async (t) => {
const f = await setup(t),
c = f.store.create(f.principal.user.id);
let calls = 0;
f.assistant.runtime.chat = async () => {
calls++;
return '{}';
};
const schedules = await NativeSchedules.create({ assistant: f.assistant });
const task = await schedules.createTask(c.id, f.principal, {
confirm: true,
request: { title: 'Admission audit failure', prompt: 'Read', after_seconds: 3600 },
});
const record = f.assistant.record.bind(f.assistant);
t.mock.method(f.assistant, 'record', (...args) => {
if (args[2] === 'context') throw Error('audit failed after turn');
return record(...args);
});
enqueue(schedules, c, task, f.principal.user.id);
await until(
async () => (await schedules.list(c.id, f.principal)).history[0]?.status === 'needs_review',
);
const h = (await schedules.list(c.id, f.principal)).history[0];
assert.ok(h.turnId);
assert.equal(h.retryable, false);
assert.equal(calls, 0);
await assert.rejects(
schedules.retryDelivery(c.id, h.id, f.principal, { confirm: true }),
(e) => e.status === 409,
);
});
test('a lost native stop acknowledgement remains stopped after restart and preserves its reminder', async (t) => {
const f = await setup(t);
f.store.saveSettings({ version: f.store.settings().version, dailyLimit: 1000 });
// Create the rule in a free conversation, then fill it before the reminder is delivered.
const target = f.store.create(f.principal.user.id);
let schedules = await NativeSchedules.create({ assistant: f.assistant });
const task = await schedules.createTask(target.id, f.principal, {
confirm: true,
request: { title: 'Stop receipt lost', prompt: 'Read', every_seconds: 60 },
});
for (let i = 0; i < 100; i++) {
const turn = f.store.beginTurn(target.id, f.principal.user.id, 'old ' + i);
f.store.updateTurn(turn.id, 'success', { reply: 'done' });
}
const nativeDelete = schedules.native.delete.bind(schedules.native);
t.mock.method(schedules.native, 'delete', async (args) => {
await nativeDelete(args);
throw Object.assign(Error('lost receipt'), { status: 502 });
});
enqueue(schedules, target, task, f.principal.user.id);
await until(
async () =>
(await schedules.list(target.id, f.principal)).history[0]?.status === 'waiting_capacity',
);
await until(() => !schedules.tasks.has(task.id));
await schedules.stop();
schedules = await NativeSchedules.create({ assistant: f.assistant });
const result = await schedules.list(target.id, f.principal);
assert.equal(result.tasks[0].status, 'capacity_stopped');
assert.equal(result.history[0].status, 'waiting_capacity');
assert.equal(result.history[0].retryable, true);
});
test('a recovered queue clears the current global error without deleting historical delivery errors', async (t) => {
const f = await setup(t),
schedules = await NativeSchedules.create({ assistant: f.assistant });
await schedules.draining;
let broken = true;
const drain = schedules.drain.bind(schedules);
t.mock.method(schedules, 'drain', async () => {
if (broken) throw Error('temporary queue read failure');
return drain();
});
schedules.kick();
await schedules.draining;
assert.ok((await schedules.catalog(f.principal)).error);
broken = false;
schedules.kick();
await schedules.draining;
assert.equal((await schedules.catalog(f.principal)).error, '');
});
test('capacity-stop UI keeps the native rule stopped while moving only the accepted message', async (t) => {
const f = await setup(t, true),
c = f.store.create(f.principal.user.id);
f.store.saveSettings({ version: f.store.settings().version, dailyLimit: 1000 });
const schedules = await NativeSchedules.create({ assistant: f.assistant });
const task = await schedules.createTask(c.id, f.principal, {
confirm: true,
request: { title: 'Capacity UI reminder', prompt: 'Read only', every_seconds: 60 },
});
for (let i = 0; i < 100; i++) {
const turn = f.store.beginTurn(c.id, f.principal.user.id, 'old ' + i);
f.store.updateTurn(turn.id, 'success', { reply: 'done' });
}
enqueue(schedules, c, task, f.principal.user.id);
await until(
async () => (await schedules.list(c.id, f.principal)).tasks[0]?.status === 'capacity_stopped',
);
const target = f.store.create(f.principal.user.id);
let calls = 0;
f.assistant.runtime.chat = async () => {
calls++;
return JSON.stringify({ reply: 'Capacity continuation received', actions: [] });
};
await f.open();
await f.page.getByRole('button', { name: '定时任务', exact: true }).click();
const manager = f.page.getByRole('region', { name: '定时任务管理', exact: true });
await manager
.locator('[data-task-list]')
.getByText('容量已满,已停止', { exact: true })
.waitFor();
await manager
.getByRole('button', { name: '重试定时消息 Capacity UI reminder', exact: true })
.click();
const dialog = f.page.getByRole('dialog', { name: '重试定时消息', exact: true });
assert.match(await dialog.innerText(), /周期任务请在新会话重新安排/);
await dialog.getByLabel('目标会话', { exact: true }).selectOption(target.id);
await dialog.getByRole('checkbox', { name: '我确认仅重试这条未调用消息', exact: true }).check();
await dialog.getByRole('button', { name: '确认重试', exact: true }).click();
await until(
async () => (await schedules.list(target.id, f.principal)).history[0]?.status === 'success',
);
assert.equal(calls, 1);
assert.equal((await schedules.native.list({ sessionId: c.id })).length, 0);
const output = path.resolve(__dirname, '../../output/verification/review-4');
fs.mkdirSync(output, { recursive: true });
await f.page.screenshot({ path: path.join(output, 'capacity-recovered.png'), fullPage: true });
});
test('retry HTTP rejects missing confirmation, forged state and cross-owner targets', async (t) => {
const f = await setup(t),
c = f.store.create(f.principal.user.id),
schedules = await NativeSchedules.create({ assistant: f.assistant });
const task = await schedules.createTask(c.id, f.principal, {
confirm: true,
request: { title: 'HTTP recovery', prompt: 'Read', after_seconds: 3600 },
});
const disabled = f.store.providers.read('deepseek');
f.store.providers.save({ ...disabled, enabled: false });
enqueue(schedules, c, task, f.principal.user.id);
await until(
async () => (await schedules.list(c.id, f.principal)).history[0]?.status === 'not_started',
);
const row = (await schedules.list(c.id, f.principal)).history[0];
const cookie = await f.login(),
viewer = await f.login('viewer'),
endpoint = '/assistant/conversations/' + c.id + '/schedules/deliveries/' + row.id + '/retry';
assert.equal((await f.call(endpoint, {}, cookie)).status, 403);
assert.equal((await f.call(endpoint, { confirm: true, status: 'queued' }, cookie)).status, 400);
assert.equal((await f.call(endpoint, { confirm: true }, viewer)).status, 404);
const foreign = f.store.create(f.users.users().find((u) => u.username === 'viewer').id);
assert.equal(
(await f.call(endpoint, { confirm: true, targetConversationId: foreign.id }, cookie)).status,
404,
);
f.store.providers.save({ ...f.store.providers.read('deepseek'), enabled: true });
f.assistant.runtime.chat = async () => JSON.stringify({ reply: 'Recovered API', actions: [] });
assert.equal((await f.call(endpoint, { confirm: true }, cookie)).status, 200);
await until(
async () => (await schedules.list(c.id, f.principal)).history[0]?.status === 'success',
);
});