From 73ca4edee5d33cc9f480f60fc20be7f3ce1f0c3e Mon Sep 17 00:00:00 2001 From: EdgeMLHacker <126883848+Apageoflove@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:53:55 -0700 Subject: [PATCH] =?UTF-8?q?fix(minimax-bot):=20=E5=8F=B0=E8=B4=A6=E8=AE=B0?= =?UTF-8?q?=E4=B8=8B=E5=B4=A9=E6=8E=89=E7=9A=84=E9=82=A3=E4=B8=80=E5=9B=9E?= =?UTF-8?q?=E5=90=88=EF=BC=8C=E9=9D=A2=E6=9D=BF=E4=B8=8A=E7=9A=84=E6=95=B0?= =?UTF-8?q?=E5=AD=97=E4=B8=8D=E5=86=8D=E8=87=AA=E5=B7=B1=E5=86=99=E6=AD=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../minimax-bot/.minimax-plugin/plugin.json | 2 +- plugins/minimax-ai/minimax-bot/README.md | 6 + .../minimax-bot/miniapp/client/core.js | 19 ++ .../minimax-bot/miniapp/client/panels.js | 7 +- .../miniapp/client/strings-panels.js | 4 + .../minimax-bot/miniapp/node/engine.mjs | 74 ++++-- .../minimax-bot/miniapp/node/mcp.mjs | 2 +- .../minimax-bot/miniapp/node/server.mjs | 17 +- .../minimax-bot/miniapp/node/store.mjs | 76 +++++- plugins/minimax-ai/minimax-bot/package.json | 2 +- .../minimax-bot/tests/note-limits-ui.test.mjs | 85 +++++++ .../minimax-bot/tests/usage-honest.test.mjs | 237 ++++++++++++++++++ 12 files changed, 502 insertions(+), 29 deletions(-) create mode 100644 plugins/minimax-ai/minimax-bot/tests/note-limits-ui.test.mjs create mode 100644 plugins/minimax-ai/minimax-bot/tests/usage-honest.test.mjs diff --git a/plugins/minimax-ai/minimax-bot/.minimax-plugin/plugin.json b/plugins/minimax-ai/minimax-bot/.minimax-plugin/plugin.json index 74347484..8eb17dec 100644 --- a/plugins/minimax-ai/minimax-bot/.minimax-plugin/plugin.json +++ b/plugins/minimax-ai/minimax-bot/.minimax-plugin/plugin.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "name": "minimax-bot", "displayName": "MiniMax Bot", - "version": "0.7.14", + "version": "0.7.17", "description": "Turn MiniMax Code into an always-on team of Bots. A Chief Bot helps you build the team; in group chats the admin answers and @mentions hand work to other members. Every Bot has its own octopus avatar and memory, with scheduled tasks and usage stats.", "author": "MiniMax Code", "icon": "icon.png", diff --git a/plugins/minimax-ai/minimax-bot/README.md b/plugins/minimax-ai/minimax-bot/README.md index 64a2a0c5..85497535 100644 --- a/plugins/minimax-ai/minimax-bot/README.md +++ b/plugins/minimax-ai/minimax-bot/README.md @@ -36,6 +36,12 @@ Text that travels from one Bot into another Bot's prompt is cleaned on the way. Only the tag goes — the words around it stay — and the plugin's own delivery format (``) is not touched, so a report that names a file still reaches the Bot that has to hand it over. +## 0.7.17 The panel and the ledger say what happened + +The usage ledger is the record of what the model really did. A turn cut off by a crash or a restart is written down as aborted, with no duration — it never reported how long it ran — and with the characters and tool calls it did make; the panel's average is taken over the turns that have a known duration, so a lost turn neither vanishes from the count nor counts as zero seconds. Every row says how many times the model was called for that turn (a retry is still one turn), and the panel adds that count when it is higher than the number of turns. A turn that ran is on the ledger even when a later step of the same turn fails to write. + +The note section of the group panel reads the note's limits (6 entries and 1200 characters) from the server, so the numbers it shows are the numbers the plugin uses. + ## 0.7.10 Optional group notes - A group can keep a note of what it said. After every round the group records the closing lines of the replies, and every Bot of the group gets that record in its standing brief. diff --git a/plugins/minimax-ai/minimax-bot/miniapp/client/core.js b/plugins/minimax-ai/minimax-bot/miniapp/client/core.js index 1a08b7d3..2545a484 100644 --- a/plugins/minimax-ai/minimax-bot/miniapp/client/core.js +++ b/plugins/minimax-ai/minimax-bot/miniapp/client/core.js @@ -60,6 +60,9 @@ export const state = { user: { name: '', systemName: '' }, /** Effective language resolved by the server. */ locale: systemLocale(), + /** Limits the server applies to a group note; `null` until the server states them. */ + /** @type {{ entries: number, chars: number } | null} */ + noteLimits: null, }; /** Navigation callbacks registered by app.js so panels can drive the main view. */ @@ -255,6 +258,19 @@ export async function api(method, path, body) { return /** @type {any} */ (data); } +/** + * The note limits as the server states them, or `null` when it does not state a usable pair: + * the panel then says nothing at all rather than half a line. + * @param {any} limits + * @returns {{ entries: number, chars: number } | null} + */ +function noteLimitsOf(limits) { + if (!limits || typeof limits !== 'object') return null; + const { entries, chars } = limits; + if (!Number.isFinite(entries) || !Number.isFinite(chars)) return null; + return { entries, chars }; +} + export async function loadState() { const data = await api('GET', '/api/state'); state.bots = data.bots; @@ -271,6 +287,9 @@ export async function loadState() { state.settings = data.settings ?? state.settings; state.user = data.user ?? state.user; state.locale = data.locale ?? state.locale; + // The server owns the note limits: the panel states them instead of writing down numbers of + // its own, which would drift from the ones the plugin actually applies. + state.noteLimits = noteLimitsOf(data.noteLimits); setLocale(state.locale); if (!state.current || !chatExists(state.current)) { const remembered = prefs.lastChat && chatExists(prefs.lastChat) ? prefs.lastChat : ''; diff --git a/plugins/minimax-ai/minimax-bot/miniapp/client/panels.js b/plugins/minimax-ai/minimax-bot/miniapp/client/panels.js index bb249a31..1ad94444 100644 --- a/plugins/minimax-ai/minimax-bot/miniapp/client/panels.js +++ b/plugins/minimax-ai/minimax-bot/miniapp/client/panels.js @@ -1473,13 +1473,16 @@ function usagePage(body, o) { .then((data) => { if (state.overlay?.kind !== 'settings' || state.overlay.page !== 'usage') return; const totals = data.totals; + // A turn that was cut off has no known duration, so the average counts the turns that + // really ran; the count of model calls says how often the model was really asked. + const modelCalls = totals.modelCalls ?? totals.turns; const kpis = h( 'div', { class: 'kpis' }, - h('div', { class: 'kpi big' }, h('span', { class: 'k-label' }, days === 1 ? t('panels.usage.last24h') : t('panels.usage.lastDays', { n: days })), h('span', { class: 'k-value' }, fmtNum(totals.turns), h('span', { class: 'k-unit' }, t('panels.usage.turnsUnit', { count: totals.turns })))), + h('div', { class: 'kpi big' }, h('span', { class: 'k-label' }, days === 1 ? t('panels.usage.last24h') : t('panels.usage.lastDays', { n: days })), h('span', { class: 'k-value' }, fmtNum(totals.turns), h('span', { class: 'k-unit' }, t('panels.usage.turnsUnit', { count: totals.turns }), modelCalls > totals.turns ? ` · ${t('panels.usage.modelCalls', { n: modelCalls, count: modelCalls })}` : ''))), h('div', { class: 'kpi' }, h('span', { class: 'k-label' }, t('panels.usage.replyChars')), h('span', { class: 'k-value' }, fmtNum(totals.chars))), h('div', { class: 'kpi' }, h('span', { class: 'k-label' }, t('panels.usage.toolCalls')), h('span', { class: 'k-value' }, fmtNum(totals.tools))), - h('div', { class: 'kpi' }, h('span', { class: 'k-label' }, t('panels.usage.avgDuration')), h('span', { class: 'k-value' }, totals.turns ? fmtDuration(totals.durationMs / totals.turns) : '—')), + h('div', { class: 'kpi' }, h('span', { class: 'k-label' }, t('panels.usage.avgDuration')), h('span', { class: 'k-value' }, totals.avgDurationMs != null ? fmtDuration(totals.avgDurationMs) : '—'), totals.durationSamples < totals.turns ? h('small', { class: 'muted' }, t('panels.usage.avgDurationNote')) : null), ); holder.replaceChildren( kpis, diff --git a/plugins/minimax-ai/minimax-bot/miniapp/client/strings-panels.js b/plugins/minimax-ai/minimax-bot/miniapp/client/strings-panels.js index 91a4a808..becd24ae 100644 --- a/plugins/minimax-ai/minimax-bot/miniapp/client/strings-panels.js +++ b/plugins/minimax-ai/minimax-bot/miniapp/client/strings-panels.js @@ -346,9 +346,11 @@ export default { 'panels.usage.last24h': '最近 24 小时', 'panels.usage.lastDays': '最近 {n} 天', 'panels.usage.turnsUnit': '轮对话', + 'panels.usage.modelCalls': '{n} 次模型调用', 'panels.usage.replyChars': '回复字数', 'panels.usage.toolCalls': '工具调用', 'panels.usage.avgDuration': '平均耗时', + 'panels.usage.avgDurationNote': '仅统计有已知耗时的轮次', 'panels.usage.turnsTitle': '对话轮次', 'panels.usage.hourly': '按小时', 'panels.usage.daily': '按天', @@ -709,9 +711,11 @@ export default { 'panels.usage.last24h': 'Last 24 hours', 'panels.usage.lastDays': 'Last {n} days', 'panels.usage.turnsUnit': (p) => pl(p.count, 'turn', 'turns'), + 'panels.usage.modelCalls': '{n} model calls', 'panels.usage.replyChars': 'Reply characters', 'panels.usage.toolCalls': 'Tool calls', 'panels.usage.avgDuration': 'Avg. duration', + 'panels.usage.avgDurationNote': 'Turns with a known duration only', 'panels.usage.turnsTitle': 'Chat turns', 'panels.usage.hourly': 'Hourly', 'panels.usage.daily': 'Daily', diff --git a/plugins/minimax-ai/minimax-bot/miniapp/node/engine.mjs b/plugins/minimax-ai/minimax-bot/miniapp/node/engine.mjs index 7f156249..c12f13cf 100644 --- a/plugins/minimax-ai/minimax-bot/miniapp/node/engine.mjs +++ b/plugins/minimax-ai/minimax-bot/miniapp/node/engine.mjs @@ -47,6 +47,11 @@ const HELPER_ACTIVE = new Set(['queued', 'running']); * Model capacity errors (429/529) and timeouts usually clear within a minute. */ export const TURN_RETRY_DELAYS_MS = Object.freeze([8000, 20000]); +/** + * How often a running turn writes its progress to disk. A crash must leave the work the model + * already did in the ledger, so the turn's known state cannot live only in memory. + */ +const LIVE_PERSIST_MS = 2000; /** Failure reasons a retry cannot fix; anything else (or no reason from an older Host) may be retried. */ export const NO_RETRY_REASONS = new Set(['auth', 'quota', 'context_too_long', 'blocked', 'invalid_request']); /** The turn may already have run, so resending it could repeat a reply or an action. */ @@ -578,11 +583,14 @@ export class Engine { const startedAt = Date.now(); let lastFlush = 0; + let lastPersist = 0; let rootSessionId = this.store.getSession(chatId, bot.id)?.sessionId; let toolStarted = false; /** Whether any attempt started a turn, i.e. the session already holds this input. */ let reachedHost = false; let invoked = false; + /** How many times the model was called for this turn, retries included. */ + let modelCalls = 0; let preparedInput = input; let acceptedGroupCursor; /** @type {{ type: 'text', text: string, live?: boolean } | undefined} */ @@ -593,6 +601,13 @@ export class Engine { lastFlush = now; message.text = textOf(message.parts); this.broadcast({ type: 'message.upsert', chatId, message }); + // Keep the turn's progress on disk: a crash must leave the work the model already did + // in the ledger, not just a message that says it was still running. + if (now - lastPersist >= LIVE_PERSIST_MS) { + lastPersist = now; + void this.store.persistMessages(chatId).catch((error) => + this.logger.warn('minimax_bot.turn_persist_failed', { code: error?.code ?? 'unknown' })); + } }; const ensureLiveText = () => { if (!liveText || message.parts.at(-1) !== liveText) { @@ -709,6 +724,8 @@ export class Engine { if (attempt === 0) preparedInput = await this.prepareGroupInput(input); const attemptInput = attempt === 0 || !reachedHost ? preparedInput : { ...preparedInput, text: t('retry.continue') }; invoked = true; + modelCalls += 1; + message.modelCalls = modelCalls; result = await this.invoke(attemptInput, rootSessionId, controller.signal, onEvent) ?? skippedTurn(); } catch (error) { if (errorCode(error) === 'SESSION_NOT_FOUND' && !input.answer && attempt === 0) { @@ -716,6 +733,8 @@ export class Engine { await this.store.clearSession(chatId, bot.id); try { preparedInput = await this.prepareGroupInput(input); + modelCalls += 1; + message.modelCalls = modelCalls; result = await this.invoke(preparedInput, undefined, controller.signal, onEvent) ?? skippedTurn(); } catch (retryError) { failure = describeAgentError(retryError); @@ -814,31 +833,37 @@ export class Engine { request: result.pendingQuestion.request, }; } - await this.store.updateMessage(chatId, message.id, {}); - if (result?.sessionId) { - await this.store.setSession(chatId, bot.id, { sessionId: result.sessionId }); - } - if (acceptedGroupCursor !== undefined) { - const previous = this.store.getSession(chatId, bot.id)?.lastSeenSeq ?? 0; - await this.store.setSession(chatId, bot.id, { lastSeenSeq: Math.max(previous, acceptedGroupCursor) }); + const tools = message.parts.filter((part) => part.type === 'tool').length; + try { + await this.store.updateMessage(chatId, message.id, {}); + if (result?.sessionId) { + await this.store.setSession(chatId, bot.id, { sessionId: result.sessionId }); + } + if (acceptedGroupCursor !== undefined) { + const previous = this.store.getSession(chatId, bot.id)?.lastSeenSeq ?? 0; + await this.store.setSession(chatId, bot.id, { lastSeenSeq: Math.max(previous, acceptedGroupCursor) }); + } + } finally { + // The model ran: the ledger must say so even when a later step of this turn fails. + await this.store.recordUsage({ + ts: Date.now(), + turnId: message.id, + botId: bot.id, + botName: bot.name, + chatId, + kind: input.kind, + status, + durationMs: message.durationMs, + chars: finalText.length, + tools, + // One turn, but the model may have been called more than once to finish it. + attempts: modelCalls, + }); + this.broadcast({ type: 'usage.changed' }); } this.broadcast({ type: 'message.upsert', chatId, message }); this.emitBusy(chatId); this.broadcastSummary(chatId); - - const tools = message.parts.filter((part) => part.type === 'tool').length; - await this.store.recordUsage({ - ts: Date.now(), - botId: bot.id, - botName: bot.name, - chatId, - kind: input.kind, - status, - durationMs: message.durationMs, - chars: finalText.length, - tools, - }); - this.broadcast({ type: 'usage.changed' }); return { status, text: finalText, @@ -1755,6 +1780,10 @@ export class Engine { // A background job calls the same tools a Bot's own turn does. Count them, or the // turn it spends looks like it cost nothing. turnTools += 1; + // Keep the count on disk: a job cut off by a restart must still show the tools it + // really called in the ledger. + helper.turnTools = turnTools; + void this.store.save().catch((error) => this.logger.warn('minimax_bot.helper_save_failed', { helperId: helper.id, code: error?.code ?? 'unknown' })); } else if (event.type === 'permission.requested') { // The user approves a helper's tool call in its Bot's chat, like the Bot's own. const approval = { @@ -1833,6 +1862,7 @@ export class Engine { // counts fewer turns than the model actually ran. await this.store.recordUsage({ ts: Date.now(), + turnId: helper.id, botId: helper.botId, botName: turnBotName, chatId: helper.chatId, @@ -1841,6 +1871,8 @@ export class Engine { durationMs: turnMs, chars: turnText.length, tools: turnTools, + // A background job runs the model once; it has no retry loop of its own. + attempts: 1, }); this.broadcast({ type: 'usage.changed' }); } diff --git a/plugins/minimax-ai/minimax-bot/miniapp/node/mcp.mjs b/plugins/minimax-ai/minimax-bot/miniapp/node/mcp.mjs index 4cf80a03..1fb19293 100644 --- a/plugins/minimax-ai/minimax-bot/miniapp/node/mcp.mjs +++ b/plugins/minimax-ai/minimax-bot/miniapp/node/mcp.mjs @@ -586,7 +586,7 @@ export function createMcpHandler({ team, engine }) { result = { protocolVersion: PROTOCOL_VERSIONS.includes(requested) ? requested : PROTOCOL_VERSIONS[0], capabilities: { tools: { listChanged: false } }, - serverInfo: { name: 'minimax-bot', version: '0.7.14' }, + serverInfo: { name: 'minimax-bot', version: '0.7.17' }, instructions: t('mcp.instructions'), }; break; diff --git a/plugins/minimax-ai/minimax-bot/miniapp/node/server.mjs b/plugins/minimax-ai/minimax-bot/miniapp/node/server.mjs index ce8f55aa..a5511bb1 100644 --- a/plugins/minimax-ai/minimax-bot/miniapp/node/server.mjs +++ b/plugins/minimax-ai/minimax-bot/miniapp/node/server.mjs @@ -213,7 +213,7 @@ export async function start(context) { } /** @type {Record} */ const byBot = {}; - const totals = { turns: 0, chars: 0, tools: 0, durationMs: 0, errors: 0 }; + const totals = { turns: 0, chars: 0, tools: 0, durationMs: 0, durationSamples: 0, modelCalls: 0, errors: 0 }; for (const row of rows) { const bucket = buckets[keyOf(row.ts)]; const live = store.getBot(row.botId); @@ -228,6 +228,8 @@ export async function start(context) { chars: 0, tools: 0, durationMs: 0, + durationSamples: 0, + modelCalls: 0, errors: 0, }); if (bucket) { @@ -238,13 +240,24 @@ export async function start(context) { target.turns += 1; target.chars += row.chars ?? 0; target.tools += row.tools ?? 0; - target.durationMs += row.durationMs ?? 0; + // A turn that was cut off has no known duration: it must not pull the average down, + // so only the turns that really ran are counted. + if (Number.isFinite(row.durationMs)) { + target.durationMs += row.durationMs; + target.durationSamples += 1; + } + // How many times the model really ran for this turn; a row without the field is one call. + target.modelCalls += row.attempts ?? 1; } if (row.status === 'error') { bot.errors += 1; totals.errors += 1; } } + // The average covers the turns whose duration is known; a turn that was cut off has none. + for (const target of [totals, ...Object.values(byBot)]) { + target.avgDurationMs = target.durationSamples ? target.durationMs / target.durationSamples : null; + } return { days: hourly ? 1 : span, hourly, diff --git a/plugins/minimax-ai/minimax-bot/miniapp/node/store.mjs b/plugins/minimax-ai/minimax-bot/miniapp/node/store.mjs index 6dc837ed..ce5017ad 100644 --- a/plugins/minimax-ai/minimax-bot/miniapp/node/store.mjs +++ b/plugins/minimax-ai/minimax-bot/miniapp/node/store.mjs @@ -203,6 +203,15 @@ function previewOf(message, nameOf) { } } +/** The text a stored turn had produced by the time it was cut off. */ +function knownText(message) { + if (typeof message.text === 'string' && message.text) return message.text; + return (message.parts ?? []) + .filter((part) => part.type === 'text' && part.text) + .map((part) => part.text) + .join('\n\n'); +} + export class Store { /** @param {string} dataDir */ constructor(dataDir) { @@ -216,6 +225,11 @@ export class Store { this.messageCache = new Map(); /** @type {Map>} cold chats being read from disk */ this.messageLoads = new Map(); + /** Turn ids already in the ledger, so a recovered turn is never counted twice. */ + /** @type {Set} */ + this.usageIds = new Set(); + /** @type {Promise | undefined} */ + this.usageIdsReady = undefined; /** @type {Promise} */ this.writeChain = Promise.resolve(); } @@ -230,9 +244,26 @@ export class Store { this.state.helpers = Array.isArray(this.state.helpers) ? this.state.helpers : []; for (const helper of this.state.helpers) { if (helper.status === 'queued' || helper.status === 'running') { + const wasQueued = helper.status === 'queued'; helper.status = 'interrupted'; helper.endedAt = helper.endedAt ?? Date.now(); migrated = true; + // The run was cut off, so the model never reported how long it took: the ledger row + // says "aborted" with no duration rather than inventing one. + await this.recordUsageOnce({ + ts: helper.startedAt ?? helper.createdAt ?? helper.endedAt, + turnId: helper.id, + botId: helper.botId, + botName: this.getBot(helper.botId)?.name ?? '', + chatId: helper.chatId, + kind: 'helper-run', + status: 'aborted', + durationMs: null, + chars: 0, + tools: helper.turnTools ?? 0, + // A queued job never reached the model; a running one had already called it. + attempts: wasQueued ? 0 : 1, + }); } } this.state.taskBatches = Array.isArray(this.state.taskBatches) ? this.state.taskBatches : []; @@ -403,6 +434,24 @@ export class Store { message.status = 'interrupted'; message.live = undefined; dirty = true; + // The turn the model was in the middle of is real work: it belongs in the ledger. + // The run died, so its duration is unknown and stays null. + if (message.role === 'bot') { + await this.recordUsageOnce({ + ts: message.startedAt ?? message.ts, + turnId: message.id, + botId: message.botId, + botName: this.getBot(message.botId)?.name ?? '', + chatId, + kind: message.kind ?? 'private', + status: 'aborted', + durationMs: null, + chars: knownText(message).length, + tools: (message.parts ?? []).filter((part) => part.type === 'tool').length, + // The model was already answering; a turn that never reached it is not streaming. + attempts: message.modelCalls ?? 1, + }); + } } if (message.role === 'user' && ['adjusting', 'processing'].includes(message.dispatchState)) { message.dispatchState = message.dispatchState === 'adjusting' ? 'held' @@ -570,7 +619,32 @@ export class Store { // ---------- usage ---------- recordUsage(entry) { - return this.enqueue(() => appendFile(this.usagePath, `${JSON.stringify(entry)}\n`, 'utf8')); + return this.enqueue(async () => { + await appendFile(this.usagePath, `${JSON.stringify(entry)}\n`, 'utf8'); + if (entry.turnId) this.usageIds.add(entry.turnId); + }); + } + + /** + * Writes a row only when the turn has none yet. A restart recovers the turns that were in + * flight; a turn the engine already metered must not be counted a second time. + * @param {Record} entry + * @returns {Promise} whether a row was written + */ + async recordUsageOnce(entry) { + await this.loadUsageIds(); + if (entry.turnId && this.usageIds.has(entry.turnId)) return false; + await this.recordUsage(entry); + return true; + } + + /** The turn ids the ledger already holds, read once per process. */ + async loadUsageIds() { + this.usageIdsReady ??= (async () => { + for (const row of await this.readUsage(0)) if (row.turnId) this.usageIds.add(row.turnId); + })(); + await this.usageIdsReady; + return this.usageIds; } async readUsage(sinceTs) { diff --git a/plugins/minimax-ai/minimax-bot/package.json b/plugins/minimax-ai/minimax-bot/package.json index e34b05e9..387b263a 100644 --- a/plugins/minimax-ai/minimax-bot/package.json +++ b/plugins/minimax-ai/minimax-bot/package.json @@ -1,6 +1,6 @@ { "name": "minimax-bot", - "version": "0.7.14", + "version": "0.7.17", "private": true, "type": "module", "mcode": { diff --git a/plugins/minimax-ai/minimax-bot/tests/note-limits-ui.test.mjs b/plugins/minimax-ai/minimax-bot/tests/note-limits-ui.test.mjs new file mode 100644 index 00000000..aa768ed9 --- /dev/null +++ b/plugins/minimax-ai/minimax-bot/tests/note-limits-ui.test.mjs @@ -0,0 +1,85 @@ +// @ts-check +// The note limits the server states, as the client shows them. +// +// The point of this file: the numbers in the group panel are the server's numbers. The server +// says how many entries and how many characters a group note keeps; the client carries those +// into the panel instead of writing down numbers of its own that would drift from the ones the +// plugin actually applies. A panel that states no numbers at all is better than a panel that +// states the wrong ones. + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { JSDOM } from 'jsdom'; + +const dom = new JSDOM( + '
', + { url: 'http://localhost/bot' }, +); +for (const key of ['window', 'document', 'navigator', 'localStorage', 'HTMLElement', 'Node', 'FormData']) { + Object.defineProperty(globalThis, key, { value: dom.window[key], configurable: true }); +} +window.matchMedia = () => ({ matches: false }); +dom.window.HTMLDialogElement.prototype.showModal = function () { this.open = true; }; + +/** What the server sends on /api/state; the cases below change it. */ +let serverState = { + bots: [{ id: 'b1', name: 'Rob', role: 'Coordinator', skin: 'bubble', isMain: true }], + groups: [{ + id: 'g1', name: 'Team', notice: '', adminId: 'b1', memberIds: ['b1'], replyMode: 'admin', + notes: true, noteEntries: [], + }], + noteLimits: { entries: 6, chars: 1200 }, + schedules: [], helpers: [], tasks: [], busyChats: [], turns: [], chats: {}, approvals: [], sessions: {}, + agentAvailable: true, locale: 'en', settings: { language: 'en', userName: 'Test' }, user: { name: 'Test', systemName: 'Test' }, +}; + +globalThis.fetch = async (url) => { + if (String(url).includes('/api/state')) return { ok: true, status: 200, json: async () => structuredClone(serverState) }; + if (String(url).includes('/api/usage')) return { ok: true, status: 200, json: async () => ({ byBot: [] }) }; + return { ok: true, status: 200, json: async () => ({}) }; +}; + +const { state, loadState } = await import('../miniapp/client/core.js'); +const { setLocale } = await import('../miniapp/client/i18n.js'); +const { renderDrawer } = await import('../miniapp/client/panels.js'); + +setLocale('en'); +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); + +/** Open the panel of the group and hand back what it shows. */ +async function groupPanel() { + state.current = 'group:g1'; + state.drawer = { open: true, view: 'overview', memTab: 'self', dirty: false }; + renderDrawer(); + await tick(); + return document.getElementById('drawer').textContent; +} + +test('the client keeps the note limits the server states', async () => { + await loadState(); + assert.deepEqual(state.noteLimits, { entries: 6, chars: 1200 }); +}); + +test('the group panel states the limits the server sent', async () => { + await loadState(); + const text = await groupPanel(); + assert.match(text, /6 entries and 1200 characters/); +}); + +test('the panel follows the server, it does not write the numbers down itself', async () => { + serverState = { ...serverState, noteLimits: { entries: 3, chars: 400 } }; + await loadState(); + assert.deepEqual(state.noteLimits, { entries: 3, chars: 400 }); + const text = await groupPanel(); + assert.match(text, /3 entries and 400 characters/); + assert.doesNotMatch(text, /6 entries and 1200 characters/); +}); + +test('a server that states no limits leaves the panel without a limit line', async () => { + const { noteLimits, ...withoutLimits } = serverState; + serverState = withoutLimits; + await loadState(); + assert.equal(state.noteLimits, null); + const text = await groupPanel(); + assert.doesNotMatch(text, /entries and .* characters/); +}); diff --git a/plugins/minimax-ai/minimax-bot/tests/usage-honest.test.mjs b/plugins/minimax-ai/minimax-bot/tests/usage-honest.test.mjs new file mode 100644 index 00000000..fc52962c --- /dev/null +++ b/plugins/minimax-ai/minimax-bot/tests/usage-honest.test.mjs @@ -0,0 +1,237 @@ +// The usage ledger must say what the model really did: +// - a turn a crash or a restart cut off is still a turn: aborted, with no invented duration, +// - a retried turn stays one turn but says how many times the model was called, +// - a turn whose bookkeeping failed afterwards is still on the ledger, +// - the average duration covers the turns that ran, so the panel never shows NaN. +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { createServer } from 'node:net'; +import { JSDOM } from 'jsdom'; +import { Store } from '../miniapp/node/store.mjs'; +import { MemoryStore } from '../miniapp/node/memory.mjs'; +import { Engine } from '../miniapp/node/engine.mjs'; +import { start } from '../miniapp/node/server.mjs'; +import { setLocale } from '../miniapp/node/i18n.mjs'; + +const dom = new JSDOM('
', { url: 'http://localhost/bot' }); +for (const key of ['window', 'document', 'navigator', 'localStorage', 'HTMLElement', 'Node', 'FormData']) Object.defineProperty(globalThis, key, { value: dom.window[key], configurable: true }); +window.matchMedia = () => ({ matches: false }); +dom.window.HTMLDialogElement.prototype.showModal = function () { this.open = true; }; +const { state } = await import('../miniapp/client/core.js'); +const { setLocale: setClientLocale } = await import('../miniapp/client/i18n.js'); +const { openOverlay, closeOverlay } = await import('../miniapp/client/panels.js'); +setClientLocale('en'); + +const tick = () => new Promise((done) => setTimeout(done, 0)); + +/** A data directory with a bot and no turn of its own. */ +async function fixture(t, prefix) { + const dir = await mkdtemp(join(tmpdir(), prefix)); + const store = new Store(dir); + const memory = new MemoryStore(dir); + await store.init(); + await memory.init(); + setLocale('en'); + const engine = new Engine({ store, memory, logger: { warn() {} }, broadcast() {}, retryDelaysMs: [] }); + t.after(async () => { + engine.dispose(); + await store.writeChain; + await rm(dir, { recursive: true, force: true }); + }); + return { dir, store, memory, engine }; +} + +test('a turn a restart cut off is still on the ledger: aborted, with no invented duration', async (t) => { + const dir = await mkdtemp(join(tmpdir(), 'bot-usage-honest-restart-')); + t.after(() => rm(dir, { recursive: true, force: true })); + const chatId = 'bot:bot_a'; + const store = new Store(dir); + await store.init(); + store.state.bots = [{ id: 'bot_a', name: 'Alice', role: 'Writer' }]; + // A Bot turn that was still streaming when the process stopped. + await store.appendMessage(chatId, { + id: 'msg_stream_1', + role: 'bot', + botId: 'bot_a', + text: 'Half an answer', + status: 'streaming', + parts: [ + { type: 'text', text: 'Half an answer' }, + { type: 'tool', name: 'read_file', state: 'running' }, + ], + kind: 'private', + startedAt: Date.now() - 90_000, + modelCalls: 1, + }); + // A background job that was running when the process stopped. + store.state.helpers = [{ + id: 'help_1', + botId: 'bot_a', + chatId, + title: 'research', + task: 'look it up', + status: 'running', + createdAt: Date.now() - 118_000, + startedAt: Date.now() - 118_000, + turnTools: 2, + }]; + await store.save(); + + // The process restarts: a new Store reads the same data directory. + const restarted = new Store(dir); + await restarted.init(); + await restarted.getMessages(chatId); + const rows = await restarted.readUsage(0); + assert.equal(rows.length, 2, 'the work that was in flight left no trace in the ledger'); + + const turn = rows.find((row) => row.turnId === 'msg_stream_1'); + assert.ok(turn, 'the interrupted turn left no row in the ledger'); + assert.equal(turn.status, 'aborted'); + assert.equal(turn.durationMs, null, 'a turn that died has no known duration'); + assert.equal(turn.chars, 'Half an answer'.length); + assert.equal(turn.tools, 1); + assert.equal(turn.attempts, 1); + + const job = rows.find((row) => row.turnId === 'help_1'); + assert.ok(job, 'the interrupted background job left no row in the ledger'); + assert.equal(job.status, 'aborted'); + assert.equal(job.durationMs, null, 'a job that died has no known duration'); + assert.equal(job.tools, 2); + assert.equal(job.kind, 'helper-run'); +}); + +test('a turn the engine already metered is not counted twice by the recovery', async (t) => { + const dir = await mkdtemp(join(tmpdir(), 'bot-usage-honest-once-')); + t.after(() => rm(dir, { recursive: true, force: true })); + const chatId = 'bot:bot_a'; + const store = new Store(dir); + await store.init(); + store.state.bots = [{ id: 'bot_a', name: 'Alice', role: 'Writer' }]; + // The turn was metered, but its message never left the 'streaming' state on disk. + await store.appendMessage(chatId, { + id: 'msg_stream_2', + role: 'bot', + botId: 'bot_a', + text: 'Done', + status: 'streaming', + parts: [{ type: 'text', text: 'Done' }], + kind: 'private', + startedAt: Date.now() - 5000, + }); + await store.save(); + await writeFile(join(dir, 'usage.jsonl'), `${JSON.stringify({ + ts: Date.now() - 4000, + turnId: 'msg_stream_2', + botId: 'bot_a', + botName: 'Alice', + chatId, + kind: 'private', + status: 'done', + durationMs: 1200, + chars: 4, + tools: 0, + attempts: 1, + })}\n`, 'utf8'); + + const restarted = new Store(dir); + await restarted.init(); + await restarted.getMessages(chatId); + const rows = await restarted.readUsage(0); + assert.equal(rows.length, 1, 'one turn is one row, however often the process restarts'); + assert.equal(rows[0].status, 'done'); +}); + +test('a retried turn stays one turn, and says how many times the model was called', async (t) => { + const { store, engine } = await fixture(t, 'bot-usage-honest-retry-'); + engine.retryDelays = [1, 1]; + const calls = { count: 0 }; + engine.agent = { + async run(input, options) { + calls.count += 1; + if (calls.count < 3) throw Object.assign(new Error('Unavailable'), { code: 'AGENT_UNAVAILABLE', retryable: true }); + options.onEvent({ type: 'turn.started', sessionId: 's1' }); + return { sessionId: 's1', status: 'finished', text: 'Done' }; + }, + }; + const bot = store.state.bots[0]; + const chatId = `bot:${bot.id}`; + await engine.runBotTurn({ bot, chatId, text: 'Hello', kind: 'private' }); + assert.equal(calls.count, 3, 'the model was called three times'); + const rows = await store.readUsage(0); + assert.equal(rows.length, 1, 'a retried turn is still one turn'); + assert.equal(rows[0].attempts, 3, 'the ledger does not say how often the model ran'); + assert.equal(rows[0].status, 'done'); +}); + +test('a turn whose bookkeeping fails afterwards is still on the ledger', async (t) => { + const { store, engine } = await fixture(t, 'bot-usage-honest-late-'); + const calls = { count: 0 }; + engine.agent = { + async run(input, options) { + calls.count += 1; + options.onEvent({ type: 'turn.started', sessionId: 's1' }); + return { sessionId: 's1', status: 'finished', text: 'Done' }; + }, + }; + // The turn's own bookkeeping fails after the model ran: the message cannot be stored. + store.updateMessage = async () => { throw Object.assign(new Error('disk full'), { code: 'EIO' }); }; + const bot = store.state.bots[0]; + const chatId = `bot:${bot.id}`; + await assert.rejects(() => engine.runBotTurn({ bot, chatId, text: 'Hello', kind: 'private' }), /disk full/); + assert.equal(calls.count, 1, 'the model ran once'); + const rows = await store.readUsage(0); + assert.equal(rows.length, 1, 'the turn the model ran left no row in the ledger'); + assert.equal(rows[0].attempts, 1); +}); + +test('the average duration covers the turns that ran, and the panel never shows NaN', async (t) => { + const dir = await mkdtemp(join(tmpdir(), 'bot-usage-honest-avg-')); + t.after(() => rm(dir, { recursive: true, force: true })); + const now = Date.now(); + const chatId = 'bot:bot_a'; + const rows = [ + { ts: now, turnId: 'm1', botId: 'bot_a', botName: 'Alice', chatId, kind: 'private', status: 'done', durationMs: 1000, chars: 10, tools: 1, attempts: 1 }, + { ts: now, turnId: 'm2', botId: 'bot_a', botName: 'Alice', chatId, kind: 'private', status: 'done', durationMs: 3000, chars: 10, tools: 0, attempts: 3 }, + { ts: now, turnId: 'm3', botId: 'bot_a', botName: 'Alice', chatId, kind: 'private', status: 'aborted', durationMs: null, chars: 0, tools: 1, attempts: 1 }, + ]; + await writeFile(join(dir, 'usage.jsonl'), `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8'); + + const probe = createServer(); + await new Promise((done) => probe.listen(0, '127.0.0.1', done)); + const port = probe.address().port; + await new Promise((done) => probe.close(done)); + const app = await start({ + pluginRoot: resolve(import.meta.dirname, '..'), + dataDir: dir, + listen: { host: '127.0.0.1', port }, + signal: new AbortController().signal, + logger: { info() {}, warn() {}, error() {} }, + }); + t.after(async () => { await app.dispose(); }); + const usage = await (await fetch(`http://127.0.0.1:${port}/api/usage?days=1`)).json(); + assert.equal(usage.totals.turns, 3); + assert.equal(usage.totals.durationSamples, 2, 'the cut-off turn has no duration to average'); + assert.equal(usage.totals.durationMs, 4000); + // (1000 + 3000) / 2: the average of the turns that ran, not 4000 / 3 and not NaN. + assert.equal(usage.totals.avgDurationMs, 2000); + assert.ok(Number.isFinite(usage.totals.avgDurationMs)); + assert.equal(usage.totals.modelCalls, 5, 'the model really ran five times'); + + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + if (String(url).includes('/api/usage')) return { ok: true, json: async () => usage }; + return { ok: true, json: async () => ({}) }; + }; + t.after(() => { globalThis.fetch = originalFetch; }); + openOverlay({ kind: 'settings', page: 'usage' }); + await tick(); + await tick(); + const panel = document.querySelector('.kpis')?.textContent ?? ''; + assert.doesNotMatch(panel, /NaN|undefined/); + assert.match(panel, /2s/, 'the panel shows the average of the turns that ran'); + assert.match(panel, /5 model calls/, 'the panel says how often the model really ran'); + closeOverlay(); +});