Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 112 additions & 38 deletions packages/pi/src/runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,21 @@ export interface PiOptions extends PiModelOptions {
/** The internal action a scheduled wake calls. */
const WAKE_ACTION = "pi.wake";

/**
* How far ahead the recovery wake is scheduled while Pi work is active. An
* actor whose process dies during a run resumes within about this long, plus
* the time the engine takes to find the process gone.
*/
const RECOVERY_WAKE_MS = 15_000;

interface OpenHarness {
harness: Harness;
/** Calls `listener` whenever Pi's wake value changes. Returns a function that stops it. */
onWakeChange: (listener: () => void) => () => void;
/** Resumes Pi's due sleeps, for the scheduled `pi.wake`. */
resumeDue: () => void;
/** Resolves once the wake for the work a call started is written. */
settleWake: () => Promise<void>;
stopWake: () => void;
}

Expand Down Expand Up @@ -218,10 +227,12 @@ export async function withHarness<T>(
const release = countCall(runtime);
try {
const context = withCallerSpan(c.actorId, BACKGROUND_CONTEXT);
return await run(harness, context, (wait) => {
const result = await run(harness, context, (wait) => {
release();
return wait();
});
await settleWake(runtime);
return result;
} catch (error) {
await recoverHarness(runtime.actor ?? c, runtime, options);
throw toClientError(error);
Expand All @@ -230,6 +241,15 @@ export async function withHarness<T>(
}
}

/**
* Waits until the wake for the work a call started is written, so a crash
* right after the reply still resumes that work.
*/
async function settleWake(runtime: PiRuntime): Promise<void> {
const open = await runtime.harness?.catch(() => undefined);
await open?.settleWake();
}

/**
* Runs a wait on a run, such as `submission.wait`, without holding a sleep.
* While the run is busy, the run itself holds the sleep.
Expand Down Expand Up @@ -283,7 +303,9 @@ export async function withPiAccess<T>(
Object.defineProperty(c, "pi", { value: harness, configurable: true });
const release = countCall(runtime);
try {
return await body();
const result = await body();
await settleWake(runtime);
return result;
} catch (error) {
await recoverHarness(runtime.actor ?? c, runtime, options);
throw toClientError(error);
Expand Down Expand Up @@ -371,6 +393,7 @@ async function openHarness(
harness,
onWakeChange: wake.subscribe,
resumeDue: wake.resumeDue,
settleWake: wake.settle,
stopWake: wake.stop,
};
}
Expand Down Expand Up @@ -527,11 +550,16 @@ function reportBlockedTasks(c: PiContext, inspection: HarnessInspection): void {
}

/**
* Keeps the actor awake while Pi Durable can run work, and schedules a wake
* for its earliest sleeping task, such as a retry backoff. Pi calls `onWake`
* after open and whenever the value changes. With `onWake`, Pi keeps no timer
* for sleeps, so the scheduled `pi.wake` action resumes them. While other work
* is active the actor cannot sleep, so an in-process timer resumes them.
* Keeps the actor awake while Pi Durable can run work, and keeps one `pi.wake`
* scheduled, as Pi Durable's host contract asks (`HarnessOptions.onWake`). Pi
* calls `onWake` after open and whenever `harness.wake()` changes: `active`
* while some task can run now or is running, and `at`, the earliest sleep.
*
* The wake is at the earliest sleep, and while work is active no later than
* `RECOVERY_WAKE_MS` ahead. Keep-awake only lasts while the process does, so
* that recovery wake is what resumes work a crash cut off. While work runs,
* the wake fires on the live actor and `resumeDue` arms the next one. A stop
* keeps the wake, so a sleep that cut work off resumes it too.
*
* Pi runs a sleep only once its time has passed, and reports a wake only when
* the value changes. So a wake that fires early, even by a millisecond,
Expand All @@ -545,16 +573,18 @@ function wakeActor(
attach: (harness: Harness) => void;
subscribe: (listener: () => void) => () => void;
resumeDue: () => void;
settle: () => Promise<void>;
stop: () => void;
} {
const listeners = new Set<() => void>();
let harness: Harness | undefined;
let latest: Wake = { active: false };
let release: (() => void) | undefined;
let scheduledAt: number | undefined;
let scheduling = false;
/** The one `pi.wake` this generation keeps. `id` resolves once it is written, or to `undefined` if it failed. */
let armed: { at: number; id: Promise<string | undefined> } | undefined;
/** When a schedule write last failed. Until `RECOVERY_WAKE_MS` later, a timer stands in for the wake. */
let failedAt: number | undefined;
let timer: { at: number; handle: ReturnType<typeof setTimeout> } | undefined;
let scheduleFailedAt: number | undefined;
let stopped = false;

const stayAwake = () => {
Expand All @@ -570,7 +600,6 @@ function wakeActor(
const clearTimer = () => {
if (timer) clearTimeout(timer.handle);
timer = undefined;
scheduleFailedAt = undefined;
};
const armTimer = (at: number) => {
if (timer?.at === at) return;
Expand All @@ -579,55 +608,88 @@ function wakeActor(
const delay = Math.min(Math.max(0, at - now()), 2 ** 31 - 1);
timer = { at, handle: setTimeout(resumeDue, delay) };
};
/** Replaces the kept wake. The new one is written before the old one is cancelled, so work never goes without one. */
const rearm = (at: number | undefined) => {
const previous = armed?.id;
const next =
at === undefined
? undefined
: c.schedule.at(at, WAKE_ACTION).then(
(id) => {
failedAt = undefined;
return id;
},
(error: unknown) => {
c.log.warn({
msg: "pi durable could not schedule its wake, so the actor stays awake and a timer resumes its work",
error,
});
failedAt = Date.now();
if (armed?.id === next) armed = undefined;
apply();
return undefined;
},
);
armed = next && at !== undefined ? { at, id: next } : undefined;
void (async () => {
await next;
const old = await previous;
if (old !== undefined) await c.schedule.cancel(old).catch(() => false);
})();
};
const apply = () => {
if (stopped) return;
const { active, at } = latest;
if (timer?.at !== at) clearTimer();
if (active) {
const heartbeat = active ? Date.now() + RECOVERY_WAKE_MS : undefined;
const target =
heartbeat === undefined || at === undefined
? (heartbeat ?? at)
: Math.min(heartbeat, at);
if (active) stayAwake();
if (target === undefined) {
clearTimer();
if (armed) rearm(undefined);
return letGo();
}
if (failedAt !== undefined && Date.now() - failedAt < RECOVERY_WAKE_MS) {
stayAwake();
if (at !== undefined) armTimer(at);
return;
}
if (at === undefined || at === scheduledAt) return letGo();
clearTimer();
// While work runs, an earlier wake still ahead is enough: it fires, and `resumeDue` arms the next one.
const kept = active
? armed && armed.at <= target && armed.at > Date.now()
: armed?.at === target;
if (!kept) rearm(target);
if (active) return;
// The actor may sleep only once its wake is written.
stayAwake();
if (scheduling || scheduleFailedAt === at) return;
scheduling = true;
c.schedule
.at(at, WAKE_ACTION)
.then(
() => {
scheduledAt = at;
},
(error: unknown) => {
c.log.warn({
msg: "pi durable could not schedule its wake, so the actor stays awake until it",
error,
});
armTimer(at);
scheduleFailedAt = at;
},
)
.finally(() => {
scheduling = false;
apply();
});
const written = armed;
void written?.id.then((id) => {
if (id !== undefined && armed === written && !latest.active && !stopped)
letGo();
});
};
const resumeDue = () => {
if (stopped || !harness) return;
harness.resume();
clearTimer();
scheduledAt = undefined;
// RivetKit removes a wake before it runs it, so one that came due is gone.
if (armed && armed.at <= Date.now()) armed = undefined;
harness.resume();
latest = harness.wake();
apply();
};
const stop = () => {
stopped = true;
clearTimer();
// The wake stays, so work a stop cut off resumes.
letGo();
};
// Keep-awake only blocks idle sleep. Once shutdown starts it would only delay it.
c.abortSignal.addEventListener("abort", stop, { once: true });
// A reopen during a drain gets a signal that is already aborted.
if (c.abortSignal.aborted) stop();
else c.abortSignal.addEventListener("abort", stop, { once: true });
return {
onWake: (wake) => {
latest = wake;
Expand All @@ -642,9 +704,21 @@ function wakeActor(
return () => listeners.delete(listener);
},
resumeDue,
settle: async () => {
// Pi reports a change in a microtask after its commit, so read the value now.
if (stopped || !harness || !isUsable(harness)) return;
latest = harness.wake();
apply();
await armed?.id;
},
stop: () => {
c.abortSignal.removeEventListener("abort", stop);
stop();
// Closing with nothing left to wake for, such as after a drain, needs no wake.
// A failed harness refuses `wake()`; its wake stays, and the reopen arms a new one.
if (!harness || !isUsable(harness)) return;
const { active, at } = harness.wake();
if (!active && at === undefined && armed) rearm(undefined);
},
};
}
Expand Down
64 changes: 60 additions & 4 deletions packages/pi/tests/actor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ const PI_DURABLE_TABLES_051 = [
/** Tool calls of the crash test. A tool's first run blocks until the stop aborts it; a rerun returns at once. */
const toolRuns = { safe: 0, unsafe: 0 };

/** Runs of `poison_tool`. Its first run commits a document that `failCommitsOfPoison` makes storage reject. */
let poisonRuns = 0;

/** Sandboxes the unreachable provider was asked to create. */
let sandboxCreates = 0;

Expand Down Expand Up @@ -131,11 +134,30 @@ function buildRegistry(mock: MockModel, root: string) {
return { output: [{ type: "text", text: `${name} done` }] };
},
});
const poisonTool = defineTool({
name: "poison_tool",
description: "A safe tool whose first run makes a storage commit fail",
parameters: obj({}),
replay: "safe",
execute: async (_args, api, context) => {
poisonRuns += 1;
if (poisonRuns === 1) {
await api.commit(async (tx) => {
(await tx.doc(Poison, api.conversationId)).value = "x";
}, context);
}
return { output: [{ type: "text", text: "poison done" }] };
},
});
const tools = createRegistry();
tools.install(
defineExtension({
name: "tools",
tools: [waitingTool("safe", "safe"), waitingTool("unsafe", "unsafe")],
tools: [
waitingTool("safe", "safe"),
waitingTool("unsafe", "unsafe"),
poisonTool,
],
}),
);

Expand Down Expand Up @@ -356,6 +378,11 @@ beforeAll(async () => {
toolCall("lookup", {}),
fauxAssistantMessage("it is 42"),
);
mockModel.reply(
"poison the run",
toolCall("poison_tool", {}),
fauxAssistantMessage("recovered from storage"),
);
mockModel.reply(
"create hello.txt",
toolCall("write", {
Expand Down Expand Up @@ -411,7 +438,7 @@ afterAll(async () => {
});

describe("pi actor", () => {
test("a run stopped at the end of the grace period resumes on wake: safe tools rerun, unsafe tools report the interruption", async (c) => {
test("a run stopped at the end of the grace period resumes by itself: safe tools rerun, unsafe tools report the interruption", async (c) => {
const { client } = await setupTest(c, registry);
const key = ["crash", randomUUID()];
const handle = client.agent.getOrCreate(key);
Expand All @@ -426,9 +453,13 @@ describe("pi actor", () => {
await handle.nap();
await sleeps.waitFor(key, 1);

// Call nothing: a call would wake the actor and resume the run itself.
await vi.waitFor(() => expect(toolRuns).toEqual({ safe: 2, unsafe: 1 }), {
timeout: 45_000,
interval: 250,
});
const settled = await handle.submission.wait(submission.id);
expect(settled.status).toBe("done");
expect(toolRuns).toEqual({ safe: 2, unsafe: 1 });
const { messages } = await handle.conversation.context(root.id);
const results = messages.filter((message) => message.role === "toolResult");
expect(
Expand All @@ -448,7 +479,7 @@ describe("pi actor", () => {
),
);
expect(answers).toHaveLength(1);
});
}, 90_000);

test("an app action writes a document through c.pi, and clients read it by kind after sleep", async (c) => {
const { client } = await setupTest(c, registry);
Expand Down Expand Up @@ -501,6 +532,31 @@ describe("pi actor", () => {
expect(result).toMatchObject({ status: "done", text: "Hi there!" });
});

test("a commit that fails during a run reopens the harness at once, and the run finishes with no call", async (c) => {
const { client } = await setupTest(c, registry);
const handle = client.agent.getOrCreate(["poison-run", randomUUID()]);
const root = await handle.harness.root();
await handle.failCommitsOfPoison();
const submission = await handle.conversation.submit(root.id, {
type: "input",
content: "poison the run",
});

// Nothing calls the actor until the tool reruns. The recovery wake would
// resume the run only about 15 seconds after it started.
await vi.waitFor(() => expect(poisonRuns).toBe(2), { timeout: 10_000 });
expect(await handle.submission.wait(submission.id)).toMatchObject({
status: "done",
});
const { messages } = await handle.conversation.context(root.id);
expect(
messages.find((message) => message.role === "toolResult"),
).toMatchObject({
isError: false,
content: [{ type: "text", text: "poison done" }],
});
});

test("a watching connection gets a fresh view when a failed commit reopens the harness, then the next run's frames", async (c) => {
const { client } = await setupTest(c, registry);
const conn = client.agent
Expand Down
Loading
Loading