Faster quiz generation: planned parallel batches, room to reason
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s

- Output limits leave room for reasoning (answer plus 8k-64k by effort, up to
  the model maximum); a call cut off before answering is retried with double
  the limit. At "max" effort, batches were being cut off at 16k tokens.
- Quizzes are planned first (one distinct angle per question), then written
  in waves of four batches; med math runs alongside the first wave.
- Replacements for removed questions are planned and written in parallel.
- The Python check and the second review run at the same time each round.

Measured live (DeepSeek flash, max reasoning, 50 questions): 7 min 21 s with
no duplicates, against 20 min and 29 replaced duplicates without planning.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 11:00:27 -07:00
parent 1a212bb8ba
commit 2454ab6e95
8 changed files with 213 additions and 51 deletions

View file

@ -41,11 +41,13 @@ import { hybridSearch } from '../services/semantic.ts';
import { weakTopics } from '../services/stats.ts';
import { getSetting, setSetting } from '../settings.ts';
import { newId, now, parseJson } from '../util.ts';
import { generateStructured, resolveModel, runChat, runChatWithMcp } from './service.ts';
import { proofreadQuiz, verifyQuizDraft, type QuizCheck } from './verify.ts';
import { generateStructured, outputBudget, resolveModel, runChat, runChatWithMcp } from './service.ts';
import { applyQuizFixes, proofreadQuiz, verifyQuizDraft, type QuizCheck } from './verify.ts';
/** Most quiz checks per generation: the first, then re-checks after fixes and replacements. */
const VERIFY_ROUNDS = 3;
/** Writing batches run at the same time, this many per wave. */
const WRITE_WAVE = 4;
import { AIError } from './types.ts';
/* ------------------------------ drafts ------------------------------ */
@ -903,7 +905,7 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
// Broad topic labels, shared with the class's existing quizzes so results group sensibly.
const knownTopics = isQuiz && req.classId ? db.all<{ topic: string }>(`SELECT DISTINCT q.topic FROM questions q JOIN quizzes z ON z.id = q.quiz_id WHERE z.class_id = ? AND q.topic != '' LIMIT 60`, req.classId).map((r) => r.topic) : [];
const runBatch = async (index: number, want: number, math = false): Promise<number> => {
const runBatch = async (index: number, want: number, math = false, parallel = false, angles: string[] = []): Promise<number> => {
const written = items().map((x) => `- ${('stem' in x ? x.stem : x.front).replace(/\s+/g, ' ').slice(0, 140)}`);
const sataCount = Math.round(want * sata);
// With requested math, the regular batches write none; the math batches write only those.
@ -936,6 +938,8 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
'Do not write hints.',
];
if (written.length > 0) ask.push(`Already written (do not repeat or closely paraphrase these):\n${written.slice(-80).join('\n')}`);
if (angles.length > 0 && !math) ask.push(`Write exactly one question for each of these planned angles, in this order (other angles are covered by other questions, so stay on each one):\n${angles.map((a, k) => `${k + 1}. ${a}`).join('\n')}`);
else if (parallel && !math) ask.push('Other questions are being written from other excerpts at the same time: base every item on your own excerpts so the sets do not overlap.');
// Med math is standard material: the excerpts would only cost tokens.
const user = [math ? '' : sourceBlock(sources, index), ask.filter(Boolean).join('\n')].filter(Boolean).join('\n\n======\n\n');
@ -948,7 +952,7 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
toolName: isQuiz ? 'submit_questions' : 'submit_flashcards',
toolDescription: isQuiz ? 'Submit the finished questions.' : 'Submit the finished flashcards.',
schema: isQuiz ? QUESTION_SCHEMA : CARD_SCHEMA,
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
maxOutputTokens: outputBudget(resolved, 16_000),
signal,
mcpArea: 'generate',
});
@ -979,20 +983,107 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
// Regular items written so far (with requested math, calculations are counted separately).
const regularDone = () => (forcedMath === null ? items().length : items().length - numericCount());
let failures = 0;
for (let i = 0; i < Math.ceil(regularCount / perBatch) && regularDone() < regularCount; i += 1) {
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
const want = Math.min(perBatch, regularCount - regularDone());
setProgress(db, draftId, 0.05 + (0.9 * items().length) / count, `Writing ${isQuiz ? 'questions' : 'cards'} ${items().length + 1}-${Math.min(count, items().length + want)} of ${count}`, draft);
/**
* Plans `n` distinct angles (client situation + what is tested) before writing, so batches that
* run at the same time, and replacements, cover different ground instead of repeating each
* other. Empty when planning fails (batches then fall back to their own excerpts).
*/
const planAngles = async (n: number, avoid: string[]): Promise<string[]> => {
if (!isQuiz || n <= 0) return [];
try {
await runBatch(i, want);
const result = await generateStructured(app, {
task: 'generate',
purpose: 'planning questions',
modelId: req.modelId,
system: `${baseSystem(db, req, hasSources)}\n\nYou plan NCLEX-style quizzes. List distinct angles, one per question: each names the client situation and the single decision or fact it tests, in one line (for example "Continuous IV heparin, aPTT 3x control with gum bleeding: first action"). No two angles may test the same decision, fact or teaching point, even in a different scenario. Spread them over the client-needs areas the material supports (pharmacology, safety, prioritisation, delegation, teaching, lab values, complications, psychosocial). When the material is short, vary the situations and the area tested rather than repeating its points.`,
messages: [
{
role: 'user',
content: [
{
type: 'text',
text: [
hasSources ? `MATERIAL:\n${sourcesText(sources).slice(0, 60_000)}` : '',
`Write ${n} angles${req.focus ? ` about: ${req.focus}` : ''}. Difficulty: ${difficulty}. Calculation questions are planned separately; do not include any.`,
avoid.length ? `Already in the quiz (plan different angles from all of these):\n${avoid.map((x) => `- ${x.replace(/\s+/g, ' ').slice(0, 140)}`).join('\n')}` : '',
]
.filter(Boolean)
.join('\n\n'),
},
],
},
],
toolName: 'plan_questions',
toolDescription: 'Submit the planned angles.',
schema: { type: 'object', properties: { angles: { type: 'array', items: { type: 'string' } } }, required: ['angles'] },
maxOutputTokens: outputBudget(resolved, 8_000),
signal,
});
const list = (result.data as { angles?: unknown }).angles;
return Array.isArray(list) ? list.filter((a): a is string => typeof a === 'string' && a.trim() !== '').map((a) => a.trim().slice(0, 300)).slice(0, n) : [];
} catch (err) {
if (signal.aborted) throw err;
app.log('warn', 'Planning quiz angles failed; writing without a plan', err instanceof Error ? err.message : err);
return [];
}
};
/** Runs batches for `wants` at once; the planned angles are shared out in order. */
const runWave = (wants: number[], indexFrom: number, angles: string[]) => {
let at = 0;
return Promise.allSettled(
wants.map((want, k) => {
const mine = angles.slice(at, at + want);
at += want;
return runBatch(indexFrom + k, want, false, wants.length > 1, mine.length === want ? mine : []);
}),
);
};
// Requested medication math needs no excerpts, so it is written alongside the first wave.
const writeMath = async () => {
if (!forcedMath || draft.kind !== 'quiz') return;
for (let attempt = 0; attempt < Math.ceil(forcedMath / perBatch) + 2 && numericCount() < forcedMath; attempt += 1) {
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
const want = Math.min(perBatch, forcedMath - numericCount());
try {
if ((await runBatch(batches + 10 + attempt, want, true)) === 0 && attempt > 0) break;
} catch (err) {
if (err instanceof AIError && err.code === 'aborted') throw err;
notes.push(`A med math batch failed: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
};
const mathJob = writeMath();
// Batches run in waves of WRITE_WAVE at a time. Each batch has its own excerpts, and each wave
// is told what the earlier waves wrote, so the parallel writing does not repeat itself.
const plan: number[] = [];
for (let left = regularCount; left > 0; left -= perBatch) plan.push(Math.min(perBatch, left));
if (plan.length > 1) setProgress(db, draftId, 0.04, 'Planning the questions', draft);
const angles = plan.length > 1 ? await planAngles(regularCount, []) : [];
let angleAt = 0;
let failures = 0;
let firstError: unknown = null;
for (let w = 0; w < plan.length && regularDone() < regularCount; w += WRITE_WAVE) {
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
const wave = plan.slice(w, w + WRITE_WAVE);
const from = regularDone() + 1;
const to = Math.min(regularCount, regularDone() + wave.reduce((n, x) => n + x, 0));
setProgress(db, draftId, 0.05 + (0.85 * items().length) / count, `Writing ${isQuiz ? 'questions' : 'cards'} ${from}-${to} of ${count}`, draft);
const results = await runWave(wave, w, angles.slice(angleAt));
angleAt += wave.reduce((n, x) => n + x, 0);
for (const r of results) {
if (r.status === 'fulfilled') continue;
const err = r.reason;
if (err instanceof AIError && err.code === 'aborted') throw err;
// One bad batch should not sink a long job that already has good content.
failures += 1;
if (items().length === 0 || failures > 2 || (err instanceof AIError && !['invalid_response', 'server', 'rate_limit', 'timeout', 'network'].includes(err.code))) throw err;
firstError ??= err;
if (err instanceof AIError && !['invalid_response', 'server', 'rate_limit', 'timeout', 'network'].includes(err.code)) throw err;
notes.push(`A batch failed and was skipped: ${err instanceof Error ? err.message : 'unknown error'}`);
}
if ((regularDone() === 0 && failures > 0) || failures > Math.max(2, Math.floor(plan.length / 2))) throw firstError;
}
await mathJob;
// Top up when validation or de-duplication left the draft short.
for (let extra = 0; extra < 2 && regularDone() < regularCount && items().length > 0; extra += 1) {
if (signal.aborted) break;
@ -1004,19 +1095,7 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
break;
}
}
// Requested medication math, in batches of up to eight, then spread evenly through the quiz.
if (forcedMath && draft.kind === 'quiz') {
for (let attempt = 0; attempt < Math.ceil(forcedMath / perBatch) + 2 && numericCount() < forcedMath; attempt += 1) {
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
const want = Math.min(perBatch, forcedMath - numericCount());
setProgress(db, draftId, 0.9, `Writing ${want} med math question${want === 1 ? '' : 's'}`, draft);
try {
if ((await runBatch(batches + 10 + attempt, want, true)) === 0 && attempt > 0) break;
} catch (err) {
if (err instanceof AIError && err.code === 'aborted') throw err;
notes.push(`A med math batch failed: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
draft.questions = spreadEvenly(draft.questions, (q) => q.type === 'numeric');
if (numericCount() < forcedMath) notes.push(`Wrote ${numericCount()} of the ${forcedMath} med math questions requested.`);
}
@ -1045,21 +1124,42 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
rounds = round;
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
setProgress(db, draftId, 0.96, round === 1 ? 'Checking the questions with Python' : `Checking the questions again (round ${round})`, quiz);
last = await verifyQuizDraft(app, quiz, { count, style, sataShare: sata, difficulty, numericCap, numericRequested: forcedMath !== null }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, round === 1 ? label : `${label} · round ${round}`) });
quiz.questions = last.draft.questions;
if (last.incomplete) break;
history.push(...last.applied.map((a) => (round > 1 ? `Round ${round}: ${a}` : a)));
// A second, independent read of the whole quiz for badly written questions.
setProgress(db, draftId, 0.98, round === 1 ? 'Second review of the questions' : `Second review · round ${round}`, quiz);
const second = await proofreadQuiz(app, quiz, { modelId: req.modelId, signal });
quiz.questions = second.draft.questions;
secondReview = second;
history.push(...second.applied.map((a) => `${round > 1 ? `Round ${round}: ` : ''}Second review: ${a}`));
if (second.refused.length) history.push(...second.refused.map((r) => `Second review: ${r}`));
// The Python check and a second, independent read of the whole quiz (for badly written
// questions) run at the same time on the same questions.
const snapshot = quiz.questions;
const [check, second] = await Promise.all([
verifyQuizDraft(app, quiz, { count, style, sataShare: sata, difficulty, numericCap, numericRequested: forcedMath !== null }, { modelId: req.modelId, signal, onStep: (label) => setProgress(db, draftId, 0.97, round === 1 ? label : `${label} · round ${round}`) }),
proofreadQuiz(app, quiz, { modelId: req.modelId, signal }),
]);
last = check;
quiz.questions = check.draft.questions;
if (check.incomplete) break;
history.push(...check.applied.map((a) => (round > 1 ? `Round ${round}: ${a}` : a)));
// The second review's fixes, moved onto the checked questions. A question the check already
// rewrote or removed is left to the next round.
const moved = second.fixes.flatMap((f) => {
const stem = snapshot[f.question - 1]?.stem;
const at = stem === undefined ? -1 : quiz.questions.findIndex((q) => q.stem === stem);
return at >= 0 ? [{ ...f, question: at + 1 }] : [];
});
const reviewed = applyQuizFixes(quiz.questions, moved);
quiz.questions = reviewed.questions;
secondReview = { ...second, applied: reviewed.applied, refused: reviewed.refused };
history.push(...reviewed.applied.map((a) => `${round > 1 ? `Round ${round}: ` : ''}Second review: ${a}`));
if (reviewed.refused.length) history.push(...reviewed.refused.map((r) => `Second review: ${r}`));
// Replace what the check removed (duplicates, unfixable items) with new questions.
const missing = count - quiz.questions.length;
if (missing > 0) {
setProgress(db, draftId, 0.96, `Replacing ${missing} question${missing === 1 ? '' : 's'}`, quiz);
// Planned replacements, written in parallel, then single batches for anything still missing.
const mathShort = forcedMath ? Math.max(0, forcedMath - numericCount()) : 0;
const regularShort = missing - mathShort;
if (regularShort > perBatch) {
const fresh = await planAngles(regularShort, quiz.questions.map((q) => q.stem));
const wants: number[] = [];
for (let left = regularShort; left > 0; left -= perBatch) wants.push(Math.min(perBatch, left));
await Promise.all([runWave(wants, batches + 40 + round * 10, fresh), mathShort > 0 ? runBatch(batches + 39 + round * 10, mathShort, true).catch(() => 0) : Promise.resolve(0)]);
}
for (let attempt = 0; attempt < 3 && quiz.questions.length < count; attempt += 1) {
const wantMath = forcedMath ? Math.min(forcedMath - numericCount(), count - quiz.questions.length) : 0;
try {
@ -1073,7 +1173,7 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ
history.push(`Round ${round}: ${missing} removed question${missing === 1 ? ' was' : 's were'} replaced with new ones${round === VERIFY_ROUNDS ? ' (after the last check)' : ''}.`);
}
// Done when nothing is left to fix and nothing new (rewritten or replacement questions) needs checking.
const done = missing <= 0 && last.rewritten === 0 && second.applied.length === 0 && last.failed.length === 0 && last.refused.length === 0;
const done = missing <= 0 && check.rewritten === 0 && reviewed.applied.length === 0 && check.failed.length === 0 && check.refused.length === 0;
if (done || round === VERIFY_ROUNDS) break;
}
if (last) {
@ -1204,7 +1304,7 @@ async function generateNote(app: AppContext, draftId: string, req: GenerateReque
if (passes.length <= 1) {
setProgress(db, draftId, 0.2, 'Writing');
const body = [passes[0]?.length ? `SOURCE EXCERPTS:\n\n${formatExcerpts(passes[0])}` : '', sources.inline, ask].filter(Boolean).join('\n\n======\n\n');
const res = await runChatWithMcp(app, { task: 'generate', purpose: 'study guide', modelId: req.modelId, system, messages: [{ role: 'user', content: [{ type: 'text', text: body }] }], maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000, signal }, 'generate');
const res = await runChatWithMcp(app, { task: 'generate', purpose: 'study guide', modelId: req.modelId, system, messages: [{ role: 'user', content: [{ type: 'text', text: body }] }], maxOutputTokens: outputBudget(resolved, 16_000), signal }, 'generate');
content = res.text;
} else {
// Long material: condense each part, then write the final piece from the condensed notes.
@ -1232,7 +1332,7 @@ async function generateNote(app: AppContext, draftId: string, req: GenerateReque
modelId: req.modelId,
system,
messages: [{ role: 'user', content: [{ type: 'text', text: `CONDENSED NOTES FROM THE FULL MATERIAL:\n\n${partials.join('\n\n---\n\n')}\n\n======\n\n${sources.inline ? `${sources.inline}\n\n======\n\n` : ''}${ask}` }] }],
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
maxOutputTokens: outputBudget(resolved, 16_000),
signal,
},
'generate',
@ -1328,7 +1428,7 @@ Return the questions in the same order, one output question per input question.$
toolName: 'submit_questions',
toolDescription: 'Submit the improved questions in the original order.',
schema: QUESTION_SCHEMA,
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
maxOutputTokens: outputBudget(resolved, 16_000),
signal: controller.signal,
mcpArea: 'generate',
});

View file

@ -638,6 +638,23 @@ export async function runChatWithMcp(app: AppContext, opts: ChatOptions, area: M
}
}
/** Extra output room for the model's reasoning at each effort (reasoning counts as output). */
const REASONING_ROOM: Partial<Record<ReasoningEffort, number>> = { low: 8_000, medium: 16_000, high: 32_000, xhigh: 48_000, max: 64_000 };
/**
* The output limit for a call that must fit `answerTokens` of answer: the user's own limit when
* they set one, otherwise the answer plus room for reasoning at the configured effort, within
* the model's maximum. A reasoning model at a high effort can otherwise spend the whole limit
* thinking and be cut off before it answers.
*/
export function outputBudget(resolved: ResolvedModel, answerTokens: number): number {
if (resolved.params.maxOutputTokens) return resolved.params.maxOutputTokens;
const effort = resolved.params.reasoningEffort;
const thinks = resolved.model.capabilities.reasoning && effort !== 'none' && effort !== 'minimal';
const room = !thinks ? 0 : effort ? (REASONING_ROOM[effort] ?? 16_000) : 16_000;
return Math.min(answerTokens + room, resolved.model.maxOutput ?? answerTokens + room);
}
export interface StructuredOptions extends Omit<ChatOptions, 'tools' | 'toolChoice' | 'jsonMode'> {
/** Name + description of the "function" whose arguments are the result. */
toolName: string;
@ -662,13 +679,18 @@ export async function generateStructured(app: AppContext, opts: StructuredOption
};
if (!resolved.model.capabilities.tools) return promptJson();
const chatOpts: ChatOptions = { ...opts, tools: [tool], toolChoice: { name: opts.toolName } };
const once = async () => {
const res = opts.mcpArea ? await runChatWithMcp(app, chatOpts, opts.mcpArea) : await runChat(app, chatOpts);
let cutOff = false;
const once = async (maxOutputTokens = chatOpts.maxOutputTokens) => {
const callOpts = { ...chatOpts, maxOutputTokens };
const res = opts.mcpArea ? await runChatWithMcp(app, callOpts, opts.mcpArea) : await runChat(app, callOpts);
const call = res.message.content.find((p) => p.type === 'tool_call');
if (call && call.type === 'tool_call') return { data: call.input as unknown, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
if (res.text.trim()) return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
if (res.stopReason === 'refusal') throw new AIError('content_filter', 'The model declined this request.');
if (res.stopReason === 'max_tokens') throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
if (res.stopReason === 'max_tokens') {
cutOff = true;
throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
}
throw new AIError('invalid_response', 'The model returned nothing usable.', { retryable: true });
};
try {
@ -676,6 +698,12 @@ export async function generateStructured(app: AppContext, opts: StructuredOption
} catch (err) {
if (err instanceof AIError && err.code === 'unsupported') return promptJson();
if (!(err instanceof AIError) || err.code !== 'invalid_response' || opts.signal?.aborted) throw err;
// Cut off before answering: asking again with the same limit fails the same way, so double it.
if (cutOff) {
const doubled = Math.min((chatOpts.maxOutputTokens ?? 16_000) * 2, resolved.model.maxOutput ?? Infinity);
app.log('warn', `${opts.toolName} ran out of output space; retrying with ${doubled} tokens`);
return await once(doubled);
}
// Malformed output (jsonrepair could not fix it): the model repairs its own JSON when we
// have it, otherwise it gets one more try at the same request.
if (err.rawOutput) {

View file

@ -12,7 +12,7 @@ import { validateDraftQuestion, type DraftQuestion, type QuizDraft } from '../..
import type { AppContext } from '../http.ts';
import { runPython } from './sandbox.ts';
import { loadMcpTools, mcpPromptNote, type LoadedMcp } from './mcp.ts';
import { resolveModel, runChat } from './service.ts';
import { outputBudget, resolveModel, runChat } from './service.ts';
import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts';
/* ------------------------------ data handed to Python ------------------------------ */
@ -110,6 +110,7 @@ const reviewSchema = z.object({
fixes: z.array(z.unknown()).optional().default([]).transform((list) => list.flatMap((f) => { const r = fixSchema.safeParse(f); return r.success ? [r.data] : []; })),
});
export type QuizReview = z.infer<typeof reviewSchema>;
export type QuizFix = z.infer<typeof fixSchema>;
const SUBMIT_REVIEW_QUESTION_TEXT = {
type: 'object',
@ -313,7 +314,7 @@ export async function verifyQuizDraft(
// MCP tools stay listed at wrap-up (the transcript may hold their calls) but submit_review is forced.
tools: wrapUp ? [SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)] : [RUN_PYTHON_TOOL, SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)],
toolChoice: wrapUp ? { name: 'submit_review' } : 'auto',
maxOutputTokens: 16_000,
maxOutputTokens: outputBudget(resolved, 16_000),
signal: opts.signal,
});
messages.push(res.message);
@ -471,8 +472,8 @@ export async function proofreadQuiz(
app: AppContext,
draft: QuizDraft,
opts: { modelId?: string | null; signal?: AbortSignal } = {},
): Promise<{ draft: QuizDraft; flagged: number; applied: string[]; refused: string[]; rewritten: number; removed: number; incomplete: boolean }> {
const none = { draft, flagged: 0, applied: [], refused: [], rewritten: 0, removed: 0 };
): Promise<{ draft: QuizDraft; flagged: number; fixes: QuizFix[]; applied: string[]; refused: string[]; rewritten: number; removed: number; incomplete: boolean }> {
const none = { draft, flagged: 0, fixes: [], applied: [], refused: [], rewritten: 0, removed: 0 };
const resolved = resolveModel(app, 'generate', opts.modelId);
if (!resolved.model.capabilities.tools) return { ...none, incomplete: true };
const data = quizForPython(draft);
@ -490,7 +491,7 @@ export async function proofreadQuiz(
messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
tools: [SUBMIT_PROOFREAD],
toolChoice: { name: 'submit_proofread' },
maxOutputTokens: 16_000,
maxOutputTokens: outputBudget(resolved, 16_000),
signal: opts.signal,
});
const call = res.message.content.find((p) => p.type === 'tool_call');
@ -502,6 +503,7 @@ export async function proofreadQuiz(
return {
draft: { ...draft, questions },
flagged: issues.length,
fixes,
applied,
refused,
rewritten: applied.filter((a) => /: rewritten/.test(a)).length,