Faster quiz generation: planned parallel batches, room to reason
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s

- Output limits leave room for reasoning (answer plus 8k-64k by effort, up to
  the model maximum); a call cut off before answering is retried with double
  the limit. At "max" effort, batches were being cut off at 16k tokens.
- Quizzes are planned first (one distinct angle per question), then written
  in waves of four batches; med math runs alongside the first wave.
- Replacements for removed questions are planned and written in parallel.
- The Python check and the second review run at the same time each round.

Measured live (DeepSeek flash, max reasoning, 50 questions): 7 min 21 s with
no duplicates, against 20 min and 29 replaced duplicates without planning.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 11:00:27 -07:00
parent 1a212bb8ba
commit 2454ab6e95
8 changed files with 213 additions and 51 deletions

View file

@ -638,6 +638,23 @@ export async function runChatWithMcp(app: AppContext, opts: ChatOptions, area: M
}
}
/** Extra output room for the model's reasoning at each effort (reasoning counts as output). */
const REASONING_ROOM: Partial<Record<ReasoningEffort, number>> = { low: 8_000, medium: 16_000, high: 32_000, xhigh: 48_000, max: 64_000 };
/**
* The output limit for a call that must fit `answerTokens` of answer: the user's own limit when
* they set one, otherwise the answer plus room for reasoning at the configured effort, within
* the model's maximum. A reasoning model at a high effort can otherwise spend the whole limit
* thinking and be cut off before it answers.
*/
export function outputBudget(resolved: ResolvedModel, answerTokens: number): number {
if (resolved.params.maxOutputTokens) return resolved.params.maxOutputTokens;
const effort = resolved.params.reasoningEffort;
const thinks = resolved.model.capabilities.reasoning && effort !== 'none' && effort !== 'minimal';
const room = !thinks ? 0 : effort ? (REASONING_ROOM[effort] ?? 16_000) : 16_000;
return Math.min(answerTokens + room, resolved.model.maxOutput ?? answerTokens + room);
}
export interface StructuredOptions extends Omit<ChatOptions, 'tools' | 'toolChoice' | 'jsonMode'> {
/** Name + description of the "function" whose arguments are the result. */
toolName: string;
@ -662,13 +679,18 @@ export async function generateStructured(app: AppContext, opts: StructuredOption
};
if (!resolved.model.capabilities.tools) return promptJson();
const chatOpts: ChatOptions = { ...opts, tools: [tool], toolChoice: { name: opts.toolName } };
const once = async () => {
const res = opts.mcpArea ? await runChatWithMcp(app, chatOpts, opts.mcpArea) : await runChat(app, chatOpts);
let cutOff = false;
const once = async (maxOutputTokens = chatOpts.maxOutputTokens) => {
const callOpts = { ...chatOpts, maxOutputTokens };
const res = opts.mcpArea ? await runChatWithMcp(app, callOpts, opts.mcpArea) : await runChat(app, callOpts);
const call = res.message.content.find((p) => p.type === 'tool_call');
if (call && call.type === 'tool_call') return { data: call.input as unknown, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
if (res.text.trim()) return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
if (res.stopReason === 'refusal') throw new AIError('content_filter', 'The model declined this request.');
if (res.stopReason === 'max_tokens') throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
if (res.stopReason === 'max_tokens') {
cutOff = true;
throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
}
throw new AIError('invalid_response', 'The model returned nothing usable.', { retryable: true });
};
try {
@ -676,6 +698,12 @@ export async function generateStructured(app: AppContext, opts: StructuredOption
} catch (err) {
if (err instanceof AIError && err.code === 'unsupported') return promptJson();
if (!(err instanceof AIError) || err.code !== 'invalid_response' || opts.signal?.aborted) throw err;
// Cut off before answering: asking again with the same limit fails the same way, so double it.
if (cutOff) {
const doubled = Math.min((chatOpts.maxOutputTokens ?? 16_000) * 2, resolved.model.maxOutput ?? Infinity);
app.log('warn', `${opts.toolName} ran out of output space; retrying with ${doubled} tokens`);
return await once(doubled);
}
// Malformed output (jsonrepair could not fix it): the model repairs its own JSON when we
// have it, otherwise it gets one more try at the same request.
if (err.rawOutput) {