Faster quiz generation: planned parallel batches, room to reason
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s

- Output limits leave room for reasoning (answer plus 8k-64k by effort, up to
  the model maximum); a call cut off before answering is retried with double
  the limit. At "max" effort, batches were being cut off at 16k tokens.
- Quizzes are planned first (one distinct angle per question), then written
  in waves of four batches; med math runs alongside the first wave.
- Replacements for removed questions are planned and written in parallel.
- The Python check and the second review run at the same time each round.

Measured live (DeepSeek flash, max reasoning, 50 questions): 7 min 21 s with
no duplicates, against 20 min and 29 replaced duplicates without planning.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 11:00:27 -07:00
parent 1a212bb8ba
commit 2454ab6e95
8 changed files with 213 additions and 51 deletions

View file

@ -12,7 +12,7 @@ import { validateDraftQuestion, type DraftQuestion, type QuizDraft } from '../..
import type { AppContext } from '../http.ts';
import { runPython } from './sandbox.ts';
import { loadMcpTools, mcpPromptNote, type LoadedMcp } from './mcp.ts';
import { resolveModel, runChat } from './service.ts';
import { outputBudget, resolveModel, runChat } from './service.ts';
import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts';
/* ------------------------------ data handed to Python ------------------------------ */
@ -110,6 +110,7 @@ const reviewSchema = z.object({
fixes: z.array(z.unknown()).optional().default([]).transform((list) => list.flatMap((f) => { const r = fixSchema.safeParse(f); return r.success ? [r.data] : []; })),
});
export type QuizReview = z.infer<typeof reviewSchema>;
export type QuizFix = z.infer<typeof fixSchema>;
const SUBMIT_REVIEW_QUESTION_TEXT = {
type: 'object',
@ -313,7 +314,7 @@ export async function verifyQuizDraft(
// MCP tools stay listed at wrap-up (the transcript may hold their calls) but submit_review is forced.
tools: wrapUp ? [SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)] : [RUN_PYTHON_TOOL, SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)],
toolChoice: wrapUp ? { name: 'submit_review' } : 'auto',
maxOutputTokens: 16_000,
maxOutputTokens: outputBudget(resolved, 16_000),
signal: opts.signal,
});
messages.push(res.message);
@ -471,8 +472,8 @@ export async function proofreadQuiz(
app: AppContext,
draft: QuizDraft,
opts: { modelId?: string | null; signal?: AbortSignal } = {},
): Promise<{ draft: QuizDraft; flagged: number; applied: string[]; refused: string[]; rewritten: number; removed: number; incomplete: boolean }> {
const none = { draft, flagged: 0, applied: [], refused: [], rewritten: 0, removed: 0 };
): Promise<{ draft: QuizDraft; flagged: number; fixes: QuizFix[]; applied: string[]; refused: string[]; rewritten: number; removed: number; incomplete: boolean }> {
const none = { draft, flagged: 0, fixes: [], applied: [], refused: [], rewritten: 0, removed: 0 };
const resolved = resolveModel(app, 'generate', opts.modelId);
if (!resolved.model.capabilities.tools) return { ...none, incomplete: true };
const data = quizForPython(draft);
@ -490,7 +491,7 @@ export async function proofreadQuiz(
messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
tools: [SUBMIT_PROOFREAD],
toolChoice: { name: 'submit_proofread' },
maxOutputTokens: 16_000,
maxOutputTokens: outputBudget(resolved, 16_000),
signal: opts.signal,
});
const call = res.message.content.find((p) => p.type === 'tool_call');
@ -502,6 +503,7 @@ export async function proofreadQuiz(
return {
draft: { ...draft, questions },
flagged: issues.length,
fixes,
applied,
refused,
rewritten: applied.filter((a) => /: rewritten/.test(a)).length,