Faster quiz generation: planned parallel batches, room to reason
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s
- Output limits leave room for reasoning (answer plus 8k-64k by effort, up to the model maximum); a call cut off before answering is retried with double the limit. At "max" effort, batches were being cut off at 16k tokens. - Quizzes are planned first (one distinct angle per question), then written in waves of four batches; med math runs alongside the first wave. - Replacements for removed questions are planned and written in parallel. - The Python check and the second review run at the same time each round. Measured live (DeepSeek flash, max reasoning, 50 questions): 7 min 21 s with no duplicates, against 20 min and 29 replaced duplicates without planning. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
1a212bb8ba
commit
2454ab6e95
8 changed files with 213 additions and 51 deletions
|
|
@ -12,7 +12,7 @@ import { validateDraftQuestion, type DraftQuestion, type QuizDraft } from '../..
|
|||
import type { AppContext } from '../http.ts';
|
||||
import { runPython } from './sandbox.ts';
|
||||
import { loadMcpTools, mcpPromptNote, type LoadedMcp } from './mcp.ts';
|
||||
import { resolveModel, runChat } from './service.ts';
|
||||
import { outputBudget, resolveModel, runChat } from './service.ts';
|
||||
import { AIError, type ChatMessage, type ContentPart, type ToolDef } from './types.ts';
|
||||
|
||||
/* ------------------------------ data handed to Python ------------------------------ */
|
||||
|
|
@ -110,6 +110,7 @@ const reviewSchema = z.object({
|
|||
fixes: z.array(z.unknown()).optional().default([]).transform((list) => list.flatMap((f) => { const r = fixSchema.safeParse(f); return r.success ? [r.data] : []; })),
|
||||
});
|
||||
export type QuizReview = z.infer<typeof reviewSchema>;
|
||||
export type QuizFix = z.infer<typeof fixSchema>;
|
||||
|
||||
const SUBMIT_REVIEW_QUESTION_TEXT = {
|
||||
type: 'object',
|
||||
|
|
@ -313,7 +314,7 @@ export async function verifyQuizDraft(
|
|||
// MCP tools stay listed at wrap-up (the transcript may hold their calls) but submit_review is forced.
|
||||
tools: wrapUp ? [SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)] : [RUN_PYTHON_TOOL, SUBMIT_REVIEW, ...mcp.tools.map((t) => t.def)],
|
||||
toolChoice: wrapUp ? { name: 'submit_review' } : 'auto',
|
||||
maxOutputTokens: 16_000,
|
||||
maxOutputTokens: outputBudget(resolved, 16_000),
|
||||
signal: opts.signal,
|
||||
});
|
||||
messages.push(res.message);
|
||||
|
|
@ -471,8 +472,8 @@ export async function proofreadQuiz(
|
|||
app: AppContext,
|
||||
draft: QuizDraft,
|
||||
opts: { modelId?: string | null; signal?: AbortSignal } = {},
|
||||
): Promise<{ draft: QuizDraft; flagged: number; applied: string[]; refused: string[]; rewritten: number; removed: number; incomplete: boolean }> {
|
||||
const none = { draft, flagged: 0, applied: [], refused: [], rewritten: 0, removed: 0 };
|
||||
): Promise<{ draft: QuizDraft; flagged: number; fixes: QuizFix[]; applied: string[]; refused: string[]; rewritten: number; removed: number; incomplete: boolean }> {
|
||||
const none = { draft, flagged: 0, fixes: [], applied: [], refused: [], rewritten: 0, removed: 0 };
|
||||
const resolved = resolveModel(app, 'generate', opts.modelId);
|
||||
if (!resolved.model.capabilities.tools) return { ...none, incomplete: true };
|
||||
const data = quizForPython(draft);
|
||||
|
|
@ -490,7 +491,7 @@ export async function proofreadQuiz(
|
|||
messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
|
||||
tools: [SUBMIT_PROOFREAD],
|
||||
toolChoice: { name: 'submit_proofread' },
|
||||
maxOutputTokens: 16_000,
|
||||
maxOutputTokens: outputBudget(resolved, 16_000),
|
||||
signal: opts.signal,
|
||||
});
|
||||
const call = res.message.content.find((p) => p.type === 'tool_call');
|
||||
|
|
@ -502,6 +503,7 @@ export async function proofreadQuiz(
|
|||
return {
|
||||
draft: { ...draft, questions },
|
||||
flagged: issues.length,
|
||||
fixes,
|
||||
applied,
|
||||
refused,
|
||||
rewritten: applied.filter((a) => /: rewritten/.test(a)).length,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue