Faster quiz generation: planned parallel batches, room to reason
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s

- Output limits leave room for reasoning (answer plus 8k-64k by effort, up to
  the model maximum); a call cut off before answering is retried with double
  the limit. At "max" effort, batches were being cut off at 16k tokens.
- Quizzes are planned first (one distinct angle per question), then written
  in waves of four batches; med math runs alongside the first wave.
- Replacements for removed questions are planned and written in parallel.
- The Python check and the second review run at the same time each round.

Measured live (DeepSeek flash, max reasoning, 50 questions): 7 min 21 s with
no duplicates, against 20 min and 29 replaced duplicates without planning.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 11:00:27 -07:00
parent 1a212bb8ba
commit 2454ab6e95
8 changed files with 213 additions and 51 deletions

View file

@ -54,7 +54,7 @@ async function sse(url: string, body: unknown): Promise<any[]> {
const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review');
/** The quick call that groups topic labels after a quiz is written. */
const isGrouping = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_topics');
const isNaming = (r: { body: any }) => ['name_set', 'submit_proofread'].includes(r.body?.tool_choice?.function?.name);
const isNaming = (r: { body: any }) => ['name_set', 'submit_proofread', 'plan_questions'].includes(r.body?.tool_choice?.function?.name);
async function waitForDraft(id: string) {
for (let i = 0; i < 200; i += 1) {
@ -209,11 +209,17 @@ describe('generation', () => {
const checks = all.filter(isCheck);
expect(checks.length).toBeGreaterThanOrEqual(2);
const pythonResult = JSON.parse(checks[1].body.messages.find((m: any) => m.role === 'tool').content);
expect(pythonResult).toMatchObject({ ok: true, stdout: '12 12 4 [1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2]\n' });
// 12 questions against a request for 12, then the select-all count and correct answers per question.
expect(pythonResult.ok).toBe(true);
expect(pythonResult.stdout).toMatch(/^12 12 \d+ \[(?:[12], ){11}[12]\]\n$/);
expect(draft.payload.questions[1].options.map((o: any) => o.correct)).toEqual([false, true, false, false]);
expect(draft.meta.notes.join(' ')).toMatch(/Checked with Python \(1 run\): 2 of 2 checks passed, 1 fix applied/);
expect(JSON.stringify(calls[0].body.messages)).toMatch(/SOURCE EXCERPTS/);
expect(JSON.stringify(calls[1].body.messages)).toMatch(/Already written/);
// The questions are planned first; both batches then run at the same time, each with its own planned angles.
expect(all.filter((r) => r.body?.tool_choice?.function?.name === 'plan_questions')).toHaveLength(1);
expect(JSON.stringify(calls[0].body.messages)).toMatch(/Mock angle 1\b/);
expect(JSON.stringify(calls[1].body.messages)).toMatch(/SOURCE EXCERPTS/);
expect(JSON.stringify(calls[1].body.messages)).toMatch(/Mock angle 9\b/);
expect(calls[0].body.tool_choice).toEqual({ type: 'function', function: { name: 'submit_questions' } });
// Nothing is in the library until the draft is saved.

View file

@ -70,6 +70,8 @@ function questions(n: number, refs: string[], offset = 0) {
});
}
let parallelSeq = 0;
function toolArgs(name: string, userText: string, all: string): Record<string, unknown> {
const count = Number(/(?:Write|write)\s+(\d+)/.exec(userText)?.[1] ?? /(\d+)[- ](?:question|card|flashcard)/.exec(userText)?.[1] ?? 3);
// Only refs from the user's turn count: the system prompt contains an example id.
@ -77,7 +79,11 @@ function toolArgs(name: string, userText: string, all: string): Record<string, u
// Continue numbering after anything already written, so later batches are not duplicates.
const written = userText.split('Already written')[1] ?? '';
// Continue after the highest number written, so a refill after a removal is never a repeat.
const offset = Math.max((written.match(/^- /gm) ?? []).length, ...[...written.matchAll(/Mock (?:question|math|term) (\d+)/g)].map((m) => Number(m[1])));
// Batches written at the same time each get their own range, as their different excerpts would.
// A batch with planned angles is numbered from its first angle, so batches never collide.
const angle = /Mock angle (\d+)/.exec(userText);
const parallel = angle ? (Number(angle[1]) - 1) * 3 + 3000 : /at the same time/.test(userText) ? (parallelSeq = (parallelSeq + 1) % 1000) * 300 : 0; // a multiple of 3 keeps the type pattern
const offset = parallel + Math.max((written.match(/^- /gm) ?? []).length, ...[...written.matchAll(/Mock (?:question|math|term) (\d+)/g)].map((m) => Number(m[1])));
void all;
switch (name) {
case 'submit_questions':
@ -97,6 +103,8 @@ function toolArgs(name: string, userText: string, all: string): Record<string, u
return { questions: questions(count, refs, offset) };
case 'submit_flashcards':
return { cards: Array.from({ length: count }, (_, i) => ({ front: `Mock term ${i + 1 + offset}`, back: `Mock definition ${i + 1 + offset}`, tags: ['mock'] })) };
case 'plan_questions':
return { angles: Array.from({ length: count }, (_, k) => `Mock angle ${k + 1 + offset}`) };
case 'name_set':
return /quiz of/i.test(userText) ? { title: 'Heart Failure and Pharmacology', description: 'Covers heart failure care and the drugs used for it.' } : { title: 'Mock Terms', description: 'Covers the mock terms.' };
case 'generate_quiz':

View file

@ -169,7 +169,8 @@ describe('the quiz check', () => {
const proofreads = chatCalls().filter((r) => r.body.tool_choice?.function?.name === 'submit_proofread');
expect(proofreads.length).toBeGreaterThanOrEqual(2);
// The leftover was pointed out to the reviewer, and the question no longer has it.
expect(JSON.stringify(proofreads[0].body.messages)).toMatch(/looks like a leftover/);
// (The first second-review runs alongside the check that planted it, so a later one sees it.)
expect(proofreads.some((p) => /looks like a leftover/.test(JSON.stringify(p.body.messages)))).toBe(true);
expect(draft.payload.questions.some((q: any) => /\?\s*No\b/.test(q.stem))).toBe(false);
expect(draft.meta.notes.join(' | ')).toMatch(/Second review: Question \d+: rewritten/);
});