Faster quiz generation: planned parallel batches, room to reason
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s
- Output limits leave room for reasoning (answer plus 8k-64k by effort, up to the model maximum); a call cut off before answering is retried with double the limit. At "max" effort, batches were being cut off at 16k tokens. - Quizzes are planned first (one distinct angle per question), then written in waves of four batches; med math runs alongside the first wave. - Replacements for removed questions are planned and written in parallel. - The Python check and the second review run at the same time each round. Measured live (DeepSeek flash, max reasoning, 50 questions): 7 min 21 s with no duplicates, against 20 min and 29 replaced duplicates without planning. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
1a212bb8ba
commit
2454ab6e95
8 changed files with 213 additions and 51 deletions
|
|
@ -54,7 +54,7 @@ async function sse(url: string, body: unknown): Promise<any[]> {
|
|||
const isCheck = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_review');
|
||||
/** The quick call that groups topic labels after a quiz is written. */
|
||||
const isGrouping = (r: { body: any }) => (r.body?.tools ?? []).some((t: any) => t.function?.name === 'submit_topics');
|
||||
const isNaming = (r: { body: any }) => ['name_set', 'submit_proofread'].includes(r.body?.tool_choice?.function?.name);
|
||||
const isNaming = (r: { body: any }) => ['name_set', 'submit_proofread', 'plan_questions'].includes(r.body?.tool_choice?.function?.name);
|
||||
|
||||
async function waitForDraft(id: string) {
|
||||
for (let i = 0; i < 200; i += 1) {
|
||||
|
|
@ -209,11 +209,17 @@ describe('generation', () => {
|
|||
const checks = all.filter(isCheck);
|
||||
expect(checks.length).toBeGreaterThanOrEqual(2);
|
||||
const pythonResult = JSON.parse(checks[1].body.messages.find((m: any) => m.role === 'tool').content);
|
||||
expect(pythonResult).toMatchObject({ ok: true, stdout: '12 12 4 [1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2]\n' });
|
||||
// 12 questions against a request for 12, then the select-all count and correct answers per question.
|
||||
expect(pythonResult.ok).toBe(true);
|
||||
expect(pythonResult.stdout).toMatch(/^12 12 \d+ \[(?:[12], ){11}[12]\]\n$/);
|
||||
expect(draft.payload.questions[1].options.map((o: any) => o.correct)).toEqual([false, true, false, false]);
|
||||
expect(draft.meta.notes.join(' ')).toMatch(/Checked with Python \(1 run\): 2 of 2 checks passed, 1 fix applied/);
|
||||
expect(JSON.stringify(calls[0].body.messages)).toMatch(/SOURCE EXCERPTS/);
|
||||
expect(JSON.stringify(calls[1].body.messages)).toMatch(/Already written/);
|
||||
// The questions are planned first; both batches then run at the same time, each with its own planned angles.
|
||||
expect(all.filter((r) => r.body?.tool_choice?.function?.name === 'plan_questions')).toHaveLength(1);
|
||||
expect(JSON.stringify(calls[0].body.messages)).toMatch(/Mock angle 1\b/);
|
||||
expect(JSON.stringify(calls[1].body.messages)).toMatch(/SOURCE EXCERPTS/);
|
||||
expect(JSON.stringify(calls[1].body.messages)).toMatch(/Mock angle 9\b/);
|
||||
expect(calls[0].body.tool_choice).toEqual({ type: 'function', function: { name: 'submit_questions' } });
|
||||
|
||||
// Nothing is in the library until the draft is saved.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue