Faster quiz generation: planned parallel batches, room to reason
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s
- Output limits leave room for reasoning (answer plus 8k-64k by effort, up to the model maximum); a call cut off before answering is retried with double the limit. At "max" effort, batches were being cut off at 16k tokens. - Quizzes are planned first (one distinct angle per question), then written in waves of four batches; med math runs alongside the first wave. - Replacements for removed questions are planned and written in parallel. - The Python check and the second review run at the same time each round. Measured live (DeepSeek flash, max reasoning, 50 questions): 7 min 21 s with no duplicates, against 20 min and 29 replaced duplicates without planning. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
1a212bb8ba
commit
2454ab6e95
8 changed files with 213 additions and 51 deletions
|
|
@ -638,6 +638,23 @@ export async function runChatWithMcp(app: AppContext, opts: ChatOptions, area: M
|
|||
}
|
||||
}
|
||||
|
||||
/** Extra output room for the model's reasoning at each effort (reasoning counts as output). */
|
||||
const REASONING_ROOM: Partial<Record<ReasoningEffort, number>> = { low: 8_000, medium: 16_000, high: 32_000, xhigh: 48_000, max: 64_000 };
|
||||
|
||||
/**
|
||||
* The output limit for a call that must fit `answerTokens` of answer: the user's own limit when
|
||||
* they set one, otherwise the answer plus room for reasoning at the configured effort, within
|
||||
* the model's maximum. A reasoning model at a high effort can otherwise spend the whole limit
|
||||
* thinking and be cut off before it answers.
|
||||
*/
|
||||
export function outputBudget(resolved: ResolvedModel, answerTokens: number): number {
|
||||
if (resolved.params.maxOutputTokens) return resolved.params.maxOutputTokens;
|
||||
const effort = resolved.params.reasoningEffort;
|
||||
const thinks = resolved.model.capabilities.reasoning && effort !== 'none' && effort !== 'minimal';
|
||||
const room = !thinks ? 0 : effort ? (REASONING_ROOM[effort] ?? 16_000) : 16_000;
|
||||
return Math.min(answerTokens + room, resolved.model.maxOutput ?? answerTokens + room);
|
||||
}
|
||||
|
||||
export interface StructuredOptions extends Omit<ChatOptions, 'tools' | 'toolChoice' | 'jsonMode'> {
|
||||
/** Name + description of the "function" whose arguments are the result. */
|
||||
toolName: string;
|
||||
|
|
@ -662,13 +679,18 @@ export async function generateStructured(app: AppContext, opts: StructuredOption
|
|||
};
|
||||
if (!resolved.model.capabilities.tools) return promptJson();
|
||||
const chatOpts: ChatOptions = { ...opts, tools: [tool], toolChoice: { name: opts.toolName } };
|
||||
const once = async () => {
|
||||
const res = opts.mcpArea ? await runChatWithMcp(app, chatOpts, opts.mcpArea) : await runChat(app, chatOpts);
|
||||
let cutOff = false;
|
||||
const once = async (maxOutputTokens = chatOpts.maxOutputTokens) => {
|
||||
const callOpts = { ...chatOpts, maxOutputTokens };
|
||||
const res = opts.mcpArea ? await runChatWithMcp(app, callOpts, opts.mcpArea) : await runChat(app, callOpts);
|
||||
const call = res.message.content.find((p) => p.type === 'tool_call');
|
||||
if (call && call.type === 'tool_call') return { data: call.input as unknown, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
||||
if (res.text.trim()) return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
||||
if (res.stopReason === 'refusal') throw new AIError('content_filter', 'The model declined this request.');
|
||||
if (res.stopReason === 'max_tokens') throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
|
||||
if (res.stopReason === 'max_tokens') {
|
||||
cutOff = true;
|
||||
throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
|
||||
}
|
||||
throw new AIError('invalid_response', 'The model returned nothing usable.', { retryable: true });
|
||||
};
|
||||
try {
|
||||
|
|
@ -676,6 +698,12 @@ export async function generateStructured(app: AppContext, opts: StructuredOption
|
|||
} catch (err) {
|
||||
if (err instanceof AIError && err.code === 'unsupported') return promptJson();
|
||||
if (!(err instanceof AIError) || err.code !== 'invalid_response' || opts.signal?.aborted) throw err;
|
||||
// Cut off before answering: asking again with the same limit fails the same way, so double it.
|
||||
if (cutOff) {
|
||||
const doubled = Math.min((chatOpts.maxOutputTokens ?? 16_000) * 2, resolved.model.maxOutput ?? Infinity);
|
||||
app.log('warn', `${opts.toolName} ran out of output space; retrying with ${doubled} tokens`);
|
||||
return await once(doubled);
|
||||
}
|
||||
// Malformed output (jsonrepair could not fix it): the model repairs its own JSON when we
|
||||
// have it, otherwise it gets one more try at the same request.
|
||||
if (err.rawOutput) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue