Study/tests/mockAi.ts
Elijah 517278be71
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 30s
Varied question openers, readable rationales, plain practice score
- Stems no longer all open the same way: the writing rules, batch prompt and
  planner ask for varied openings (age openings stay part of the mix), and when
  one opener covers more than a quarter of a quiz the excess stems have only
  their opening reworded, keeping every number, before the check.
- Rationales are asked for as Markdown (a bold key point, then short bullets);
  older one-paragraph rationales are laid out the same way on display.
- The practice score in the quiz header is a plain "x/y correct", with no
  percentage or colour.
- Fix a flaky assertion: the mock's answer-key fix is a no-op when the shuffle
  already put the answer in place.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 12:05:46 -07:00

318 lines
18 KiB
TypeScript

/**
* A small OpenAI-compatible server with scripted behaviour, for tests and for trying the
* AI features locally without a real provider (see scripts/mock-ai.mjs).
*
* - Forced tool call -> returns that tool call with plausible arguments.
* - Tools + user turn -> picks a tool from keywords in the message.
* - Tool result turn -> answers in text, citing the first [cN] ref it was given.
* - No tools -> streams a short Markdown answer.
* - /embeddings -> deterministic word-hash vectors (see mockEmbedding).
*/
import http from 'node:http';
import type { AddressInfo } from 'node:net';
export interface MockAi {
url: string;
requests: { path: string; headers: http.IncomingHttpHeaders; body: any }[];
/** Force the next N chat calls to fail with this status. */
failNext: (status: number, times?: number) => void;
/** Delay every chat call by this many milliseconds (0 turns it off). */
delayChat: (ms: number) => void;
/** The next N calls of this tool send arguments that are not valid JSON (beyond jsonrepair). */
badJsonNext: (tool: string, times?: number) => void;
/** Arguments for the next submit_review calls, in order (then the default review). */
scriptReviews: (reviews: Record<string, unknown>[]) => void;
close: () => Promise<void>;
}
const text = (content: unknown): string =>
typeof content === 'string' ? content : Array.isArray(content) ? content.map((p: any) => (typeof p?.text === 'string' ? p.text : '')).join('\n') : '';
/**
* Deterministic "embeddings": hashed bag of words over 64 dimensions, with a few nursing
* synonyms folded together so a paraphrased query lands near the right passage.
*/
const SYNONYMS: Record<string, string> = {
lasix: 'furosemide', loop: 'furosemide', diuretic: 'furosemide', diuretics: 'furosemide',
banana: 'potassium', bananas: 'potassium', hypokalemia: 'potassium', k: 'potassium',
halos: 'digoxin', lanoxin: 'digoxin', glycoside: 'digoxin',
};
export function mockEmbedding(input: string, dims = 64): number[] {
const v = new Array(dims).fill(0);
for (const raw of input.toLowerCase().match(/[a-z]+/g) ?? []) {
if (raw.length < 3 && raw !== 'k') continue;
const word = SYNONYMS[raw] ?? raw;
let h = 2166136261;
for (const ch of word) h = Math.imul(h ^ ch.charCodeAt(0), 16777619) >>> 0;
v[h % dims] += 1;
}
return v;
}
function questions(n: number, refs: string[], offset = 0) {
return Array.from({ length: n }, (_, k) => {
const i = k + offset;
const multi = i % 3 === 2;
return {
type: multi ? 'multi' : 'single',
stem: `Mock question ${i + 1}: which action is the priority for the client described in case ${i + 1}?`,
options: [
{ text: `Correct action ${i + 1}`, correct: true, why: 'This addresses the immediate risk.' },
{ text: `Plausible distractor ${i + 1}a`, correct: multi, why: multi ? 'Also indicated.' : 'Lower priority.' },
{ text: `Plausible distractor ${i + 1}b`, correct: false, why: 'Not indicated.' },
{ text: `Plausible distractor ${i + 1}c`, correct: false, why: 'Contraindicated.' },
],
rationale: `Rationale for question ${i + 1}.`,
topic: i % 2 === 0 ? 'Heart failure' : 'Pharmacology',
difficulty: 'medium',
source: refs.length ? refs[i % refs.length] : '',
};
});
}
let parallelSeq = 0;
function toolArgs(name: string, userText: string, all: string): Record<string, unknown> {
const count = Number(/(?:Write|write)\s+(\d+)/.exec(userText)?.[1] ?? /(\d+)[- ](?:question|card|flashcard)/.exec(userText)?.[1] ?? 3);
// Only refs from the user's turn count: the system prompt contains an example id.
const refs = [...new Set([...userText.matchAll(/\[(c\d+)\]/g)].map((m) => m[1]))];
// Continue numbering after anything already written, so later batches are not duplicates.
const written = userText.split('Already written')[1] ?? '';
// Continue after the highest number written, so a refill after a removal is never a repeat.
// Batches written at the same time each get their own range, as their different excerpts would.
// A batch with planned angles is numbered from its first angle, so batches never collide.
const angle = /Mock angle (\d+)/.exec(userText);
const parallel = angle ? (Number(angle[1]) - 1) * 3 + 3000 : /at the same time/.test(userText) ? (parallelSeq = (parallelSeq + 1) % 1000) * 300 : 0; // a multiple of 3 keeps the type pattern
const offset = parallel + Math.max((written.match(/^- /gm) ?? []).length, ...[...written.matchAll(/Mock (?:question|math|term) (\d+)/g)].map((m) => Number(m[1])));
void all;
switch (name) {
case 'submit_questions':
// A medication math batch: calculations only.
if (/medication math/i.test(userText)) {
return {
questions: Array.from({ length: count }, (_, k) => ({
type: 'numeric',
stem: `Mock math ${k + 1 + offset}: give ${k + 2} mg of a drug supplied as 1 mg per mL. How many mL? Round to the nearest tenth.`,
options: [{ text: `${k + 2} mL`, correct: true, why: `${k + 2} mg / 1 mg/mL = ${k + 2} mL` }],
rationale: 'Desired over have.',
topic: 'Dosage calculation',
difficulty: 'medium',
})),
};
}
return { questions: questions(count, refs, offset) };
case 'submit_flashcards':
return { cards: Array.from({ length: count }, (_, i) => ({ front: `Mock term ${i + 1 + offset}`, back: `Mock definition ${i + 1 + offset}`, tags: ['mock'] })) };
case 'plan_questions':
return { angles: Array.from({ length: count }, (_, k) => `Mock angle ${k + 1 + offset}`) };
case 'submit_stems':
// Rewords each listed stem's opening, keeping the rest of it.
return { stems: [...userText.matchAll(/^Q(\d+): (.+)$/gm)].map((m) => ({ question: Number(m[1]), stem: `On the unit, ${m[2]}` })) };
case 'name_set':
return /quiz of/i.test(userText) ? { title: 'Heart Failure and Pharmacology', description: 'Covers heart failure care and the drugs used for it.' } : { title: 'Mock Terms', description: 'Covers the mock terms.' };
case 'generate_quiz':
return { count: Math.min(count, 5), topic: 'heart failure', style: 'nclex', use_all_class_documents: /document/i.test(userText), from_weak_areas: /wrong|weak|miss/i.test(userText) };
case 'generate_flashcards':
return { count: Math.min(count, 6), topic: 'medications', use_all_class_documents: /document/i.test(userText) };
case 'search_documents':
return { query: userText.replace(/[^\w\s]/g, ' ').split(/\s+/).filter((w) => w.length > 3).slice(0, 6).join(' ') };
case 'run_python':
return { code: 'qs = draft["questions"]\nprint(len(qs), request["count"], sum(1 for q in qs if q["type"] == "multi"), [sum(o["correct"] for o in q["options"]) for q in qs])' };
case 'submit_review': {
// Second question (if any) gets a "corrected" key so tests can see fixes applied.
const fix = /Q2 \[single\]/.test(all) ? [{ question: 2, action: 'set_correct', correct_options: [2], reason: 'mock: option 2 is the right answer' }] : [];
return { summary: 'Checked by the mock.', checks: [{ name: 'Question count', passed: true, detail: 'matches the request' }, { name: 'Correct answers per type', passed: true }], fixes: fix };
}
case 'submit_proofread': {
// Rewrites a question whose stem has a writer's change of mind in it ("? No —").
const m = /^Q(\d+) \[\w+\] .*\?\s*No\b/m.exec(userText);
if (!m) return { issues: [] };
return {
issues: [
{
question: Number(m[1]),
problem: 'The stem gives two weights.',
action: 'rewrite',
question_text: { type: 'numeric', stem: 'An 80 kg client receives norepinephrine 4 mg in 250 mL at 15 mL/h. What dose is the client receiving in mcg/kg/min? Round to the nearest hundredth.', options: [{ text: '0.05 mcg/kg/min', correct: true, why: '4000 mcg / 250 mL x 15 mL/h / 60 / 80 kg = 0.05' }], rationale: 'Convert mg to mcg, find mcg/h, then per minute and per kg.' },
},
],
};
}
case 'submit_topics': {
// Labels of three or more words are broadened to their first word, like a real model would.
const labels = [...(userText.split('Labels:')[1] ?? '').matchAll(/^- (.+)$/gm)].map((m) => m[1].trim());
return { topics: labels.map((label) => ({ label, topic: label.split(/\s+/).length >= 3 ? label.split(/\s+/)[0] : label })) };
}
case 'get_performance':
case 'list_classes':
case 'list_materials':
return {};
case 'propose_changes': {
const m = /(delete|rename)\s+(quiz|deck|note|document)\s+([\w-]{8,})/i.exec(userText);
return { reason: 'Requested by the user.', changes: [{ op: m?.[1]?.toLowerCase() ?? 'delete', kind: m?.[2]?.toLowerCase() ?? 'quiz', id: m?.[3] ?? 'missing', title: 'Renamed by mock' }] };
}
case 'ask_user':
return { questions: [{ id: 'exam', question: 'Which exam is this for?', options: [{ label: 'Exam 1' }, { label: 'Exam 2' }], defaults: ['Exam 2'] }, { id: 'count', question: 'How many questions?', input: 'number', defaults: ['20'] }] };
case 'web_search':
return { objective: 'Current sepsis bundle guidance', search_queries: ['sepsis bundle 2026', 'surviving sepsis hour-1 bundle'] };
default:
return name.startsWith('mcp_') ? { drug: 'furosemide' } : {};
}
}
function chooseTool(userText: string, names: string[]): string | null {
const pick = (name: string) => (names.includes(name) ? name : null);
// "the web" in the message: the built-in web search.
if (/\bthe web\b/i.test(userText) && names.includes('web_search')) return 'web_search';
// "mcp" in the message: use the first tool from an MCP server.
if (/\bmcp\b/i.test(userText)) return names.find((n) => n.startsWith('mcp_')) ?? null;
// "ask me" in the message: clarifying questions first.
if (/\bask me\b/i.test(userText)) return pick('ask_user');
if (/\b(delete|rename)\b/i.test(userText)) return pick('propose_changes');
if (/flashcard/i.test(userText)) return pick('generate_flashcards');
if (/\bquiz\b|practice questions/i.test(userText)) return pick('generate_quiz');
if (/should i study|performance|how am i doing/i.test(userText)) return pick('get_performance');
if (/\?|what|explain|summar/i.test(userText)) return pick('search_documents');
return null;
}
export function startMockAi(port = 0): Promise<MockAi> {
const requests: MockAi['requests'] = [];
let fail = { status: 0, times: 0 };
let delayMs = 0;
let badJson: { tool: string; times: number } | null = null;
let reviews: Record<string, unknown>[] = [];
const server = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on('data', (c) => chunks.push(c));
req.on('end', async () => {
const raw = Buffer.concat(chunks).toString('utf8');
let body: any = null;
try {
body = raw ? JSON.parse(raw) : null;
} catch {
/* ignore */
}
const path = req.url ?? '';
requests.push({ path, headers: req.headers, body });
const json = (status: number, data: unknown) => {
res.writeHead(status, { 'content-type': 'application/json' });
res.end(JSON.stringify(data));
};
if (req.headers.authorization !== 'Bearer mock-key') return json(401, { error: { message: 'Incorrect API key provided: ' + String(req.headers.authorization), type: 'invalid_request_error' } });
if (path.endsWith('/models')) return json(200, { object: 'list', data: [{ id: 'mock-large' }, { id: 'mock-small' }, { id: 'mock-embedding-1' }] });
if (path.endsWith('/embeddings')) {
const inputs: string[] = Array.isArray(body?.input) ? body.input : [String(body?.input ?? '')];
const dims = Number(body?.dimensions) || 64;
return json(200, { object: 'list', model: body?.model, data: inputs.map((t, index) => ({ object: 'embedding', index, embedding: mockEmbedding(t, dims) })), usage: { prompt_tokens: Math.ceil(inputs.join(' ').length / 4), total_tokens: Math.ceil(inputs.join(' ').length / 4) } });
}
if (!path.endsWith('/chat/completions')) return json(404, { error: { message: 'not found' } });
if (delayMs > 0) await new Promise((r) => setTimeout(r, delayMs));
if (fail.times > 0) {
fail.times -= 1;
const kind = fail.status === 401 ? ['Incorrect API key provided', 'invalid_request_error'] : fail.status === 429 ? ['Rate limit reached', 'rate_limit_error'] : ['The server had an error', 'server_error'];
return json(fail.status, { error: { message: kind[0], type: kind[1] } });
}
const messages: any[] = body?.messages ?? [];
const tools: any[] = body?.tools ?? [];
const names = tools.map((t) => t.function?.name);
const last = messages[messages.length - 1] ?? {};
const lastUser = [...messages].reverse().find((m) => m.role === 'user');
const userText = text(lastUser?.content);
const all = messages.map((m) => text(m.content)).join('\n');
res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' });
const send = (delta: Record<string, unknown>, finish: string | null = null) =>
res.write(`data: ${JSON.stringify({ id: 'mock', object: 'chat.completion.chunk', model: body?.model, choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`);
const end = (finish: string, outTokens: number) => {
send({}, finish);
res.write(`data: ${JSON.stringify({ id: 'mock', object: 'chat.completion.chunk', choices: [], usage: { prompt_tokens: Math.ceil(raw.length / 4), completion_tokens: outTokens } })}\n\n`);
res.write('data: [DONE]\n\n');
res.end();
};
const callTool = (name: string) => {
let args = JSON.stringify(name === 'submit_review' && reviews.length ? reviews.shift() : toolArgs(name, userText, all));
if (badJson && badJson.tool === name && badJson.times > 0) {
badJson.times -= 1;
args = '{"cards": [{"front": "Broken" "back": ]]]} :: not json';
}
send({ role: 'assistant', content: null, tool_calls: [{ index: 0, id: `call_${Math.random().toString(36).slice(2, 10)}`, type: 'function', function: { name, arguments: '' } }] });
// Arguments arrive in fragments, like a real provider.
for (let i = 0; i < args.length; i += 37) send({ tool_calls: [{ index: 0, function: { arguments: args.slice(i, i + 37) } }] });
end('tool_calls', Math.ceil(args.length / 4));
};
const say = (content: string) => {
send({ role: 'assistant', content: '' });
for (const word of content.match(/\S+\s*/g) ?? []) send({ content: word });
end('stop', Math.ceil(content.length / 4));
};
// Document reading: answer with one "## Page N" section per page of the attached batch.
if (/Transcribe this study material/.test(userText)) {
const file = (Array.isArray(lastUser?.content) ? lastUser.content : []).find((p: any) => p?.type === 'file');
if (!file?.file?.file_data?.startsWith('data:application/pdf;base64,')) return say('[no readable file attached]');
const range = /contains pages? (\d+)(?:-(\d+))? of a/i.exec(userText);
const from = Number(range?.[1] ?? 1);
const to = Number(range?.[2] ?? range?.[1] ?? 1);
const pages = Array.from({ length: to - from + 1 }, (_, k) => `## Page ${from + k}\n\nScanned lecture text for page ${from + k}: nursing priority ${from + k}.`);
return say(pages.join('\n\n'));
}
const forced = body?.tool_choice?.function?.name as string | undefined;
// The quiz checker: run Python once, then submit the review.
if (names.includes('submit_review')) {
const ranPython = messages.some((m) => m.role === 'tool');
return callTool(forced === 'submit_review' || ranPython ? 'submit_review' : 'run_python');
}
if (forced) return callTool(forced);
// "Some tool" (generation with MCP tools offered): an MCP tool first when asked for, then the real one.
if (body?.tool_choice === 'required' && names.length) {
const mcpTool = names.find((n) => n.startsWith('mcp_'));
if (last.role !== 'tool' && mcpTool && /\bmcp\b/i.test(userText)) return callTool(mcpTool);
return callTool(names.find((n) => !n.startsWith('mcp_')) ?? names[0]);
}
if (last.role === 'tool') {
const ref = /"ref":"(c\d+)"/.exec(text(last.content))?.[1];
if (/draft_ready|revisions_ready/.test(text(last.content))) return say('The draft is ready. Review it and save it when you are happy with it.');
if (/awaiting_user_confirmation/.test(text(last.content))) return say('I have prepared the change. Press **Confirm** to apply it.');
return say(ref ? `Based on your documents, loop diuretics relieve congestion and need potassium monitoring [${ref}].` : 'Here is what I found in your library.');
}
if (names.length > 0 && body?.tool_choice !== 'none') {
const tool = chooseTool(userText, names);
if (tool) return callTool(tool);
}
if (/JSON Schema/.test(text(messages[0]?.content))) {
// Prompt-JSON fallback for models flagged as not supporting tools.
return say(JSON.stringify(/cards/.test(text(messages[0]?.content)) ? toolArgs('submit_flashcards', userText, all) : toolArgs('submit_questions', userText, all)));
}
const ref = /\[(c\d+)\]/.exec(userText)?.[1];
return say(`## Overview\n\nThis is a mock study guide.\n\n- Key point one${ref ? ` [${ref}]` : ''}\n- Key point two\n\n## Check yourself\n\n1. What is the priority?`);
});
});
return new Promise((resolve) => {
server.listen(port, '127.0.0.1', () => {
const { port: actual } = server.address() as AddressInfo;
resolve({
url: `http://127.0.0.1:${actual}/v1`,
requests,
failNext: (status, times = 1) => {
fail = { status, times };
},
delayChat: (ms) => {
delayMs = ms;
},
badJsonNext: (tool, times = 1) => {
badJson = { tool, times };
},
scriptReviews: (list) => {
reviews = [...list];
},
close: () => new Promise((done) => server.close(() => done())),
});
});
});
}