Study/shared/importer.ts
Elijah 3e74ecd843 Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-11 08:16:41 -07:00

561 lines
22 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* Lenient import parser.
*
* Accepts Study Desk V2 exports, V1 exports, loosely-shaped AI output, plain-text
* question blocks and delimited flashcard lists, and normalises all of them into
* editable drafts plus a list of issues. Nothing is saved from here: the preview UI
* shows the drafts, lets the user fix problems, and only then posts strict payloads.
*/
import { jsonrepair } from 'jsonrepair';
import {
DIFFICULTIES,
type ContentDraft,
type DeckDraft,
type Difficulty,
type DraftCard,
type DraftOption,
type DraftQuestion,
type Issue,
type NoteDraft,
type QuizDraft,
validateDraft,
} from './content.ts';
export const EXPORT_FORMAT = 'studydesk';
export const EXPORT_VERSION = 2;
export interface ImportResult {
items: ContentDraft[];
/** Problems that are not tied to one item (unparseable input, skipped entries...). */
notices: Issue[];
/** True when the JSON had to be repaired before it could be parsed. */
repaired: boolean;
/** How the input was interpreted, for display. */
detected: 'json' | 'text-quiz' | 'text-cards' | 'none';
}
type Raw = Record<string, unknown>;
const isObj = (v: unknown): v is Raw => typeof v === 'object' && v !== null && !Array.isArray(v);
const str = (v: unknown): string => (typeof v === 'string' ? v : typeof v === 'number' || typeof v === 'boolean' ? String(v) : '');
const pick = (o: Raw, ...keys: string[]): unknown => {
for (const k of keys) if (o[k] !== undefined && o[k] !== null) return o[k];
return undefined;
};
function toTags(v: unknown): string[] {
if (Array.isArray(v)) return v.map(str).map((s) => s.trim()).filter(Boolean).slice(0, 12);
if (typeof v === 'string') return v.split(/[,;]/).map((s) => s.trim()).filter(Boolean).slice(0, 12);
return [];
}
function toDifficulty(v: unknown): Difficulty {
const s = str(v).toLowerCase().trim();
if ((DIFFICULTIES as readonly string[]).includes(s)) return s as Difficulty;
if (['1', 'beginner', 'basic', 'low'].includes(s)) return 'easy';
if (['3', 'advanced', 'difficult', 'high', 'challenging'].includes(s)) return 'hard';
return 'medium';
}
function truthy(v: unknown): boolean {
if (typeof v === 'boolean') return v;
if (typeof v === 'number') return v !== 0;
const s = str(v).trim().toLowerCase();
return s === 'true' || s === 'yes' || s === 'y' || s === '1' || s === 'correct' || s === 'x';
}
/** Maps "B", "b", "2", 2 or the option text to option indexes. */
function resolveAnswerRefs(answer: unknown, options: DraftOption[]): number[] {
const refs = Array.isArray(answer) ? answer : typeof answer === 'string' ? answer.split(/[,;&]|\band\b/i) : [answer];
const out = new Set<number>();
for (const ref of refs) {
if (typeof ref === 'number' && Number.isInteger(ref)) {
// Ambiguous 0/1-based: treat as 0-based only when 0 appears, otherwise 1-based is far more common in hand-written keys.
const idx = ref >= 1 && ref <= options.length ? ref - 1 : ref;
if (idx >= 0 && idx < options.length) out.add(idx);
continue;
}
const s = str(ref).trim();
if (!s) continue;
const byText = options.findIndex((o) => o.text.trim().toLowerCase() === s.toLowerCase());
if (byText >= 0) {
out.add(byText);
continue;
}
const letter = s.replace(/[().\s]/g, '').toUpperCase();
if (/^[A-J]$/.test(letter)) {
const idx = letter.charCodeAt(0) - 65;
if (idx < options.length) out.add(idx);
} else if (/^\d+$/.test(letter)) {
const idx = Number(letter) - 1;
if (idx >= 0 && idx < options.length) out.add(idx);
}
}
return [...out];
}
function normalizeOptions(raw: unknown): DraftOption[] {
if (Array.isArray(raw)) {
return raw
.map((o): DraftOption | null => {
if (typeof o === 'string' || typeof o === 'number') return { text: String(o).trim(), correct: false, why: '' };
if (!isObj(o)) return null;
return {
text: str(pick(o, 'text', 'option', 'answer', 'label', 'value', 'content')).trim(),
correct: truthy(pick(o, 'correct', 'isCorrect', 'is_correct', 'right', 'answer_correct')),
why: str(pick(o, 'why', 'rationale', 'explanation', 'feedback')).trim(),
};
})
.filter((o): o is DraftOption => o !== null);
}
// { "A": "text", "B": "text" }
if (isObj(raw)) {
return Object.keys(raw)
.sort()
.map((k) => ({ text: str(raw[k]).trim(), correct: false, why: '' }));
}
return [];
}
function normalizeQuestion(raw: unknown, notices: Issue[], index: number): DraftQuestion | null {
if (!isObj(raw)) {
notices.push({ level: 'warning', index, message: `Question ${index + 1} is not an object and was skipped` });
return null;
}
const options = normalizeOptions(pick(raw, 'options', 'choices', 'answers', 'answerOptions', 'answer_options'));
const answerKey = pick(raw, 'correctAnswers', 'correct_answers', 'correctAnswer', 'correct_answer', 'answer', 'correct', 'key');
if (answerKey !== undefined && !options.some((o) => o.correct)) {
for (const idx of resolveAnswerRefs(answerKey, options)) options[idx].correct = true;
}
const correctCount = options.filter((o) => o.correct).length;
const rawType = str(pick(raw, 'type', 'questionType', 'question_type', 'kind')).toLowerCase().replace(/[\s-]/g, '_');
let type: DraftQuestion['type'];
if (['multi', 'sata', 'select_all', 'select_all_that_apply', 'multiple_response', 'multiple_select', 'multi_select', 'checkbox'].includes(rawType)) {
type = 'multi';
} else if (['single', 'multiple_choice', 'mcq', 'mc', 'single_choice', 'true_false', 'radio'].includes(rawType)) {
type = 'single';
} else {
type = correctCount > 1 ? 'multi' : 'single';
}
// A "multiple choice" question with several keyed answers can only be select-all.
if (type === 'single' && correctCount > 1) type = 'multi';
return {
type,
stem: str(pick(raw, 'stem', 'prompt', 'question', 'text', 'q')).trim(),
options,
rationale: str(pick(raw, 'rationale', 'explanation', 'reasoning', 'feedback')).trim(),
topic: str(pick(raw, 'topic', 'category', 'subject', 'concept')).trim().slice(0, 80),
difficulty: toDifficulty(pick(raw, 'difficulty', 'level')),
tags: toTags(pick(raw, 'tags', 'keywords')),
source: str(pick(raw, 'source', 'citation', 'reference')).trim() || undefined,
};
}
function normalizeCard(raw: unknown, notices: Issue[], index: number): DraftCard | null {
if (Array.isArray(raw) && raw.length >= 2) return { front: str(raw[0]).trim(), back: str(raw[1]).trim(), hint: '', tags: [] };
if (!isObj(raw)) {
notices.push({ level: 'warning', index, message: `Card ${index + 1} is not an object and was skipped` });
return null;
}
return {
front: str(pick(raw, 'front', 'term', 'question', 'prompt', 'q', 'word')).trim(),
back: str(pick(raw, 'back', 'definition', 'answer', 'a', 'response', 'meaning')).trim(),
hint: str(pick(raw, 'hint', 'clue')).trim(),
tags: toTags(pick(raw, 'tags', 'keywords')),
};
}
function quizFrom(raw: Raw, notices: Issue[]): QuizDraft {
const list = pick(raw, 'questions', 'items', 'quiz') as unknown;
const questions = (Array.isArray(list) ? list : [])
.map((q, i) => normalizeQuestion(q, notices, i))
.filter((q): q is DraftQuestion => q !== null);
return {
kind: 'quiz',
title: str(pick(raw, 'title', 'quizName', 'quiz_name', 'name')).trim(),
description: str(pick(raw, 'description', 'summary')).trim(),
questions,
};
}
function deckFrom(raw: Raw, notices: Issue[]): DeckDraft {
const list = pick(raw, 'cards', 'flashcards', 'items') as unknown;
const cards = (Array.isArray(list) ? list : [])
.map((c, i) => normalizeCard(c, notices, i))
.filter((c): c is DraftCard => c !== null);
return {
kind: 'deck',
title: str(pick(raw, 'title', 'deckName', 'deck_name', 'name')).trim(),
description: str(pick(raw, 'description', 'summary')).trim(),
cards,
};
}
function noteFrom(raw: Raw): NoteDraft {
return { kind: 'note', title: str(pick(raw, 'title', 'name')).trim(), content: str(pick(raw, 'content', 'markdown', 'body', 'text')) };
}
function looksLikeQuestion(v: unknown): boolean {
return isObj(v) && pick(v, 'options', 'choices', 'answers', 'answerOptions') !== undefined;
}
function looksLikeCard(v: unknown): boolean {
return isObj(v) && pick(v, 'front', 'term') !== undefined && pick(v, 'back', 'definition') !== undefined;
}
function collect(value: unknown, items: ContentDraft[], notices: Issue[], depth = 0): void {
if (depth > 4) return;
if (Array.isArray(value)) {
if (value.length === 0) return;
if (value.every(looksLikeQuestion)) {
items.push(quizFrom({ questions: value }, notices));
return;
}
if (value.every((v) => looksLikeCard(v) || (Array.isArray(v) && v.length === 2))) {
items.push(deckFrom({ cards: value }, notices));
return;
}
for (const v of value) collect(v, items, notices, depth + 1);
return;
}
if (!isObj(value)) return;
const kind = str(pick(value, 'kind', 'type')).toLowerCase();
// Bundles / backups: { classes: [{ quizzes, decks, notes }] } or { items: [...] }
if (Array.isArray(value.classes)) {
for (const cls of value.classes) collect(cls, items, notices, depth + 1);
return;
}
const containers = ['quizzes', 'decks', 'notes', 'quizSets'].filter((k) => Array.isArray(value[k]));
if (containers.length > 0 && !Array.isArray(value.questions) && !Array.isArray(value.cards)) {
for (const k of containers) {
for (const entry of value[k] as unknown[]) {
if (!isObj(entry)) continue;
if (k === 'notes') items.push(noteFrom(entry));
else if (k === 'decks') items.push(deckFrom(entry, notices));
else items.push(quizFrom(entry, notices));
}
}
return;
}
if (kind === 'bundle' && Array.isArray(value.items)) {
for (const entry of value.items) collect(entry, items, notices, depth + 1);
return;
}
if (kind === 'note' || kind === 'guide') {
items.push(noteFrom(value));
return;
}
if (kind === 'deck' || kind === 'flashcards' || Array.isArray(value.cards) || Array.isArray(value.flashcards)) {
items.push(deckFrom(value, notices));
return;
}
if (kind === 'quiz' || Array.isArray(value.questions)) {
items.push(quizFrom(value, notices));
return;
}
if (looksLikeQuestion(value)) {
items.push(quizFrom({ questions: [value] }, notices));
return;
}
if (looksLikeCard(value)) {
items.push(deckFrom({ cards: [value] }, notices));
return;
}
// Unknown wrapper: look one level down for something recognisable.
for (const v of Object.values(value)) if (typeof v === 'object' && v !== null) collect(v, items, notices, depth + 1);
}
/** Strips a Markdown code fence and any prose around the outermost JSON value. */
function extractJsonText(text: string): string | null {
let t = text.trim();
const fence = /```(?:json|JSON)?\s*\n([\s\S]*?)```/.exec(t);
if (fence) t = fence[1].trim();
const start = t.search(/[[{]/);
if (start < 0) return null;
const open = t[start];
const close = open === '{' ? '}' : ']';
const end = t.lastIndexOf(close);
return end > start ? t.slice(start, end + 1) : t.slice(start);
}
/* ------------------------------------------------------------------ */
/* Plain-text formats */
/* ------------------------------------------------------------------ */
const OPTION_LINE = /^\s*(\*)?\s*(?:\(?([A-Ja-j])[.)]|[-•])\s+(.*)$/;
const ANSWER_LINE = /^\s*(?:correct\s+)?answers?\s*[:=]\s*(.+)$/i;
const RATIONALE_LINE = /^\s*(?:rationale|explanation|why)\s*[:=]\s*(.*)$/i;
const TOPIC_LINE = /^\s*(?:topic|category)\s*[:=]\s*(.*)$/i;
/**
* Parses hand-typed question blocks separated by blank lines:
*
* 1. Which finding requires immediate action?
* A. Heart rate 88
* *B. Oxygen saturation 84%
* C. Temperature 37.2
* Answer: B (alternative to the * marker)
* Rationale: ...
* Topic: Respiratory
*/
export function parseTextQuestions(text: string): DraftQuestion[] {
const blocks = text.replace(/\r\n?/g, '\n').split(/\n\s*\n/);
const questions: DraftQuestion[] = [];
for (const block of blocks) {
const lines = block.split('\n').filter((l) => l.trim());
if (lines.length < 3) continue;
const stem: string[] = [];
const options: DraftOption[] = [];
let rationale = '';
let topic = '';
let answer: string | null = null;
let section: 'stem' | 'options' | 'rationale' = 'stem';
for (const line of lines) {
let m: RegExpExecArray | null;
if ((m = ANSWER_LINE.exec(line))) {
answer = m[1];
continue;
}
if ((m = RATIONALE_LINE.exec(line))) {
rationale = m[1].trim();
section = 'rationale';
continue;
}
if ((m = TOPIC_LINE.exec(line))) {
topic = m[1].trim();
continue;
}
if ((m = OPTION_LINE.exec(line)) && section !== 'rationale') {
let body = m[3].trim();
let correct = Boolean(m[1]);
const tail = /\s*(?:\*|\(correct\)|✓|✔)\s*$/i.exec(body);
if (tail) {
correct = true;
body = body.slice(0, tail.index).trim();
}
options.push({ text: body, correct, why: '' });
section = 'options';
continue;
}
if (section === 'stem') stem.push(line.replace(/^\s*(?:Q?\d+[.):]|Q[.:])\s*/i, '').trim());
else if (section === 'rationale') rationale += `\n${line.trim()}`;
else if (options.length) options[options.length - 1].text += ` ${line.trim()}`;
}
if (options.length < 2 || stem.length === 0) continue;
if (answer && !options.some((o) => o.correct)) for (const idx of resolveAnswerRefs(answer, options)) options[idx].correct = true;
const correctCount = options.filter((o) => o.correct).length;
questions.push({
type: correctCount > 1 ? 'multi' : 'single',
stem: stem.join('\n'),
options,
rationale: rationale.trim(),
topic,
difficulty: 'medium',
tags: [],
});
}
return questions;
}
/** Parses "term <sep> definition" lines; the separator is detected from the input. */
export function parseTextCards(text: string): DraftCard[] {
const lines = text.replace(/\r\n?/g, '\n').split('\n').map((l) => l.trim()).filter(Boolean);
if (lines.length === 0) return [];
const seps = ['\t', ' :: ', '::', ' | ', ' — ', ' – ', ' - ', ';', ','];
const sep = seps.find((s) => lines.filter((l) => l.includes(s)).length >= Math.ceil(lines.length * 0.8));
if (!sep) return [];
const cards: DraftCard[] = [];
for (const line of lines) {
const at = line.indexOf(sep);
if (at <= 0) continue;
const front = line.slice(0, at).trim().replace(/^"|"$/g, '');
const back = line.slice(at + sep.length).trim().replace(/^"|"$/g, '');
if (front && back) cards.push({ front, back, hint: '', tags: [] });
}
return cards;
}
/* ------------------------------------------------------------------ */
/* Entry point */
/* ------------------------------------------------------------------ */
export function parseImport(text: string, opts: { prefer?: 'quiz' | 'deck'; fallbackTitle?: string } = {}): ImportResult {
const notices: Issue[] = [];
const items: ContentDraft[] = [];
let repaired = false;
let detected: ImportResult['detected'] = 'none';
const trimmed = text.trim();
if (!trimmed) return { items, notices: [{ level: 'error', message: 'Nothing to import yet. Paste content or choose a file.' }], repaired, detected };
const jsonText = /^[\s`]*(?:```|[[{])/.test(trimmed) || /^[^\n]{0,200}[[{]/.test(trimmed) ? extractJsonText(trimmed) : null;
if (jsonText) {
let parsed: unknown;
let ok = false;
try {
parsed = JSON.parse(jsonText);
ok = true;
} catch {
try {
parsed = JSON.parse(jsonrepair(jsonText));
ok = true;
repaired = true;
} catch (e) {
notices.push({ level: 'error', message: `The JSON could not be read: ${e instanceof Error ? e.message : 'syntax error'}` });
}
}
if (ok) {
detected = 'json';
collect(parsed, items, notices);
if (items.length === 0) notices.push({ level: 'error', message: 'No quizzes or flashcards were found in that JSON. Download a template to see the expected shape.' });
if (repaired) notices.push({ level: 'warning', message: 'The JSON had syntax problems that were repaired automatically. Check the preview carefully.' });
}
}
if (items.length === 0 && detected !== 'json') {
const questions = opts.prefer === 'deck' ? [] : parseTextQuestions(trimmed);
const cards = questions.length > 0 ? [] : parseTextCards(trimmed);
if (questions.length > 0) {
detected = 'text-quiz';
items.push({ kind: 'quiz', title: '', description: '', questions });
} else if (cards.length > 0) {
detected = 'text-cards';
items.push({ kind: 'deck', title: '', description: '', cards });
} else if (notices.length === 0) {
notices.push({
level: 'error',
message: 'Could not recognise this content. Use JSON, question blocks with lettered choices, or one "term [tab] definition" per line.',
});
}
}
for (const item of items) {
if (!item.title && opts.fallbackTitle) item.title = opts.fallbackTitle;
}
return { items, notices, repaired, detected };
}
/** Issues for one draft, including a friendly summary count. */
export function summarizeIssues(draft: ContentDraft): { errors: number; warnings: number; issues: Issue[] } {
const issues = validateDraft(draft);
return {
errors: issues.filter((i) => i.level === 'error').length,
warnings: issues.filter((i) => i.level === 'warning').length,
issues,
};
}
/* ------------------------------------------------------------------ */
/* Export shapes + templates */
/* ------------------------------------------------------------------ */
export function exportQuiz(q: { title: string; description: string; questions: DraftQuestion[] }) {
return {
format: EXPORT_FORMAT,
version: EXPORT_VERSION,
kind: 'quiz' as const,
title: q.title,
description: q.description,
questions: q.questions.map((x) => ({
type: x.type,
stem: x.stem,
options: x.options.map((o) => (o.why ? { text: o.text, correct: o.correct, why: o.why } : { text: o.text, correct: o.correct })),
rationale: x.rationale,
topic: x.topic,
difficulty: x.difficulty,
tags: x.tags,
})),
};
}
export function exportDeck(d: { title: string; description: string; cards: DraftCard[] }) {
return {
format: EXPORT_FORMAT,
version: EXPORT_VERSION,
kind: 'deck' as const,
title: d.title,
description: d.description,
cards: d.cards.map((c) => ({ front: c.front, back: c.back, ...(c.hint ? { hint: c.hint } : {}), tags: c.tags })),
};
}
export const QUIZ_JSON_TEMPLATE = JSON.stringify(
exportQuiz({
title: 'Example quiz',
description: 'Optional description',
questions: [
{
type: 'single',
stem: 'A client with heart failure reports a 1.4 kg weight gain overnight. Which action should the nurse take first?',
options: [
{ text: 'Assess lung sounds and oxygen saturation', correct: true, why: 'Rapid weight gain signals fluid retention; assess for pulmonary congestion first.' },
{ text: 'Document the weight and continue to monitor', correct: false },
{ text: 'Encourage increased oral fluid intake', correct: false },
{ text: 'Schedule a dietitian consult for next week', correct: false },
],
rationale: 'Assessment comes first. A gain of more than 1 kg in a day suggests fluid volume overload.',
topic: 'Heart failure',
difficulty: 'medium',
tags: ['cardiac', 'prioritization'],
},
{
type: 'multi',
stem: 'Which findings are expected with hypokalemia? Select all that apply.',
options: [
{ text: 'Muscle weakness', correct: true },
{ text: 'Flattened T waves', correct: true },
{ text: 'Peaked T waves', correct: false },
{ text: 'Decreased bowel sounds', correct: true },
],
rationale: 'Low potassium slows smooth and skeletal muscle activity and flattens T waves. Peaked T waves indicate hyperkalemia.',
topic: 'Electrolytes',
difficulty: 'medium',
tags: [],
},
],
}),
null,
2,
);
export const DECK_JSON_TEMPLATE = JSON.stringify(
exportDeck({
title: 'Example deck',
description: 'Optional description',
cards: [
{ front: 'Furosemide: drug class', back: 'Loop diuretic', hint: 'Acts on the loop of Henle', tags: ['pharm'] },
{ front: 'Normal serum potassium', back: '3.5 to 5.0 mEq/L', tags: ['labs'] },
],
}),
null,
2,
);
export const TEXT_QUIZ_TEMPLATE = `1. Which electrolyte imbalance causes peaked T waves?
A. Hypokalemia
*B. Hyperkalemia
C. Hyponatremia
D. Hypercalcemia
Rationale: High potassium speeds repolarisation, producing tall, peaked T waves.
Topic: Electrolytes
2. Which are signs of digoxin toxicity? Select all that apply.
*A. Visual halos
*B. Nausea and vomiting
C. Hypertension
*D. Bradycardia
Rationale: Early toxicity is gastrointestinal and visual; bradycardia and dysrhythmias follow.
Topic: Pharmacology`;
export const TEXT_CARDS_TEMPLATE = `Furosemide\tLoop diuretic; monitor potassium
Metoprolol\tBeta-1 selective blocker; hold for HR below 60
Warfarin\tVitamin K antagonist; monitor INR`;
/** Normalises already-parsed data (for example the arguments of an AI tool call) into drafts. */
export function draftsFromValue(value: unknown): { items: ContentDraft[]; notices: Issue[] } {
const items: ContentDraft[] = [];
const notices: Issue[] = [];
collect(value, items, notices);
return { items, notices };
}