diff --git a/.forgejo/workflows/build-v2.yml b/.forgejo/workflows/build-v2.yml deleted file mode 100644 index 2941eb8..0000000 --- a/.forgejo/workflows/build-v2.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Verify and publish V2 container - -# Publishes ONLY the `v2` and `v2-` tags. It never touches `latest`, which V1's -# workflow on `main` publishes and the current production container may track. -on: - push: - branches: - - v2 - workflow_dispatch: - -jobs: - build-and-push: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Resolve image names - run: | - IMAGE_PATH=$(echo "git.elijahkuntz.com/${{ gitea.actor }}/${{ github.event.repository.name }}" | tr '[:upper:]' '[:lower:]') - SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-12) - echo "IMAGE_PATH=$IMAGE_PATH" >> "$GITHUB_ENV" - echo "IMAGE_SHA=$IMAGE_PATH:v2-$SHORT_SHA" >> "$GITHUB_ENV" - echo "IMAGE_V2=$IMAGE_PATH:v2" >> "$GITHUB_ENV" - - # Typecheck and tests run inside the Node 24 image the runtime uses, so the runner's - # own Node version does not matter. - - name: Typecheck and test - run: docker build --target test -t "study-v2-test:${{ github.sha }}" . - - - name: Build production image - run: docker build -t "$IMAGE_SHA" . - - - name: Smoke-test the production image - run: | - SMOKE="study-v2-smoke-$(openssl rand -hex 6)" - cleanup() { docker rm -f "$SMOKE" > /dev/null 2>&1 || true; } - trap cleanup EXIT - docker run -d --name "$SMOKE" "$IMAGE_SHA" - for attempt in $(seq 1 30); do - STATUS=$(docker inspect --format='{{.State.Health.Status}}' "$SMOKE") - if [ "$STATUS" = "healthy" ]; then break; fi - if [ "$(docker inspect --format='{{.State.Running}}' "$SMOKE")" != "true" ]; then break; fi - sleep 2 - done - if [ "$(docker inspect --format='{{.State.Health.Status}}' "$SMOKE")" != "healthy" ]; then - docker logs "$SMOKE" - exit 1 - fi - # Runs as the non-root node user, data volume writable, first-run setup code created. - test "$(docker exec "$SMOKE" id -u)" != "0" - docker exec "$SMOKE" test -s /data/setup-code.txt - docker exec "$SMOKE" test -s /data/study.db - # Web app served; fresh install reports setup needed; private API denied. - docker exec "$SMOKE" node -e "fetch('http://127.0.0.1:3726/').then(async r => { const t = await r.text(); if (!r.ok || !t.includes('id=\"root\"')) process.exit(1) }).catch(() => process.exit(1))" - docker exec "$SMOKE" node -e "fetch('http://127.0.0.1:3726/api/auth/status').then(async r => { const b = await r.json(); if (!r.ok || b.configured !== false || b.authenticated !== false) process.exit(1) }).catch(() => process.exit(1))" - docker exec "$SMOKE" node -e "fetch('http://127.0.0.1:3726/api/classes').then(r => { if (r.status !== 401) process.exit(1) }).catch(() => process.exit(1))" - docker logs "$SMOKE" 2>&1 | grep -vi 'setup code' | tail -n 20 - - - name: Log into registry - run: | - echo "${{ secrets.FORGEJO_PAT }}" | docker login git.elijahkuntz.com -u "${{ gitea.actor }}" --password-stdin - - - name: Publish v2 tags - run: | - docker tag "$IMAGE_SHA" "$IMAGE_V2" - docker push "$IMAGE_SHA" - docker push "$IMAGE_V2" diff --git a/Dockerfile b/Dockerfile index 3e5847a..4fda957 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,9 @@ # syntax=docker/dockerfile:1 -FROM node:24-bookworm-slim AS deps +FROM node:24-bookworm-slim AS build WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci COPY . . - -# CI target: `docker build --target test .` runs typecheck and the full test suite on the -# same Node version the runtime image uses. Not part of the default image build. -FROM deps AS test -RUN npm run typecheck && npm test - -FROM deps AS build RUN npm run build && npm prune --omit=dev FROM node:24-bookworm-slim AS runtime diff --git a/PROGRESS.md b/PROGRESS.md index b0c1e1e..d7a9ee4 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,6 +1,6 @@ # Study Desk V2 — progress and resume notes -Updated 2026-11-12. Branch: **v2**. Original V1: **main**, preserved at `7bd0a9a`. +Updated 2026-11-11. Branch: **v2**. Original V1: **main**, preserved at `7bd0a9a`. The owner authorized publishing progress and `v2` now tracks Forgejo `origin/v2`. No production deployment has been performed. @@ -36,36 +36,11 @@ type-grouped materials, copyable import instructions and assistant library links timer submissions after a failed request. - Preserved the V1 card elapsed-days scheduling field during import. -## 2026-11-12 additions - -- `.forgejo/workflows/build-v2.yml`: pushes to `v2` run typecheck + tests inside the Node 24 - image (new Dockerfile `test` stage), build and smoke-test the production image, then publish - `study:v2` and `study:v2-`. Never `:latest`. Verified locally with Docker Desktop; the - workflow itself has not run on the Forgejo runner yet (nothing pushed). -- Drag-to-reorder (`web/src/ui/Sortable.tsx`, pointer events so touch works, keyboard with - arrows/Home/End and live announcements) for class materials, folders, questions and cards. - New endpoint `POST /api/classes/:id/items/reorder`. -- Timed exams use a wall-clock `deadlineAt` stored in attempt settings. The server refuses late - answers (409 `time_up`), submits overdue exams lazily and before every authenticated GET, - and caps time used at the limit. Shared exams use a local wall-clock deadline too. -- Shared quiz sessions freeze their questions (with option order) in browser storage, with a - fingerprint to tell the visitor when the owner has since edited the quiz. -- Scanned PDFs are read in 8-page batches split with pdf-lib, cached per batch, with real page - numbers. OpenAI-compatible adapter now sends PDFs as Chat Completions `file` parts when the - model is marked as PDF-capable. -- Semantic search: migration `2_embeddings` (`chunk_vectors`), embedding support in the - OpenAI/Gemini/Ollama/compatible adapters, `server/services/semantic.ts` (background indexing, - in-memory cosine search, reciprocal-rank fusion with FTS), Settings → AI section. -- Browser QA on an isolated `.test-data/qa` instance (port 3742, `study-qa` launch config): - drag and keyboard reorder persisted; 375px class page has no overflow; semantic search - enabled from Settings and indexed; a 1-minute exam counted down on the wall clock and - auto-submitted at 60 s; a shared session kept original questions after the owner edited them. - ## Verification status Baseline: 162 tests passed, TypeScript passed and production web/server builds passed. -Added six regression tests. Full `npm run check` passes: **181 tests** (2026-11-12), TypeScript -and both production builds. The Docker `test` stage runs the same 181 tests on Node 24. Tests were run with loopback access for the mock provider. +Added six regression tests. Full `npm run check` passes: **168 tests**, TypeScript and both +production builds. Tests were run with loopback access for the mock provider. The earlier agent manually exercised quiz completion/results, flashcards, streak effects, mock AI generation through the assistant, editable drafts, and a 375px home page without diff --git a/README.md b/README.md index d0e41a3..c5c9c8f 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,5 @@ account-specific model access, Linux container behavior and Unraid proxy configu be checked in their target environments. [PROGRESS.md](PROGRESS.md) records completed work, remaining verification, decisions and the -resume procedure. V1 remains on `main` and in Git history; V2 work is on the `v2` branch. -Pushing `v2` to Forgejo runs CI and publishes the `study:v2` image (never `:latest`); see -[deployment](docs/DEPLOYMENT.md). Nothing deploys automatically. +resume procedure. V1 remains on `main` and in Git history; all V2 work is on local `v2`. +Nothing is automatically pushed or published. diff --git a/docs/AI.md b/docs/AI.md index e7175ec..acf8acb 100644 --- a/docs/AI.md +++ b/docs/AI.md @@ -13,31 +13,9 @@ container itself, not your workstation or Unraid host. Use a reachable LAN addre Keys are encrypted with AES-256-GCM in SQLite using `/data/secret.key` or `APP_SECRET`. Only the server decrypts them. Keep the key with backups; changing it breaks decryption. Documents are sent to the selected provider when you request AI processing; extraction and -full-text indexing otherwise run locally. Review generated nursing content and source -references before using it for study. - -## Reading scanned PDFs (OCR) - -A PDF with little selectable text is read by the "Vision and OCR" model, which must accept -PDFs (OpenAI, Anthropic, Gemini; OpenAI-compatible servers when the model is marked "PDF"). -Long files are split with pdf-lib into batches of 8 pages (smaller if a batch exceeds 24 MB) -and sent one batch per request, so there is no whole-file size cap. Each batch keeps its real -page numbers for citations. Finished batches are cached beside the upload -(`uploads/.ocr.json`), so a restart or a failed batch does not repeat paid work; "Read -with AI" on the document forces a fresh read. A batch that fails twice is skipped with a -warning on the document. At most 600 pages are read per file. - -## Semantic search - -Settings → AI → Semantic search chooses an embeddings provider and model (suggested: -OpenAI `text-embedding-3-small`, Gemini `gemini-embedding-001`, Ollama `nomic-embed-text`; -Anthropic has no embeddings API). Every document passage is embedded in the background and -stored as a vector in SQLite (`chunk_vectors`). Search, the assistant's document search and -focused generation then merge keyword (FTS5) and meaning-based rankings with reciprocal-rank -fusion, so exact terms still match while paraphrased questions find the right passage. If the -embedding provider fails, search silently falls back to keywords. Changing the model clears -and rebuilds the index. Costs appear under Usage as "semantic indexing" / "semantic search". -Quiz questions, flashcards and notes are still searched by keyword. +full-text indexing otherwise run locally. OCR requires a compatible vision/PDF model. +Retrieval uses chunked SQLite FTS search; it is lexical retrieval, not embedding similarity. +Review generated nursing content and source references before using it for study. ## ChatGPT account investigation (2026-11-11) @@ -58,9 +36,7 @@ ChatGPT browser session tokens or a Codex client's registration/credentials into Mock protocol tests cover request translation, streams, tools, usage and error handling. Integration tests use a local OpenAI-compatible mock to generate/edit/save drafts, search -documents with citations and exercise confirmed assistant actions, batched OCR of a 20-page -scanned PDF and semantic indexing/search (the mock returns deterministic word-hash vectors). -They cost no API credits. +documents with citations and exercise confirmed assistant actions. They cost no API credits. No successful real-provider inference or real ChatGPT OAuth authorization is claimed. For manual local AI testing: `npx tsx scripts/mock-ai.ts` starts a scripted mock at diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index d298811..2e3854e 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -2,25 +2,8 @@ ## Unraid -### Getting the image - -Every push to the `v2` branch runs `.forgejo/workflows/build-v2.yml`. It typechecks and runs the -test suite inside the Node 24 image (`docker build --target test`), builds the production image, -smoke-tests it as a fresh non-root container, then publishes two tags: - -- `git.elijahkuntz.com/elijah/study:v2` (moves with each successful `v2` push) -- `git.elijahkuntz.com/elijah/study:v2-<12-char commit>` (fixed; use it to pin or roll back) - -It **never** publishes `:latest`, which V1's `main` workflow owns. Keep the V1 container on its -existing tag. Do not merge `v2` into `main` until V2 replaces V1: that merge would remove V1's -workflow, and this one only triggers on `v2`. The workflow can also be run manually. - -Alternatively build locally from this checkout with `docker build -t study-desk:v2-local .`. - -### Container settings - -Create a **new** container named `study-v2`, image `git.elijahkuntz.com/elijah/study:v2` -(or your local tag): +Build from this checkout with `docker build -t study-desk:v2-local .`. +Create a **new** container named `study-v2`, image `study-desk:v2-local`: | Setting | Value | | --- | --- | @@ -33,15 +16,11 @@ Create a **new** container named `study-v2`, image `git.elijahkuntz.com/elijah/s | TRUST_PROXY | true only behind a trusted proxy | | COOKIE_SECURE | true with HTTPS; auto for local HTTP testing | -The image uses UID/GID **1000:1000**. Unraid appdata folders are normally owned by -`nobody:users` (**99:100**), which that user cannot write to. Pick one: - -- Create the dedicated folder and hand it to 1000: `mkdir -p /mnt/user/appdata/study-v2` then - `chown 1000:1000 /mnt/user/appdata/study-v2`. Only change permissions on this new folder. -- Or keep Unraid's ownership and add `--user 99:100` to the container's Extra Parameters. - -If the container restarts repeatedly with a permissions error in its log, this is the cause. -Never mount V1's database or data folder. +The image uses UID/GID **1000:1000**. Create a dedicated host directory writable by that user +before starting. For example, on Unraid: `mkdir -p /mnt/user/appdata/study-v2` then +`chown 1000:1000 /mnt/user/appdata/study-v2`. Only change permissions on this new folder. +Alternatively select an appropriate non-root UID/GID using Docker's `--user` option and grant +that identity access to the dedicated directory. Never mount V1's database or data folder. The supplied `compose.yaml` defaults to `./data-v2` and host port 3727. Override `STUDY_DATA_DIR` for Unraid. Do not reuse a V1 container name, mount or proxy destination. diff --git a/docs/V2-COMPARISON.md b/docs/V2-COMPARISON.md index 0ef154a..9277d56 100644 --- a/docs/V2-COMPARISON.md +++ b/docs/V2-COMPARISON.md @@ -10,7 +10,7 @@ establish real-provider compatibility, and sampled desktop/mobile QA is not exha | --- | --- | | New architecture and UI | Rebuilt: React/Vite, Hono, built-in SQLite; distinct responsive light/dark design | | Single administrator and public visitors | Implemented: secure local login, revocable links, isolated visitor progress | -| Classes and organization | Classes/folders, edit/move/search/delete, quiz/deck duplication, drag-to-reorder (mouse, touch, keyboard) for materials, folders, questions and cards | +| Classes and organization | Classes/folders, edit/move/search/delete, quiz/deck duplication; manual ordering and duplication are not equally exposed for every content type | | Quizzes | MC/SATA, per-option explanations, question rationale, partial credit, practice/exam, shuffle, resume, missed review and history implemented | | Flashcards | Decks, browse/self-assessment, persistent FSRS, reviews, undo and class/deck scoping implemented | | Imports/templates/export | Preview/validation/editing, JSON/text bulk input, reusable templates and exports implemented | @@ -31,10 +31,9 @@ exam engine are not implemented; the brief listed additional types as optional. independent schedules and new-card limits. V2 schedules each card once, with a global daily new-card limit and deck/class/all review scopes. A practical simplification, but it cannot reproduce separate course/exam review sets or independent per-set schedules. -- **Manual material order:** restored on 2026-11-12. The class page's default "Your order" - sort shows grip handles: drag (mouse or touch) or use arrow keys/Home/End on a handle to - reorder quizzes, decks, documents and notes within their group, and to reorder folders. - Quiz questions and deck cards are reorderable the same way. +- **Manual material order:** V1 supported drag/drop ordering of decks within material groups. + V2 class lists sort by recent update or title; the visible drag/drop is for file upload. + Some reorder APIs exist without matching UI. Quiz questions do have move-up/down controls. - **Migration fidelity:** V2 imports source content, completed attempts and one selected schedule per card. V1 interrupted StudyProgress sessions, named sets and their independent schedules are not recreated. Historical reviewSnapshotJson is not consumed: imported @@ -74,18 +73,18 @@ Screenshot: ignored local `.test-data/rationale-check/correct-inline.png`. ## Remaining usability/feature priorities -Done on 2026-11-12: drag-to-reorder; wall-clock exam deadlines enforced by the server -(answers after the deadline plus a 5 s grace are refused and overdue exams are submitted with -what was saved, even if nobody reopens them); frozen shared-quiz sessions (a visitor's session -keeps the questions as they were at the start and notes later edits); batched, cached OCR for -long scanned PDFs; semantic (embedding) search merged with keyword search. - 1. Restore selected-deck named review collections if replacing that V1 workflow is important. -2. Better import reporting/preservation for V1 history and interrupted sessions. -3. Scanned PDFs need a vision model that accepts PDFs; vision-only models (e.g. most Ollama - vision models) cannot read scanned PDFs yet because pages are not rendered to images. -4. Semantic search covers document passages; questions, cards and notes are keyword-only. -5. Full offline private studying, rich-text WYSIWYG authoring, study-plan generation and +2. Expose consistent ordering and duplication controls across content types. +3. Make timed exams use an explicit wall-clock deadline. The current runner is an active-study + clock: it pauses when hidden, and server completion does not enforce a deadline. +4. Preserve shared quiz question snapshots if scores must remain stable across owner edits; + public saved sessions currently rebuild against the latest published quiz content. +5. Add better import reporting/preservation for V1 history and interrupted sessions. +6. Improve large scanned-document OCR: currently sent as one native vision/PDF request with + a 30 MB scanned-PDF limit and an output cap, rather than paginated OCR. Lexical retrieval + and sampled long-document generation are implemented; comprehensive textbook coverage + and semantic matching are not guaranteed. +7. Full offline private studying, rich-text WYSIWYG authoring, study-plan generation and additional case/question types remain optional future enhancements. No obvious missing basic flip/streak/quiz-answer feedback remains by source review. Visual diff --git a/package-lock.json b/package-lock.json index c711a90..f45a0b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,6 @@ "jsonrepair": "^3.15.0", "lucide-react": "^1.71.0", "mammoth": "^1.13.0", - "pdf-lib": "^1.17.1", "radix-ui": "^1.8.1", "react": "^19.3.1", "react-dom": "^19.3.1", @@ -650,24 +649,6 @@ "url": "https://github.com/sponsors/oxc-project" } }, - "node_modules/@pdf-lib/standard-fonts": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@pdf-lib/standard-fonts/-/standard-fonts-1.0.0.tgz", - "integrity": "sha512-hU30BK9IUN/su0Mn9VdlVKsWBS6GyhVfqjwl1FjZN4TxP6cCw0jP2w7V3Hf5uX7M0AZJ16vey9yE0ny7Sa59ZA==", - "license": "MIT", - "dependencies": { - "pako": "^1.0.6" - } - }, - "node_modules/@pdf-lib/upng": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@pdf-lib/upng/-/upng-1.0.1.tgz", - "integrity": "sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==", - "license": "MIT", - "dependencies": { - "pako": "^1.0.10" - } - }, "node_modules/@radix-ui/number": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.3.tgz", @@ -5431,24 +5412,6 @@ "node": ">=14.0.0" } }, - "node_modules/pdf-lib": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/pdf-lib/-/pdf-lib-1.17.1.tgz", - "integrity": "sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==", - "license": "MIT", - "dependencies": { - "@pdf-lib/standard-fonts": "^1.0.0", - "@pdf-lib/upng": "^1.0.1", - "pako": "^1.0.11", - "tslib": "^1.11.1" - } - }, - "node_modules/pdf-lib/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "license": "0BSD" - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", diff --git a/package.json b/package.json index 4fe52ae..2393579 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,6 @@ "jsonrepair": "^3.15.0", "lucide-react": "^1.71.0", "mammoth": "^1.13.0", - "pdf-lib": "^1.17.1", "radix-ui": "^1.8.1", "react": "^19.3.1", "react-dom": "^19.3.1", diff --git a/server/ai/assistant.ts b/server/ai/assistant.ts index f63652c..bd8430f 100644 --- a/server/ai/assistant.ts +++ b/server/ai/assistant.ts @@ -16,7 +16,6 @@ import { getAppSettings } from '../settings.ts'; import { createAttempt, missedQuestionIds } from '../services/attempts.ts'; import { createClass, createFolder, getClass, getDeck, getQuiz, listCards, listQuestions, resolveFolder } from '../services/content.ts'; import { deleteDocument, listChunks, searchChunks, sourceLabel } from '../services/documents.ts'; -import { hybridSearch } from '../services/semantic.ts'; import { dueCounts } from '../services/srs.ts'; import { topicStats, totals } from '../services/stats.ts'; import { newId, now, parseJson } from '../util.ts'; @@ -168,11 +167,11 @@ const TOOLS: ToolSpec[] = [ ), }, label: (i) => `Searching documents for "${clip(String(i.query ?? ''), 50)}"`, - run: async (input, env) => { + run: (input, env) => { const docIds = (Array.isArray(input.document_ids) && input.document_ids.length ? input.document_ids : env.context.documentIds) as string[] | undefined; const scope = docIds?.length ? { documentIds: docIds } : { classId: classIdOr(env, input.class_id) ?? null }; const limit = Math.max(1, Math.min(12, Number(input.limit) || 6)); - const hits = await hybridSearch(env.app, String(input.query ?? ''), scope, limit); + const hits = searchChunks(env.app.db, String(input.query ?? ''), scope, limit); if (hits.length === 0) return { output: { results: [], note: 'No matching passages. Try different keywords, or list_materials to check which documents exist.' }, summary: 'No matches' }; return { output: { results: hits.map((h) => ({ ref: `c${h.id}`, document_id: h.documentId, source: sourceLabel(h), heading: h.heading || undefined, text: clip(h.text, 2400) })) }, diff --git a/server/ai/generate.ts b/server/ai/generate.ts index 025265c..57a226e 100644 --- a/server/ai/generate.ts +++ b/server/ai/generate.ts @@ -37,7 +37,6 @@ import { updateQuestion, } from '../services/content.ts'; import { chunkById, formatExcerpts, searchChunks, selectContext, sourceLabel, type ChunkHit } from '../services/documents.ts'; -import { hybridSearch } from '../services/semantic.ts'; import { weakTopics } from '../services/stats.ts'; import { newId, now, parseJson } from '../util.ts'; import { generateStructured, resolveModel, runChat } from './service.ts'; @@ -404,7 +403,7 @@ interface Sources { totalTokens: number; } -async function gatherSources(app: AppContext, req: GenerateRequest, budgetTokens: number, groups: number): Promise { +function gatherSources(app: AppContext, req: GenerateRequest, budgetTokens: number, groups: number): Sources { const { db } = app; const src = req.source; const out: Sources = { groups: [], inline: '', docs: [], refs: new Map(), totalTokens: 0 }; @@ -424,9 +423,7 @@ async function gatherSources(app: AppContext, req: GenerateRequest, budgetTokens throw badRequest(rows.some((r) => r.status !== 'ready') ? 'Those documents are still being processed. Try again in a moment.' : 'Those documents have no readable text yet.'); } out.docs = usable.map((r) => ({ documentId: r.id, title: r.title })); - const scope = { documentIds: usable.map((r) => r.id) }; - const ranked = req.focus?.trim() ? await hybridSearch(app, req.focus, scope, 400) : undefined; - out.groups = selectContext(db, scope, { focus: req.focus, budgetTokens, groups, ranked }); + out.groups = selectContext(db, { documentIds: usable.map((r) => r.id) }, { focus: req.focus, budgetTokens, groups }); } if (src.text?.trim()) inline.push(`Notes provided by the student:\n${src.text.trim()}`); @@ -469,7 +466,7 @@ async function gatherSources(app: AppContext, req: GenerateRequest, budgetTokens // Pull supporting passages from the class documents for those topics. if (req.classId && out.groups.length === 0) { const query = [...topics.map((t) => t.topic), ...missed.slice(0, 10).map((q) => q.topic)].filter(Boolean).join(' '); - const hits = query ? await hybridSearch(app, query, { classId: req.classId }, 24) : []; + const hits = query ? searchChunks(db, query, { classId: req.classId }, 24) : []; if (hits.length > 0) { const perGroup = Math.max(1, Math.ceil(hits.length / groups)); for (let i = 0; i < hits.length; i += perGroup) out.groups.push(hits.slice(i, i + perGroup)); @@ -586,7 +583,7 @@ async function runGeneration(app: AppContext, draftId: string, req: GenerateRequ count = Math.min(Math.max(count, 1), 100); } const batches = Math.ceil(count / perBatch); - const sources = await gatherSources(app, req, budgetTokens, batches); + const sources = gatherSources(app, req, budgetTokens, batches); const hasSources = sources.groups.length > 0 || Boolean(sources.inline); if (!hasSources && !req.focus && !req.instructions) throw badRequest('Choose source material or describe a topic to generate from.'); @@ -699,7 +696,7 @@ async function generateNote(app: AppContext, draftId: string, req: GenerateReque const style = req.noteStyle ?? 'study_guide'; // A guide reads the material in as few passes as possible; allow a bigger slice per pass than item generation. const perPass = Math.max(budgetTokens, Math.min(Math.floor(window * 0.55), 90_000)); - const sources = await gatherSources(app, req, perPass, 12); + const sources = gatherSources(app, req, perPass, 12); const hasSources = sources.groups.length > 0 || Boolean(sources.inline); if (!hasSources && !req.focus) throw badRequest('Choose source material or describe a topic to write about.'); const title = defaultTitle(app, req, sources); diff --git a/server/ai/providers/common.ts b/server/ai/providers/common.ts index 5b9c5aa..89f9731 100644 --- a/server/ai/providers/common.ts +++ b/server/ai/providers/common.ts @@ -49,21 +49,6 @@ export function newToolCallId(prefix: string): string { return `${prefix}_${globalThis.crypto.randomUUID().replace(/-/g, '').slice(0, 24)}`; } -/** Parses an OpenAI-style `/embeddings` response ({ data: [{ index, embedding }], usage }). */ -export function parseEmbeddings(res: unknown, expected: number, label: string): { vectors: number[][]; inputTokens: number } { - const body = res as { data?: { index?: number; embedding?: unknown }[]; usage?: { prompt_tokens?: number; total_tokens?: number } }; - const rows = Array.isArray(body?.data) ? body.data : []; - const vectors: number[][] = new Array(expected); - rows.forEach((row, i) => { - const at = typeof row.index === 'number' ? row.index : i; - if (Array.isArray(row.embedding)) vectors[at] = row.embedding.map(Number); - }); - if (rows.length !== expected || vectors.some((v) => !v || v.length === 0)) { - throw new AIError('invalid_response', `${label} returned ${rows.length} embeddings for ${expected} inputs.`); - } - return { vectors, inputTokens: num(body?.usage?.prompt_tokens ?? body?.usage?.total_tokens) }; -} - export function dataUrl(mime: string, base64: string): string { return `data:${mime};base64,${base64}`; } diff --git a/server/ai/providers/gemini.ts b/server/ai/providers/gemini.ts index d834fa5..911bbe6 100644 --- a/server/ai/providers/gemini.ts +++ b/server/ai/providers/gemini.ts @@ -16,7 +16,7 @@ * numeric `thinkingBudget` on 2.5; an unsupported config is dropped and retried. * - A response can be blocked (`promptFeedback.blockReason`, finishReason SAFETY). */ -import { type EmbedRequest, type EmbedResult, +import { AIError, type ChatMessage, type ChatRequest, @@ -296,30 +296,6 @@ async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator { - const name = req.model.startsWith('models/') ? req.model : `models/${req.model}`; - const requests = req.inputs.map((text) => ({ - model: name, - content: { parts: [{ text }] }, - taskType: req.purpose === 'query' ? 'RETRIEVAL_QUERY' : 'RETRIEVAL_DOCUMENT', - ...(req.dimensions ? { outputDimensionality: req.dimensions } : {}), - })); - const res = await requestJson<{ embeddings?: { values?: number[] }[] }>(`${root(cfg)}/v1beta/${name}:batchEmbedContents`, { - ...errorContext(cfg), - headers: headers(cfg), - body: { requests }, - signal: req.signal, - firstByteTimeoutMs: 60_000, - totalTimeoutMs: 120_000, - }); - const vectors = (res.embeddings ?? []).map((e) => (Array.isArray(e.values) ? e.values.map(Number) : [])); - if (vectors.length !== req.inputs.length || vectors.some((v) => v.length === 0)) { - throw new AIError('invalid_response', `${LABEL} returned ${vectors.length} embeddings for ${req.inputs.length} inputs.`); - } - // Gemini does not report token use for embeddings; estimate it for the usage log. - return { vectors, inputTokens: Math.ceil(req.inputs.reduce((n, t) => n + t.length, 0) / 4) }; -} - async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise { const found: DiscoveredModel[] = []; let pageToken = ''; @@ -362,6 +338,5 @@ export const geminiAdapter: ProviderAdapter = { }, stream, listModels: (cfg, signal) => listModels(cfg, signal), - embed, test, }; diff --git a/server/ai/providers/ollama.ts b/server/ai/providers/ollama.ts index f057c94..607e0ba 100644 --- a/server/ai/providers/ollama.ts +++ b/server/ai/providers/ollama.ts @@ -9,7 +9,7 @@ import { AIError, type DiscoveredModel, type ModelCapabilities, type ProviderAdapter, type ProviderConfig, type TestResult } from '../types.ts'; import { num, runTest, str, TEST_TIMEOUT_MS } from './common.ts'; import { isRecord, requestJson, trimBaseUrl, type ErrorContext } from './http.ts'; -import { compatEmbed, streamChatCompletions, type CompatOptions } from './openaiCompatible.ts'; +import { streamChatCompletions, type CompatOptions } from './openaiCompatible.ts'; const LABEL = 'Ollama'; const DEFAULT_BASE_URL = 'http://localhost:11434'; @@ -98,7 +98,5 @@ export const ollamaAdapter: ProviderAdapter = { }, stream: (cfg, req) => streamChatCompletions(compat(cfg), req), listModels: (cfg, signal) => listModels(cfg, signal), - // Ollama serves OpenAI-style embeddings at /v1/embeddings (e.g. nomic-embed-text). - embed: (cfg, req) => compatEmbed(compat(cfg), req), test, }; diff --git a/server/ai/providers/openai.ts b/server/ai/providers/openai.ts index ce78ff8..01f5d9c 100644 --- a/server/ai/providers/openai.ts +++ b/server/ai/providers/openai.ts @@ -14,7 +14,7 @@ * rejects the chosen value the request is retried once with a safe one. * - Tool-call arguments arrive as deltas keyed by output item. */ -import { type EmbedRequest, type EmbedResult, +import { AIError, type ChatMessage, type ChatRequest, @@ -28,7 +28,7 @@ import { type EmbedRequest, type EmbedResult, type TestResult, } from '../types.ts'; import { lookupModelDefaults } from './catalog.ts'; -import { parseEmbeddings, dataUrl, isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts'; +import { dataUrl, isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts'; import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts'; type Json = Record; @@ -292,13 +292,6 @@ async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: return withCatalogDefaults('openai', found); } -async function embed(cfg: ProviderConfig, req: EmbedRequest): Promise { - const body: Json = { model: req.model, input: req.inputs, encoding_format: 'float' }; - if (req.dimensions && /text-embedding-3/.test(req.model)) body.dimensions = req.dimensions; - const res = await requestJson(`${base(cfg)}/embeddings`, { ...errorContext(cfg), headers: headers(cfg), body, signal: req.signal, firstByteTimeoutMs: 60_000, totalTimeoutMs: 120_000 }); - return parseEmbeddings(res, req.inputs.length, LABEL); -} - async function test(cfg: ProviderConfig): Promise { return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length })); } @@ -316,6 +309,5 @@ export const openaiAdapter: ProviderAdapter = { }, stream, listModels: (cfg, signal) => listModels(cfg, signal), - embed, test, }; diff --git a/server/ai/providers/openaiCompatible.ts b/server/ai/providers/openaiCompatible.ts index d8b543d..099b73d 100644 --- a/server/ai/providers/openaiCompatible.ts +++ b/server/ai/providers/openaiCompatible.ts @@ -13,7 +13,7 @@ * - reasoning text may arrive as `reasoning_content`, `reasoning` or `thinking`; * - the last chunk may lack a `finish_reason`. */ -import { type EmbedRequest, type EmbedResult, +import { AIError, type ChatMessage, type ChatRequest, @@ -27,7 +27,7 @@ import { type EmbedRequest, type EmbedResult, type StreamEvent, type TestResult, } from '../types.ts'; -import { parseEmbeddings, dataUrl, isChatModelId, makeUsage, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts'; +import { dataUrl, isChatModelId, makeUsage, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts'; import { ensureOk, errorFromProvider, hostOf, httpRequest, isLocalUrl, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts'; type Json = Record; @@ -62,8 +62,6 @@ function userContent(parts: ContentPart[], caps: Partial): st else blocks.push({ type: 'image_url', image_url: { url: dataUrl(p.mime, p.data) } }); } else if (p.type === 'file') { if (p.mime.startsWith('text/')) blocks.push({ type: 'text', text: Buffer.from(p.data, 'base64').toString('utf8') }); - // Chat Completions "file" parts (OpenAI, OpenRouter and others) for models marked as reading PDFs. - else if (p.mime === 'application/pdf' && caps.pdf) blocks.push({ type: 'file', file: { filename: p.filename ?? 'document.pdf', file_data: dataUrl(p.mime, p.data) } }); else blocks.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` }); } } @@ -321,14 +319,6 @@ export async function listCompatModels(o: CompatOptions, signal?: AbortSignal, t return withCatalogDefaults(o.kind, found); } -/** OpenAI-style `/embeddings`, shared with Ollama's compatible endpoint. */ -export async function compatEmbed(o: CompatOptions, req: EmbedRequest): Promise { - const body: Record = { model: req.model, input: req.inputs, encoding_format: 'float' }; - if (req.dimensions && /text-embedding-3/.test(req.model)) body.dimensions = req.dimensions; - const res = await requestJson(`${normalizeCompatBase(o.baseUrl)}/embeddings`, { ...errorContext(o), headers: authHeaders(o), body, signal: req.signal, firstByteTimeoutMs: 120_000, totalTimeoutMs: 300_000 }); - return parseEmbeddings(res, req.inputs.length, o.label); -} - function options(cfg: ProviderConfig): CompatOptions { if (!cfg.baseUrl?.trim()) throw new AIError('not_configured', `Add the server URL for "${cfg.name}" in Settings.`); return { kind: 'openai_compatible', label: cfg.name || hostOf(cfg.baseUrl), baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, headers: cfg.headers }; @@ -370,5 +360,4 @@ export const openaiCompatibleAdapter: ProviderAdapter = { stream: (cfg, req) => streamChatCompletions(options(cfg), req), listModels: (cfg, signal) => listCompatModels(options(cfg), signal), test, - embed: (cfg, req) => compatEmbed(options(cfg), req), }; diff --git a/server/ai/service.ts b/server/ai/service.ts index 94ccb23..95fa7dc 100644 --- a/server/ai/service.ts +++ b/server/ai/service.ts @@ -157,13 +157,6 @@ function providerConfig(app: AppContext, row: ProviderRow): ProviderConfig { }; } -/** Decrypted connection details for one enabled provider (server-side use only, never sent to clients). */ -export function providerConfigById(app: AppContext, id: string): { cfg: ProviderConfig; kind: ProviderKind; name: string } { - const row = providerRow(app.db, id); - if (!row.enabled) throw new AIError('not_configured', `The provider "${row.name}" is turned off. Turn it on in Settings.`); - return { cfg: providerConfig(app, row), kind: row.kind, name: row.name }; -} - function providerRow(db: DB, id: string): ProviderRow { const row = db.get('SELECT * FROM ai_providers WHERE id = ?', id); if (!row) throw notFound('Provider'); diff --git a/server/ai/types.ts b/server/ai/types.ts index b8fe834..51a52b8 100644 --- a/server/ai/types.ts +++ b/server/ai/types.ts @@ -118,21 +118,6 @@ export interface TestResult { modelCount?: number; } -export interface EmbedRequest { - model: string; - inputs: string[]; - /** Some providers embed search queries and stored passages differently. */ - purpose: 'document' | 'query'; - /** Requested vector size, for models that can shorten their output (OpenAI v3, Gemini). */ - dimensions?: number; - signal?: AbortSignal; -} - -export interface EmbedResult { - vectors: number[][]; - inputTokens: number; -} - export interface ProviderAdapter { kind: ProviderKind; /** Human label and defaults used by the settings UI. */ @@ -151,8 +136,6 @@ export interface ProviderAdapter { listModels(cfg: ProviderConfig, signal?: AbortSignal): Promise; /** Cheap credential / connectivity check. Never throws. */ test(cfg: ProviderConfig, model?: string): Promise; - /** Text embeddings for semantic search. Absent when the provider has no embeddings API. */ - embed?(cfg: ProviderConfig, req: EmbedRequest): Promise; } export type AIErrorCode = diff --git a/server/app.ts b/server/app.ts index 1f2c70a..57eeba9 100644 --- a/server/app.ts +++ b/server/app.ts @@ -12,7 +12,6 @@ import { documentRoutes, mediaRoute } from './routes/documents.ts'; import { libraryRoutes } from './routes/library.ts'; import { publicRoutes, shareRoutes } from './routes/shares.ts'; import { studyRoutes } from './routes/study.ts'; -import { expireOverdueAttempts } from './services/attempts.ts'; const MIME: Record = { '.html': 'text/html; charset=utf-8', @@ -114,18 +113,6 @@ export function createApp(app: AppContext): Hono { const secured = new Hono(); secured.use('*', requireAuth(app)); - // Timed exams end on the wall clock even if nobody has the session open: submit any that are - // overdue before reading data, so lists never offer to resume an expired exam. - secured.use('*', async (c, next) => { - if (c.req.method === 'GET') { - try { - expireOverdueAttempts(app.db); - } catch (err) { - app.log('warn', 'Could not submit overdue timed sessions', err); - } - } - await next(); - }); secured.route('/', libraryRoutes(app)); secured.route('/', studyRoutes(app)); secured.route('/', shareRoutes(app)); diff --git a/server/documents/pdfSplit.ts b/server/documents/pdfSplit.ts deleted file mode 100644 index f2cf578..0000000 --- a/server/documents/pdfSplit.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Splits a PDF into smaller PDFs of consecutive pages, so a long scanned document can be read by - * an AI model in batches instead of one oversized request. - */ -import { PDFDocument } from 'pdf-lib'; - -export interface PdfBatch { - /** 1-based page numbers, inclusive. */ - start: number; - end: number; - bytes: Uint8Array; -} - -export async function pdfPageCount(bytes: Uint8Array): Promise { - const doc = await PDFDocument.load(bytes, { ignoreEncryption: true, updateMetadata: false }); - return doc.getPageCount(); -} - -/** - * Yields batches of up to `pagesPerBatch` pages. A batch larger than `maxBytes` is split again - * (down to a single page); a single page that is still too large is yielded as-is for the caller - * to skip with a warning. - */ -export async function* splitPdf(bytes: Uint8Array, pagesPerBatch: number, maxBytes: number): AsyncGenerator { - const src = await PDFDocument.load(bytes, { ignoreEncryption: true, updateMetadata: false }); - const total = src.getPageCount(); - - async function build(start: number, end: number): Promise { - const out = await PDFDocument.create({ updateMetadata: false }); - const indices = Array.from({ length: end - start + 1 }, (_, i) => start - 1 + i); - for (const page of await out.copyPages(src, indices)) out.addPage(page); - return out.save({ useObjectStreams: true }); - } - - async function* emit(start: number, end: number): AsyncGenerator { - const part = await build(start, end); - if (part.length <= maxBytes || start === end) { - yield { start, end, bytes: part }; - return; - } - const mid = Math.floor((start + end) / 2); - yield* emit(start, mid); - yield* emit(mid + 1, end); - } - - for (let start = 1; start <= total; start += pagesPerBatch) { - yield* emit(start, Math.min(total, start + pagesPerBatch - 1)); - } -} diff --git a/server/main.ts b/server/main.ts index a1928b8..6417f64 100644 --- a/server/main.ts +++ b/server/main.ts @@ -8,7 +8,6 @@ import type { AppContext } from './http.ts'; import { APP_VERSION } from './routes/data.ts'; import { startBackupSchedule } from './services/backup.ts'; import { resumeQueue } from './services/documents.ts'; -import { scheduleIndexing } from './services/semantic.ts'; import { seedTemplates } from './services/templates.ts'; function log(level: 'info' | 'warn' | 'error', message: string, extra?: unknown): void { @@ -36,7 +35,6 @@ async function main(): Promise { tidyDrafts(db); const resumed = resumeQueue(app); if (resumed > 0) log('info', `Resuming processing of ${resumed} document(s)`); - scheduleIndexing(app); const stopBackups = startBackupSchedule(app); const server = serve({ fetch: createApp(app).fetch, port: config.port, hostname: config.host }, (info) => { diff --git a/server/migrations.ts b/server/migrations.ts index 9bf0d88..fb5d7ec 100644 --- a/server/migrations.ts +++ b/server/migrations.ts @@ -369,18 +369,4 @@ CREATE TABLE ai_actions ( ); `; -/** Semantic search: one embedding per document chunk, for the configured embedding model. */ -const embeddings = ` -CREATE TABLE chunk_vectors ( - chunk_id INTEGER PRIMARY KEY REFERENCES chunks(id) ON DELETE CASCADE, - model_key TEXT NOT NULL, - dims INTEGER NOT NULL, - vector BLOB NOT NULL -); -CREATE INDEX chunk_vectors_model ON chunk_vectors(model_key); -`; - -export const MIGRATIONS: Migration[] = [ - { id: 1, name: 'init', sql: init }, - { id: 2, name: 'embeddings', sql: embeddings }, -]; +export const MIGRATIONS: Migration[] = [{ id: 1, name: 'init', sql: init }]; diff --git a/server/routes/ai.ts b/server/routes/ai.ts index eb227b7..1c40639 100644 --- a/server/routes/ai.ts +++ b/server/routes/ai.ts @@ -37,7 +37,6 @@ import { testProvider, usageSummary, } from '../ai/service.ts'; -import { embedTexts, embeddingSettingsSchema, getEmbeddingSettings, embeddingStatus, rebuildIndex, saveEmbeddingSettings, scheduleIndexing } from '../services/semantic.ts'; import { AIError } from '../ai/types.ts'; import { badRequest, idParam, notFound, readBody, type AppContext } from '../http.ts'; @@ -109,10 +108,7 @@ export function aiRoutes(app: AppContext): Hono { r.post('/ai/providers', async (c) => c.json({ provider: saveProvider(app, null, await readBody(c, providerInputSchema)) }, 201)); r.patch('/ai/providers/:id', async (c) => c.json({ provider: saveProvider(app, idParam(c), await readBody(c, providerInputSchema)) })); r.delete('/ai/providers/:id', (c) => { - const id = idParam(c); - deleteProvider(app, id); - // Semantic search cannot keep using a provider that no longer exists. - if (getEmbeddingSettings(db).providerId === id) saveEmbeddingSettings(app, { providerId: null, model: '', dimensions: null }); + deleteProvider(app, idParam(c)); return c.json({ ok: true }); }); r.post('/ai/providers/:id/test', async (c) => c.json({ result: await testProvider(app, idParam(c)) })); @@ -271,27 +267,5 @@ export function aiRoutes(app: AppContext): Hono { }); }); - /* ------------------------------ semantic search ------------------------------ */ - - r.get('/ai/embeddings', (c) => c.json(embeddingStatus(app))); - r.put('/ai/embeddings', async (c) => { - saveEmbeddingSettings(app, await readBody(c, embeddingSettingsSchema)); - return c.json(embeddingStatus(app)); - }); - /** Embeds one sentence to prove the model works before indexing everything. */ - r.post('/ai/embeddings/test', async (c) => { - const started = Date.now(); - const [v] = await embedTexts(app, ['Furosemide is a loop diuretic.'], 'query'); - return c.json({ ok: true, dimensions: v.length, latencyMs: Date.now() - started }); - }); - r.post('/ai/embeddings/rebuild', (c) => { - rebuildIndex(app); - return c.json(embeddingStatus(app)); - }); - r.post('/ai/embeddings/resume', (c) => { - scheduleIndexing(app); - return c.json(embeddingStatus(app)); - }); - return r; } diff --git a/server/routes/library.ts b/server/routes/library.ts index cbe9a16..1f69978 100644 --- a/server/routes/library.ts +++ b/server/routes/library.ts @@ -116,7 +116,7 @@ export function libraryRoutes(app: AppContext): Hono { ); const decks = db.all(`SELECT d.*, (SELECT COUNT(*) FROM cards c WHERE c.deck_id = d.id) AS cn FROM decks d WHERE d.class_id = ?`, cls.id); const documents = db.all>( - 'SELECT id, folder_id, title, filename, kind, size, status, error, progress, page_count, token_count, needs_ocr, sort_order, created_at, updated_at FROM documents WHERE class_id = ?', + 'SELECT id, folder_id, title, filename, kind, size, status, error, progress, page_count, token_count, needs_ocr, created_at, updated_at FROM documents WHERE class_id = ?', cls.id, ); const notes = db.all('SELECT * FROM notes WHERE class_id = ?', cls.id); @@ -128,7 +128,6 @@ export function libraryRoutes(app: AppContext): Hono { title: q.title, description: q.description, folderId: q.folder_id, - sortOrder: q.sort_order, createdAt: q.created_at, updatedAt: q.updated_at, shared: shared('quiz', q.id), @@ -146,7 +145,6 @@ export function libraryRoutes(app: AppContext): Hono { title: d.title, description: d.description, folderId: d.folder_id, - sortOrder: d.sort_order, createdAt: d.created_at, updatedAt: d.updated_at, shared: shared('deck', d.id), @@ -162,7 +160,6 @@ export function libraryRoutes(app: AppContext): Hono { title: d.title as string, description: '', folderId: d.folder_id as string | null, - sortOrder: d.sort_order as number, createdAt: d.created_at as number, updatedAt: d.updated_at as number, shared: shared('document', d.id), @@ -182,7 +179,6 @@ export function libraryRoutes(app: AppContext): Hono { title: n.title, description: n.content.replace(/[#*_>`|-]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 160), folderId: n.folder_id, - sortOrder: n.sort_order, createdAt: n.created_at, updatedAt: n.updated_at, shared: shared('note', n.id), @@ -282,15 +278,6 @@ export function libraryRoutes(app: AppContext): Hono { const TABLES = { quiz: 'quizzes', deck: 'decks', note: 'notes', document: 'documents' } as const; - /** Manual order for one kind of material in a class. Ids from other classes are ignored. */ - r.post('/classes/:id/items/reorder', async (c) => { - const cls = getClass(db, idParam(c)); - const body = await readBody(c, z.object({ kind: z.enum(['quiz', 'deck', 'note', 'document']), ids: z.array(z.string()).max(2000) })); - const table = TABLES[body.kind]; - db.tx(() => body.ids.forEach((id, i) => db.run(`UPDATE ${table} SET sort_order = ? WHERE id = ? AND class_id = ?`, i, id, cls.id))); - return c.json({ ok: true }); - }); - r.post('/items/move', async (c) => { const body = await readBody( c, diff --git a/server/routes/study.ts b/server/routes/study.ts index d2adcd9..b2a9edc 100644 --- a/server/routes/study.ts +++ b/server/routes/study.ts @@ -1,5 +1,4 @@ import { Hono } from 'hono'; -import { getEmbeddingSettings, semanticSearch } from '../services/semantic.ts'; import { z } from 'zod'; import { badRequest, idParam, readBody, type AppContext } from '../http.ts'; import { getAppSettings } from '../settings.ts'; @@ -126,14 +125,14 @@ export function studyRoutes(app: AppContext): Hono { /* ------------------------------ search ------------------------------ */ - r.get('/search', async (c) => { + r.get('/search', (c) => { const q = (c.req.query('q') ?? '').trim().slice(0, 120); if (q.length < 2) return c.json({ query: q, results: [] }); const like = `%${q.replace(/[\\%_]/g, (m) => `\\${m}`)}%`; const classId = c.req.query('classId') || null; const scoped = (col: string) => (classId ? `AND ${col} = ?` : ''); const p = (...base: string[]) => (classId ? [...base, classId] : base); - type Hit = { kind: string; id: string; title: string; snippet: string; classId: string; className: string; classColor: string; parentId?: string; page?: number | null; semantic?: boolean }; + type Hit = { kind: string; id: string; title: string; snippet: string; classId: string; className: string; classColor: string; parentId?: string; page?: number | null }; const results: Hit[] = []; for (const row of db.all>(`SELECT id, name, color FROM classes WHERE name LIKE ? ESCAPE '\\' OR code LIKE ? ESCAPE '\\' LIMIT 5`, like, like)) { @@ -185,25 +184,6 @@ export function studyRoutes(app: AppContext): Hono { /* a malformed FTS query should never break search */ } } - // Passages that match the meaning rather than the words (when semantic search is set up). - // Only for phrases or longer terms: single short words are already well served by keywords. - if ((/\s/.test(q) || q.length >= 8) && getEmbeddingSettings(db).providerId) { - try { - const seen = new Set(results.filter((r) => r.kind === 'passage').map((r) => r.id)); - for (const hit of await semanticSearch(app, q, { classId }, 6, 0.35)) { - if (seen.has(String(hit.id))) continue; - const doc = db.get>( - 'SELECT k.id AS class_id, k.name AS class_name, k.color AS class_color FROM documents d JOIN classes k ON k.id = d.class_id WHERE d.id = ?', - hit.documentId, - ); - if (!doc) continue; - const text = hit.text.replace(/\s+/g, ' ').trim(); - results.push({ kind: 'passage', id: String(hit.id), parentId: hit.documentId, page: hit.pageStart, title: hit.documentTitle, snippet: text.length > 160 ? `${text.slice(0, 160)} …` : text, classId: doc.class_id, className: doc.class_name, classColor: doc.class_color, semantic: true }); - } - } catch (err) { - app.log('warn', `Semantic search failed: ${err instanceof Error ? err.message : String(err)}`); - } - } return c.json({ query: q, results }); }); diff --git a/server/services/attempts.ts b/server/services/attempts.ts index de7dcc4..3887d91 100644 --- a/server/services/attempts.ts +++ b/server/services/attempts.ts @@ -6,7 +6,7 @@ import { z } from 'zod'; import { isFullyCorrect, scoreQuestion, seededShuffle, type Option, type QuestionType, type SataScoring } from '../../shared/content.ts'; import type { DB } from '../db.ts'; -import { HttpError, badRequest, notFound } from '../http.ts'; +import { badRequest, notFound } from '../http.ts'; import { newId, now, parseJson } from '../util.ts'; import { getClass, getQuiz, mapQuestion, type QuestionRow } from './content.ts'; @@ -34,19 +34,8 @@ export interface AttemptSettings { shuffleOptions: boolean; timeLimitSec: number | null; sataScoring: SataScoring; - /** - * Wall-clock end of a timed session (epoch ms), fixed when it starts. The clock keeps - * running while the session is closed; answers after it are refused and the session is - * submitted with what was saved. Sessions from before this field use the active clock. - */ - deadlineAt?: number | null; } -/** Saves sent a few seconds after the deadline (network delay, the final autosave) still count. */ -export const DEADLINE_GRACE_MS = 5_000; - -const DEFAULT_SETTINGS: AttemptSettings = { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' }; - interface AttemptRow { id: string; quiz_id: string | null; @@ -218,16 +207,14 @@ export function createAttempt(db: DB, input: AttemptCreate, sataScoring: SataSco if (opts.count && opts.count < questions.length) questions = questions.slice(0, opts.count); if (opts.shuffleOptions) questions = questions.map((q, i) => ({ ...q, options: shuffleOptions(q.options, seed + i + 1) })); - const id = newId(); - const at = now(); - const timeLimitSec = opts.timeLimitMin ? Math.round(opts.timeLimitMin * 60) : null; const settings: AttemptSettings = { shuffleQuestions: opts.shuffleQuestions, shuffleOptions: opts.shuffleOptions, - timeLimitSec, + timeLimitSec: opts.timeLimitMin ? Math.round(opts.timeLimitMin * 60) : null, sataScoring, - deadlineAt: timeLimitSec ? at + timeLimitSec * 1000 : null, }; + const id = newId(); + const at = now(); db.run( `INSERT INTO attempts (id, quiz_id, class_id, title, mode, kind, status, settings, snapshot, question_count, started_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?)`, @@ -246,38 +233,12 @@ export function createAttempt(db: DB, input: AttemptCreate, sataScoring: SataSco return id; } -function loadRaw(db: DB, id: string): AttemptRow { +function loadAttempt(db: DB, id: string): AttemptRow { const row = db.get('SELECT * FROM attempts WHERE id = ?', id); if (!row) throw notFound('Session'); return row; } -/** Loads a session, first submitting it if its deadline has passed. */ -function loadAttempt(db: DB, id: string): AttemptRow { - const row = loadRaw(db, id); - if (isOverdue(row)) { - completeAttempt(db, row.id); - return db.get('SELECT * FROM attempts WHERE id = ?', id)!; - } - return row; -} - -function isOverdue(row: AttemptRow, at = now()): boolean { - if (row.status !== 'active') return false; - const deadline = parseJson(row.settings, DEFAULT_SETTINGS).deadlineAt; - return typeof deadline === 'number' && at > deadline + DEADLINE_GRACE_MS; -} - -/** Submits every timed session whose deadline has passed. Returns how many were submitted. */ -export function expireOverdueAttempts(db: DB): number { - const rows = db.all<{ id: string }>( - `SELECT id FROM attempts WHERE status = 'active' AND json_extract(settings, '$.deadlineAt') IS NOT NULL AND json_extract(settings, '$.deadlineAt') < ?`, - now() - DEADLINE_GRACE_MS, - ); - for (const r of rows) completeAttempt(db, r.id); - return rows.length; -} - function recordResponse(db: DB, row: AttemptRow, q: SnapshotQuestion, selected: string[], sata: SataScoring, at: number): void { const score = scoreQuestion(q, selected, sata); db.run( @@ -295,7 +256,7 @@ function recordResponse(db: DB, row: AttemptRow, q: SnapshotQuestion, selected: } export function mapAttempt(row: AttemptRow) { - const settings = parseJson(row.settings, DEFAULT_SETTINGS); + const settings = parseJson(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' }); const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] }); const answers = parseJson>(row.answers, {}); const results = @@ -328,8 +289,6 @@ export function mapAttempt(row: AttemptRow) { updatedAt: row.updated_at, completedAt: row.completed_at, results, - /** Server clock when this was produced, so clients can correct for clock skew. */ - serverNow: now(), }; } export type AttemptDto = ReturnType; @@ -372,12 +331,10 @@ export const attemptPatchSchema = z.object({ export function patchAttempt(db: DB, id: string, input: z.input): AttemptDto { const patch = attemptPatchSchema.parse(input); - const overdue = isOverdue(loadRaw(db, id)); const row = loadAttempt(db, id); - if (overdue) throw new HttpError(409, 'time_up', 'Time is up. The exam was submitted with the answers saved before the deadline.'); if (row.status !== 'active') throw badRequest('This session is already finished'); const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] }); - const settings = parseJson(row.settings, DEFAULT_SETTINGS); + const settings = parseJson(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' }); const answers = parseJson>(row.answers, {}); const at = now(); db.tx(() => { @@ -414,17 +371,12 @@ export function patchAttempt(db: DB, id: string, input: z.input(row.snapshot, { questions: [] }); - const settings = parseJson(row.settings, DEFAULT_SETTINGS); + const settings = parseJson(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' }); const answers = parseJson>(row.answers, {}); const at = now(); - let elapsed = elapsedSec !== undefined ? Math.max(row.elapsed_sec, elapsedSec) : row.elapsed_sec; - if (settings.deadlineAt && settings.timeLimitSec) { - // Timed sessions run on the wall clock: time used is time since the start, capped at the limit. - elapsed = Math.min(settings.timeLimitSec, Math.max(0, Math.round((at - row.started_at) / 1000))); - } let score = 0; let max = 0; db.tx(() => { @@ -443,7 +395,7 @@ export function completeAttempt(db: DB, id: string, elapsedSec?: number): Attemp max, at, at, - elapsed, + elapsedSec !== undefined ? Math.max(row.elapsed_sec, elapsedSec) : row.elapsed_sec, id, ); }); diff --git a/server/services/documents.ts b/server/services/documents.ts index bde4cac..46ef26e 100644 --- a/server/services/documents.ts +++ b/server/services/documents.ts @@ -10,12 +10,10 @@ import { AIError, type ContentPart } from '../ai/types.ts'; import type { DB } from '../db.ts'; import { chunkSections, estimateTokens } from '../documents/chunk.ts'; import { detectKind, extractDocument } from '../documents/extract.ts'; -import { pdfPageCount, splitPdf } from '../documents/pdfSplit.ts'; import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts'; import { HttpError, notFound, type AppContext } from '../http.ts'; import { ftsQuery, newId, now, parseJson } from '../util.ts'; import { getClass, resolveFolder } from './content.ts'; -import { scheduleIndexing } from './semantic.ts'; export interface DocumentRow { id: string; @@ -152,7 +150,6 @@ export function deleteDocument(app: AppContext, id: string): void { app.db.run('DELETE FROM documents WHERE id = ?', id); }); fs.rmSync(documentPath(app, row), { force: true }); - fs.rmSync(ocrCachePath(app, row), { force: true }); } /* ------------------------------ processing queue ------------------------------ */ @@ -211,136 +208,43 @@ const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown. - If something is illegible, write [illegible]. Return only the transcription.`; -/** Long scanned PDFs are read this many pages at a time. */ -export const OCR_PAGES_PER_BATCH = 8; -/** Largest single request sent to a model (providers cap inline files around 32 MB). */ -const OCR_MAX_BATCH_BYTES = 24 * 1024 * 1024; -/** Upper bound on pages read with AI for one document, to keep costs predictable. */ -export const OCR_MAX_PAGES = 600; - -interface OcrCache { - modelId: string; - /** Finished batches keyed by "start-end", so an interrupted read resumes instead of paying twice. */ - batches: Record; -} - -const ocrCachePath = (app: AppContext, row: Pick) => `${documentPath(app, row)}.ocr.json`; - -/** Splits a "## Page N" transcription into page sections; text without markers goes to `fallbackPage`. */ -export function splitTranscription(text: string, fallbackPage: number | undefined, range?: { start: number; end: number }): ExtractedSection[] { - const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im); - if (pages.length < 3) return text.trim() ? [{ text: text.trim(), page: fallbackPage }] : []; - const sections: ExtractedSection[] = []; - if (pages[0].trim()) sections.push({ text: pages[0].trim(), page: fallbackPage }); - for (let i = 1; i < pages.length; i += 2) { - const body = (pages[i + 1] ?? '').trim(); - let page = Number(pages[i]); - // Models sometimes number a batch from 1; map those back onto the batch's real pages. - if (range && (page < range.start || page > range.end) && page >= 1 && page <= range.end - range.start + 1) page = range.start + page - 1; - if (body) sections.push({ text: body, page }); +/** Uses a vision-capable model to read an image or a scanned PDF. Returns null when no such model is configured. */ +async function transcribeWithVision(app: AppContext, row: DocumentRow, bytes: Uint8Array): Promise { + const data = Buffer.from(bytes).toString('base64'); + const part: ContentPart = row.kind === 'image' ? { type: 'image', mime: row.mime, data } : { type: 'file', mime: 'application/pdf', data, filename: row.filename }; + if (row.kind === 'pdf' && bytes.length > 30 * 1024 * 1024) { + throw new AIError('bad_request', 'This scanned PDF is too large to send to an AI model for reading (30 MB limit). Split it into smaller files.'); } - return sections; -} - -interface Transcription { - sections: ExtractedSection[]; - warnings: string[]; -} - -/** - * Uses a vision-capable model to read an image or a scanned PDF. PDFs are sent in batches of a - * few pages, with finished batches cached beside the upload so a restart or a failed batch does - * not repeat paid work. Returns null when no vision model is configured. - */ -async function transcribeWithVision( - app: AppContext, - row: DocumentRow, - bytes: Uint8Array, - opts: { fresh: boolean; onProgress?: (fraction: number) => void }, -): Promise { - let vision; + let result; try { - vision = resolveModel(app, 'vision'); + const vision = resolveModel(app, 'vision'); + if (!vision.model.capabilities.vision) return null; + if (row.kind === 'pdf' && !vision.model.capabilities.pdf) { + throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`); + } + result = await runChat(app, { + task: 'vision', + purpose: 'document transcription', + messages: [{ role: 'user', content: [part, { type: 'text', text: TRANSCRIBE_PROMPT }] }], + maxOutputTokens: 32000, + reasoningEffort: 'low', + }); } catch (err) { if (err instanceof AIError && err.code === 'not_configured') return null; throw err; } - if (!vision.model.capabilities.vision) return null; - - const ask = async (part: ContentPart, prompt: string) => { - const result = await runChat(app, { - task: 'vision', - purpose: 'document transcription', - messages: [{ role: 'user', content: [part, { type: 'text', text: prompt }] }], - maxOutputTokens: 32000, - reasoningEffort: 'low', - }); - return result.text.trim(); - }; - - if (row.kind === 'image') { - const text = await ask({ type: 'image', mime: row.mime, data: Buffer.from(bytes).toString('base64') }, TRANSCRIBE_PROMPT); - return { sections: text ? [{ text }] : [], warnings: [] }; - } - - if (!vision.model.capabilities.pdf) { - throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`); - } - - const cacheFile = ocrCachePath(app, row); - let cache: OcrCache = { modelId: vision.model.id, batches: {} }; - if (opts.fresh) fs.rmSync(cacheFile, { force: true }); - else { - try { - const saved = JSON.parse(fs.readFileSync(cacheFile, 'utf8')) as OcrCache; - if (saved.modelId === vision.model.id && saved.batches) cache = saved; - } catch { - /* no cache yet */ - } - } - const save = () => fs.writeFileSync(cacheFile, JSON.stringify(cache)); - - const warnings: string[] = []; - let total: number; - try { - total = await pdfPageCount(bytes); - } catch { - throw new AIError('bad_request', 'This PDF could not be split into pages for reading. It may be damaged or encrypted.'); - } - const limit = Math.min(total, OCR_MAX_PAGES); - if (total > OCR_MAX_PAGES) warnings.push(`Only the first ${OCR_MAX_PAGES} of ${total} pages were read with AI. Split the file to read the rest.`); - + const text = result.text.trim(); + if (!text) return []; + // Split on the "## Page N" markers the prompt asks for so citations keep page numbers. + const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im); + if (pages.length < 3) return [{ text, page: row.kind === 'image' ? undefined : 1 }]; const sections: ExtractedSection[] = []; - for await (const batch of splitPdf(bytes, OCR_PAGES_PER_BATCH, OCR_MAX_BATCH_BYTES)) { - if (batch.start > limit) break; - const key = `${batch.start}-${batch.end}`; - const label = batch.start === batch.end ? `Page ${batch.start}` : `Pages ${batch.start}-${batch.end}`; - let got: ExtractedSection[] | undefined = cache.batches[key]; - if (!got) { - if (batch.bytes.length > OCR_MAX_BATCH_BYTES) { - warnings.push(`${label} is too large to send to an AI model and was skipped.`); - } else { - const prompt = `${TRANSCRIBE_PROMPT}\n\nThis file contains ${label.toLowerCase()} of a ${total}-page document. Number the "## Page N" headings with those real page numbers (${batch.start} to ${batch.end}).`; - const part: ContentPart = { type: 'file', mime: 'application/pdf', data: Buffer.from(batch.bytes).toString('base64'), filename: `${row.filename.replace(/\.pdf$/i, '')}-p${key}.pdf` }; - for (let attempt = 1; attempt <= 2 && !got; attempt++) { - try { - got = splitTranscription(await ask(part, prompt), batch.start, batch); - } catch (err) { - // Configuration problems will not fix themselves on the next batch. - if (err instanceof AIError && ['not_configured', 'unsupported', 'quota'].includes(err.code)) throw err; - if (attempt === 2) warnings.push(`${label} could not be read: ${err instanceof Error ? err.message : 'unknown error'}`); - } - } - if (got) { - cache.batches[key] = got; - save(); - } - } - } - if (got) sections.push(...got); - opts.onProgress?.(Math.min(limit, batch.end) / limit); + if (pages[0].trim()) sections.push({ text: pages[0].trim() }); + for (let i = 1; i < pages.length; i += 2) { + const body = (pages[i + 1] ?? '').trim(); + if (body) sections.push({ text: body, page: Number(pages[i]) }); } - return { sections, warnings }; + return sections; } export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise { @@ -375,16 +279,15 @@ export async function processDocument(app: AppContext, id: string, opts: { force setProgress(0.82); if (row.kind === 'image' || row.kind === 'pdf') { try { - const transcribed = await transcribeWithVision(app, row, bytes, { fresh: Boolean(opts.forceOcr), onProgress: (f) => setProgress(0.82 + f * 0.16) }); - if (transcribed) warnings.push(...transcribed.warnings); + const transcribed = await transcribeWithVision(app, row, bytes); if (transcribed === null) { warnings.push( row.kind === 'image' ? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".' : 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".', ); - } else if (transcribed.sections.length > 0) { - sections = transcribed.sections; + } else if (transcribed.length > 0) { + sections = transcribed; needsOcr = false; } else { warnings.push('The AI model did not find any readable text in this file.'); @@ -422,12 +325,11 @@ export async function processDocument(app: AppContext, id: string, opts: { force tokens, chunks.length, needsOcr ? 1 : 0, - JSON.stringify([...new Set(warnings)].slice(0, 12)), + JSON.stringify([...new Set(warnings)].slice(0, 8)), now(), id, ); }); - scheduleIndexing(app); } catch (err) { const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.'; if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err); @@ -530,7 +432,7 @@ export function sourceLabel(hit: Pick= 3) { const keep = new Set(); let used = 0; diff --git a/server/services/semantic.ts b/server/services/semantic.ts deleted file mode 100644 index 8a3637c..0000000 --- a/server/services/semantic.ts +++ /dev/null @@ -1,320 +0,0 @@ -/** - * Semantic search over document passages. - * - * Each chunk gets an embedding from the configured provider (OpenAI, Gemini, Ollama or any - * OpenAI-compatible server). Vectors are normalised Float32 BLOBs in `chunk_vectors`, keyed by - * provider + model + size, and searched by brute-force cosine similarity over an in-memory - * copy: a personal library has thousands of passages, not millions, so no index is needed. - * - * Search is hybrid: keyword (FTS5 BM25) and semantic rankings are merged with reciprocal-rank - * fusion, so exact terms (drug names, lab values) still win while paraphrased questions - * ("why do patients on furosemide need bananas?") also find the right passage. - */ -import { z } from 'zod'; -import { getAdapter } from '../ai/providers/index.ts'; -import { providerConfigById } from '../ai/service.ts'; -import { AIError, type ProviderKind } from '../ai/types.ts'; -import type { DB } from '../db.ts'; -import { badRequest, type AppContext } from '../http.ts'; -import { getSetting, setSetting } from '../settings.ts'; -import { now } from '../util.ts'; -import { chunkById, searchChunks, type ChunkHit, type ChunkScope } from './documents.ts'; - -/* ------------------------------ settings ------------------------------ */ - -export interface EmbeddingSettings { - providerId: string | null; - model: string; - /** Requested vector size for models that support shortening (OpenAI v3, Gemini). Null = native. */ - dimensions: number | null; -} - -const SETTINGS_KEY = 'embeddings'; -const EMPTY: EmbeddingSettings = { providerId: null, model: '', dimensions: null }; - -/** Suggested embedding model per provider type. Anthropic has no embeddings API. */ -export const DEFAULT_EMBEDDING_MODELS: Record = { - openai: 'text-embedding-3-small', - gemini: 'gemini-embedding-001', - ollama: 'nomic-embed-text', - openai_compatible: '', - anthropic: null, -}; - -/** USD per million input tokens, for the usage log. */ -const PRICES: Record = { - 'text-embedding-3-small': 0.02, - 'text-embedding-3-large': 0.13, - 'text-embedding-ada-002': 0.1, - 'gemini-embedding-001': 0.15, -}; - -export const embeddingSettingsSchema = z.object({ - providerId: z.string().nullable(), - model: z.string().trim().max(200).default(''), - dimensions: z.number().int().min(64).max(4096).nullable().default(null), -}); - -export function getEmbeddingSettings(db: DB): EmbeddingSettings { - return { ...EMPTY, ...getSetting>(db, SETTINGS_KEY, EMPTY) }; -} - -const modelKey = (s: EmbeddingSettings) => `${s.providerId}:${s.model}:${s.dimensions ?? 'native'}`; -const isConfigured = (s: EmbeddingSettings) => Boolean(s.providerId && s.model); - -export function saveEmbeddingSettings(app: AppContext, input: z.input): EmbeddingSettings { - const next = embeddingSettingsSchema.parse(input); - if (next.providerId) { - const { kind } = providerConfigById(app, next.providerId); - if (!getAdapter(kind).embed) throw badRequest('This provider has no embeddings API. Choose OpenAI, Gemini, Ollama or an OpenAI-compatible server.'); - if (!next.model) throw badRequest('Enter the embedding model to use.'); - } - const prev = getEmbeddingSettings(app.db); - setSetting(app.db, SETTINGS_KEY, next); - if (modelKey(prev) !== modelKey(next)) { - // Vectors from different models are not comparable: start over. - app.db.run('DELETE FROM chunk_vectors WHERE model_key != ?', modelKey(next)); - cache = null; - state.lastError = ''; - } - if (isConfigured(next)) scheduleIndexing(app); - return next; -} - -/* ------------------------------ embedding calls ------------------------------ */ - -/** Inputs per request and a cap on characters per input (about 2k tokens). */ -const BATCH = 64; -const MAX_INPUT_CHARS = 8000; - -function normalise(v: number[]): Float32Array { - const out = new Float32Array(v.length); - let sum = 0; - for (const x of v) sum += x * x; - const len = Math.sqrt(sum) || 1; - for (let i = 0; i < v.length; i++) out[i] = v[i] / len; - return out; -} - -export async function embedTexts(app: AppContext, texts: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise { - const s = getEmbeddingSettings(app.db); - if (!isConfigured(s)) throw new AIError('not_configured', 'Semantic search is not set up. Choose an embedding model in Settings → AI.'); - const { cfg, kind, name } = providerConfigById(app, s.providerId!); - const adapter = getAdapter(kind); - if (!adapter.embed) throw new AIError('unsupported', `${name} has no embeddings API.`); - const out: Float32Array[] = []; - for (let i = 0; i < texts.length; i += BATCH) { - const inputs = texts.slice(i, i + BATCH).map((t) => t.slice(0, MAX_INPUT_CHARS) || ' '); - const started = now(); - try { - const res = await adapter.embed(cfg, { model: s.model, inputs, purpose, dimensions: s.dimensions ?? undefined, signal }); - out.push(...res.vectors.map(normalise)); - logUsage(app.db, s, name, purpose, res.inputTokens, started); - } catch (err) { - logUsage(app.db, s, name, purpose, 0, started, err instanceof Error ? err.message : String(err)); - throw err; - } - } - return out; -} - -function logUsage(db: DB, s: EmbeddingSettings, providerName: string, purpose: string, tokens: number, started: number, error?: string) { - try { - const price = PRICES[s.model.replace(/^models\//, '')]; - db.run( - `INSERT INTO ai_usage (at, provider_id, provider_name, model, task, input_tokens, output_tokens, cache_read_tokens, cost, ok, error, duration_ms) - VALUES (?, ?, ?, ?, ?, ?, 0, 0, ?, ?, ?, ?)`, - now(), - s.providerId, - providerName, - s.model, - purpose === 'query' ? 'semantic search' : 'semantic indexing', - tokens, - price !== undefined ? (tokens * price) / 1_000_000 : null, - error ? 0 : 1, - error ? error.slice(0, 500) : '', - now() - started, - ); - } catch { - /* usage logging must never break search */ - } -} - -/* ------------------------------ background indexing ------------------------------ */ - -const state = { running: false, lastError: '', lastRunAt: 0 }; - -/** Embeds every chunk that has no vector for the current model. Safe to call often. */ -export function scheduleIndexing(app: AppContext): void { - if (state.running || !isConfigured(getEmbeddingSettings(app.db))) return; - state.running = true; - state.lastError = ''; - void (async () => { - try { - for (;;) { - const s = getEmbeddingSettings(app.db); - if (!isConfigured(s)) break; - const key = modelKey(s); - const rows = app.db.all<{ id: number; text: string; heading: string; title: string }>( - `SELECT ch.id, ch.text, ch.heading, d.title FROM chunks ch JOIN documents d ON d.id = ch.document_id - LEFT JOIN chunk_vectors v ON v.chunk_id = ch.id AND v.model_key = ? - WHERE v.chunk_id IS NULL ORDER BY ch.id LIMIT ?`, - key, - BATCH, - ); - if (rows.length === 0) break; - // The document title and heading give a passage its context ("Digoxin" under "Cardiac drugs"). - const vectors = await embedTexts(app, rows.map((r) => [r.title, r.heading, r.text].filter(Boolean).join('\n')), 'document'); - if (modelKey(getEmbeddingSettings(app.db)) !== key) continue; // settings changed mid-batch - app.db.tx(() => { - rows.forEach((r, i) => { - const v = vectors[i]; - // The chunk may have been deleted while it was being embedded. - if (app.db.get('SELECT 1 FROM chunks WHERE id = ?', r.id)) { - app.db.run('INSERT OR REPLACE INTO chunk_vectors (chunk_id, model_key, dims, vector) VALUES (?, ?, ?, ?)', r.id, key, v.length, new Uint8Array(v.buffer, v.byteOffset, v.byteLength)); - } - }); - }); - } - } catch (err) { - state.lastError = err instanceof Error ? err.message : String(err); - app.log('warn', `Semantic indexing paused: ${state.lastError}`); - } finally { - state.running = false; - state.lastRunAt = now(); - } - })(); -} - -/** Waits for the current indexing run. Used by tests. */ -export async function indexingIdle(): Promise { - while (state.running) await new Promise((r) => setTimeout(r, 10)); -} - -export function rebuildIndex(app: AppContext): void { - app.db.run('DELETE FROM chunk_vectors'); - cache = null; - state.lastError = ''; - scheduleIndexing(app); -} - -export function embeddingStatus(app: AppContext) { - const s = getEmbeddingSettings(app.db); - const total = app.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM chunks')?.n ?? 0; - const indexed = isConfigured(s) ? (app.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM chunk_vectors WHERE model_key = ?', modelKey(s))?.n ?? 0) : 0; - return { settings: s, configured: isConfigured(s), indexed, total, running: state.running, lastError: state.lastError, defaults: DEFAULT_EMBEDDING_MODELS }; -} - -/* ------------------------------ vector search ------------------------------ */ - -interface VectorCache { - key: string; - count: number; - maxId: number; - dims: number; - ids: Int32Array; - docIds: string[]; - classIds: string[]; - matrix: Float32Array; -} -let cache: VectorCache | null = null; - -function loadVectors(db: DB, key: string): VectorCache | null { - const sig = db.get<{ n: number; m: number | null }>('SELECT COUNT(*) AS n, MAX(chunk_id) AS m FROM chunk_vectors WHERE model_key = ?', key)!; - if (sig.n === 0) return null; - if (cache && cache.key === key && cache.count === sig.n && cache.maxId === (sig.m ?? 0)) return cache; - const rows = db.all<{ chunk_id: number; document_id: string; class_id: string; dims: number; vector: Uint8Array }>( - `SELECT v.chunk_id, ch.document_id, ch.class_id, v.dims, v.vector FROM chunk_vectors v JOIN chunks ch ON ch.id = v.chunk_id WHERE v.model_key = ?`, - key, - ); - const dims = rows[0]?.dims ?? 0; - const usable = rows.filter((r) => r.dims === dims && r.vector.byteLength === dims * 4); - const matrix = new Float32Array(usable.length * dims); - usable.forEach((r, i) => matrix.set(new Float32Array(r.vector.slice().buffer), i * dims)); - cache = { - key, - count: sig.n, - maxId: sig.m ?? 0, - dims, - ids: Int32Array.from(usable.map((r) => r.chunk_id)), - docIds: usable.map((r) => r.document_id), - classIds: usable.map((r) => r.class_id), - matrix, - }; - return cache; -} - -const queryCache = new Map(); - -async function queryVector(app: AppContext, key: string, query: string): Promise { - const k = `${key}\n${query}`; - const hit = queryCache.get(k); - if (hit) return hit; - const [v] = await embedTexts(app, [query], 'query'); - queryCache.set(k, v); - if (queryCache.size > 200) queryCache.delete(queryCache.keys().next().value!); - return v; -} - -/** Passages closest in meaning to the query, best first. Empty when semantic search is off. */ -export async function semanticSearch(app: AppContext, query: string, scope: ChunkScope, limit = 8, minScore = 0.2): Promise<(ChunkHit & { similarity: number })[]> { - const s = getEmbeddingSettings(app.db); - if (!isConfigured(s) || !query.trim()) return []; - const key = modelKey(s); - const vectors = loadVectors(app.db, key); - if (!vectors) return []; - const q = await queryVector(app, key, query.trim().slice(0, 2000)); - if (q.length !== vectors.dims) return []; - const docs = scope.documentIds?.length ? new Set(scope.documentIds) : null; - const top: { i: number; score: number }[] = []; - const { dims, matrix } = vectors; - for (let i = 0; i < vectors.ids.length; i++) { - if (docs ? !docs.has(vectors.docIds[i]) : scope.classId && vectors.classIds[i] !== scope.classId) continue; - let dot = 0; - const off = i * dims; - for (let d = 0; d < dims; d++) dot += matrix[off + d] * q[d]; - if (dot < minScore) continue; - if (top.length < limit) { - top.push({ i, score: dot }); - top.sort((a, b) => b.score - a.score); - } else if (dot > top[top.length - 1].score) { - top[top.length - 1] = { i, score: dot }; - top.sort((a, b) => b.score - a.score); - } - } - const out: (ChunkHit & { similarity: number })[] = []; - for (const t of top) { - const hit = chunkById(app.db, vectors.ids[t.i]); - if (hit) out.push({ ...hit, similarity: t.score }); - } - return out; -} - -/** - * Keyword + semantic search merged by reciprocal-rank fusion. Falls back to keyword search alone - * when semantic search is off or its provider fails, so search never breaks. - */ -export async function hybridSearch(app: AppContext, query: string, scope: ChunkScope, limit = 8): Promise { - const pool = Math.max(limit * 3, 30); - const lexical = searchChunks(app.db, query, scope, pool); - let semantic: ChunkHit[] = []; - try { - semantic = await semanticSearch(app, query, scope, pool); - } catch (err) { - app.log('warn', `Semantic search unavailable, using keywords only: ${err instanceof Error ? err.message : String(err)}`); - } - if (semantic.length === 0) return lexical.slice(0, limit); - const K = 60; - const fused = new Map(); - const add = (list: ChunkHit[]) => - list.forEach((hit, rank) => { - const prev = fused.get(hit.id); - fused.set(hit.id, { hit: prev?.hit ?? hit, score: (prev?.score ?? 0) + 1 / (K + rank + 1) }); - }); - add(lexical); - add(semantic); - return [...fused.values()] - .sort((a, b) => b.score - a.score) - .slice(0, limit) - .map((x) => ({ ...x.hit, score: x.score })); -} diff --git a/tests/ai.test.ts b/tests/ai.test.ts index c3beb6c..2b847a2 100644 --- a/tests/ai.test.ts +++ b/tests/ai.test.ts @@ -375,56 +375,3 @@ describe('assistant', () => { expect(bogus.at(-1).type).toBe('done'); }); }); - -describe('reading long scanned PDFs', () => { - async function scannedPdf(pages: number) { - const { PDFDocument } = await import('pdf-lib'); - const doc = await PDFDocument.create(); - for (let i = 0; i < pages; i++) doc.addPage([612, 792]); - return doc.save(); - } - const ocrCalls = () => mock.requests.filter((r) => r.path.endsWith('/chat/completions') && JSON.stringify(r.body?.messages ?? '').includes('Transcribe this study material')); - - it('reads the PDF in page batches, keeps real page numbers and caches finished batches', async () => { - const { models } = await connectAi(); - for (const m of models) await t.api('PATCH', `/api/ai/models/${m.id}`, { capabilities: { vision: true, pdf: true } }); - const cls = (await t.api('POST', '/api/classes', { name: 'Med-Surg' })).body.class; - - const up = await upload(cls.id, 'scanned-textbook.pdf', await scannedPdf(20), 'application/pdf'); - expect(up.status).toBe(201); - const id = up.body.documents[0].id; - const doc = (await t.api('GET', `/api/documents/${id}`)).body.document; - expect(doc).toMatchObject({ status: 'ready', needsOcr: false, pageCount: 20 }); - - // 20 pages at 8 per batch: pages 1-8, 9-16, 17-20, each sent as its own PDF. - const calls = ocrCalls(); - expect(calls).toHaveLength(3); - const prompts = calls.map((c) => JSON.stringify(c.body.messages)); - expect(prompts[0]).toContain('pages 1-8 of a 20-page document'); - expect(prompts[2]).toContain('pages 17-20 of a 20-page document'); - expect(prompts[1]).toContain('data:application/pdf;base64,'); - - const chunks = (await t.api('GET', `/api/documents/${id}/chunks`)).body.chunks as { text: string; pageStart: number | null; pageEnd: number | null }[]; - const text = chunks.map((c) => c.text).join('\n'); - for (const n of [1, 8, 9, 20]) expect(text).toContain(`page ${n}:`); - expect(Math.min(...chunks.map((c) => c.pageStart ?? 99))).toBe(1); - expect(Math.max(...chunks.map((c) => c.pageEnd ?? 0))).toBe(20); - expect(fs.readdirSync(path.join(t.dir, 'uploads')).some((f) => f.endsWith('.ocr.json'))).toBe(true); - - // Reprocessing without asking for a fresh read reuses the cached batches: no new AI calls. - mock.requests.length = 0; - await t.api('POST', `/api/documents/${id}/reprocess`, {}); - await queueIdle(t.ctx); - expect(ocrCalls()).toHaveLength(0); - expect((await t.api('GET', `/api/documents/${id}`)).body.document.needsOcr).toBe(false); - - // "Read with AI" asks for a fresh read and sends every batch again. - await t.api('POST', `/api/documents/${id}/reprocess`, { ocr: true }); - await queueIdle(t.ctx); - expect(ocrCalls()).toHaveLength(3); - - // Deleting the document removes the cache with the file. - await t.api('DELETE', `/api/documents/${id}`); - expect(fs.readdirSync(path.join(t.dir, 'uploads')).some((f) => f.endsWith('.ocr.json'))).toBe(false); - }); -}); diff --git a/tests/mockAi.ts b/tests/mockAi.ts index 5d16ff3..8d21337 100644 --- a/tests/mockAi.ts +++ b/tests/mockAi.ts @@ -6,7 +6,6 @@ * - Tools + user turn -> picks a tool from keywords in the message. * - Tool result turn -> answers in text, citing the first [cN] ref it was given. * - No tools -> streams a short Markdown answer. - * - /embeddings -> deterministic word-hash vectors (see mockEmbedding). */ import http from 'node:http'; import type { AddressInfo } from 'node:net'; @@ -22,27 +21,6 @@ export interface MockAi { const text = (content: unknown): string => typeof content === 'string' ? content : Array.isArray(content) ? content.map((p: any) => (typeof p?.text === 'string' ? p.text : '')).join('\n') : ''; -/** - * Deterministic "embeddings": hashed bag of words over 64 dimensions, with a few nursing - * synonyms folded together so a paraphrased query lands near the right passage. - */ -const SYNONYMS: Record = { - lasix: 'furosemide', loop: 'furosemide', diuretic: 'furosemide', diuretics: 'furosemide', - banana: 'potassium', bananas: 'potassium', hypokalemia: 'potassium', k: 'potassium', - halos: 'digoxin', lanoxin: 'digoxin', glycoside: 'digoxin', -}; -export function mockEmbedding(input: string, dims = 64): number[] { - const v = new Array(dims).fill(0); - for (const raw of input.toLowerCase().match(/[a-z]+/g) ?? []) { - if (raw.length < 3 && raw !== 'k') continue; - const word = SYNONYMS[raw] ?? raw; - let h = 2166136261; - for (const ch of word) h = Math.imul(h ^ ch.charCodeAt(0), 16777619) >>> 0; - v[h % dims] += 1; - } - return v; -} - function questions(n: number, refs: string[], offset = 0) { return Array.from({ length: n }, (_, k) => { const i = k + offset; @@ -129,11 +107,6 @@ export function startMockAi(port = 0): Promise { if (req.headers.authorization !== 'Bearer mock-key') return json(401, { error: { message: 'Incorrect API key provided: ' + String(req.headers.authorization), type: 'invalid_request_error' } }); if (path.endsWith('/models')) return json(200, { object: 'list', data: [{ id: 'mock-large' }, { id: 'mock-small' }, { id: 'mock-embedding-1' }] }); - if (path.endsWith('/embeddings')) { - const inputs: string[] = Array.isArray(body?.input) ? body.input : [String(body?.input ?? '')]; - const dims = Number(body?.dimensions) || 64; - return json(200, { object: 'list', model: body?.model, data: inputs.map((t, index) => ({ object: 'embedding', index, embedding: mockEmbedding(t, dims) })), usage: { prompt_tokens: Math.ceil(inputs.join(' ').length / 4), total_tokens: Math.ceil(inputs.join(' ').length / 4) } }); - } if (!path.endsWith('/chat/completions')) return json(404, { error: { message: 'not found' } }); if (fail.times > 0) { fail.times -= 1; @@ -171,17 +144,6 @@ export function startMockAi(port = 0): Promise { end('stop', Math.ceil(content.length / 4)); }; - // Document reading: answer with one "## Page N" section per page of the attached batch. - if (/Transcribe this study material/.test(userText)) { - const file = (Array.isArray(lastUser?.content) ? lastUser.content : []).find((p: any) => p?.type === 'file'); - if (!file?.file?.file_data?.startsWith('data:application/pdf;base64,')) return say('[no readable file attached]'); - const range = /contains pages? (\d+)(?:-(\d+))? of a/i.exec(userText); - const from = Number(range?.[1] ?? 1); - const to = Number(range?.[2] ?? range?.[1] ?? 1); - const pages = Array.from({ length: to - from + 1 }, (_, k) => `## Page ${from + k}\n\nScanned lecture text for page ${from + k}: nursing priority ${from + k}.`); - return say(pages.join('\n\n')); - } - const forced = body?.tool_choice?.function?.name as string | undefined; if (forced) return callTool(forced); if (last.role === 'tool') { diff --git a/tests/ordering-deadlines.test.ts b/tests/ordering-deadlines.test.ts deleted file mode 100644 index b2bf371..0000000 --- a/tests/ordering-deadlines.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { createTestApp, sampleQuestions, type TestApp } from './helpers.ts'; - -let t: TestApp; -beforeEach(async () => { - t = await createTestApp(); -}); -afterEach(() => { - vi.restoreAllMocks(); - t.close(); -}); - -async function makeClass(name = 'Pharmacology') { - return (await t.api('POST', '/api/classes', { name })).body.class as { id: string }; -} - -describe('manual ordering', () => { - it('stores a custom order per kind and exposes it on the class', async () => { - const cls = await makeClass(); - const ids: string[] = []; - for (const title of ['A', 'B', 'C']) ids.push((await t.api('POST', '/api/quizzes', { classId: cls.id, title, questions: sampleQuestions })).body.quiz.id); - const other = await makeClass('Other'); - const foreign = (await t.api('POST', '/api/quizzes', { classId: other.id, title: 'Foreign', questions: sampleQuestions })).body.quiz.id; - - const res = await t.api('POST', `/api/classes/${cls.id}/items/reorder`, { kind: 'quiz', ids: [ids[2], foreign, ids[0], ids[1]] }); - expect(res.status).toBe(200); - const items = (await t.api('GET', `/api/classes/${cls.id}`)).body.items as { id: string; kind: string; title: string; sortOrder: number }[]; - const order = items.filter((i) => i.kind === 'quiz').sort((a, b) => a.sortOrder - b.sortOrder).map((i) => i.title); - expect(order).toEqual(['C', 'A', 'B']); - // An id from another class is ignored, not moved. - const otherItems = (await t.api('GET', `/api/classes/${other.id}`)).body.items as { id: string; sortOrder: number }[]; - expect(otherItems.find((i) => i.id === foreign)?.sortOrder).toBe(0); - }); - - it('rejects unknown kinds', async () => { - const cls = await makeClass(); - expect((await t.api('POST', `/api/classes/${cls.id}/items/reorder`, { kind: 'class', ids: [] })).status).toBe(422); - }); -}); - -describe('timed exam deadlines', () => { - async function startExam(minutes: number) { - const cls = await makeClass(); - const quiz = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Timed', questions: sampleQuestions })).body.quiz; - const res = await t.api('POST', '/api/attempts', { mode: 'exam', source: { type: 'quiz', quizId: quiz.id }, timeLimitMin: minutes, shuffleQuestions: false, shuffleOptions: false }); - expect(res.status).toBe(201); - return { attempt: res.body.attempt, quiz }; - } - - it('fixes a wall-clock deadline when the exam starts', async () => { - const before = Date.now(); - const { attempt } = await startExam(10); - expect(attempt.settings.deadlineAt).toBeGreaterThanOrEqual(before + 600_000); - expect(attempt.settings.deadlineAt).toBeLessThanOrEqual(Date.now() + 600_000); - expect(typeof attempt.serverNow).toBe('number'); - }); - - it('accepts answers before the deadline and refuses them after, submitting what was saved', async () => { - const { attempt } = await startExam(1); - const q = attempt.questions[0]; - const right = q.options.find((o: { correct: boolean }) => o.correct).id; - expect((await t.api('PATCH', `/api/attempts/${attempt.id}`, { answers: { [q.id]: { selected: [right] } } })).status).toBe(200); - - const start = attempt.settings.deadlineAt - 60_000; - vi.spyOn(Date, 'now').mockReturnValue(attempt.settings.deadlineAt + 30_000); - const q2 = attempt.questions[1]; - const late = await t.api('PATCH', `/api/attempts/${attempt.id}`, { answers: { [q2.id]: { selected: [q2.options[0].id] } } }); - expect(late.status).toBe(409); - expect(late.body.error.code).toBe('time_up'); - - const after = (await t.api('GET', `/api/attempts/${attempt.id}`)).body.attempt; - expect(after.status).toBe('completed'); - expect(after.answers[q2.id]).toBeUndefined(); - expect(after.score).toBe(1); - // Time used is capped at the limit even though the clock ran on. - expect(after.elapsedSec).toBe(60); - expect(after.startedAt).toBe(start); - }); - - it('allows a short grace period for saves in flight at the deadline', async () => { - const { attempt } = await startExam(1); - const q = attempt.questions[0]; - vi.spyOn(Date, 'now').mockReturnValue(attempt.settings.deadlineAt + 2_000); - expect((await t.api('PATCH', `/api/attempts/${attempt.id}`, { answers: { [q.id]: { selected: [q.options[0].id] } } })).status).toBe(200); - }); - - it('submits overdue exams before lists are read, so they are not offered for resuming', async () => { - const { attempt, quiz } = await startExam(1); - vi.spyOn(Date, 'now').mockReturnValue(attempt.settings.deadlineAt + 60_000); - const active = (await t.api('GET', '/api/attempts?status=active')).body.attempts; - expect(active).toHaveLength(0); - const cls = (await t.api('GET', `/api/classes/${quiz.classId}`)).body; - expect(cls.items.find((i: { id: string }) => i.id === quiz.id).activeAttemptId).toBeNull(); - }); - - it('leaves untimed sessions alone', async () => { - const cls = await makeClass(); - const quiz = (await t.api('POST', '/api/quizzes', { classId: cls.id, title: 'Untimed', questions: sampleQuestions })).body.quiz; - const attempt = (await t.api('POST', '/api/attempts', { mode: 'exam', source: { type: 'quiz', quizId: quiz.id } })).body.attempt; - expect(attempt.settings.deadlineAt).toBeNull(); - vi.spyOn(Date, 'now').mockReturnValue(Date.now() + 86_400_000); - expect((await t.api('GET', `/api/attempts/${attempt.id}`)).body.attempt.status).toBe('active'); - }); -}); diff --git a/tests/semantic.test.ts b/tests/semantic.test.ts deleted file mode 100644 index dbf7ab3..0000000 --- a/tests/semantic.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; -import { queueIdle } from '../server/services/documents.ts'; -import { hybridSearch, indexingIdle } from '../server/services/semantic.ts'; -import { createTestApp, type TestApp } from './helpers.ts'; -import { startMockAi, type MockAi } from './mockAi.ts'; - -let mock: MockAi; -let t: TestApp; - -beforeAll(async () => { - mock = await startMockAi(); -}); -afterAll(() => mock.close()); -beforeEach(async () => { - t = await createTestApp(); - mock.requests.length = 0; -}); -afterEach(async () => { - await indexingIdle(); - t.close(); -}); - -const DIURETICS = `# Loop diuretics - -Furosemide acts on the ascending limb of the loop of Henle. Watch for hypokalemia: encourage potassium-rich foods and check serum levels before each dose.`; -const DIGOXIN = `# Cardiac glycosides - -Digoxin strengthens contraction and slows the heart rate. Toxicity shows as nausea, vomiting and yellow-green halos around lights. Take the apical pulse for one minute first.`; - -async function setup() { - const provider = (await t.api('POST', '/api/ai/providers', { name: 'Mock', kind: 'openai_compatible', baseUrl: mock.url, apiKey: 'mock-key' })).body.provider; - const cls = (await t.api('POST', '/api/classes', { name: 'Pharmacology' })).body.class; - for (const [title, text] of [ - ['Diuretics lecture', DIURETICS], - ['Cardiac glycosides lecture', DIGOXIN], - ]) { - expect((await t.api('POST', '/api/documents/text', { classId: cls.id, title, text })).status).toBe(201); - } - await queueIdle(t.ctx); - return { provider, cls }; -} - -async function enable(providerId: string, model = 'mock-embedding-1') { - const res = await t.api('PUT', '/api/ai/embeddings', { providerId, model, dimensions: null }); - expect(res.status).toBe(200); - await indexingIdle(); - return (await t.api('GET', '/api/ai/embeddings')).body; -} - -describe('semantic search', () => { - it('is off by default and search falls back to keywords', async () => { - const { cls } = await setup(); - const status = (await t.api('GET', '/api/ai/embeddings')).body; - expect(status).toMatchObject({ configured: false, indexed: 0 }); - expect(status.total).toBeGreaterThan(0); - const hits = await hybridSearch(t.ctx, 'digoxin', { classId: cls.id }, 3); - expect(hits[0].documentTitle).toBe('Cardiac glycosides lecture'); - expect(mock.requests.filter((r) => r.path.endsWith('/embeddings'))).toHaveLength(0); - }); - - it('indexes every passage once and finds paraphrased questions that share no keywords', async () => { - const { provider, cls } = await setup(); - const status = await enable(provider.id); - expect(status).toMatchObject({ configured: true, running: false, lastError: '' }); - expect(status.indexed).toBe(status.total); - - // No word of this query appears in the passage, so keyword search alone finds nothing. - const query = 'why should lasix patients eat bananas'; - const search = (await t.api('GET', `/api/search?q=${encodeURIComponent(query)}&classId=${cls.id}`)).body.results as { kind: string; title: string; semantic?: boolean }[]; - const passages = search.filter((r) => r.kind === 'passage'); - expect(passages[0]).toMatchObject({ title: 'Diuretics lecture', semantic: true }); - - const hits = await hybridSearch(t.ctx, query, { classId: cls.id }, 2); - expect(hits[0].documentTitle).toBe('Diuretics lecture'); - - // Usage is logged, and a second index run embeds nothing new. - const usage = t.ctx.db.all<{ task: string }>("SELECT task FROM ai_usage WHERE task LIKE 'semantic%'"); - expect(usage.some((u) => u.task === 'semantic indexing')).toBe(true); - const before = mock.requests.length; - await t.api('POST', '/api/ai/embeddings/resume'); - await indexingIdle(); - expect(mock.requests.length).toBe(before); - }); - - it('indexes documents added later and drops vectors when the model changes', async () => { - const { provider, cls } = await setup(); - await enable(provider.id); - await t.api('POST', '/api/documents/text', { classId: cls.id, title: 'Insulin notes', text: '# Insulin\n\nRapid-acting insulin starts within 15 minutes; give it with a meal.' }); - await queueIdle(t.ctx); - await indexingIdle(); - let status = (await t.api('GET', '/api/ai/embeddings')).body; - expect(status.indexed).toBe(status.total); - - status = await enable(provider.id, 'mock-embedding-2'); - expect(status.indexed).toBe(status.total); - const keys = t.ctx.db.all<{ model_key: string }>('SELECT DISTINCT model_key FROM chunk_vectors'); - expect(keys).toHaveLength(1); - expect(keys[0].model_key).toContain('mock-embedding-2'); - }); - - it('refuses providers without an embeddings API and clears settings when the provider is deleted', async () => { - const { provider } = await setup(); - const claude = (await t.api('POST', '/api/ai/providers', { name: 'Claude', kind: 'anthropic', apiKey: 'sk-ant-test' })).body.provider; - const refused = await t.api('PUT', '/api/ai/embeddings', { providerId: claude.id, model: 'anything', dimensions: null }); - expect(refused.status).toBe(400); - - await enable(provider.id); - await t.api('DELETE', `/api/ai/providers/${provider.id}`); - const status = (await t.api('GET', '/api/ai/embeddings')).body; - expect(status).toMatchObject({ configured: false, indexed: 0 }); - }); - - it('keeps working on keywords when the embedding provider fails', async () => { - const { provider, cls } = await setup(); - await enable(provider.id); - await t.api('PATCH', `/api/ai/providers/${provider.id}`, { apiKey: 'wrong-key' }); - const hits = await hybridSearch(t.ctx, 'apical pulse before digoxin', { classId: cls.id }, 2); - expect(hits[0].documentTitle).toBe('Cardiac glycosides lecture'); - }); -}); diff --git a/web/src/app/Shell.tsx b/web/src/app/Shell.tsx index e4b4368..56c2c9e 100644 --- a/web/src/app/Shell.tsx +++ b/web/src/app/Shell.tsx @@ -177,7 +177,7 @@ function CommandPalette({ open, onOpenChange, onNewClass }: { open: boolean; onO key: `${h.kind}:${h.id}`, icon: HIT_ICON[h.kind], title: h.title, - meta: h.semantic ? 'Related passage' : HIT_LABEL[h.kind], + meta: HIT_LABEL[h.kind], sub: [h.snippet, h.kind !== 'class' ? h.className : ''].filter(Boolean).join(' · '), color: h.classColor, run: go(hitUrl(h)), diff --git a/web/src/features/AiSettings.tsx b/web/src/features/AiSettings.tsx index f14161b..e625702 100644 --- a/web/src/features/AiSettings.tsx +++ b/web/src/features/AiSettings.tsx @@ -4,7 +4,6 @@ import { useEffect, useMemo, useState } from 'react'; import { toast } from 'sonner'; import { api, errorMessage } from '@/lib/api'; import { qk } from '@/lib/queries'; -import { SemanticSettings } from './SemanticSettings'; import type { AdapterMeta, AiModel, AiStatus, Provider, ProviderKind, TaskConfig, TaskName, TaskSettings } from '@/lib/types'; import { cx, formatCost, formatTokens, plural, timeAgo } from '@/lib/util'; import { Badge, Button, Callout, Dialog, EmptyState, Field, IconButton, Input, Menu, MenuItem, MenuSeparator, Section, Select, Skeleton, Switch, Textarea, Tooltip, useConfirm } from '@/ui'; @@ -699,8 +698,6 @@ export default function AiSettings() { )} - {providers.data.length > 0 && } - {providers.data.length > 0 && } setDialog((d) => ({ ...d, open: false }))} adapters={meta.data.adapters} existing={dialog.existing} onSaved={afterSave} /> diff --git a/web/src/features/SemanticSettings.tsx b/web/src/features/SemanticSettings.tsx deleted file mode 100644 index bdd683e..0000000 --- a/web/src/features/SemanticSettings.tsx +++ /dev/null @@ -1,180 +0,0 @@ -import { useQuery, useQueryClient } from '@tanstack/react-query'; -import { CircleAlert, RefreshCw, Sparkles, Zap } from 'lucide-react'; -import { useEffect, useState } from 'react'; -import { toast } from 'sonner'; -import { api, errorMessage } from '@/lib/api'; -import type { Provider, ProviderKind } from '@/lib/types'; -import { plural } from '@/lib/util'; -import { Badge, Button, Callout, Field, Input, ProgressBar, Section, Select, useConfirm } from '@/ui'; - -interface EmbeddingStatus { - settings: { providerId: string | null; model: string; dimensions: number | null }; - configured: boolean; - indexed: number; - total: number; - running: boolean; - lastError: string; - defaults: Record; -} - -const KEY = ['ai', 'embeddings']; - -/** - * Semantic search settings: which provider and embedding model turn document passages into - * vectors, and how far indexing has got. - */ -export function SemanticSettings({ providers }: { providers: Provider[] }) { - const qc = useQueryClient(); - const confirm = useConfirm(); - const q = useQuery({ - queryKey: KEY, - queryFn: () => api.get('/api/ai/embeddings'), - // Poll while indexing so the progress bar moves. - refetchInterval: (query) => (query.state.data?.running ? 1500 : false), - }); - const data = q.data; - const eligible = providers.filter((p) => p.kind !== 'anthropic' && p.enabled); - const [providerId, setProviderId] = useState(''); - const [model, setModel] = useState(''); - const [dimensions, setDimensions] = useState(''); - const [busy, setBusy] = useState<'save' | 'test' | 'rebuild' | null>(null); - - useEffect(() => { - if (!data) return; - setProviderId(data.settings.providerId ?? ''); - setModel(data.settings.model); - setDimensions(data.settings.dimensions ? String(data.settings.dimensions) : ''); - }, [data]); - - if (!data) return null; - const chosen = providers.find((p) => p.id === providerId); - const suggested = chosen ? data.defaults[chosen.kind] : null; - const dirty = providerId !== (data.settings.providerId ?? '') || model.trim() !== data.settings.model || dimensions !== (data.settings.dimensions ? String(data.settings.dimensions) : ''); - const reload = () => qc.invalidateQueries({ queryKey: KEY }); - - const save = async () => { - if (data.configured && data.indexed > 0 && providerId && (providerId !== data.settings.providerId || model.trim() !== data.settings.model)) { - const ok = await confirm({ title: 'Switch embedding model?', message: `Vectors from different models cannot be mixed, so all ${plural(data.indexed, 'passage')} will be indexed again with the new model.`, confirmLabel: 'Switch and re-index' }); - if (!ok) return; - } - setBusy('save'); - try { - await api.put('/api/ai/embeddings', { providerId: providerId || null, model: providerId ? model.trim() : '', dimensions: dimensions ? Number(dimensions) : null }); - toast.success(providerId ? 'Semantic search is on. Indexing your documents…' : 'Semantic search is off'); - await reload(); - } catch (e) { - toast.error(errorMessage(e)); - } finally { - setBusy(null); - } - }; - const test = async () => { - setBusy('test'); - try { - const r = await api.post<{ dimensions: number; latencyMs: number }>('/api/ai/embeddings/test'); - toast.success(`Working: ${r.dimensions}-dimension vectors in ${r.latencyMs} ms`); - } catch (e) { - toast.error(errorMessage(e)); - } finally { - setBusy(null); - } - }; - const act = async (path: 'rebuild' | 'resume') => { - if (path === 'rebuild' && !(await confirm({ title: 'Rebuild the index?', message: 'Every passage is embedded again. This repeats the embedding cost.', confirmLabel: 'Rebuild' }))) return; - setBusy('rebuild'); - try { - await api.post(`/api/ai/embeddings/${path}`); - await reload(); - } catch (e) { - toast.error(errorMessage(e)); - } finally { - setBusy(null); - } - }; - - const pct = data.total > 0 ? data.indexed / data.total : 0; - return ( -
= data.total ? 'good' : 'accent'}>{data.indexed >= data.total ? 'On' : 'Indexing'} : Off} - > -
-

- Finds passages by meaning as well as by words, so "why do patients on Lasix need bananas?" finds the section on furosemide and potassium. Used by search, the assistant and - focused generation. Keyword search keeps working without it. -

- {eligible.length === 0 ? ( - } className="mt-4" title="Needs an embeddings provider"> - Connect OpenAI, Gemini, Ollama or an OpenAI-compatible server above. Anthropic does not offer embeddings. - - ) : ( -
- - - - - setModel(e.target.value)} placeholder={suggested || 'text-embedding-3-small'} disabled={!providerId} /> - - - setDimensions(e.target.value)} placeholder="Native" disabled={!providerId} className="sm:w-28" /> - -
- )} - {eligible.length > 0 && ( -
- - {data.configured && !dirty && ( - - )} -
- )} - - {data.configured && ( -
-
- - {data.indexed.toLocaleString()} of {data.total.toLocaleString()} passages indexed - {data.running ? ' · working…' : ''} - - - {!data.running && data.indexed < data.total && ( - - )} - - -
- - {data.lastError && ( - } className="mt-3" title="Indexing paused"> - {data.lastError} - - )} -
- )} -
-

Embedding costs are small (OpenAI's small model is about $0.02 per million tokens) and appear in Usage. Local Ollama models are free.

-
- ); -} diff --git a/web/src/features/quiz.tsx b/web/src/features/quiz.tsx index 97845eb..de8e5bb 100644 --- a/web/src/features/quiz.tsx +++ b/web/src/features/quiz.tsx @@ -110,11 +110,6 @@ export interface RunnerProps { index: number; elapsedSec: number; timeLimitSec: number | null; - /** - * Fixed end time on this device's clock. When set, the countdown follows the wall clock - * and keeps running while the page is hidden or closed. Without it the clock pauses. - */ - deadlineAt?: number | null; sataScoring: SataScoring; onAnswer: (questionId: string, selected: string[], checked: boolean) => void; onFlags: (flags: string[]) => void; @@ -136,6 +131,7 @@ export function QuizRunner(p: RunnerProps) { const topRef = useRef(null); const tick = useLatest(p.onTick); const finish = useLatest(p.onFinish); + const elapsedRef = useLatest(elapsed); const autoSubmitted = useRef(false); const total = p.questions.length; @@ -160,36 +156,24 @@ export function QuizRunner(p: RunnerProps) { } return n > 0 ? { n, points, percent: Math.round((points / n) * 100) } : null; }, [p.mode, p.questions, p.answers, p.sataScoring]); - const [wallNow, setWallNow] = useState(() => Date.now()); - const deadline = p.deadlineAt && p.timeLimitSec !== null ? p.deadlineAt : null; - const remaining = deadline !== null ? Math.max(0, Math.ceil((deadline - wallNow) / 1000)) : p.timeLimitSec !== null ? Math.max(0, p.timeLimitSec - elapsed) : null; - // Time used: wall clock for deadline exams, otherwise the active-study clock. - const clock = deadline !== null && p.timeLimitSec !== null ? Math.max(elapsed, p.timeLimitSec - (remaining ?? 0)) : elapsed; - const clockRef = useLatest(clock); + const remaining = p.timeLimitSec !== null ? Math.max(0, p.timeLimitSec - elapsed) : null; + // Session clock: pauses while the tab is hidden. useEffect(() => { const timer = setInterval(() => { - setWallNow(Date.now()); - // The untimed/legacy clock pauses while the tab is hidden. if (document.visibilityState === 'visible') setElapsed((e) => e + 1); }, 1000); - // Catch up immediately when returning to a deadline exam. - const onVisible = () => document.visibilityState === 'visible' && setWallNow(Date.now()); - document.addEventListener('visibilitychange', onVisible); - return () => { - clearInterval(timer); - document.removeEventListener('visibilitychange', onVisible); - }; + return () => clearInterval(timer); }, []); useEffect(() => { - if (elapsed > 0 && elapsed % 10 === 0) tick.current?.(clockRef.current); - }, [elapsed, tick, clockRef]); + if (elapsed > 0 && elapsed % 10 === 0) tick.current?.(elapsed); + }, [elapsed, tick]); useEffect(() => { if (remaining === 0 && !p.finishing && !autoSubmitted.current) { autoSubmitted.current = true; - finish.current(clockRef.current); + finish.current(elapsedRef.current); } - }, [remaining, p.finishing, finish, clockRef]); + }, [remaining, p.finishing, finish, elapsedRef]); useEffect(() => { topRef.current?.scrollIntoView({ block: 'start' }); }, [index]); @@ -213,7 +197,7 @@ export function QuizRunner(p: RunnerProps) { ); if (!ok) return; } - p.onFinish(clock); + p.onFinish(elapsed); }; const primary = () => { @@ -264,7 +248,7 @@ export function QuizRunner(p: RunnerProps) { )}
- {formatDuration(remaining ?? clock)} + {formatDuration(remaining ?? elapsed)}
- ); - return render( - item, - { - ref: (el) => { - if (el) nodes.current.set(key, el); - else nodes.current.delete(key); - }, - style, - dragging, - handle, - }, - i, - ); - })} - {liveAs === 'li' ? ( -
  • - {announce} -
  • - ) : ( -
    - {announce} -
    - )} - - ); -}