Restore a full backup from Settings
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 30s

Settings → Data → Import gets "Restore a full backup" in place of the V1
import row. The zip from "Download backup" is uploaded and replaces the
database, every setting and the uploaded files on the running server:

- the staged database is upgraded to this schema (newer backups are refused),
  saved keys are re-encrypted for this server's key, interrupted jobs are
  marked, and the restoring session is carried over;
- a pre-restore snapshot is written, running generations stop, and the file
  is swapped under the open DB handle (DB.replaceWith); uploads and media are
  copied into place and background work restarts.

The /api/import/v1 endpoint stays for the V1 cutover (docs/MIGRATION.md).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-16 12:33:42 -07:00
parent e0bc7b580e
commit 7125dec3e0
13 changed files with 521 additions and 72 deletions

View file

@ -615,6 +615,14 @@ export function cancelGeneration(draftId: string): void {
running.delete(draftId);
}
/** Stops every running generation (before a full backup replaces the database). */
export function cancelAllGenerations(): number {
const n = running.size;
for (const c of running.values()) c.abort();
running.clear();
return n;
}
/** Listeners for a running job's progress labels (the assistant shows them on its tool step). */
const progressListeners = new Map<string, (label: string) => void>();

View file

@ -366,6 +366,11 @@ function forget(serverId: string): void {
sessions.delete(serverId);
}
/** Drops every open MCP session (after a restore the servers and their credentials may differ). */
export function forgetMcpSessions(): void {
sessions.clear();
}
export class McpError extends Error {
expired: boolean;
/** The server answered 401: the credentials are missing, wrong or expired. */

View file

@ -97,6 +97,12 @@ function createSession(app: AppContext, c: Context): void {
setCookie(c, COOKIE, token, { httpOnly: true, sameSite: 'Lax', secure, path: '/', maxAge: Math.floor(SESSION_MS / 1000) });
}
/** The stored hash of this request's session token, or null when there is no session cookie. */
export function sessionHash(c: Context): string | null {
const token = getCookie(c, COOKIE);
return token ? sha256(token) : null;
}
export function isAuthenticated(app: AppContext, c: Context): boolean {
const token = getCookie(c, COOKIE);
if (!token) return false;

View file

@ -14,7 +14,7 @@ function bind(params: Param[]): (string | number | bigint | null | Uint8Array)[]
* exactly what a one-user app in one container needs, with no native add-on to build.
*/
export class DB {
readonly raw: DatabaseSync;
private handle: DatabaseSync;
readonly path: string;
private statements = new Map<string, StatementSync>();
private txDepth = 0;
@ -23,11 +23,53 @@ export class DB {
constructor(file: string) {
this.path = file;
this.raw = new DatabaseSync(file);
this.raw.exec('PRAGMA journal_mode = WAL');
this.raw.exec('PRAGMA synchronous = NORMAL');
this.raw.exec('PRAGMA foreign_keys = ON');
this.raw.exec('PRAGMA busy_timeout = 5000');
this.handle = DB.open(file);
}
private static open(file: string): DatabaseSync {
const raw = new DatabaseSync(file);
raw.exec('PRAGMA journal_mode = WAL');
raw.exec('PRAGMA synchronous = NORMAL');
raw.exec('PRAGMA foreign_keys = ON');
raw.exec('PRAGMA busy_timeout = 5000');
return raw;
}
/** The node:sqlite handle. Changes when a full backup is restored (`replaceWith`). */
get raw(): DatabaseSync {
return this.handle;
}
/**
* Swaps the database file for `file` (a complete, closed SQLite file in the same folder) under
* this same object, so every part of the app keeps working with the restored data. If the
* swap fails, the current database is reopened untouched.
*/
replaceWith(file: string): void {
if (this.txDepth > 0) throw new Error('A database transaction is still open.');
if (this.path === ':memory:') throw new Error('An in-memory database cannot be replaced.');
this.statements.clear();
try {
this.handle.exec('PRAGMA wal_checkpoint(TRUNCATE)');
} catch {
/* closing writes it anyway */
}
this.handle.close();
try {
const aside = `${this.path}.replaced`;
fs.rmSync(aside, { force: true });
fs.renameSync(this.path, aside);
try {
fs.renameSync(file, this.path);
} catch (e) {
fs.renameSync(aside, this.path);
throw e;
}
for (const ext of ['-wal', '-shm']) fs.rmSync(`${this.path}${ext}`, { force: true });
fs.rmSync(aside, { force: true });
} finally {
this.handle = DB.open(this.path);
}
}
private stmt(sql: string): StatementSync {

View file

@ -11,10 +11,14 @@ import { appSettingsSchema, deleteSetting, getAppSettings, getSetting, setSettin
import { createSnapshot, exportBundle, fullBackupStream, listBackups } from '../services/backup.ts';
import { createClass, createDeck, createFolder, createNote, createQuiz, draftCardToInput, draftQuestionToInput, getClass, getDeck, getQuiz, listCards, listQuestions } from '../services/content.ts';
import { createTemplate, deleteTemplate, listTemplates, resetTemplate, templateInputSchema, updateTemplate } from '../services/templates.ts';
import { restoreFullBackup } from '../services/restore.ts';
import { importV1 } from '../services/v1import.ts';
import { sessionHash } from '../auth.ts';
import { slugify } from '../util.ts';
export const APP_VERSION = '2.0.0';
/** The largest full backup that can be restored through the app. */
const RESTORE_MAX_BYTES = 20 * 1024 ** 3;
function dirSize(dir: string): number {
try {
@ -176,7 +180,45 @@ export function dataRoutes(app: AppContext): Hono {
return c.json({ summary }, 201);
});
/** Migration from a V1 database file. `?preview=1` only reports what would be imported. */
/**
* Restores a full backup zip (from "Download backup"): the database with every setting, and the
* uploaded files. The zip is the raw request body, streamed to disk. See services/restore.ts.
*/
r.post('/backup/restore', async (c) => {
const max = RESTORE_MAX_BYTES;
const declared = Number(c.req.header('content-length') ?? 0);
if (declared > max) throw new HttpError(413, 'too_large', 'The backup is larger than this server accepts.');
const body = c.req.raw.body;
if (!body) throw badRequest('Choose a backup zip');
const tmp = path.join(app.config.dataDir, `.restore-upload-${Date.now()}.zip`);
try {
let size = 0;
const out = fs.createWriteStream(tmp);
try {
for await (const chunk of body as unknown as AsyncIterable<Uint8Array>) {
size += chunk.length;
if (size > max) throw new HttpError(413, 'too_large', 'The backup is larger than this server accepts.');
if (!out.write(chunk)) await new Promise((r) => out.once('drain', r));
}
} finally {
await new Promise<void>((resolve, reject) => out.end((err?: Error | null) => (err ? reject(err) : resolve())));
}
if (size < 22) throw badRequest('Choose a backup zip');
const head = Buffer.alloc(4);
const fd = fs.openSync(tmp, 'r');
fs.readSync(fd, head, 0, 4, 0);
fs.closeSync(fd);
if (head.readUInt32LE(0) !== 0x04034b50) throw badRequest('That file is not a zip. Choose a file made with "Download backup".');
return c.json({ summary: await restoreFullBackup(app, tmp, { sessionHash: sessionHash(c) }) });
} finally {
fs.rmSync(tmp, { force: true });
}
});
/**
* Migration from a V1 database file. `?preview=1` only reports what would be imported. Not in
* Settings any more (full-backup restore took its place); see docs/MIGRATION.md.
*/
r.post('/import/v1', bodyLimit({ maxSize: 500 * 1024 * 1024 }), async (c) => {
const form = await c.req.parseBody().catch(() => {
throw badRequest('Upload could not be read');

222
server/services/restore.ts Normal file
View file

@ -0,0 +1,222 @@
/**
* Restoring a full backup zip ("Download backup" in Settings → Data) into the running app.
*
* 1. The zip is unpacked into a staging folder inside the data folder, one file at a time
* (only study.db, secret.key, manifest.json, uploads/* and media/* are accepted).
* 2. The staged database is opened and upgraded to this version's schema (a backup from a
* newer version is refused), saved AI keys are re-encrypted for this server's key, jobs that
* were running when the backup was taken are marked interrupted, and the signed-in session
* is carried over so the person restoring stays signed in.
* 3. A snapshot of the current database is written to the backups folder, running generations
* stop, and the database file is swapped under the open handle (`DB.replaceWith`).
* 4. Uploaded files and images from the backup are copied into place (files already here that
* the backup does not have are left alone), and background work restarts on the new data.
*/
import fs from 'node:fs';
import path from 'node:path';
import { Unzip, UnzipInflate, type UnzipFile } from 'fflate';
import { decryptSecret, encryptSecret } from '../crypto.ts';
import { DB, migrate } from '../db.ts';
import { badRequest, type AppContext } from '../http.ts';
import { cancelAllGenerations, tidyDrafts } from '../ai/generate.ts';
import { forgetMcpSessions, warmMcpServers } from '../ai/mcp.ts';
import { createSnapshot } from './backup.ts';
import { resumeQueue } from './documents.ts';
import { notifyChange } from './events.ts';
import { resetVectorCache, scheduleIndexing } from './semantic.ts';
export interface RestoreSummary {
backupCreatedAt: string | null;
classes: number;
quizzes: number;
decks: number;
notes: number;
documents: number;
attempts: number;
files: number;
/** Saved keys (AI providers, MCP, web search) re-encrypted for this server. */
keysMoved: number;
/** Saved keys that could not be read: the backup's secret.key or APP_SECRET is not available. */
keysUnreadable: number;
/** The snapshot of the database as it was before the restore. */
safetySnapshot: string;
/** Generations that were running and were stopped. */
stoppedJobs: number;
}
const ACCEPTED = /^(?:study\.db|secret\.key|manifest\.json|(?:uploads|media)\/[^/\\]+)$/;
/** An encrypted value: `v1.<12-byte iv>.<16-byte tag>.<ciphertext>` in base64url. */
const SEALED = /v1\.[A-Za-z0-9_-]{16}\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]+/g;
/** Unpacks the accepted entries of a zip file into `dir` without holding the whole zip in memory. */
async function unpack(zipFile: string, dir: string, maxBytes: number): Promise<string[]> {
const written: string[] = [];
let failure: Error | null = null;
let total = 0;
const unzip = new Unzip((file: UnzipFile) => {
const name = file.name.replace(/\\/g, '/');
if (!ACCEPTED.test(name) || name.split('/').some((p) => p === '..' || p === '.' || p === '')) return;
const target = path.join(dir, ...name.split('/'));
fs.mkdirSync(path.dirname(target), { recursive: true });
const fd = fs.openSync(target, 'w');
file.ondata = (err, chunk, final) => {
if (failure) return;
if (err) failure = err;
else {
total += chunk.length;
if (total > maxBytes) failure = badRequest('The backup is larger than this server accepts.');
else fs.writeSync(fd, chunk);
}
if (final || failure) {
fs.closeSync(fd);
if (final && !failure) written.push(name);
}
};
file.start();
});
unzip.register(UnzipInflate);
for await (const chunk of fs.createReadStream(zipFile, { highWaterMark: 1 << 20 })) {
unzip.push(chunk as Uint8Array);
if (failure) break;
}
if (!failure) unzip.push(new Uint8Array(0), true);
if (failure) throw failure;
return written;
}
/** The 32-byte key in a backup's secret.key, or null when missing or damaged. */
function readKey(file: string): Buffer | null {
try {
const raw = Buffer.from(fs.readFileSync(file, 'utf8').trim(), 'base64');
return raw.length === 32 ? raw : null;
} catch {
return null;
}
}
/**
* Re-encrypts every sealed value in the staged database (wherever it is stored: provider
* columns, settings JSON) from the backup's key to this server's key. Values this server can
* already read are left as they are.
*/
function rekey(db: DB, from: Buffer | null, to: Buffer): { moved: number; unreadable: number } {
let moved = 0;
let unreadable = 0;
const tables = db.all<{ name: string; sql: string | null }>("SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'");
const virtual = tables.filter((t) => /^CREATE VIRTUAL TABLE/i.test(t.sql ?? '')).map((t) => t.name);
for (const t of tables) {
if (virtual.includes(t.name) || virtual.some((v) => t.name.startsWith(`${v}_`)) || /WITHOUT ROWID/i.test(t.sql ?? '')) continue;
const quoted = `"${t.name.replace(/"/g, '""')}"`;
const columns = db.all<{ name: string; type: string }>(`PRAGMA table_info(${quoted})`).filter((c) => /TEXT|^$/i.test(c.type));
for (const col of columns) {
const c = `"${col.name.replace(/"/g, '""')}"`;
const rows = db.all<{ rid: number; v: string }>(`SELECT rowid AS rid, ${c} AS v FROM ${quoted} WHERE typeof(${c}) = 'text' AND ${c} LIKE '%v1.%'`);
for (const row of rows) {
let changed = false;
const next = row.v.replace(SEALED, (sealed) => {
if (decryptSecret(sealed, to) !== null) return sealed;
const plain = from ? decryptSecret(sealed, from) : null;
if (plain === null) {
unreadable += 1;
return sealed;
}
moved += 1;
changed = true;
return encryptSecret(plain, to);
});
if (changed) db.run(`UPDATE ${quoted} SET ${c} = ? WHERE rowid = ?`, next, row.rid);
}
}
}
return { moved, unreadable };
}
/** Copies the signed-in session into the restored database so the person restoring stays signed in. */
function carrySession(from: DB, to: DB, tokenHash: string | null): void {
if (!tokenHash) return;
const row = from.get<Record<string, unknown>>('SELECT * FROM sessions WHERE token_hash = ?', tokenHash);
if (!row) return;
const known = new Set(to.all<{ name: string }>('PRAGMA table_info(sessions)').map((c) => c.name));
const cols = Object.keys(row).filter((k) => known.has(k));
to.run(`INSERT OR REPLACE INTO sessions (${cols.join(', ')}) VALUES (${cols.map(() => '?').join(', ')})`, ...cols.map((k) => row[k] as string | number | null));
}
export async function restoreFullBackup(app: AppContext, zipFile: string, opts: { sessionHash?: string | null; maxBytes?: number } = {}): Promise<RestoreSummary> {
const { config } = app;
const stage = path.join(config.dataDir, `.restore-${Date.now()}`);
fs.mkdirSync(stage, { recursive: true });
try {
const names = await unpack(zipFile, stage, opts.maxBytes ?? 20 * 1024 ** 3);
if (!names.includes('study.db')) throw badRequest('This is not a Study Desk full backup: it has no study.db. Use "Download backup" to make one.');
const manifest = (() => {
try {
return JSON.parse(fs.readFileSync(path.join(stage, 'manifest.json'), 'utf8')) as { format?: string; createdAt?: string };
} catch {
return null;
}
})();
if (manifest && manifest.format !== 'studydesk-backup') throw badRequest('This zip is not a Study Desk full backup.');
const stagedDb = path.join(stage, 'study.db');
const head = Buffer.alloc(16);
const fd = fs.openSync(stagedDb, 'r');
fs.readSync(fd, head, 0, 16, 0);
fs.closeSync(fd);
if (head.toString('latin1', 0, 15) !== 'SQLite format 3') throw badRequest('The database in this backup is damaged.');
// Upgrade the backup to this version's schema, then prepare it while it is still offline.
const staged = new DB(stagedDb);
try {
migrate(staged);
} catch (e) {
staged.close();
throw badRequest(e instanceof Error && /newer version/.test(e.message) ? 'This backup was made by a newer version of Study Desk. Update this server first.' : 'The database in this backup could not be opened.');
}
let keys: { moved: number; unreadable: number };
const count = (table: string) => staged.get<{ n: number }>(`SELECT COUNT(*) AS n FROM ${table}`)?.n ?? 0;
let counts: Pick<RestoreSummary, 'classes' | 'quizzes' | 'decks' | 'notes' | 'documents' | 'attempts'>;
try {
keys = staged.tx(() => rekey(staged, readKey(path.join(stage, 'secret.key')), config.secretKey));
tidyDrafts(staged);
carrySession(app.db, staged, opts.sessionHash ?? null);
counts = { classes: count('classes'), quizzes: count('quizzes'), decks: count('decks'), notes: count('notes'), documents: count('documents'), attempts: count('attempts') };
} finally {
staged.close();
}
// Point of no return: keep a copy of what is here now, stop work, and swap.
const safety = createSnapshot(app, 'pre-restore');
const stoppedJobs = cancelAllGenerations();
forgetMcpSessions();
app.db.replaceWith(stagedDb);
let files = 0;
for (const [sub, target] of [
['uploads', config.uploadsDir],
['media', config.mediaDir],
] as const) {
const from = path.join(stage, sub);
if (!fs.existsSync(from)) continue;
fs.mkdirSync(target, { recursive: true });
for (const f of fs.readdirSync(from)) {
const dest = path.join(target, f);
fs.rmSync(dest, { force: true });
fs.renameSync(path.join(from, f), dest);
files += 1;
}
}
resetVectorCache();
resumeQueue(app);
scheduleIndexing(app);
warmMcpServers(app);
for (const topic of ['library', 'drafts', 'documents'] as const) notifyChange(topic);
app.log('info', `Full backup restored (${counts.classes} classes, ${files} files); previous database kept as ${safety.name}`);
return { backupCreatedAt: manifest?.createdAt ?? null, ...counts, files, keysMoved: keys.moved, keysUnreadable: keys.unreadable, safetySnapshot: safety.name, stoppedJobs };
} finally {
try {
fs.rmSync(stage, { recursive: true, force: true });
} catch (e) {
app.log('warn', `Could not remove the restore staging folder ${stage}`, e instanceof Error ? e.message : e);
}
}
}

View file

@ -251,6 +251,12 @@ interface VectorCache {
}
let cache: VectorCache | null = null;
/** Drops the in-memory vectors (after a restore replaced the database). */
export function resetVectorCache(): void {
cache = null;
queryCache.clear();
}
function loadVectors(db: DB, key: string): VectorCache | null {
const sig = db.get<{ n: number; m: number | null }>('SELECT COUNT(*) AS n, MAX(chunk_id) AS m FROM chunk_vectors WHERE model_key = ?', key)!;
if (sig.n === 0) return null;