From 631314dea8e472c50fc02ac2a42599cb4b5bfe88 Mon Sep 17 00:00:00 2001 From: Yasindu20 Date: Thu, 22 Jan 2026 12:25:21 +0530 Subject: [PATCH] change the buffer to stream --- src/db/db.ts | 51 +++++++++++++++++----- src/pages/download.tsx | 68 +++++++++++++++++++++++++++--- src/storage/LocalStorageAdapter.ts | 12 +++++- src/storage/S3StorageAdapter.ts | 10 ++++- src/storage/index.ts | 17 +++++--- 5 files changed, 134 insertions(+), 24 deletions(-) diff --git a/src/db/db.ts b/src/db/db.ts index a8478ee..bc70be3 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -4,8 +4,21 @@ import { Database } from "bun:sqlite"; mkdirSync("./data", { recursive: true }); const db = new Database("./data/mydb.sqlite", { create: true }); -if (!db.query("SELECT * FROM sqlite_master WHERE type='table'").get()) { - db.exec(` +function getTableInfo(tableName: string) { + try { + return db.query(`PRAGMA table_info('${tableName}')`).all() as Array<{ name: string}>; + } catch (error) { + console.error(`Error getting table info for ${tableName}:`, error) + return []; + } +} + +function hasColumn(tableName: string, columnName: string) { + const info = getTableInfo(tableName); + return info.some((c) => c.name === columnName); +} + +db.exec(` CREATE TABLE IF NOT EXISTS users ( id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL, @@ -17,6 +30,7 @@ CREATE TABLE IF NOT EXISTS file_names ( file_name TEXT NOT NULL, output_file_name TEXT NOT NULL, status TEXT DEFAULT 'not started', + storage_key TEXT, -- v2 column FOREIGN KEY (job_id) REFERENCES jobs(id) ); CREATE TABLE IF NOT EXISTS jobs ( @@ -27,6 +41,7 @@ CREATE TABLE IF NOT EXISTS jobs ( num_files INTEGER DEFAULT 0, FOREIGN KEY (user_id) REFERENCES users(id) ); +-- Ensure storage_metadata exists for legacy DBs CREATE TABLE IF NOT EXISTS storage_metadata ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id INTEGER NOT NULL, @@ -36,17 +51,33 @@ CREATE TABLE IF NOT EXISTS storage_metadata ( FOREIGN KEY (job_id) REFERENCES jobs(id), FOREIGN KEY (user_id) REFERENCES users(id) ); -PRAGMA user_version = 2;`); -} +`); -const dbVersion = (db.query("PRAGMA user_version").get() as { user_version?: number }).user_version!; -if (dbVersion < 2) { - db.exec("ALTER TABLE file_names ADD COLUMN storage_key TEXT;"); - db.exec("PRAGMA user_version = 2;"); - console.log("Updated database to version 2."); +try { + if (!hasColumn("file_names", "status")) { + db.exec("ALTER TABLE file_names ADD COLUMN status TEXT DEFAULT 'not started';"); + console.log("Added column file_names.status"); + } + + if (!hasColumn("file_names", "storage_key")) { + db.exec("ALTER TABLE file_names ADD COLUMN storage_key TEXT;"); + console.log("Added column file_names.storage_key"); + } + + const currentVersion = (db.query("PRAGMA user_version").get() as { user_version?: number}).user_version ?? 0; + if (currentVersion < 2) { + db.exec("PRAGMA user_version = 2;"); + console.log(`Updated database to version 2 (was ${currentVersion}).`); + } +} catch (error) { + console.error("Error running migrations: ", error); } // enable WAL mode -db.exec("PRAGMA journal_mode = WAL;"); +try { + db.exec("PRAGMA journal_mode = WAL;"); +} catch (error) { + console.warn("Could not enable WAL mode: ", error); +} export default db; diff --git a/src/pages/download.tsx b/src/pages/download.tsx index 470f5e1..2d513a8 100644 --- a/src/pages/download.tsx +++ b/src/pages/download.tsx @@ -3,7 +3,10 @@ import sanitize from "sanitize-filename"; import db from "../db/db"; import { WEBROOT } from "../helpers/env"; import { userService } from "./user"; -import { getStorage } from "../storage/index"; +import { getStorage, getStorageType } from "../storage/index"; +import { outputDir } from ".."; +import path from "path"; +import * as tar from "tar"; export const download = new Elysia() .use(userService) @@ -22,27 +25,78 @@ export const download = new Elysia() const fileRow = db .query(` - SELECT storage_key FROM file_names + SELECT storage_key, file_name + FROM file_names WHERE job_id = ? AND file_name = ? `, ) - .get(params.jobId, fileName) as { storage_key: string } | undefined; - + .get(params.jobId, fileName) as { storage_key?: string; file_name?: string } | undefined; if (!fileRow) { return redirect(`${WEBROOT}/results`, 302); } const storage = getStorage(); - const fileBuffer = await storage.get(fileRow.storage_key); + const stream = await storage.getStream(fileRow.storage_key!); - return new Response(fileBuffer, { + return new Response(stream, { headers: { "Content-Type": "application/octet-stream", - "Content-Disposition": `attachment; filename="${fileName}"`, + "Content-Disposition": `attachment; filename="${fileRow.file_name ?? fileName}"`, }, }); }, { auth: true, }, + ) + + .get( + "/archive/:jobId", + async ({ params, redirect, user }) => { + const job = await db + .query("SELECT * FROM jobs WHERE user_id = ? AND id = ?") + .get(user.id, params.jobId); + + if(!job) { + return redirect(`${WEBROOT}/results`, 302); + } + + const storageType = getStorageType(); + if (storageType === "local") { + const userId = user.id; + const jobId = decodeURIComponent(params.jobId); + const outputPath = `${outputDir}${userId}/${jobId}`; + const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`); + + await tar.create( + { + file: outputTar, + cwd: outputPath, + filter: (path) => { + return !path.match(".*\\.tar"); + }, + }, + ["."], + ); + + return Bun.file(outputTar); + } + + return new Response( + JSON.stringify({ + ok: false, + message: + "Archive download is not supported when object storage is enabled. This is intentional it avoid 404s - please use per-file downloads or request a follow-up for server-side archiving", + }), + { + status: 501, + headers: { + "Content-Type": "application/json", + }, + }, + ); + }, + { + auth: true, + }, ); diff --git a/src/storage/LocalStorageAdapter.ts b/src/storage/LocalStorageAdapter.ts index a640e6a..f849fa1 100644 --- a/src/storage/LocalStorageAdapter.ts +++ b/src/storage/LocalStorageAdapter.ts @@ -23,6 +23,16 @@ export class LocalStorageAdapter implements IStorageAdapter { async delete(key: string): Promise { const fullPath = path.join(this.baseDir, key); - await fs.unlink(fullPath); + try { + await fs.unlink(fullPath); + } catch { + //ignore error if file does not exist + } + } + + getStream(key: string): ReadableStream { + const fullPath = path.join(this.baseDir, key); + const file = Bun.file(fullPath); + return file.stream(); } } \ No newline at end of file diff --git a/src/storage/S3StorageAdapter.ts b/src/storage/S3StorageAdapter.ts index 5e66b97..a3fd916 100644 --- a/src/storage/S3StorageAdapter.ts +++ b/src/storage/S3StorageAdapter.ts @@ -23,7 +23,8 @@ export class S3StorageAdapter implements IStorageAdapter { bucket: this.bucket, }); - return Buffer.from(await file.bytes()); + const buf = await file.bytes(); + return Buffer.from(buf); } async delete(key: string): Promise { @@ -33,4 +34,11 @@ export class S3StorageAdapter implements IStorageAdapter { await file.delete(); } + + getStream(key: string): ReadableStream { + const file: S3File = s3.file(key, { + bucket: this.bucket + }); + return file.stream(); + } } \ No newline at end of file diff --git a/src/storage/index.ts b/src/storage/index.ts index e822184..9803768 100644 --- a/src/storage/index.ts +++ b/src/storage/index.ts @@ -5,16 +5,23 @@ export interface IStorageAdapter { save(key: string, data: Buffer): Promise; get(key: string): Promise; delete(key: string): Promise; + getStream(key: string): ReadableStream; +} + +export function getStorageType(): "local" | "s3" { + return process.env.STORAGE_TYPE === "s3" ? "s3" : "local"; } export function getStorage(): IStorageAdapter { - if (process.env.STORAGE_BACKEND === "s3") { - if (!process.env.S3_BUCKET) { - throw new Error("S3_BUCKET must be set when STORAGE_BACKEND=s3"); + if (getStorageType() === "s3") { + const bucket = process.env.S3_BUCKET_NAME; + if (!bucket) { + throw new Error("S3_BUCKET_NAME must be set when STORAGE_TYPE=s3"); } - return new S3StorageAdapter(process.env.S3_BUCKET); + return new S3StorageAdapter(bucket); } - return new LocalStorageAdapter("./data"); + const baseDir = process.env.LOCAL_STORAGE_PATH || "./data/storage"; + return new LocalStorageAdapter(baseDir); } \ No newline at end of file