From b93c840ee29b782cfc5d487a854241cee863f7f9 Mon Sep 17 00:00:00 2001 From: Yasindu20 Date: Wed, 21 Jan 2026 10:16:54 +0530 Subject: [PATCH] Add optional S3-based storage --- src/db/db.ts | 21 ++++++++---- src/pages/download.tsx | 51 ++++++++++-------------------- src/pages/upload.tsx | 25 +++++++++++---- src/storage/LocalStorageAdapter.ts | 28 ++++++++++++++++ src/storage/S3StorageAdapter.ts | 36 +++++++++++++++++++++ src/storage/index.ts | 20 ++++++++++++ 6 files changed, 135 insertions(+), 46 deletions(-) create mode 100644 src/storage/LocalStorageAdapter.ts create mode 100644 src/storage/S3StorageAdapter.ts create mode 100644 src/storage/index.ts diff --git a/src/db/db.ts b/src/db/db.ts index de57268..a8478ee 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -27,14 +27,23 @@ CREATE TABLE IF NOT EXISTS jobs ( num_files INTEGER DEFAULT 0, FOREIGN KEY (user_id) REFERENCES users(id) ); -PRAGMA user_version = 1;`); +CREATE TABLE IF NOT EXISTS storage_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + job_id INTEGER NOT NULL, + file_name TEXT NOT NULL, + storage_key TEXT NOT NULL, + FOREIGN KEY (job_id) REFERENCES jobs(id), + FOREIGN KEY (user_id) REFERENCES users(id) +); +PRAGMA user_version = 2;`); } -const dbVersion = (db.query("PRAGMA user_version").get() as { user_version?: number }).user_version; -if (dbVersion === 0) { - db.exec("ALTER TABLE file_names ADD COLUMN status TEXT DEFAULT 'not started';"); - db.exec("PRAGMA user_version = 1;"); - console.log("Updated database to version 1."); +const dbVersion = (db.query("PRAGMA user_version").get() as { user_version?: number }).user_version!; +if (dbVersion < 2) { + db.exec("ALTER TABLE file_names ADD COLUMN storage_key TEXT;"); + db.exec("PRAGMA user_version = 2;"); + console.log("Updated database to version 2."); } // enable WAL mode diff --git a/src/pages/download.tsx b/src/pages/download.tsx index f10fec4..470f5e1 100644 --- a/src/pages/download.tsx +++ b/src/pages/download.tsx @@ -1,18 +1,15 @@ -import path from "node:path"; import { Elysia } from "elysia"; import sanitize from "sanitize-filename"; -import * as tar from "tar"; -import { outputDir } from ".."; import db from "../db/db"; import { WEBROOT } from "../helpers/env"; import { userService } from "./user"; +import { getStorage } from "../storage/index"; export const download = new Elysia() .use(userService) .get( "/download/:userId/:jobId/:fileName", async ({ params, redirect, user }) => { - const userId = user.id; const job = await db .query("SELECT * FROM jobs WHERE user_id = ? AND id = ?") .get(user.id, params.jobId); @@ -20,44 +17,30 @@ export const download = new Elysia() if (!job) { return redirect(`${WEBROOT}/results`, 302); } - // parse from URL encoded string - const jobId = decodeURIComponent(params.jobId); + const fileName = sanitize(decodeURIComponent(params.fileName)); - const filePath = `${outputDir}${userId}/${jobId}/${fileName}`; - return Bun.file(filePath); - }, - { - auth: true, - }, - ) - .get( - "/archive/:jobId", - async ({ params, redirect, user }) => { - const userId = user.id; - const job = await db - .query("SELECT * FROM jobs WHERE user_id = ? AND id = ?") - .get(user.id, params.jobId); + const fileRow = db + .query(` + SELECT storage_key FROM file_names + WHERE job_id = ? AND file_name = ? + `, + ) + .get(params.jobId, fileName) as { storage_key: string } | undefined; - if (!job) { + if (!fileRow) { return redirect(`${WEBROOT}/results`, 302); } - const jobId = decodeURIComponent(params.jobId); - const outputPath = `${outputDir}${userId}/${jobId}`; - const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`); + const storage = getStorage(); + const fileBuffer = await storage.get(fileRow.storage_key); - await tar.create( - { - file: outputTar, - cwd: outputPath, - filter: (path) => { - return !path.match(".*\\.tar"); - }, + return new Response(fileBuffer, { + headers: { + "Content-Type": "application/octet-stream", + "Content-Disposition": `attachment; filename="${fileName}"`, }, - ["."], - ); - return Bun.file(outputTar); + }); }, { auth: true, diff --git a/src/pages/upload.tsx b/src/pages/upload.tsx index 0c00009..bb488f1 100644 --- a/src/pages/upload.tsx +++ b/src/pages/upload.tsx @@ -1,9 +1,10 @@ import { Elysia, t } from "elysia"; import db from "../db/db"; import { WEBROOT } from "../helpers/env"; -import { uploadsDir } from "../index"; import { userService } from "./user"; import sanitize from "sanitize-filename"; +import { getStorage } from "../storage"; +import crypto from "node:crypto"; export const upload = new Elysia().use(userService).post( "/upload", @@ -12,6 +13,8 @@ export const upload = new Elysia().use(userService).post( return redirect(`${WEBROOT}/`, 302); } + const jobIdValue = jobId.value; + const existingJob = await db .query("SELECT * FROM jobs WHERE id = ? AND user_id = ?") .get(jobId.value, user.id); @@ -20,17 +23,27 @@ export const upload = new Elysia().use(userService).post( return redirect(`${WEBROOT}/`, 302); } - const userUploadsDir = `${uploadsDir}${user.id}/${jobId.value}/`; + const storage = getStorage(); + + const saveFile = async (file: File) => { + const sanitizedFileName = sanitize(file.name); + const storageKey = `${user.id}/${jobId.value}/${crypto.randomUUID()}`; + const buffer = Buffer.from(await file.arrayBuffer()); + await storage.save(storageKey, buffer); + + db.query(` + INSERT INTO file_names (job_id, file_name, storage_key) + VALUES (?, ?, ?) + `).run(jobIdValue, sanitizedFileName, storageKey); + }; if (body?.file) { if (Array.isArray(body.file)) { for (const file of body.file) { - const santizedFileName = sanitize(file.name); - await Bun.write(`${userUploadsDir}${santizedFileName}`, file); + await saveFile(file); } } else { - const santizedFileName = sanitize(body.file["name"]); - await Bun.write(`${userUploadsDir}${santizedFileName}`, body.file); + await saveFile(body.file);; } } diff --git a/src/storage/LocalStorageAdapter.ts b/src/storage/LocalStorageAdapter.ts new file mode 100644 index 0000000..a640e6a --- /dev/null +++ b/src/storage/LocalStorageAdapter.ts @@ -0,0 +1,28 @@ +import { IStorageAdapter } from "./index"; +import { promises as fs } from "fs"; +import path from "path"; + +export class LocalStorageAdapter implements IStorageAdapter { + baseDir: string; + + constructor(baseDir: string) { + this.baseDir = baseDir; + } + + async save(key: string, data: Buffer): Promise { + const fullPath = path.join(this.baseDir, key); + await fs.mkdir(path.dirname(fullPath), { recursive: true }); + await fs.writeFile(fullPath, data); + return key; + } + + async get(key: string): Promise { + const fullPath = path.join(this.baseDir, key); + return fs.readFile(fullPath); + } + + async delete(key: string): Promise { + const fullPath = path.join(this.baseDir, key); + await fs.unlink(fullPath); + } +} \ No newline at end of file diff --git a/src/storage/S3StorageAdapter.ts b/src/storage/S3StorageAdapter.ts new file mode 100644 index 0000000..5e66b97 --- /dev/null +++ b/src/storage/S3StorageAdapter.ts @@ -0,0 +1,36 @@ +import { s3, S3File } from "bun"; +import { IStorageAdapter } from "./index"; + +export class S3StorageAdapter implements IStorageAdapter { + private bucket: string; + + constructor(bucket: string) { + this.bucket = bucket; + } + + async save(key: string, data: Buffer): Promise { + const file: S3File = s3.file(key, { + bucket: this.bucket, + acl: "private", + }); + + await file.write(data); + return key; + } + + async get(key: string): Promise { + const file: S3File = s3.file(key, { + bucket: this.bucket, + }); + + return Buffer.from(await file.bytes()); + } + + async delete(key: string): Promise { + const file: S3File = s3.file(key, { + bucket: this.bucket, + }) + + await file.delete(); + } +} \ No newline at end of file diff --git a/src/storage/index.ts b/src/storage/index.ts new file mode 100644 index 0000000..e822184 --- /dev/null +++ b/src/storage/index.ts @@ -0,0 +1,20 @@ +import { LocalStorageAdapter } from "./LocalStorageAdapter"; +import { S3StorageAdapter } from "./S3StorageAdapter"; + +export interface IStorageAdapter { + save(key: string, data: Buffer): Promise; + get(key: string): Promise; + delete(key: string): Promise; +} + +export function getStorage(): IStorageAdapter { + if (process.env.STORAGE_BACKEND === "s3") { + if (!process.env.S3_BUCKET) { + throw new Error("S3_BUCKET must be set when STORAGE_BACKEND=s3"); + } + + return new S3StorageAdapter(process.env.S3_BUCKET); + } + + return new LocalStorageAdapter("./data"); +} \ No newline at end of file