change the buffer to stream

src/db/db.ts

@@ -4,7 +4,20 @@ import { Database } from "bun:sqlite";
 mkdirSync("./data", { recursive: true });
 const db = new Database("./data/mydb.sqlite", { create: true });
 
-if (!db.query("SELECT * FROM sqlite_master WHERE type='table'").get()) {
+function getTableInfo(tableName: string) {
+  try {
+    return db.query(`PRAGMA table_info('${tableName}')`).all() as Array<{ name: string}>;
+  } catch (error) {
+    console.error(`Error getting table info for ${tableName}:`, error)
+    return [];
+  }
+}
+
+function hasColumn(tableName: string, columnName: string) {
+  const info = getTableInfo(tableName);
+  return info.some((c) => c.name === columnName);
+}
+
 db.exec(`
 CREATE TABLE IF NOT EXISTS users (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -17,6 +30,7 @@ CREATE TABLE IF NOT EXISTS file_names (
   file_name TEXT NOT NULL,
   output_file_name TEXT NOT NULL,
   status TEXT DEFAULT 'not started',
+  storage_key TEXT, -- v2 column
   FOREIGN KEY (job_id) REFERENCES jobs(id)
 );
 CREATE TABLE IF NOT EXISTS jobs (
@@ -27,6 +41,7 @@ CREATE TABLE IF NOT EXISTS jobs (
   num_files INTEGER DEFAULT 0,
   FOREIGN KEY (user_id) REFERENCES users(id)
 );
+-- Ensure storage_metadata exists for legacy DBs
 CREATE TABLE IF NOT EXISTS storage_metadata (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   user_id INTEGER NOT NULL,
@@ -36,17 +51,33 @@ CREATE TABLE IF NOT EXISTS storage_metadata (
   FOREIGN KEY (job_id) REFERENCES jobs(id),
   FOREIGN KEY (user_id) REFERENCES users(id)
 );
-PRAGMA user_version = 2;`);
+`);
+
+try {
+  if (!hasColumn("file_names", "status")) {
+    db.exec("ALTER TABLE file_names ADD COLUMN status TEXT DEFAULT 'not started';");
+    console.log("Added column file_names.status");
   }
 
-const dbVersion = (db.query("PRAGMA user_version").get() as { user_version?: number }).user_version!;
+  if (!hasColumn("file_names", "storage_key")) {
-if (dbVersion < 2) {
     db.exec("ALTER TABLE file_names ADD COLUMN storage_key TEXT;");
+    console.log("Added column file_names.storage_key");
+  }
+
+  const currentVersion = (db.query("PRAGMA user_version").get() as { user_version?: number}).user_version ?? 0;
+  if (currentVersion < 2) {
     db.exec("PRAGMA user_version = 2;");
-  console.log("Updated database to version 2.");
+    console.log(`Updated database to version 2 (was ${currentVersion}).`);
+  }
+} catch (error) {
+  console.error("Error running migrations: ", error);
 }
 
 // enable WAL mode
+try {
   db.exec("PRAGMA journal_mode = WAL;");
+} catch (error) {
+  console.warn("Could not enable WAL mode: ", error);
+}
 
 export default db;

src/pages/download.tsx

@@ -3,7 +3,10 @@ import sanitize from "sanitize-filename";
 import db from "../db/db";
 import { WEBROOT } from "../helpers/env";
 import { userService } from "./user";
-import { getStorage } from "../storage/index";
+import { getStorage, getStorageType } from "../storage/index";
+import { outputDir } from "..";
+import path from "path";
+import * as tar from "tar";
 
 export const download = new Elysia()
   .use(userService)
@@ -22,27 +25,78 @@ export const download = new Elysia()
 
       const fileRow = db
         .query(`
-          SELECT storage_key FROM file_names
+          SELECT storage_key, file_name 
+          FROM file_names
           WHERE job_id = ? AND file_name = ?
         `,
       )
-      .get(params.jobId, fileName) as { storage_key: string } | undefined;
+      .get(params.jobId, fileName) as { storage_key?: string; file_name?: string } | undefined;
-
       if (!fileRow) {
         return redirect(`${WEBROOT}/results`, 302);
       }
 
       const storage = getStorage();
-      const fileBuffer = await storage.get(fileRow.storage_key);
+      const stream = await storage.getStream(fileRow.storage_key!);
 
-      return new Response(fileBuffer, {
+      return new Response(stream, {
         headers: {
           "Content-Type": "application/octet-stream",
-          "Content-Disposition": `attachment; filename="${fileName}"`,
+          "Content-Disposition": `attachment; filename="${fileRow.file_name ?? fileName}"`,
         },
       });
     },
     {
       auth: true,
     },
+  )
+
+  .get(
+    "/archive/:jobId",
+    async ({ params, redirect, user }) => {
+      const job = await db
+        .query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
+        .get(user.id, params.jobId);
+
+      if(!job) {
+        return redirect(`${WEBROOT}/results`, 302);
+      }
+
+      const storageType = getStorageType();
+      if (storageType === "local") {
+        const userId = user.id;
+        const jobId = decodeURIComponent(params.jobId);
+        const outputPath = `${outputDir}${userId}/${jobId}`;
+        const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`);
+
+        await tar.create(
+          {
+            file: outputTar,
+            cwd: outputPath,
+            filter: (path) => {
+              return !path.match(".*\\.tar");
+            },
+          },
+          ["."],
+        );
+
+        return Bun.file(outputTar);
+      }
+
+      return new Response(
+        JSON.stringify({
+          ok: false,
+          message:
+            "Archive download is not supported when object storage is enabled. This is intentional it avoid 404s - please use per-file downloads or request a follow-up for server-side archiving",
+        }),
+        {
+          status: 501,
+          headers: {
+            "Content-Type": "application/json",
+          },
+        },
+      );
+    },
+    {
+      auth: true,
+    },
   );

src/storage/LocalStorageAdapter.ts

@@ -23,6 +23,16 @@ export class LocalStorageAdapter implements IStorageAdapter {
 
     async delete(key: string): Promise<void> {
         const fullPath = path.join(this.baseDir, key);
+        try {
             await fs.unlink(fullPath);
+        } catch {
+            //ignore error if file does not exist
+        }
+    }
+
+    getStream(key: string): ReadableStream<Uint8Array> {
+        const fullPath = path.join(this.baseDir, key);
+        const file = Bun.file(fullPath);
+        return file.stream();
     }
 }

src/storage/S3StorageAdapter.ts

@@ -23,7 +23,8 @@ export class S3StorageAdapter implements IStorageAdapter {
             bucket: this.bucket,
         });
 
-        return Buffer.from(await file.bytes());
+        const buf = await file.bytes();
+        return Buffer.from(buf);
     }
 
     async delete(key: string): Promise<void> {
@@ -33,4 +34,11 @@ export class S3StorageAdapter implements IStorageAdapter {
 
         await file.delete();
     }
+
+    getStream(key: string): ReadableStream<Uint8Array> {
+        const file: S3File = s3.file(key, {
+            bucket: this.bucket 
+        });
+        return file.stream();
+    }
 }

src/storage/index.ts

@@ -5,16 +5,23 @@ export interface IStorageAdapter {
     save(key: string, data: Buffer): Promise<string>;
     get(key: string): Promise<Buffer>;
     delete(key: string): Promise<void>;
+    getStream(key: string): ReadableStream<Uint8Array>;
+}
+
+export function getStorageType(): "local" | "s3" {
+    return process.env.STORAGE_TYPE === "s3" ? "s3" : "local";
 }
 
 export function getStorage(): IStorageAdapter {
-    if (process.env.STORAGE_BACKEND === "s3") {
+    if (getStorageType() === "s3") {
-        if (!process.env.S3_BUCKET) {
+        const bucket = process.env.S3_BUCKET_NAME;
-            throw new Error("S3_BUCKET must be set when STORAGE_BACKEND=s3");
+        if (!bucket) {
+            throw new Error("S3_BUCKET_NAME must be set when STORAGE_TYPE=s3");
         }
 
-        return new S3StorageAdapter(process.env.S3_BUCKET);
+        return new S3StorageAdapter(bucket);
     }
     
-    return new LocalStorageAdapter("./data");
+    const baseDir = process.env.LOCAL_STORAGE_PATH || "./data/storage";
+    return new LocalStorageAdapter(baseDir);
 }