Elijah
72b1de2937
All checks were successful
Automated Container Build / build-and-push (push) Successful in 1m12s
371 lines
10 KiB
Go
371 lines
10 KiB
Go
package workers
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.elijahkuntz.com/Elijah/drive/config"
|
|
"git.elijahkuntz.com/Elijah/drive/database"
|
|
"github.com/disintegration/imaging"
|
|
"git.elijahkuntz.com/Elijah/drive/middleware"
|
|
"net/url"
|
|
"io"
|
|
"net/http"
|
|
"encoding/json"
|
|
"bytes"
|
|
"github.com/golang-jwt/jwt/v5"
|
|
)
|
|
|
|
type ThumbnailJob struct {
|
|
FilePath string
|
|
Checksum string
|
|
MimeType string
|
|
}
|
|
|
|
type ThumbnailManager struct {
|
|
Queue chan ThumbnailJob
|
|
Config *config.Config
|
|
DB *database.DB
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
var (
|
|
Instance *ThumbnailManager
|
|
once sync.Once
|
|
)
|
|
|
|
func InitThumbnailManager(cfg *config.Config, db *database.DB) {
|
|
once.Do(func() {
|
|
Instance = &ThumbnailManager{
|
|
Queue: make(chan ThumbnailJob, 1000),
|
|
Config: cfg,
|
|
DB: db,
|
|
}
|
|
Instance.Start(2) // Run 2 concurrent workers
|
|
go Instance.ScanMissingThumbnails()
|
|
})
|
|
}
|
|
|
|
func (m *ThumbnailManager) ScanMissingThumbnails() {
|
|
time.Sleep(5 * time.Second) // wait for server to settle
|
|
log.Println("Scanning for missing thumbnails...")
|
|
rows, err := m.DB.Query("SELECT path, checksum FROM files WHERE checksum IS NOT NULL AND checksum != ''")
|
|
if err != nil {
|
|
log.Printf("Failed to query files for missing thumbnails: %v\n", err)
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
|
|
count := 0
|
|
for rows.Next() {
|
|
var path, checksum string
|
|
if err := rows.Scan(&path, &checksum); err != nil {
|
|
continue
|
|
}
|
|
|
|
destPath := filepath.Join(m.Config.ThumbnailDir, checksum+".jpg")
|
|
if _, err := os.Stat(destPath); err == nil {
|
|
continue // Thumbnail already exists
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
var mimeType string
|
|
if ext == ".jpg" || ext == ".png" || ext == ".gif" || ext == ".webp" || ext == ".svg" || ext == ".jpeg" {
|
|
mimeType = "image/"
|
|
} else if ext == ".mp4" || ext == ".webm" || ext == ".ogg" || ext == ".mov" || ext == ".mkv" || ext == ".avi" {
|
|
mimeType = "video/"
|
|
} else if ext == ".mp3" || ext == ".flac" || ext == ".wav" || ext == ".m4a" || ext == ".aac" {
|
|
mimeType = "audio/"
|
|
} else if ext == ".docx" || ext == ".pptx" || ext == ".xlsx" || ext == ".pdf" {
|
|
mimeType = "application/"
|
|
} else {
|
|
continue
|
|
}
|
|
|
|
job := ThumbnailJob{
|
|
FilePath: path,
|
|
Checksum: checksum,
|
|
MimeType: mimeType,
|
|
}
|
|
|
|
// Use blocking send so we don't drop jobs, but it throttles the scanner
|
|
m.Queue <- job
|
|
count++
|
|
}
|
|
|
|
if count > 0 {
|
|
log.Printf("Enqueued %d missing thumbnails for generation\n", count)
|
|
} else {
|
|
log.Println("No missing thumbnails found.")
|
|
}
|
|
}
|
|
|
|
func (m *ThumbnailManager) Start(workers int) {
|
|
for i := 0; i < workers; i++ {
|
|
m.wg.Add(1)
|
|
go m.worker(i)
|
|
}
|
|
}
|
|
|
|
func (m *ThumbnailManager) Stop() {
|
|
close(m.Queue)
|
|
m.wg.Wait()
|
|
}
|
|
|
|
func (m *ThumbnailManager) Enqueue(job ThumbnailJob) {
|
|
if job.Checksum == "" {
|
|
return // Cannot store thumbnail without checksum
|
|
}
|
|
select {
|
|
case m.Queue <- job:
|
|
default:
|
|
log.Println("Thumbnail queue is full, dropping job for", job.FilePath)
|
|
}
|
|
}
|
|
|
|
func (m *ThumbnailManager) worker(id int) {
|
|
defer m.wg.Done()
|
|
for job := range m.Queue {
|
|
destPath := filepath.Join(m.Config.ThumbnailDir, job.Checksum+".jpg")
|
|
|
|
// Skip if thumbnail already exists
|
|
if _, err := os.Stat(destPath); err == nil {
|
|
continue
|
|
}
|
|
|
|
fullSourcePath := filepath.Join(m.Config.StorageDir, filepath.FromSlash(job.FilePath))
|
|
|
|
var err error
|
|
if strings.HasPrefix(job.MimeType, "image/") {
|
|
err = generateImageThumbnail(fullSourcePath, destPath)
|
|
} else if strings.HasPrefix(job.MimeType, "video/") {
|
|
err = generateVideoThumbnail(fullSourcePath, destPath)
|
|
} else if strings.HasPrefix(job.MimeType, "audio/") {
|
|
err = generateAudioThumbnail(fullSourcePath, destPath)
|
|
} else if strings.HasSuffix(job.FilePath, ".docx") || strings.HasSuffix(job.FilePath, ".pptx") || strings.HasSuffix(job.FilePath, ".xlsx") || strings.HasSuffix(strings.ToLower(job.FilePath), ".pdf") {
|
|
err = generateOfficeThumbnail(job.FilePath, destPath, m.Config)
|
|
} else {
|
|
continue
|
|
}
|
|
|
|
if err != nil {
|
|
log.Printf("[Worker %d] Failed to generate thumbnail for %s: %v\n", id, job.FilePath, err)
|
|
} else {
|
|
log.Printf("[Worker %d] Generated thumbnail for %s\n", id, job.FilePath)
|
|
}
|
|
}
|
|
}
|
|
|
|
func generateImageThumbnail(src, dest string) error {
|
|
img, err := imaging.Open(src, imaging.AutoOrientation(true))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Resize to fit within 256x256 while preserving aspect ratio
|
|
thumb := imaging.Fit(img, 256, 256, imaging.Lanczos)
|
|
return imaging.Save(thumb, dest, imaging.JPEGQuality(80))
|
|
}
|
|
|
|
func generateVideoThumbnail(src, dest string) error {
|
|
// Extract a frame at 1 second mark, scale to max width/height 256
|
|
cmd := exec.Command("ffmpeg", "-y", "-i", src, "-ss", "00:00:01.000", "-vframes", "1", "-vf", "scale=256:256:force_original_aspect_ratio=decrease", dest)
|
|
|
|
// Set a timeout to prevent hanging on corrupted videos
|
|
timer := time.AfterFunc(15*time.Second, func() {
|
|
if cmd.Process != nil {
|
|
cmd.Process.Kill()
|
|
}
|
|
})
|
|
|
|
err := cmd.Run()
|
|
timer.Stop()
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("ffmpeg failed: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func generateAudioThumbnail(src, dest string) error {
|
|
// Extract embedded cover art from audio file, scale to 256x256
|
|
cmd := exec.Command("ffmpeg", "-y", "-i", src, "-an", "-vcodec", "mjpeg", "-vframes", "1", "-vf", "scale=256:256:force_original_aspect_ratio=decrease", dest)
|
|
|
|
timer := time.AfterFunc(10*time.Second, func() {
|
|
if cmd.Process != nil {
|
|
cmd.Process.Kill()
|
|
}
|
|
})
|
|
|
|
err := cmd.Run()
|
|
timer.Stop()
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("ffmpeg audio cover extraction failed: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func generateOfficeThumbnail(filePath, destPath string, cfg *config.Config) error {
|
|
// Generate a download token for the Conversion API to fetch the file from our backend
|
|
token, err := middleware.GenerateDownloadToken(0, "system_worker", cfg.JWTSecret, filePath)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to generate download token: %w", err)
|
|
}
|
|
|
|
// Assuming backend runs on port from config, but we need the external URL or internal docker URL that Document Server can reach.
|
|
internalURL := cfg.InternalAPIURL
|
|
fileUrl := fmt.Sprintf("%s/public/onlyoffice/download?path=%s&token=%s", internalURL, url.QueryEscape(filePath), token)
|
|
|
|
ext := strings.TrimPrefix(filepath.Ext(filePath), ".")
|
|
if ext == "" {
|
|
ext = "docx"
|
|
}
|
|
|
|
callConvert := func(payload map[string]interface{}) (string, error) {
|
|
payloadBytes, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
req, err := http.NewRequest("POST", "https://office.elijahkuntz.com/ConvertService.ashx", bytes.NewBuffer(payloadBytes))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Accept", "application/json")
|
|
|
|
if cfg.OnlyOfficeJWT != "" {
|
|
jwtToken := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims(payload))
|
|
signedToken, err := jwtToken.SignedString([]byte(cfg.OnlyOfficeJWT))
|
|
if err == nil {
|
|
req.Header.Set("Authorization", "Bearer "+signedToken)
|
|
}
|
|
}
|
|
|
|
client := &http.Client{Timeout: 30 * time.Second}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("conversion request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("conversion request returned status: %d", resp.StatusCode)
|
|
}
|
|
|
|
var result struct {
|
|
Error *int `json:"error,omitempty"`
|
|
FileUrl string `json:"fileUrl,omitempty"`
|
|
}
|
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
|
return "", fmt.Errorf("failed to decode conversion response: %w", err)
|
|
}
|
|
|
|
if result.Error != nil && *result.Error != 0 {
|
|
return "", fmt.Errorf("conversion service returned error code: %d", *result.Error)
|
|
}
|
|
|
|
if result.FileUrl == "" {
|
|
return "", fmt.Errorf("no fileUrl in conversion response")
|
|
}
|
|
return result.FileUrl, nil
|
|
}
|
|
|
|
var finalImgUrl string
|
|
|
|
if ext == "xlsx" || ext == "xls" {
|
|
pdfPayload := map[string]interface{}{
|
|
"async": false,
|
|
"filetype": ext,
|
|
"key": fmt.Sprintf("thumb_pdf_%d", time.Now().UnixNano()),
|
|
"outputtype": "pdf",
|
|
"title": "preview.pdf",
|
|
"url": fileUrl,
|
|
"spreadsheetLayout": map[string]interface{}{
|
|
"ignorePrintArea": true,
|
|
"printGridlines": true,
|
|
"printHeadings": true,
|
|
"fitToWidth": 0,
|
|
"orientation": "landscape",
|
|
},
|
|
}
|
|
pdfUrl, err := callConvert(pdfPayload)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to convert xlsx to pdf: %w", err)
|
|
}
|
|
|
|
jpgPayload := map[string]interface{}{
|
|
"async": false,
|
|
"filetype": "pdf",
|
|
"key": fmt.Sprintf("thumb_jpg_%d", time.Now().UnixNano()),
|
|
"outputtype": "jpg",
|
|
"title": "preview.jpg",
|
|
"url": pdfUrl,
|
|
"thumbnail": map[string]interface{}{
|
|
"aspect": 1,
|
|
"first": true,
|
|
"width": 1024,
|
|
"height": 1024,
|
|
},
|
|
}
|
|
finalImgUrl, err = callConvert(jpgPayload)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to convert pdf to jpg: %w", err)
|
|
}
|
|
} else {
|
|
payload := map[string]interface{}{
|
|
"async": false,
|
|
"filetype": ext,
|
|
"key": fmt.Sprintf("thumb_%d", time.Now().UnixNano()),
|
|
"outputtype": "jpg",
|
|
"title": "preview.jpg",
|
|
"url": fileUrl,
|
|
"thumbnail": map[string]interface{}{
|
|
"aspect": 1,
|
|
"first": true,
|
|
"width": 1024,
|
|
"height": 1024,
|
|
},
|
|
}
|
|
var err error
|
|
finalImgUrl, err = callConvert(payload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Download the resulting image
|
|
client := &http.Client{Timeout: 30 * time.Second}
|
|
imgResp, err := client.Get(finalImgUrl)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to download converted image: %w", err)
|
|
}
|
|
defer imgResp.Body.Close()
|
|
|
|
if imgResp.StatusCode != http.StatusOK {
|
|
return fmt.Errorf("downloading image returned status: %d", imgResp.StatusCode)
|
|
}
|
|
|
|
// Create temp file for the downloaded image
|
|
tmpFile, err := os.CreateTemp("", "office_thumb_*.jpg")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tmpName := tmpFile.Name()
|
|
defer os.Remove(tmpName)
|
|
|
|
if _, err := io.Copy(tmpFile, imgResp.Body); err != nil {
|
|
tmpFile.Close()
|
|
return err
|
|
}
|
|
tmpFile.Close()
|
|
|
|
// Resize it using imaging to ensure it matches standard thumbnail sizes
|
|
return generateImageThumbnail(tmpName, destPath)
|
|
}
|