Compare commits

...

2 commits

Author SHA1 Message Date
97acce176b Release Kaleido Desktop 0.1.0-beta.5
All checks were successful
Validate Kaleido Desktop / validate (push) Successful in 1m27s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 19:55:53 -07:00
5b6c063ff1 Use nlmeans and FSRCNNX 16 for the Quality upscaling tier
ArtCNN is tuned for animation and looked unnatural on live action, and
its first Direct3D compile took about a minute per model. Quality now
denoises with AN3223's nlmeans (general-purpose profile, any size) and
upscales with FSRCNNX 16-0-4-1, trained on general imagery, at any
enlargement. Both compile in seconds; 1080p in a maximized 1440p window
measured 3.2 ms per frame.

The first-compile notice now covers any shader set not yet compiled on
the machine but only shows after three seconds, and a set with nothing
to verify at the current size clears it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 19:51:47 -07:00
14 changed files with 3949 additions and 7903 deletions

View file

@ -1,5 +1,12 @@
# Changelog # Changelog
## 0.1.0-beta.5
A live-action Quality upscaling tier.
- The Quality tier now uses nlmeans denoising (AN3223, made for live action, and it works at any size) ahead of the larger FSRCNNX 16 upscaler, replacing ArtCNN, which is tuned for animation and looked unnatural on live action. It also compiles in seconds rather than about a minute per model.
- The "Preparing upscaling" notice only appears if a first-time shader compile takes more than a few seconds.
## 0.1.0-beta.4 ## 0.1.0-beta.4
A Quality upscaling tier, with checks that upscaling is really running. A Quality upscaling tier, with checks that upscaling is really running.

View file

@ -72,11 +72,11 @@ transparent only while video is visible (`html.video-active`), so browsing is op
interface stays visible around it, with a matching hole masked out of the page interface stays visible around it, with a matching hole masked out of the page
(`html.player-mini`). (`html.player-mini`).
- **Upscaling** (`player/enhance.ts`): tiers set mpv's scalers and debanding and load - **Upscaling** (`player/enhance.ts`): tiers set mpv's scalers and debanding and load
shaders bundled from `src-tauri/shaders/` (FSRCNNX, ArtCNN, KrigBilateral, AMD CAS); shaders bundled from `src-tauri/shaders/` (FSRCNNX, nlmeans, KrigBilateral, AMD CAS);
each shader's `WHEN` condition decides whether it runs for the current video and size. each shader's `WHEN` condition decides whether it runs for the current video and size.
The chosen tier and the running one are tracked separately: shader errors in mpv's log The chosen tier and the running one are tracked separately: shader errors in mpv's log
or missing network passes in `vo-passes` drop to the tier below. Compiled shaders are or missing network passes in `vo-passes` drop to the tier below. Compiled shaders are
cached in the app cache folder (`gpu-shader-cache-dir`); ArtCNN's first Direct3D cached in the app cache folder (`gpu-shader-cache-dir`), so a slow first Direct3D
compile takes about a minute. compile happens once per shader set, mpv version and GPU driver.
- **Updates**: `app/updater.ts` checks the signed feed on start; installs need the - **Updates**: `app/updater.ts` checks the signed feed on start; installs need the
user's confirmation and then relaunch. user's confirmation and then relaunch.

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{ {
"name": "kaleido-desktop", "name": "kaleido-desktop",
"version": "0.1.0-beta.4", "version": "0.1.0-beta.5",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "kaleido-desktop", "name": "kaleido-desktop",
"version": "0.1.0-beta.4", "version": "0.1.0-beta.5",
"dependencies": { "dependencies": {
"@fontsource-variable/inter": "^5.3.0", "@fontsource-variable/inter": "^5.3.0",
"@tanstack/react-query": "^5.105.0", "@tanstack/react-query": "^5.105.0",

View file

@ -1,7 +1,7 @@
{ {
"name": "kaleido-desktop", "name": "kaleido-desktop",
"private": true, "private": true,
"version": "0.1.0-beta.4", "version": "0.1.0-beta.5",
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "vite", "dev": "vite",

2
src-tauri/Cargo.lock generated
View file

@ -2173,7 +2173,7 @@ dependencies = [
[[package]] [[package]]
name = "kaleido" name = "kaleido"
version = "0.1.0-beta.4" version = "0.1.0-beta.5"
dependencies = [ dependencies = [
"fs2", "fs2",
"futures-util", "futures-util",

View file

@ -1,6 +1,6 @@
[package] [package]
name = "kaleido" name = "kaleido"
version = "0.1.0-beta.4" version = "0.1.0-beta.5"
description = "Kaleido — a Plex client for Windows" description = "Kaleido — a Plex client for Windows"
authors = ["Elijah"] authors = ["Elijah"]
edition = "2021" edition = "2021"

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -1,7 +1,7 @@
{ {
"$schema": "https://schema.tauri.app/config/2", "$schema": "https://schema.tauri.app/config/2",
"productName": "Kaleido", "productName": "Kaleido",
"version": "0.1.0-beta.4", "version": "0.1.0-beta.5",
"identifier": "tv.kaleido.desktop", "identifier": "tv.kaleido.desktop",
"build": { "build": {
"beforeDevCommand": "npm run dev", "beforeDevCommand": "npm run dev",

View file

@ -150,5 +150,5 @@ export const UPSCALING: Record<Upscaling, { label: string; detail: string }> = {
off: { label: 'Off', detail: 'mpv’s standard scaling' }, off: { label: 'Off', detail: 'mpv’s standard scaling' },
performance: { label: 'Performance', detail: 'Sharp scaling, debanding and light sharpening' }, performance: { label: 'Performance', detail: 'Sharp scaling, debanding and light sharpening' },
balanced: { label: 'Balanced', detail: 'Neural upscaling (FSRCNNX), colour reconstruction, debanding and sharpening' }, balanced: { label: 'Balanced', detail: 'Neural upscaling (FSRCNNX), colour reconstruction, debanding and sharpening' },
quality: { label: 'Quality', detail: 'Larger neural upscaler (ArtCNN) with denoising, colour reconstruction, debanding and sharpening. Needs a capable GPU' }, quality: { label: 'Quality', detail: 'nlmeans denoising, larger neural upscaler (FSRCNNX 16), colour reconstruction, debanding and sharpening. Needs a capable GPU' },
}; };

View file

@ -10,7 +10,7 @@ import { win, isNative } from '../platform/native';
import { WindowControls } from '../app/TopBar'; import { WindowControls } from '../app/TopBar';
import { navigate } from '../app/router'; import { navigate } from '../app/router';
import { mpv } from './engine'; import { mpv } from './engine';
import { UPSCALING_ORDER, hasFailed, readDiagnostics, requestEnhancement, toggleBypass, type Diagnostics } from './enhance'; import { PREPARING_NOTICE_MS, UPSCALING_ORDER, hasFailed, readDiagnostics, requestEnhancement, toggleBypass, type Diagnostics } from './enhance';
import { toast } from '../ui/overlays'; import { toast } from '../ui/overlays';
import type { Item } from '../plex/types'; import type { Item } from '../plex/types';
@ -242,6 +242,17 @@ const SUB_BOXES: { value: SubtitleBox; label: string }[] = [
type SettingsView = 'main' | 'audio' | 'subtitles' | 'quality' | 'upscaling' | 'speed' | 'version' | 'chapters' | 'subSize' | 'subBox'; type SettingsView = 'main' | 'audio' | 'subtitles' | 'quality' | 'upscaling' | 'speed' | 'version' | 'chapters' | 'subSize' | 'subBox';
/** True once `flag` has stayed true for `ms` (a quick blip never shows). */
function useLasting(flag: boolean, ms: number) {
const [on, setOn] = useState(false);
useEffect(() => {
if (!flag) return setOn(false);
const t = setTimeout(() => setOn(true), ms);
return () => clearTimeout(t);
}, [flag, ms]);
return on;
}
/** Polls what mpv's renderer is doing while a panel shows it. */ /** Polls what mpv's renderer is doing while a panel shows it. */
function useDiagnostics(enabled: boolean) { function useDiagnostics(enabled: boolean) {
const [d, setD] = useState<Diagnostics | null>(null); const [d, setD] = useState<Diagnostics | null>(null);
@ -322,7 +333,7 @@ function SettingsPanel({ onClose }: { onClose: () => void }) {
<div className="panel-subhead">Stages</div> <div className="panel-subhead">Stages</div>
<Option selected={prefs.enhanceDeband} label="Debanding" detail="Smooths banding in gradients" onClick={() => prefs.set('enhanceDeband', !prefs.enhanceDeband)} /> <Option selected={prefs.enhanceDeband} label="Debanding" detail="Smooths banding in gradients" onClick={() => prefs.set('enhanceDeband', !prefs.enhanceDeband)} />
<Option selected={prefs.enhanceSharpen} label="Sharpening" detail="AMD CAS · SDR only" onClick={() => prefs.set('enhanceSharpen', !prefs.enhanceSharpen)} /> <Option selected={prefs.enhanceSharpen} label="Sharpening" detail="AMD CAS · SDR only" onClick={() => prefs.set('enhanceSharpen', !prefs.enhanceSharpen)} />
<Option selected={prefs.enhanceDenoise} label="Denoising" detail="ArtCNN’s denoising model while upscaling · Quality tier, SDR only" onClick={() => prefs.set('enhanceDenoise', !prefs.enhanceDenoise)} /> <Option selected={prefs.enhanceDenoise} label="Denoising" detail="Non-local means, at any size · Quality tier, SDR only" onClick={() => prefs.set('enhanceDenoise', !prefs.enhanceDenoise)} />
<button className={`panel-compare${st.enhance.bypass ? ' on' : ''}`} onClick={() => toggleBypass()}> <button className={`panel-compare${st.enhance.bypass ? ' on' : ''}`} onClick={() => toggleBypass()}>
{st.enhance.bypass ? 'Turn enhancement back on' : 'Compare with enhancement off'} {st.enhance.bypass ? 'Turn enhancement back on' : 'Compare with enhancement off'}
<kbd>B</kbd> <kbd>B</kbd>
@ -661,6 +672,7 @@ export function PlayerOverlay() {
const { visible, poke } = useAutoHide(st.paused || st.ended, !!panel || scrubbing || st.countdown !== null); const { visible, poke } = useAutoHide(st.paused || st.ended, !!panel || scrubbing || st.countdown !== null);
const clickTimer = useRef<ReturnType<typeof setTimeout> | null>(null); const clickTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const seekStep = useSettings((s) => s.seekStep); const seekStep = useSettings((s) => s.seekStep);
const preparing = useLasting(st.enhance.preparing, PREPARING_NOTICE_MS);
const showFlash = (icon: string) => setFlash({ icon, key: Date.now() }); const showFlash = (icon: string) => setFlash({ icon, key: Date.now() });
@ -806,13 +818,13 @@ export function PlayerOverlay() {
{!st.fullscreen && <WindowControls />} {!st.fullscreen && <WindowControls />}
</div> </div>
{(loading || st.buffering || st.seeking || st.enhance.preparing) && !st.error && ( {(loading || st.buffering || st.seeking || preparing) && !st.error && (
<div className="player-center"> <div className="player-center">
<Spinner size={44} /> <Spinner size={44} />
{st.enhance.preparing && ( {preparing && (
<p className="player-preparing"> <p className="player-preparing">
Preparing {UPSCALING[st.enhance.active].label} upscaling Preparing {UPSCALING[st.enhance.active].label} upscaling
<small>First time on this computer only · about a minute</small> <small>First time on this computer only</small>
</p> </p>
)} )}
</div> </div>

View file

@ -2,10 +2,11 @@
* Video enhancement: upscaling tiers built from mpv's scalers, debanding and * Video enhancement: upscaling tiers built from mpv's scalers, debanding and
* the shaders bundled in src-tauri/shaders. Each shader's WHEN condition * the shaders bundled in src-tauri/shaders. Each shader's WHEN condition
* decides whether it runs for the current video and window size, so one list * decides whether it runs for the current video and window size, so one list
* covers every case: the neural upscalers double luma when the video is * covers every case: FSRCNNX doubles luma when the video is enlarged (the
* enlarged (FSRCNNX above 1.3×, ArtCNN at any enlargement, so 1080p in a * small model above 1.3×, the large one at any enlargement, so 1080p in a
* maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds subsampled colour, * maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds
* CAS sharpens at native size (CAS-prescale also before upscaling). * subsampled colour, CAS sharpens at native size (CAS-prescale also before
* upscaling) and nlmeans denoises at any size.
* *
* The tier the user chose (preferences) and the one actually running (player * The tier the user chose (preferences) and the one actually running (player
* state) are kept apart: a tier whose shaders fail drops to the one below with * state) are kept apart: a tier whose shaders fail drops to the one below with
@ -28,7 +29,7 @@ let shaderDir: Promise<string> | null = null;
const NETWORK_MIN_SCALE: Partial<Record<Upscaling, number>> = { balanced: 1.3, quality: 1.0 }; const NETWORK_MIN_SCALE: Partial<Record<Upscaling, number>> = { balanced: 1.3, quality: 1.0 };
const runsNetwork = async (mode: Upscaling) => mode in NETWORK_MIN_SCALE && (await scaleFactor()) > NETWORK_MIN_SCALE[mode]!; const runsNetwork = async (mode: Upscaling) => mode in NETWORK_MIN_SCALE && (await scaleFactor()) > NETWORK_MIN_SCALE[mode]!;
const NEURAL: Partial<Record<Upscaling, string>> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'ArtCNN_C4F32.glsl' }; const NEURAL: Partial<Record<Upscaling, string>> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'FSRCNNX_x2_16-0-4-1.glsl' };
interface Plan { interface Plan {
props: Record<string, unknown>; props: Record<string, unknown>;
@ -36,10 +37,11 @@ interface Plan {
} }
/** /**
* What a tier runs. Sharpening (CAS) and denoising are tuned for SDR levels, * What a tier runs. Sharpening (CAS) and denoising (nlmeans) are tuned for
* so HDR video skips them; the upscalers and debanding stay. Denoising uses * SDR levels, so HDR video skips them; the upscalers and debanding stay.
* ArtCNN's own denoising model while upscaling: a separate denoiser (nlmeans) * Quality denoises first, so the network enlarges the picture rather than
* took over a minute to compile for Direct3D before the first frame. * the compression noise. Both FSRCNNX models are trained on general imagery,
* which suits live action.
*/ */
function plan(mode: Upscaling, hdr: boolean): Plan { function plan(mode: Upscaling, hdr: boolean): Plan {
const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState(); const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState();
@ -57,7 +59,7 @@ function plan(mode: Upscaling, hdr: boolean): Plan {
break; break;
case 'quality': case 'quality':
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 }; props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 };
shaders = [enhanceDenoise && !hdr ? 'ArtCNN_C4F32_DN.glsl' : NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])]; shaders = [...(enhanceDenoise && !hdr ? ['nlmeans.glsl'] : []), NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
break; break;
default: default:
return { props: { ...scalerDefaults }, shaders: [] }; return { props: { ...scalerDefaults }, shaders: [] };
@ -119,27 +121,29 @@ export async function scaleFactor(): Promise<number> {
/** /**
* After a tier is applied: wait for frames, then look for shader errors in * After a tier is applied: wait for frames, then look for shader errors in
* mpv's log, and, when the network should be running at this size, for its * mpv's log, and for the passes that should be running at this size (the
* passes in the renderer. Either failing drops to the tier below. * denoiser always, the network when enlarging). Either failing drops to the
* tier below.
*/ */
async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string) { async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string, shaders: string[]) {
for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250); for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250);
await sleep(1500); await sleep(1500);
if (id !== seq || !usePlayer.getState().open) return; if (id !== seq || !usePlayer.getState().open) return;
const neural = NEURAL[mode]; const expected: RegExp[] = [];
const expectNetwork = !!neural && (await runsNetwork(mode)); if (shaders.includes('nlmeans.glsl')) expected.push(/Non-local means/);
const marker = mode === 'quality' ? /ArtCNN/ : /feature map|FSRCNN/i; if (NEURAL[mode] && (await runsNetwork(mode))) expected.push(/feature map/);
let error = shaderErrors.find((e) => e.at >= appliedAt); let error = shaderErrors.find((e) => e.at >= appliedAt);
let missing = false; let missing = false;
if (!error && expectNetwork) { if (!error && expected.length) {
// A first compile can hold the renderer for over a minute; wait for the // A first compile can hold the renderer for over a minute; wait for the
// network's passes to appear (or an error) rather than judging too early. // expected passes to appear (or an error) rather than judging too early.
// Already compiled, they appear within a few frames. // Already compiled, they appear within a few frames.
missing = true; missing = true;
const patience = usePlayer.getState().enhance.preparing ? 240 : 8; const patience = usePlayer.getState().enhance.preparing ? 240 : 8;
for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) { for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) {
error = shaderErrors.find((e) => e.at >= appliedAt); error = shaderErrors.find((e) => e.at >= appliedAt);
if (error || (await renderPasses()).some((p) => marker.test(p.desc))) { const passes = await renderPasses();
if (error || expected.every((re) => passes.some((p) => re.test(p.desc)))) {
missing = false; missing = false;
break; break;
} }
@ -148,9 +152,12 @@ async function verify(id: number, mode: Upscaling, appliedAt: number, signature:
} }
if (id !== seq) return; if (id !== seq) return;
if (!error && !missing) { if (!error && !missing) {
if (expectNetwork) markWarm(signature); // Nothing compiled at this size (e.g. the network isn't needed) leaves the set unproven.
if (expected.length) markWarm(signature);
else clearPreparing();
return; return;
} }
clearPreparing();
failed.add(mode); failed.add(mode);
const fallback = usable(mode); const fallback = usable(mode);
console.warn('[upscaling]', mode, 'failed:', error?.text ?? 'network passes not running'); console.warn('[upscaling]', mode, 'failed:', error?.text ?? 'network passes not running');
@ -162,14 +169,16 @@ async function verify(id: number, mode: Upscaling, appliedAt: number, signature:
let seq = 0; let seq = 0;
let chain: Promise<unknown> = Promise.resolve(); let chain: Promise<unknown> = Promise.resolve();
/** How long preparation must take before the player mentions it. */
export const PREPARING_NOTICE_MS = 3000;
/** /**
* Shader sets that have compiled on this computer. mpv keeps compiled shaders * Shader sets that have compiled on this computer. mpv keeps compiled shaders
* on disk (gpu-shader-cache-dir), but the first ArtCNN compile for Direct3D * on disk (gpu-shader-cache-dir); a first compile for Direct3D can take a
* takes over a minute, so the player says so the first time. * while on some GPUs, so until a set is known the player can say so (after a
* short delay, so a quick compile shows nothing).
*/ */
const WARM_KEY = 'kaleido.warmShaders'; const WARM_KEY = 'kaleido.warmShaders';
const SLOW = /ArtCNN/;
function warmSet(): Set<string> { function warmSet(): Set<string> {
try { try {
return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]')); return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]'));
@ -177,6 +186,9 @@ function warmSet(): Set<string> {
return new Set(); return new Set();
} }
} }
function clearPreparing() {
if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } });
}
function markWarm(signature: string) { function markWarm(signature: string) {
const set = warmSet(); const set = warmSet();
set.add(signature); set.add(signature);
@ -227,10 +239,12 @@ async function apply(id: number, requested: Upscaling) {
// Keyed by mpv's version too: a new libmpv recompiles everything. // Keyed by mpv's version too: a new libmpv recompiles everything.
// WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile. // WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile.
const signature = `${mpvVersion}|${shaders.join('|')}`; const signature = `${mpvVersion}|${shaders.join('|')}`;
const preparing = !bypass && shaders.some((n) => SLOW.test(n)) && !warmSet().has(signature) && (await runsNetwork(mode)); const preparing = !bypass && shaders.length > 0 && !warmSet().has(signature);
usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } }); usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } });
if (preparing && usePlayer.getState().firstFrame) if (preparing && usePlayer.getState().firstFrame)
toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for about a minute.`); setTimeout(() => {
if (id === seq && usePlayer.getState().enhance.preparing) toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for a moment.`);
}, PREPARING_NOTICE_MS);
const settings = JSON.stringify({ props, shaders }); const settings = JSON.stringify({ props, shaders });
if (settings !== lastApplied) { if (settings !== lastApplied) {
@ -246,7 +260,7 @@ async function apply(id: number, requested: Upscaling) {
} }
// Errors count from when these shaders actually reached mpv, even if this // Errors count from when these shaders actually reached mpv, even if this
// re-apply changed nothing (mpv may have already reported a failure). // re-apply changed nothing (mpv may have already reported a failure).
if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature); if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature, shaders);
} }
/** /**
@ -284,8 +298,7 @@ export interface Diagnostics {
} }
const STAGES: [RegExp, string][] = [ const STAGES: [RegExp, string][] = [
[/ArtCNN C4F32 DN/, 'ArtCNN 2× with denoising'], [/Non-local means/, 'nlmeans denoising'],
[/ArtCNN C4F32 \(/, 'ArtCNN 2×'],
[/feature map|FSRCNN/i, 'FSRCNNX 2×'], [/feature map|FSRCNN/i, 'FSRCNNX 2×'],
[/KrigBilateral/, 'KrigBilateral colour'], [/KrigBilateral/, 'KrigBilateral colour'],
[/^debanding/, 'Debanding'], [/^debanding/, 'Debanding'],