Use nlmeans and FSRCNNX 16 for the Quality upscaling tier
ArtCNN is tuned for animation and looked unnatural on live action, and its first Direct3D compile took about a minute per model. Quality now denoises with AN3223's nlmeans (general-purpose profile, any size) and upscales with FSRCNNX 16-0-4-1, trained on general imagery, at any enlargement. Both compile in seconds; 1080p in a maximized 1440p window measured 3.2 ms per frame. The first-compile notice now covers any shader set not yet compiled on the machine but only shows after three seconds, and a set with nothing to verify at the current size clears it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
506c69b36e
commit
5b6c063ff1
9 changed files with 3941 additions and 7897 deletions
|
|
@ -150,5 +150,5 @@ export const UPSCALING: Record<Upscaling, { label: string; detail: string }> = {
|
|||
off: { label: 'Off', detail: 'mpv’s standard scaling' },
|
||||
performance: { label: 'Performance', detail: 'Sharp scaling, debanding and light sharpening' },
|
||||
balanced: { label: 'Balanced', detail: 'Neural upscaling (FSRCNNX), colour reconstruction, debanding and sharpening' },
|
||||
quality: { label: 'Quality', detail: 'Larger neural upscaler (ArtCNN) with denoising, colour reconstruction, debanding and sharpening. Needs a capable GPU' },
|
||||
quality: { label: 'Quality', detail: 'nlmeans denoising, larger neural upscaler (FSRCNNX 16), colour reconstruction, debanding and sharpening. Needs a capable GPU' },
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { win, isNative } from '../platform/native';
|
|||
import { WindowControls } from '../app/TopBar';
|
||||
import { navigate } from '../app/router';
|
||||
import { mpv } from './engine';
|
||||
import { UPSCALING_ORDER, hasFailed, readDiagnostics, requestEnhancement, toggleBypass, type Diagnostics } from './enhance';
|
||||
import { PREPARING_NOTICE_MS, UPSCALING_ORDER, hasFailed, readDiagnostics, requestEnhancement, toggleBypass, type Diagnostics } from './enhance';
|
||||
import { toast } from '../ui/overlays';
|
||||
import type { Item } from '../plex/types';
|
||||
|
||||
|
|
@ -242,6 +242,17 @@ const SUB_BOXES: { value: SubtitleBox; label: string }[] = [
|
|||
|
||||
type SettingsView = 'main' | 'audio' | 'subtitles' | 'quality' | 'upscaling' | 'speed' | 'version' | 'chapters' | 'subSize' | 'subBox';
|
||||
|
||||
/** True once `flag` has stayed true for `ms` (a quick blip never shows). */
|
||||
function useLasting(flag: boolean, ms: number) {
|
||||
const [on, setOn] = useState(false);
|
||||
useEffect(() => {
|
||||
if (!flag) return setOn(false);
|
||||
const t = setTimeout(() => setOn(true), ms);
|
||||
return () => clearTimeout(t);
|
||||
}, [flag, ms]);
|
||||
return on;
|
||||
}
|
||||
|
||||
/** Polls what mpv's renderer is doing while a panel shows it. */
|
||||
function useDiagnostics(enabled: boolean) {
|
||||
const [d, setD] = useState<Diagnostics | null>(null);
|
||||
|
|
@ -322,7 +333,7 @@ function SettingsPanel({ onClose }: { onClose: () => void }) {
|
|||
<div className="panel-subhead">Stages</div>
|
||||
<Option selected={prefs.enhanceDeband} label="Debanding" detail="Smooths banding in gradients" onClick={() => prefs.set('enhanceDeband', !prefs.enhanceDeband)} />
|
||||
<Option selected={prefs.enhanceSharpen} label="Sharpening" detail="AMD CAS · SDR only" onClick={() => prefs.set('enhanceSharpen', !prefs.enhanceSharpen)} />
|
||||
<Option selected={prefs.enhanceDenoise} label="Denoising" detail="ArtCNN’s denoising model while upscaling · Quality tier, SDR only" onClick={() => prefs.set('enhanceDenoise', !prefs.enhanceDenoise)} />
|
||||
<Option selected={prefs.enhanceDenoise} label="Denoising" detail="Non-local means, at any size · Quality tier, SDR only" onClick={() => prefs.set('enhanceDenoise', !prefs.enhanceDenoise)} />
|
||||
<button className={`panel-compare${st.enhance.bypass ? ' on' : ''}`} onClick={() => toggleBypass()}>
|
||||
{st.enhance.bypass ? 'Turn enhancement back on' : 'Compare with enhancement off'}
|
||||
<kbd>B</kbd>
|
||||
|
|
@ -661,6 +672,7 @@ export function PlayerOverlay() {
|
|||
const { visible, poke } = useAutoHide(st.paused || st.ended, !!panel || scrubbing || st.countdown !== null);
|
||||
const clickTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const seekStep = useSettings((s) => s.seekStep);
|
||||
const preparing = useLasting(st.enhance.preparing, PREPARING_NOTICE_MS);
|
||||
|
||||
const showFlash = (icon: string) => setFlash({ icon, key: Date.now() });
|
||||
|
||||
|
|
@ -806,13 +818,13 @@ export function PlayerOverlay() {
|
|||
{!st.fullscreen && <WindowControls />}
|
||||
</div>
|
||||
|
||||
{(loading || st.buffering || st.seeking || st.enhance.preparing) && !st.error && (
|
||||
{(loading || st.buffering || st.seeking || preparing) && !st.error && (
|
||||
<div className="player-center">
|
||||
<Spinner size={44} />
|
||||
{st.enhance.preparing && (
|
||||
{preparing && (
|
||||
<p className="player-preparing">
|
||||
Preparing {UPSCALING[st.enhance.active].label} upscaling
|
||||
<small>First time on this computer only · about a minute</small>
|
||||
<small>First time on this computer only</small>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@
|
|||
* Video enhancement: upscaling tiers built from mpv's scalers, debanding and
|
||||
* the shaders bundled in src-tauri/shaders. Each shader's WHEN condition
|
||||
* decides whether it runs for the current video and window size, so one list
|
||||
* covers every case: the neural upscalers double luma when the video is
|
||||
* enlarged (FSRCNNX above 1.3×, ArtCNN at any enlargement, so 1080p in a
|
||||
* maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds subsampled colour,
|
||||
* CAS sharpens at native size (CAS-prescale also before upscaling).
|
||||
* covers every case: FSRCNNX doubles luma when the video is enlarged (the
|
||||
* small model above 1.3×, the large one at any enlargement, so 1080p in a
|
||||
* maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds
|
||||
* subsampled colour, CAS sharpens at native size (CAS-prescale also before
|
||||
* upscaling) and nlmeans denoises at any size.
|
||||
*
|
||||
* The tier the user chose (preferences) and the one actually running (player
|
||||
* state) are kept apart: a tier whose shaders fail drops to the one below with
|
||||
|
|
@ -28,7 +29,7 @@ let shaderDir: Promise<string> | null = null;
|
|||
const NETWORK_MIN_SCALE: Partial<Record<Upscaling, number>> = { balanced: 1.3, quality: 1.0 };
|
||||
const runsNetwork = async (mode: Upscaling) => mode in NETWORK_MIN_SCALE && (await scaleFactor()) > NETWORK_MIN_SCALE[mode]!;
|
||||
|
||||
const NEURAL: Partial<Record<Upscaling, string>> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'ArtCNN_C4F32.glsl' };
|
||||
const NEURAL: Partial<Record<Upscaling, string>> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'FSRCNNX_x2_16-0-4-1.glsl' };
|
||||
|
||||
interface Plan {
|
||||
props: Record<string, unknown>;
|
||||
|
|
@ -36,10 +37,11 @@ interface Plan {
|
|||
}
|
||||
|
||||
/**
|
||||
* What a tier runs. Sharpening (CAS) and denoising are tuned for SDR levels,
|
||||
* so HDR video skips them; the upscalers and debanding stay. Denoising uses
|
||||
* ArtCNN's own denoising model while upscaling: a separate denoiser (nlmeans)
|
||||
* took over a minute to compile for Direct3D before the first frame.
|
||||
* What a tier runs. Sharpening (CAS) and denoising (nlmeans) are tuned for
|
||||
* SDR levels, so HDR video skips them; the upscalers and debanding stay.
|
||||
* Quality denoises first, so the network enlarges the picture rather than
|
||||
* the compression noise. Both FSRCNNX models are trained on general imagery,
|
||||
* which suits live action.
|
||||
*/
|
||||
function plan(mode: Upscaling, hdr: boolean): Plan {
|
||||
const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState();
|
||||
|
|
@ -57,7 +59,7 @@ function plan(mode: Upscaling, hdr: boolean): Plan {
|
|||
break;
|
||||
case 'quality':
|
||||
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 };
|
||||
shaders = [enhanceDenoise && !hdr ? 'ArtCNN_C4F32_DN.glsl' : NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
|
||||
shaders = [...(enhanceDenoise && !hdr ? ['nlmeans.glsl'] : []), NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
|
||||
break;
|
||||
default:
|
||||
return { props: { ...scalerDefaults }, shaders: [] };
|
||||
|
|
@ -119,27 +121,29 @@ export async function scaleFactor(): Promise<number> {
|
|||
|
||||
/**
|
||||
* After a tier is applied: wait for frames, then look for shader errors in
|
||||
* mpv's log, and, when the network should be running at this size, for its
|
||||
* passes in the renderer. Either failing drops to the tier below.
|
||||
* mpv's log, and for the passes that should be running at this size (the
|
||||
* denoiser always, the network when enlarging). Either failing drops to the
|
||||
* tier below.
|
||||
*/
|
||||
async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string) {
|
||||
async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string, shaders: string[]) {
|
||||
for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250);
|
||||
await sleep(1500);
|
||||
if (id !== seq || !usePlayer.getState().open) return;
|
||||
const neural = NEURAL[mode];
|
||||
const expectNetwork = !!neural && (await runsNetwork(mode));
|
||||
const marker = mode === 'quality' ? /ArtCNN/ : /feature map|FSRCNN/i;
|
||||
const expected: RegExp[] = [];
|
||||
if (shaders.includes('nlmeans.glsl')) expected.push(/Non-local means/);
|
||||
if (NEURAL[mode] && (await runsNetwork(mode))) expected.push(/feature map/);
|
||||
let error = shaderErrors.find((e) => e.at >= appliedAt);
|
||||
let missing = false;
|
||||
if (!error && expectNetwork) {
|
||||
if (!error && expected.length) {
|
||||
// A first compile can hold the renderer for over a minute; wait for the
|
||||
// network's passes to appear (or an error) rather than judging too early.
|
||||
// expected passes to appear (or an error) rather than judging too early.
|
||||
// Already compiled, they appear within a few frames.
|
||||
missing = true;
|
||||
const patience = usePlayer.getState().enhance.preparing ? 240 : 8;
|
||||
for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) {
|
||||
error = shaderErrors.find((e) => e.at >= appliedAt);
|
||||
if (error || (await renderPasses()).some((p) => marker.test(p.desc))) {
|
||||
const passes = await renderPasses();
|
||||
if (error || expected.every((re) => passes.some((p) => re.test(p.desc)))) {
|
||||
missing = false;
|
||||
break;
|
||||
}
|
||||
|
|
@ -148,9 +152,12 @@ async function verify(id: number, mode: Upscaling, appliedAt: number, signature:
|
|||
}
|
||||
if (id !== seq) return;
|
||||
if (!error && !missing) {
|
||||
if (expectNetwork) markWarm(signature);
|
||||
// Nothing compiled at this size (e.g. the network isn't needed) leaves the set unproven.
|
||||
if (expected.length) markWarm(signature);
|
||||
else clearPreparing();
|
||||
return;
|
||||
}
|
||||
clearPreparing();
|
||||
failed.add(mode);
|
||||
const fallback = usable(mode);
|
||||
console.warn('[upscaling]', mode, 'failed:', error?.text ?? 'network passes not running');
|
||||
|
|
@ -162,14 +169,16 @@ async function verify(id: number, mode: Upscaling, appliedAt: number, signature:
|
|||
|
||||
let seq = 0;
|
||||
let chain: Promise<unknown> = Promise.resolve();
|
||||
/** How long preparation must take before the player mentions it. */
|
||||
export const PREPARING_NOTICE_MS = 3000;
|
||||
|
||||
/**
|
||||
* Shader sets that have compiled on this computer. mpv keeps compiled shaders
|
||||
* on disk (gpu-shader-cache-dir), but the first ArtCNN compile for Direct3D
|
||||
* takes over a minute, so the player says so the first time.
|
||||
* on disk (gpu-shader-cache-dir); a first compile for Direct3D can take a
|
||||
* while on some GPUs, so until a set is known the player can say so (after a
|
||||
* short delay, so a quick compile shows nothing).
|
||||
*/
|
||||
const WARM_KEY = 'kaleido.warmShaders';
|
||||
const SLOW = /ArtCNN/;
|
||||
function warmSet(): Set<string> {
|
||||
try {
|
||||
return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]'));
|
||||
|
|
@ -177,6 +186,9 @@ function warmSet(): Set<string> {
|
|||
return new Set();
|
||||
}
|
||||
}
|
||||
function clearPreparing() {
|
||||
if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } });
|
||||
}
|
||||
function markWarm(signature: string) {
|
||||
const set = warmSet();
|
||||
set.add(signature);
|
||||
|
|
@ -227,10 +239,12 @@ async function apply(id: number, requested: Upscaling) {
|
|||
// Keyed by mpv's version too: a new libmpv recompiles everything.
|
||||
// WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile.
|
||||
const signature = `${mpvVersion}|${shaders.join('|')}`;
|
||||
const preparing = !bypass && shaders.some((n) => SLOW.test(n)) && !warmSet().has(signature) && (await runsNetwork(mode));
|
||||
const preparing = !bypass && shaders.length > 0 && !warmSet().has(signature);
|
||||
usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } });
|
||||
if (preparing && usePlayer.getState().firstFrame)
|
||||
toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for about a minute.`);
|
||||
setTimeout(() => {
|
||||
if (id === seq && usePlayer.getState().enhance.preparing) toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for a moment.`);
|
||||
}, PREPARING_NOTICE_MS);
|
||||
|
||||
const settings = JSON.stringify({ props, shaders });
|
||||
if (settings !== lastApplied) {
|
||||
|
|
@ -246,7 +260,7 @@ async function apply(id: number, requested: Upscaling) {
|
|||
}
|
||||
// Errors count from when these shaders actually reached mpv, even if this
|
||||
// re-apply changed nothing (mpv may have already reported a failure).
|
||||
if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature);
|
||||
if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature, shaders);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -284,8 +298,7 @@ export interface Diagnostics {
|
|||
}
|
||||
|
||||
const STAGES: [RegExp, string][] = [
|
||||
[/ArtCNN C4F32 DN/, 'ArtCNN 2× with denoising'],
|
||||
[/ArtCNN C4F32 \(/, 'ArtCNN 2×'],
|
||||
[/Non-local means/, 'nlmeans denoising'],
|
||||
[/feature map|FSRCNN/i, 'FSRCNNX 2×'],
|
||||
[/KrigBilateral/, 'KrigBilateral colour'],
|
||||
[/^debanding/, 'Debanding'],
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue