Add a Quality upscaling tier and make enhancement verifiable
Quality runs ArtCNN C4F32 (its denoising model when Denoising is on), KrigBilateral, debanding and CAS. Its network runs at any enlargement, so 1080p in a maximized 1440p window is doubled and scaled down; Balanced keeps FSRCNNX's 1.3x threshold. The enhancement pipeline moves to player/enhance.ts. Mode changes are serialized (latest wins). Shader errors in mpv's log, or network passes missing from vo-passes, drop to the tier below with a notice, and the chosen and running tiers are shown separately. HDR is judged from the decoded video. Debanding, sharpening and denoising have their own toggles, B compares with enhancement off, and the panel shows the renderer's actual stages, GPU time and dropped frames. Compiled shaders persist in the app cache (gpu-shader-cache-dir): ArtCNN's first Direct3D compile takes about a minute, which the player now announces, and later launches start immediately. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
5efcf2c5a5
commit
9fd2b0603e
11 changed files with 8400 additions and 68 deletions
|
|
@ -10,6 +10,7 @@ import { win, isNative } from '../platform/native';
|
|||
import { WindowControls } from '../app/TopBar';
|
||||
import { navigate } from '../app/router';
|
||||
import { mpv } from './engine';
|
||||
import { UPSCALING_ORDER, hasFailed, readDiagnostics, requestEnhancement, toggleBypass, type Diagnostics } from './enhance';
|
||||
import { toast } from '../ui/overlays';
|
||||
import type { Item } from '../plex/types';
|
||||
|
||||
|
|
@ -241,26 +242,34 @@ const SUB_BOXES: { value: SubtitleBox; label: string }[] = [
|
|||
|
||||
type SettingsView = 'main' | 'audio' | 'subtitles' | 'quality' | 'upscaling' | 'speed' | 'version' | 'chapters' | 'subSize' | 'subBox';
|
||||
|
||||
const UPSCALING_ORDER: Upscaling[] = ['off', 'performance', 'balanced'];
|
||||
/** Polls what mpv's renderer is doing while a panel shows it. */
|
||||
function useDiagnostics(enabled: boolean) {
|
||||
const [d, setD] = useState<Diagnostics | null>(null);
|
||||
useEffect(() => {
|
||||
if (!enabled || !isNative) return;
|
||||
let alive = true;
|
||||
const tick = () => readDiagnostics().then((x) => alive && setD(x), () => {});
|
||||
tick();
|
||||
const t = setInterval(tick, 1000);
|
||||
return () => {
|
||||
alive = false;
|
||||
clearInterval(t);
|
||||
};
|
||||
}, [enabled]);
|
||||
return d;
|
||||
}
|
||||
|
||||
/** What the current tier is doing for this video at this size, for the details list. */
|
||||
function upscalingDetail(mode: Upscaling, video: { width: number; height: number } | null, hdr: boolean) {
|
||||
if (mode === 'off') return 'Off';
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
const factor = video && video.width ? Math.min((window.innerWidth * dpr) / video.width, (window.innerHeight * dpr) / video.height) : 1;
|
||||
const parts = [UPSCALING[mode].label];
|
||||
if (factor > 1.01) parts.push(`${factor.toFixed(1)}× ${mode === 'balanced' && factor > 1.3 ? 'neural upscale' : 'upscale'}`);
|
||||
else parts.push('native size');
|
||||
if (mode === 'balanced') parts.push('colour reconstruction');
|
||||
parts.push('debanding');
|
||||
if (!hdr) parts.push('sharpening');
|
||||
return parts.join(' · ');
|
||||
/** Requested vs running tier, for the panel row. */
|
||||
function upscalingValue(requested: Upscaling, active: Upscaling, bypass: boolean) {
|
||||
if (bypass) return 'Off (comparing)';
|
||||
return active === requested ? UPSCALING[requested].label : `${UPSCALING[active].label} (${UPSCALING[requested].label} failed)`;
|
||||
}
|
||||
|
||||
function SettingsPanel({ onClose }: { onClose: () => void }) {
|
||||
const st = usePlayer();
|
||||
const prefs = useSettings();
|
||||
const [view, setView] = useState<SettingsView>('main');
|
||||
const diag = useDiagnostics(view === 'main' || view === 'upscaling');
|
||||
const speeds = [0.5, 0.75, 1, 1.25, 1.5, 1.75, 2];
|
||||
const media = st.item?.media ?? [];
|
||||
const source = media[st.mediaIndex];
|
||||
|
|
@ -306,10 +315,44 @@ function SettingsPanel({ onClose }: { onClose: () => void }) {
|
|||
);
|
||||
if (view === 'upscaling')
|
||||
return (
|
||||
<Panel title="Upscaling" onClose={onClose} onBack={back}>
|
||||
<Panel title="Upscaling" onClose={onClose} onBack={back} className="upscaling">
|
||||
{UPSCALING_ORDER.map((m) => (
|
||||
<Option key={m} selected={prefs.upscaling === m} label={UPSCALING[m].label} detail={UPSCALING[m].detail} onClick={() => (prefs.set('upscaling', m), back())} />
|
||||
<Option key={m} selected={prefs.upscaling === m} label={UPSCALING[m].label} detail={hasFailed(m) ? 'Couldn’t start on this computer' : UPSCALING[m].detail} onClick={() => prefs.set('upscaling', m)} />
|
||||
))}
|
||||
<div className="panel-subhead">Stages</div>
|
||||
<Option selected={prefs.enhanceDeband} label="Debanding" detail="Smooths banding in gradients" onClick={() => prefs.set('enhanceDeband', !prefs.enhanceDeband)} />
|
||||
<Option selected={prefs.enhanceSharpen} label="Sharpening" detail="AMD CAS · SDR only" onClick={() => prefs.set('enhanceSharpen', !prefs.enhanceSharpen)} />
|
||||
<Option selected={prefs.enhanceDenoise} label="Denoising" detail="ArtCNN’s denoising model while upscaling · Quality tier, SDR only" onClick={() => prefs.set('enhanceDenoise', !prefs.enhanceDenoise)} />
|
||||
<button className={`panel-compare${st.enhance.bypass ? ' on' : ''}`} onClick={() => toggleBypass()}>
|
||||
{st.enhance.bypass ? 'Turn enhancement back on' : 'Compare with enhancement off'}
|
||||
<kbd>B</kbd>
|
||||
</button>
|
||||
{diag && (
|
||||
<div className="panel-stats">
|
||||
<div>
|
||||
<span>Running</span>
|
||||
<b>{diag.stages.length ? diag.stages.join(' · ') : 'Standard scaling'}</b>
|
||||
</div>
|
||||
{diag.input && (
|
||||
<div>
|
||||
<span>Size</span>
|
||||
<b>
|
||||
{diag.input} → {diag.output}
|
||||
</b>
|
||||
</div>
|
||||
)}
|
||||
<div>
|
||||
<span>GPU time</span>
|
||||
<b className={diag.gpuMs > diag.budgetMs * 0.8 ? 'warn' : ''}>
|
||||
{diag.gpuMs.toFixed(1)} ms per frame <small>of {diag.budgetMs.toFixed(0)} ms</small>
|
||||
</b>
|
||||
</div>
|
||||
<div>
|
||||
<span>Dropped</span>
|
||||
<b>{diag.dropped} frames</b>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</Panel>
|
||||
);
|
||||
if (view === 'speed')
|
||||
|
|
@ -379,7 +422,7 @@ function SettingsPanel({ onClose }: { onClose: () => void }) {
|
|||
if (v && v.width > 0)
|
||||
stats.push(['Video', [`${v.width}×${v.height}`, v.codec.toUpperCase(), source?.dynamicRange, v.hwdec && v.hwdec !== 'no' ? 'Hardware decoding' : 'Software decoding'].filter(Boolean).join(' · ')]);
|
||||
if (audio) stats.push(['Audio', [audio.label, audio.detail].filter(Boolean).join(' · ')]);
|
||||
stats.push(['Upscaling', upscalingDetail(prefs.upscaling, v, !!source?.dynamicRange)]);
|
||||
if (diag && st.enhance.active !== 'off' && !st.enhance.bypass) stats.push(['Upscaling', [...diag.stages, `${diag.gpuMs.toFixed(1)} ms`].join(' · ')]);
|
||||
if (source?.bitrate && !local) stats.push(['Bitrate', st.delivery?.mode === 'transcode' ? `${bitrate(source.bitrate)} source` : bitrate(source.bitrate)]);
|
||||
if (source?.container) stats.push(['Container', source.container.toUpperCase()]);
|
||||
|
||||
|
|
@ -388,7 +431,7 @@ function SettingsPanel({ onClose }: { onClose: () => void }) {
|
|||
{row('Audio', audio ? audio.label : st.audioTracks.length ? 'Choose' : 'None', () => setView('audio'), !st.audioTracks.length)}
|
||||
{row('Subtitles', sub ? sub.label : 'Off', () => setView('subtitles'))}
|
||||
{!local && row('Quality', qualityLabel(st.quality), () => setView('quality'))}
|
||||
{row('Upscaling', UPSCALING[prefs.upscaling].label, () => setView('upscaling'))}
|
||||
{row('Upscaling', upscalingValue(prefs.upscaling, st.enhance.active, st.enhance.bypass), () => setView('upscaling'))}
|
||||
{row('Speed', st.speed === 1 ? 'Normal' : `${st.speed}×`, () => setView('speed'))}
|
||||
{media.length > 1 && !local && row('Version', `${st.mediaIndex + 1} of ${media.length}`, () => setView('version'))}
|
||||
{st.chapters.length > 1 && row('Chapters', `${st.chapters.length}`, () => setView('chapters'))}
|
||||
|
|
@ -675,9 +718,16 @@ export function PlayerOverlay() {
|
|||
const prefs = useSettings.getState();
|
||||
const next = UPSCALING_ORDER[(UPSCALING_ORDER.indexOf(prefs.upscaling) + 1) % UPSCALING_ORDER.length];
|
||||
prefs.set('upscaling', next);
|
||||
toast(`Upscaling: ${UPSCALING[next].label}`);
|
||||
// Confirm what is actually running once it's applied.
|
||||
requestEnhancement().then(() => {
|
||||
if (useSettings.getState().upscaling === next) toast(`Upscaling: ${upscalingValue(next, usePlayer.getState().enhance.active, false)}`);
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'b':
|
||||
case 'B':
|
||||
toggleBypass();
|
||||
break;
|
||||
case 'c':
|
||||
case 'C': {
|
||||
const subs = p.subTracks;
|
||||
|
|
@ -756,9 +806,15 @@ export function PlayerOverlay() {
|
|||
{!st.fullscreen && <WindowControls />}
|
||||
</div>
|
||||
|
||||
{(loading || st.buffering || st.seeking) && !st.error && (
|
||||
{(loading || st.buffering || st.seeking || st.enhance.preparing) && !st.error && (
|
||||
<div className="player-center">
|
||||
<Spinner size={44} />
|
||||
{st.enhance.preparing && (
|
||||
<p className="player-preparing">
|
||||
Preparing {UPSCALING[st.enhance.active].label} upscaling
|
||||
<small>First time on this computer only · about a minute</small>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{flash && !loading && (
|
||||
|
|
|
|||
344
src/player/enhance.ts
Normal file
344
src/player/enhance.ts
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
/**
|
||||
* Video enhancement: upscaling tiers built from mpv's scalers, debanding and
|
||||
* the shaders bundled in src-tauri/shaders. Each shader's WHEN condition
|
||||
* decides whether it runs for the current video and window size, so one list
|
||||
* covers every case: the neural upscalers double luma when the video is
|
||||
* enlarged (FSRCNNX above 1.3×, ArtCNN at any enlargement, so 1080p in a
|
||||
* maximized 1440p window is doubled and scaled down), KrigBilateral rebuilds subsampled colour,
|
||||
* CAS sharpens at native size (CAS-prescale also before upscaling).
|
||||
*
|
||||
* The tier the user chose (preferences) and the one actually running (player
|
||||
* state) are kept apart: a tier whose shaders fail drops to the one below with
|
||||
* a notice instead of silently showing plain scaling, and isn't retried.
|
||||
*/
|
||||
import { useSettings, UPSCALING, type Upscaling } from '../app/settings';
|
||||
import { isNative, resourcePath } from '../platform/native';
|
||||
import { toast } from '../ui/overlays';
|
||||
import { mpv } from './engine';
|
||||
import { usePlayer } from './store';
|
||||
|
||||
export const UPSCALING_ORDER: Upscaling[] = ['off', 'performance', 'balanced', 'quality'];
|
||||
|
||||
/** mpv's own scaler settings, read once so Off can put them back. */
|
||||
const SCALER_PROPS = ['scale', 'cscale', 'dscale', 'sigmoid-upscaling', 'deband', 'deband-iterations', 'deband-grain'] as const;
|
||||
let scalerDefaults: Partial<Record<(typeof SCALER_PROPS)[number], unknown>> | null = null;
|
||||
let shaderDir: Promise<string> | null = null;
|
||||
|
||||
/** The enlargement above which each tier's network runs (its shaders' WHEN condition). */
|
||||
const NETWORK_MIN_SCALE: Partial<Record<Upscaling, number>> = { balanced: 1.3, quality: 1.0 };
|
||||
const runsNetwork = async (mode: Upscaling) => mode in NETWORK_MIN_SCALE && (await scaleFactor()) > NETWORK_MIN_SCALE[mode]!;
|
||||
|
||||
const NEURAL: Partial<Record<Upscaling, string>> = { balanced: 'FSRCNNX_x2_8-0-4-1.glsl', quality: 'ArtCNN_C4F32.glsl' };
|
||||
|
||||
interface Plan {
|
||||
props: Record<string, unknown>;
|
||||
shaders: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* What a tier runs. Sharpening (CAS) and denoising are tuned for SDR levels,
|
||||
* so HDR video skips them; the upscalers and debanding stay. Denoising uses
|
||||
* ArtCNN's own denoising model while upscaling: a separate denoiser (nlmeans)
|
||||
* took over a minute to compile for Direct3D before the first frame.
|
||||
*/
|
||||
function plan(mode: Upscaling, hdr: boolean): Plan {
|
||||
const { enhanceDeband, enhanceSharpen, enhanceDenoise } = useSettings.getState();
|
||||
const sharpen = enhanceSharpen && !hdr;
|
||||
let props: Record<string, unknown>;
|
||||
let shaders: string[] = [];
|
||||
switch (mode) {
|
||||
case 'performance':
|
||||
props = { scale: 'spline36', cscale: 'spline36', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 1, 'deband-grain': 24 };
|
||||
if (sharpen) shaders = ['CAS-prescale.glsl'];
|
||||
break;
|
||||
case 'balanced':
|
||||
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 16 };
|
||||
shaders = [NEURAL.balanced!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
|
||||
break;
|
||||
case 'quality':
|
||||
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 12 };
|
||||
shaders = [enhanceDenoise && !hdr ? 'ArtCNN_C4F32_DN.glsl' : NEURAL.quality!, 'KrigBilateral.glsl', ...(sharpen ? ['CAS.glsl'] : [])];
|
||||
break;
|
||||
default:
|
||||
return { props: { ...scalerDefaults }, shaders: [] };
|
||||
}
|
||||
if (!enhanceDeband) props.deband = false;
|
||||
return { props, shaders };
|
||||
}
|
||||
|
||||
/** HDR as decoded (a converted stream may already be SDR), else Plex's metadata. */
|
||||
function isHdr() {
|
||||
const st = usePlayer.getState();
|
||||
if (st.videoGamma) return st.videoGamma === 'pq' || st.videoGamma === 'hlg';
|
||||
return st.delivery?.mode !== 'transcode' && !!st.item?.media[st.mediaIndex]?.dynamicRange;
|
||||
}
|
||||
|
||||
// —— Shader failures ————————————————————————————————————————————————
|
||||
|
||||
const shaderErrors: { at: number; text: string }[] = [];
|
||||
|
||||
/** Fed every mpv log line; keeps the ones that mean a user shader failed. */
|
||||
export function noteMpvLog(level: string, prefix: string, text: string) {
|
||||
if (level !== 'error' && level !== 'fatal') return;
|
||||
// A missing file is reported by mpv's file/stream layers; a broken one by the renderer.
|
||||
if (/\.glsl/i.test(text) || (/^vo/.test(prefix) && /shader|glsl|hook|compil|spirv|parse/i.test(text))) {
|
||||
shaderErrors.push({ at: Date.now(), text: text.trim() });
|
||||
if (shaderErrors.length > 20) shaderErrors.shift();
|
||||
}
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
/** Descriptions of the render passes mpv ran for the last frame (or redraw). */
|
||||
type Pass = { desc: string; avg: number; samples?: number[] };
|
||||
export async function renderPasses(): Promise<{ desc: string; avg: number }[]> {
|
||||
const passes = await mpv.get<{ fresh?: Pass[]; redraw?: Pass[] }>('vo-passes').catch(() => null);
|
||||
const list = (passes?.fresh?.length ? passes.fresh : passes?.redraw) ?? [];
|
||||
// The median of recent frames: the average includes one-off shader compiles.
|
||||
return list.map((p) => {
|
||||
const s = [...(p.samples ?? [])].sort((a, b) => a - b);
|
||||
return { desc: p.desc, avg: s.length ? s[Math.floor(s.length / 2)] : p.avg };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* How much the video is enlarged on screen. The full player fills the window,
|
||||
* which is known before mpv has drawn anything; the miniplayer asks mpv.
|
||||
*/
|
||||
export async function scaleFactor(): Promise<number> {
|
||||
const st = usePlayer.getState();
|
||||
if (!st.video?.width) return 1;
|
||||
if (!st.mini) {
|
||||
const dpr = window.devicePixelRatio || 1;
|
||||
return Math.min((window.innerWidth * dpr) / st.video.width, (window.innerHeight * dpr) / st.video.height);
|
||||
}
|
||||
const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null);
|
||||
if (!osd) return 1;
|
||||
return Math.min((osd.w - osd.ml - osd.mr) / st.video.width, (osd.h - osd.mt - osd.mb) / st.video.height);
|
||||
}
|
||||
|
||||
/**
|
||||
* After a tier is applied: wait for frames, then look for shader errors in
|
||||
* mpv's log, and, when the network should be running at this size, for its
|
||||
* passes in the renderer. Either failing drops to the tier below.
|
||||
*/
|
||||
async function verify(id: number, mode: Upscaling, appliedAt: number, signature: string) {
|
||||
for (let i = 0; i < 80 && !usePlayer.getState().firstFrame; i++) await sleep(250);
|
||||
await sleep(1500);
|
||||
if (id !== seq || !usePlayer.getState().open) return;
|
||||
const neural = NEURAL[mode];
|
||||
const expectNetwork = !!neural && (await runsNetwork(mode));
|
||||
const marker = mode === 'quality' ? /ArtCNN/ : /feature map|FSRCNN/i;
|
||||
let error = shaderErrors.find((e) => e.at >= appliedAt);
|
||||
let missing = false;
|
||||
if (!error && expectNetwork) {
|
||||
// A first compile can hold the renderer for over a minute; wait for the
|
||||
// network's passes to appear (or an error) rather than judging too early.
|
||||
// Already compiled, they appear within a few frames.
|
||||
missing = true;
|
||||
const patience = usePlayer.getState().enhance.preparing ? 240 : 8;
|
||||
for (let i = 0; i < patience && id === seq && usePlayer.getState().open; i++) {
|
||||
error = shaderErrors.find((e) => e.at >= appliedAt);
|
||||
if (error || (await renderPasses()).some((p) => marker.test(p.desc))) {
|
||||
missing = false;
|
||||
break;
|
||||
}
|
||||
await sleep(1000);
|
||||
}
|
||||
}
|
||||
if (id !== seq) return;
|
||||
if (!error && !missing) {
|
||||
if (expectNetwork) markWarm(signature);
|
||||
return;
|
||||
}
|
||||
failed.add(mode);
|
||||
const fallback = usable(mode);
|
||||
console.warn('[upscaling]', mode, 'failed:', error?.text ?? 'network passes not running');
|
||||
toast(`${UPSCALING[mode].label} upscaling couldn’t start. Using ${UPSCALING[fallback].label}.`, 'error');
|
||||
requestEnhancement();
|
||||
}
|
||||
|
||||
// —— Applying ——————————————————————————————————————————————————————
|
||||
|
||||
let seq = 0;
|
||||
let chain: Promise<unknown> = Promise.resolve();
|
||||
|
||||
/**
|
||||
* Shader sets that have compiled on this computer. mpv keeps compiled shaders
|
||||
* on disk (gpu-shader-cache-dir), but the first ArtCNN compile for Direct3D
|
||||
* takes over a minute, so the player says so the first time.
|
||||
*/
|
||||
const WARM_KEY = 'kaleido.warmShaders';
|
||||
const SLOW = /ArtCNN/;
|
||||
function warmSet(): Set<string> {
|
||||
try {
|
||||
return new Set(JSON.parse(localStorage.getItem(WARM_KEY) ?? '[]'));
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
function markWarm(signature: string) {
|
||||
const set = warmSet();
|
||||
set.add(signature);
|
||||
try {
|
||||
localStorage.setItem(WARM_KEY, JSON.stringify([...set]));
|
||||
} catch {
|
||||
/* per-device convenience only */
|
||||
}
|
||||
if (usePlayer.getState().enhance.preparing) usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, preparing: false } });
|
||||
}
|
||||
/** Tiers whose shaders failed this session. */
|
||||
const failed = new Set<Upscaling>();
|
||||
let missingWarned = false;
|
||||
|
||||
/** The chosen tier, or the nearest one below it that hasn't failed. */
|
||||
function usable(mode: Upscaling): Upscaling {
|
||||
let i = UPSCALING_ORDER.indexOf(mode);
|
||||
while (i > 0 && failed.has(UPSCALING_ORDER[i])) i--;
|
||||
return UPSCALING_ORDER[i];
|
||||
}
|
||||
|
||||
let mpvVersion = '';
|
||||
/** The last settings sent to mpv (and when), so an identical re-apply leaves mpv alone. */
|
||||
let lastApplied = '';
|
||||
let lastAppliedAt = 0;
|
||||
|
||||
async function apply(id: number, requested: Upscaling) {
|
||||
if (!isNative || id !== seq) return;
|
||||
if (!scalerDefaults) {
|
||||
scalerDefaults = {};
|
||||
for (const p of SCALER_PROPS) scalerDefaults[p] = await mpv.get(p).catch(() => undefined);
|
||||
mpvVersion = (await mpv.get<string>('mpv-version').catch(() => '')) ?? '';
|
||||
}
|
||||
const bypass = usePlayer.getState().enhance.bypass;
|
||||
let mode = usable(requested);
|
||||
let { props, shaders } = plan(bypass ? 'off' : mode, isHdr());
|
||||
const dir = await (shaderDir ??= resourcePath('shaders').catch(() => ''));
|
||||
if (shaders.length && !dir) {
|
||||
// No shaders at all: keep the scaler and debanding settings only.
|
||||
if (!missingWarned) toast('The upscaling shaders are missing from this installation.', 'error');
|
||||
missingWarned = true;
|
||||
shaders = [];
|
||||
if (mode !== 'performance') mode = 'performance';
|
||||
({ props } = plan(mode, isHdr()));
|
||||
}
|
||||
// Decided before touching mpv: while a shader compiles, mpv holds option
|
||||
// changes until it finishes, so a notice set afterwards would come too late.
|
||||
// Keyed by mpv's version too: a new libmpv recompiles everything.
|
||||
// WHEN conditions are evaluated by mpv, not compiled, so changing them needs no recompile.
|
||||
const signature = `${mpvVersion}|${shaders.join('|')}`;
|
||||
const preparing = !bypass && shaders.some((n) => SLOW.test(n)) && !warmSet().has(signature) && (await runsNetwork(mode));
|
||||
usePlayer.setState({ enhance: { ...usePlayer.getState().enhance, active: mode, preparing } });
|
||||
if (preparing && usePlayer.getState().firstFrame)
|
||||
toast(`Preparing ${UPSCALING[mode].label} upscaling for the first time. The picture may pause for about a minute.`);
|
||||
|
||||
const settings = JSON.stringify({ props, shaders });
|
||||
if (settings !== lastApplied) {
|
||||
lastApplied = settings;
|
||||
lastAppliedAt = Date.now();
|
||||
for (const [k, v] of Object.entries(props)) if (v !== undefined) await mpv.set(k, v).catch(() => {});
|
||||
if (shaders.length) {
|
||||
// mpv takes a path list: ';'-separated on Windows, ':' elsewhere.
|
||||
const windows = dir.includes('\\');
|
||||
const list = shaders.map((n) => `${dir}${windows ? '\\' : '/'}${n}`).join(windows ? ';' : ':');
|
||||
await mpv.set('glsl-shaders', list).catch((e) => shaderErrors.push({ at: Date.now(), text: String(e) }));
|
||||
} else await mpv.command('change-list', 'glsl-shaders', 'clr', '').catch(() => {});
|
||||
}
|
||||
// Errors count from when these shaders actually reached mpv, even if this
|
||||
// re-apply changed nothing (mpv may have already reported a failure).
|
||||
if (!bypass && shaders.length) verify(id, mode, lastAppliedAt, signature);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the chosen tier (and stage toggles) to the playing video. Requests
|
||||
* are serialized and only the latest runs, so quick U presses can't interleave.
|
||||
*/
|
||||
export function requestEnhancement() {
|
||||
const id = ++seq;
|
||||
const run = chain.then(() => apply(id, useSettings.getState().upscaling));
|
||||
chain = run.catch(() => {});
|
||||
return run;
|
||||
}
|
||||
|
||||
/** Whether a tier failed this session (shown in the player's choices). */
|
||||
export const hasFailed = (mode: Upscaling) => failed.has(mode);
|
||||
|
||||
/** Before/after comparison: enhancement off without changing the preference. */
|
||||
export function toggleBypass() {
|
||||
const st = usePlayer.getState();
|
||||
const bypass = !st.enhance.bypass;
|
||||
usePlayer.setState({ enhance: { ...st.enhance, bypass } });
|
||||
toast(bypass ? 'Enhancement off for comparison · press B to turn it back on' : `Enhancement on · ${UPSCALING[useSettings.getState().upscaling].label}`);
|
||||
return requestEnhancement();
|
||||
}
|
||||
|
||||
// —— Diagnostics ———————————————————————————————————————————————————
|
||||
|
||||
export interface Diagnostics {
|
||||
stages: string[];
|
||||
gpuMs: number;
|
||||
budgetMs: number;
|
||||
dropped: number;
|
||||
input: string;
|
||||
output: string;
|
||||
}
|
||||
|
||||
const STAGES: [RegExp, string][] = [
|
||||
[/ArtCNN C4F32 DN/, 'ArtCNN 2× with denoising'],
|
||||
[/ArtCNN C4F32 \(/, 'ArtCNN 2×'],
|
||||
[/feature map|FSRCNN/i, 'FSRCNNX 2×'],
|
||||
[/KrigBilateral/, 'KrigBilateral colour'],
|
||||
[/^debanding/, 'Debanding'],
|
||||
[/FidelityFX/, 'CAS sharpening'],
|
||||
];
|
||||
|
||||
/** What the renderer actually did for the last frame, read from mpv. */
|
||||
export async function readDiagnostics(): Promise<Diagnostics> {
|
||||
const passes = await renderPasses();
|
||||
const stages = STAGES.filter(([re]) => passes.some((p) => re.test(p.desc))).map(([, name]) => name);
|
||||
const scaler = passes.map((p) => /(up|down)scaling \(([\w-]+)\)/.exec(p.desc)).find(Boolean);
|
||||
if (scaler) stages.push(`${scaler[2]} ${scaler[1]}scaling`);
|
||||
const fps = (await mpv.get<number>('estimated-vf-fps').catch(() => 0)) || (await mpv.get<number>('container-fps').catch(() => 0)) || 24;
|
||||
const dropped = ((await mpv.get<number>('frame-drop-count').catch(() => 0)) || 0) + ((await mpv.get<number>('decoder-frame-drop-count').catch(() => 0)) || 0);
|
||||
const osd = await mpv.get<{ w: number; h: number; ml: number; mr: number; mt: number; mb: number }>('osd-dimensions').catch(() => null);
|
||||
const v = usePlayer.getState().video;
|
||||
return {
|
||||
stages,
|
||||
// Pass timings are in nanoseconds.
|
||||
gpuMs: passes.reduce((a, p) => a + (p.avg || 0), 0) / 1e6,
|
||||
budgetMs: 1000 / fps,
|
||||
dropped,
|
||||
input: v?.width ? `${v.width}×${v.height}` : '',
|
||||
output: osd ? `${osd.w - osd.ml - osd.mr}×${osd.h - osd.mt - osd.mb}` : '',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* While a neural tier plays, watch for dropped frames and suggest a lighter
|
||||
* tier once per title if the GPU can't keep up.
|
||||
*/
|
||||
let watchTimer: ReturnType<typeof setInterval> | null = null;
|
||||
export function watchPerformance(active: boolean) {
|
||||
if (watchTimer) clearInterval(watchTimer);
|
||||
watchTimer = null;
|
||||
if (!active || !isNative) return;
|
||||
let last = -1;
|
||||
let warned = false;
|
||||
watchTimer = setInterval(async () => {
|
||||
const st = usePlayer.getState();
|
||||
const mode = st.enhance.active;
|
||||
if (!st.open || st.paused || st.enhance.bypass || (mode !== 'quality' && mode !== 'balanced')) return void (last = -1);
|
||||
const dropped = (await mpv.get<number>('frame-drop-count').catch(() => 0)) || 0;
|
||||
if (last >= 0 && dropped - last >= 10 && !warned) {
|
||||
warned = true;
|
||||
toast(`${UPSCALING[mode].label} is dropping frames on this video. Try a lighter upscaling tier.`);
|
||||
}
|
||||
last = dropped;
|
||||
}, 5000);
|
||||
}
|
||||
|
||||
// Stage toggles and the tier apply straight away while something plays.
|
||||
useSettings.subscribe((prefs, prev) => {
|
||||
if (!isNative || !usePlayer.getState().open) return;
|
||||
if (prefs.upscaling !== prev.upscaling || prefs.enhanceDeband !== prev.enhanceDeband || prefs.enhanceSharpen !== prev.enhanceSharpen || prefs.enhanceDenoise !== prev.enhanceDenoise) requestEnhancement();
|
||||
});
|
||||
|
|
@ -4,9 +4,10 @@ import { directUrl, previewUrl, reportTimeline, stopTranscode, transcodeUrl, typ
|
|||
import type { Item, Stream } from '../plex/types';
|
||||
import { useSession } from '../app/session';
|
||||
import { useSettings, SUBTITLE_MARGIN, type Preferences, type Quality, type Upscaling } from '../app/settings';
|
||||
import { noteMpvLog, requestEnhancement, watchPerformance } from './enhance';
|
||||
import { channelLabel } from '../app/format';
|
||||
import { invalidateLibrary } from '../app/queries';
|
||||
import { keepAwake, isNative, resourcePath, win } from '../platform/native';
|
||||
import { keepAwake, isNative, win } from '../platform/native';
|
||||
import { mpv, type MpvEvent, type MpvTrack } from './engine';
|
||||
import { toast } from '../ui/overlays';
|
||||
import { stopThemeMusic } from './theme';
|
||||
|
|
@ -64,6 +65,10 @@ interface PlayerState {
|
|||
/** Playing in the corner while browsing. */
|
||||
mini: boolean;
|
||||
audioDelay: number;
|
||||
/** The upscaling tier actually running (may be below the chosen one), and the before/after bypass. */
|
||||
enhance: { active: Upscaling; bypass: boolean; preparing: boolean };
|
||||
/** Transfer function of the decoded video ('pq', 'hlg', 'bt.1886'…), once known. */
|
||||
videoGamma: string;
|
||||
|
||||
play(item: Item, opts?: PlayOptions): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
|
|
@ -203,43 +208,6 @@ async function applySubtitleStyle(prefs: Preferences = useSettings.getState()) {
|
|||
}
|
||||
}
|
||||
|
||||
/** mpv's own scaler settings, read once so Off can put them back. */
|
||||
const SCALER_PROPS = ['scale', 'cscale', 'dscale', 'sigmoid-upscaling', 'deband', 'deband-iterations', 'deband-grain'] as const;
|
||||
let scalerDefaults: Partial<Record<(typeof SCALER_PROPS)[number], unknown>> | null = null;
|
||||
let shaderDir: Promise<string> | null = null;
|
||||
|
||||
/**
|
||||
* Video enhancement tiers. The bundled shaders decide for themselves when to
|
||||
* run (their WHEN conditions), so one list covers every video and window size:
|
||||
* FSRCNNX doubles luma only when upscaling by more than 1.3×, KrigBilateral
|
||||
* rebuilds subsampled colour, CAS sharpens at native size (CAS-prescale also
|
||||
* before upscaling). CAS assumes SDR gamma, so HDR titles skip it.
|
||||
*/
|
||||
async function applyUpscaling(mode: Upscaling = useSettings.getState().upscaling) {
|
||||
if (!scalerDefaults) {
|
||||
scalerDefaults = {};
|
||||
for (const p of SCALER_PROPS) scalerDefaults[p] = await mpv.get(p).catch(() => undefined);
|
||||
}
|
||||
const st = usePlayer.getState();
|
||||
const hdr = !!st.item?.media[st.mediaIndex]?.dynamicRange;
|
||||
const dir = await (shaderDir ??= resourcePath('shaders').catch(() => ''));
|
||||
// mpv takes a path list: ';'-separated on Windows, ':' elsewhere.
|
||||
const windows = dir.includes('\\');
|
||||
const shaders = (names: string[]) => names.map((n) => `${dir}${windows ? '\\' : '/'}${n}`).join(windows ? ';' : ':');
|
||||
let props: Record<string, unknown>;
|
||||
let list: string[] = [];
|
||||
if (mode === 'performance') {
|
||||
props = { scale: 'spline36', cscale: 'spline36', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 1, 'deband-grain': 24 };
|
||||
if (!hdr) list = ['CAS-prescale.glsl'];
|
||||
} else if (mode === 'balanced') {
|
||||
props = { scale: 'ewa_lanczossharp', cscale: 'ewa_lanczossharp', dscale: 'mitchell', 'sigmoid-upscaling': true, deband: true, 'deband-iterations': 2, 'deband-grain': 16 };
|
||||
list = ['FSRCNNX_x2_8-0-4-1.glsl', 'KrigBilateral.glsl', ...(hdr ? [] : ['CAS.glsl'])];
|
||||
} else props = { ...scalerDefaults };
|
||||
for (const [k, v] of Object.entries(props)) if (v !== undefined) await mpv.set(k, v).catch(() => {});
|
||||
if (dir && list.length) await mpv.set('glsl-shaders', shaders(list)).catch((e) => console.warn('[upscaling]', e));
|
||||
else await mpv.command('change-list', 'glsl-shaders', 'clr', '').catch(() => {});
|
||||
}
|
||||
|
||||
function report(state: 'playing' | 'paused' | 'buffering' | 'stopped') {
|
||||
const s = server();
|
||||
const { item, position, delivery } = usePlayer.getState();
|
||||
|
|
@ -426,6 +394,11 @@ function onEvent(e: MpvEvent) {
|
|||
break;
|
||||
case 'video-params':
|
||||
if (v?.w) usePlayer.setState({ video: { ...(st.video ?? { hwdec: '', codec: '' }), width: v.w, height: v.h } });
|
||||
// Enhancement follows the decoded video's transfer (HDR skips SDR-tuned stages).
|
||||
if (v?.gamma && v.gamma !== st.videoGamma) {
|
||||
usePlayer.setState({ videoGamma: v.gamma });
|
||||
requestEnhancement();
|
||||
}
|
||||
break;
|
||||
case 'hwdec-current':
|
||||
usePlayer.setState({ video: { ...(st.video ?? { width: 0, height: 0, codec: '' }), hwdec: v ?? '' } });
|
||||
|
|
@ -467,6 +440,7 @@ function onEvent(e: MpvEvent) {
|
|||
} else if (e.reason === 'eof') onEnded();
|
||||
break;
|
||||
case 'log':
|
||||
noteMpvLog(e.level, e.prefix, e.text);
|
||||
if (e.level === 'error' && /http|stream|open/i.test(e.prefix + e.text) && !st.firstFrame) console.warn('[mpv]', e.prefix, e.text);
|
||||
break;
|
||||
}
|
||||
|
|
@ -530,6 +504,8 @@ export const usePlayer = create<PlayerState>((set, get) => ({
|
|||
fullscreen: false,
|
||||
mini: false,
|
||||
audioDelay: 0,
|
||||
enhance: { active: 'off', bypass: false, preparing: false },
|
||||
videoGamma: '',
|
||||
|
||||
async play(input, opts = {}) {
|
||||
const token = ++loadToken;
|
||||
|
|
@ -572,6 +548,8 @@ export const usePlayer = create<PlayerState>((set, get) => ({
|
|||
video: null,
|
||||
fullscreen: wasFullscreenBeforePlay,
|
||||
mini: false,
|
||||
videoGamma: '',
|
||||
enhance: { ...previous.enhance, bypass: false, preparing: false },
|
||||
});
|
||||
|
||||
try {
|
||||
|
|
@ -627,7 +605,8 @@ export const usePlayer = create<PlayerState>((set, get) => ({
|
|||
await mpv.set('speed', 1);
|
||||
await mpv.set('audio-delay', 0);
|
||||
await applySubtitleStyle(prefs);
|
||||
await applyUpscaling(prefs.upscaling);
|
||||
await requestEnhancement();
|
||||
watchPerformance(true);
|
||||
await mpv.set('force-media-title', item.type === 'episode' ? `${item.grandparentTitle} — ${item.title}` : item.title);
|
||||
await mpv.show(true);
|
||||
const fileOpts = [
|
||||
|
|
@ -659,6 +638,7 @@ export const usePlayer = create<PlayerState>((set, get) => ({
|
|||
const s = server();
|
||||
if (st.delivery?.session && s) stopTranscode(s, st.delivery.session);
|
||||
keepAwake(false);
|
||||
watchPerformance(false);
|
||||
if (isNative) {
|
||||
await mpv.command('stop').catch(() => {});
|
||||
await mpv.show(false).catch(() => {});
|
||||
|
|
@ -822,8 +802,3 @@ const SUBTITLE_PREFS = ['subtitleScale', 'subtitleColor', 'subtitleEdge', 'subti
|
|||
useSettings.subscribe((prefs, prev) => {
|
||||
if (isNative && usePlayer.getState().open && SUBTITLE_PREFS.some((k) => prefs[k] !== prev[k])) applySubtitleStyle(prefs);
|
||||
});
|
||||
|
||||
/** The enhancement tier, chosen in the player, applies straight away. */
|
||||
useSettings.subscribe((prefs, prev) => {
|
||||
if (isNative && usePlayer.getState().open && prefs.upscaling !== prev.upscaling) applyUpscaling(prefs.upscaling);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue