diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 99194a1..1fd921b 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -576,6 +576,85 @@ pub async fn save_playback( state.db.lock().await.save_playback(&video_id, position, duration) } +/// How much of a watch page to read. `lengthSeconds` sits in the player +/// response near the top, so there is no need to pull a megabyte of HTML. +const DURATION_PROBE_BYTES: &str = "bytes=0-262143"; +/// Deliberately small. An earlier version fetched 24 pages every 12 seconds at +/// six concurrent — about two requests a second, sustained — and YouTube +/// answered with "Sign in to confirm you're not a bot" for the whole IP, +/// breaking playback and downloads as well. Filling every length matters far +/// less than staying under the radar, so this trickles. +const DURATION_BATCH: usize = 4; +const DURATION_CONCURRENCY: usize = 1; + +/// Fills in video lengths, which the Atom feed does not carry. +/// +/// yt-dlp would cost seconds per video; a ranged GET of the watch page costs +/// about one, and only ever runs for videos whose length is still unknown. +#[tauri::command] +pub async fn fetch_durations(state: State<'_, AppState>) -> Result { + let ids = state + .db + .lock() + .await + .videos_missing_duration(DURATION_BATCH as i64)?; + if ids.is_empty() { + return Ok(0); + } + + let http = state.http.clone(); + let found = futures::stream::iter(ids.into_iter().map(|id| { + let http = http.clone(); + async move { + let secs = probe_duration(&http, &id).await; + (id, secs) + } + })) + .buffer_unordered(DURATION_CONCURRENCY) + .collect::>() + .await; + + let attempted = found.len(); + let db = state.db.lock().await; + let mut n = 0; + for (id, secs) in found { + if let Some(secs) = secs { + db.set_duration(&id, secs)?; + n += 1; + } + } + drop(db); + + // Every probe failing means YouTube is refusing us, not that these videos + // have no length. Stop asking rather than hammering a closed door. + if n == 0 && attempted > 0 { + return Err("Duration lookup is being refused; backing off.".into()); + } + Ok(n) +} + +async fn probe_duration(http: &reqwest::Client, video_id: &str) -> Option { + let body = http + .get(format!("https://www.youtube.com/watch?v={video_id}")) + .header(reqwest::header::RANGE, DURATION_PROBE_BYTES) + .send() + .await + .ok()? + .text() + .await + .ok()?; + parse_length_seconds(&body) +} + +/// Pulls `"lengthSeconds":"1315"` out of the watch page. +pub fn parse_length_seconds(body: &str) -> Option { + let needle = "\"lengthSeconds\":\""; + let at = body.find(needle)? + needle.len(); + let rest = &body[at..]; + let end = rest.find('"')?; + rest[..end].parse().ok().filter(|n| *n > 0) +} + #[tauri::command] pub async fn list_channels(state: State<'_, AppState>) -> Result, String> { state.db.lock().await.list_channels() @@ -834,6 +913,10 @@ pub async fn download_video( .await .ok_or("Download finished but the file could not be located.")?, }; + // YouTube's captions arrive pinned to the left edge and full of + // karaoke timing tags; clean them before they reach the player. + tidy_subtitles(&library, &video_id).await; + let db = state.db.lock().await; db.set_download_state(&video_id, DownloadState::Done, None)?; db.set_download_path(&video_id, &path)?; @@ -897,6 +980,62 @@ pub async fn cancel_download( Ok(()) } +/// Rewrites downloaded WebVTT so it renders as ordinary subtitles. +/// +/// YouTube's auto-captions carry `align:start position:0%` on every cue, which +/// pins them to the left edge where long lines are clipped, plus inline +/// `<00:00:12.480>word` timing tags that render as half-grey karaoke +/// text. Both are stripped; the timings themselves are untouched. +async fn tidy_subtitles(library: &std::path::Path, video_id: &str) { + let Ok(mut entries) = tokio::fs::read_dir(library).await else { + return; + }; + while let Ok(Some(entry)) = entries.next_entry().await { + let name = entry.file_name().to_string_lossy().to_string(); + if !name.contains(video_id) || !name.ends_with(".vtt") { + continue; + } + let path = entry.path(); + if let Ok(text) = tokio::fs::read_to_string(&path).await { + let _ = tokio::fs::write(&path, tidy_vtt(&text)).await; + } + } +} + +pub fn tidy_vtt(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + for line in input.lines() { + if line.contains("-->") { + // Keep the timing, drop every cue setting after it. + let end = line.find("-->").map(|i| i + 3).unwrap_or(0); + let rest = &line[end..]; + let stamp = rest.split_whitespace().next().unwrap_or(""); + out.push_str(&line[..end]); + out.push(' '); + out.push_str(stamp); + } else { + out.push_str(&strip_cue_tags(line)); + } + out.push('\n'); + } + out +} + +/// Removes `<...>` spans — both timestamps and `` wrappers. +fn strip_cue_tags(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut depth = 0usize; + for ch in line.chars() { + match ch { + '<' => depth += 1, + '>' => depth = depth.saturating_sub(1), + _ if depth == 0 => out.push(ch), + _ => {} + } + } + out +} + /// Locates a finished download by the `[]` tag in its filename. async fn find_by_video_id(library: &std::path::Path, video_id: &str) -> Option { let mut entries = tokio::fs::read_dir(library).await.ok()?; @@ -939,6 +1078,15 @@ pub async fn list_subtitles( if !name.contains(&video_id) || !name.ends_with(".vtt") { continue; } + // Downloads made before the cleanup existed still carry YouTube's + // edge-pinned cues, so tidy them the first time they are listed. + let path = entry.path(); + if let Ok(text) = tokio::fs::read_to_string(&path).await { + if text.contains("align:") || text.contains("position:") || text.contains("") { + let _ = tokio::fs::write(&path, tidy_vtt(&text)).await; + } + } + // yt-dlp names them "..vtt". let lang = name .trim_end_matches(".vtt") @@ -946,7 +1094,7 @@ pub async fn list_subtitles( .next() .unwrap_or("") .to_string(); - out.push((lang, entry.path().to_string_lossy().to_string())); + out.push((lang, path.to_string_lossy().to_string())); } out.sort(); Ok(out) @@ -1083,6 +1231,48 @@ mod tests { .to_string() } + use super::{parse_length_seconds, tidy_vtt}; + + /// Exactly the shape yt-dlp writes for YouTube auto-captions. + const RAW_VTT: &str = concat!( + "WEBVTT\nKind: captions\nLanguage: en\n\n", + "00:00:12.400 --> 00:00:26.950 align:start position:0%\n", + "Heat<00:00:12.480> up<00:00:12.480> here.\n", + ); + + #[test] + fn cue_settings_that_pin_subtitles_to_the_edge_are_removed() { + let out = tidy_vtt(RAW_VTT); + assert!(!out.contains("align:start")); + assert!(!out.contains("position:0%")); + // The timing itself must survive intact. + assert!(out.contains("00:00:12.400 --> 00:00:26.950")); + } + + #[test] + fn karaoke_timing_tags_are_stripped_leaving_plain_text() { + let out = tidy_vtt(RAW_VTT); + assert!(out.contains("Heat up here.")); + assert!(!out.contains("")); + assert!(!out.contains("00:00:12.480>")); + } + + #[test] + fn the_header_survives_or_the_file_stops_being_webvtt() { + assert!(tidy_vtt(RAW_VTT).starts_with("WEBVTT")); + } + + #[test] + fn reads_the_length_out_of_a_watch_page() { + assert_eq!( + parse_length_seconds(r#"...,"lengthSeconds":"1315","isLive"..."#), + Some(1315) + ); + assert_eq!(parse_length_seconds("nothing here"), None); + // A zero length is meaningless and must not be stored. + assert_eq!(parse_length_seconds(r#""lengthSeconds":"0""#), None); + } + #[test] fn the_original_language_becomes_the_default_track() { let out = rewrite_master(DUBBED, None, Some("en")).unwrap(); diff --git a/src-tauri/src/db.rs b/src-tauri/src/db.rs index 77924b1..c246020 100644 --- a/src-tauri/src/db.rs +++ b/src-tauri/src/db.rs @@ -86,9 +86,42 @@ impl Db { fn init(conn: Connection) -> Result { conn.execute_batch(SCHEMA) .map_err(|e| format!("Cannot create schema: {e}"))?; + // CREATE TABLE IF NOT EXISTS leaves existing databases alone, so new + // columns need adding explicitly. The error when it already exists is + // the expected case, not a failure. + let _ = conn.execute("ALTER TABLE videos ADD COLUMN duration INTEGER", []); Ok(Db { conn }) } + /// Records a video's length in seconds. + pub fn set_duration(&self, video_id: &str, seconds: i64) -> Result<(), String> { + if seconds <= 0 { + return Ok(()); + } + self.conn + .execute( + "UPDATE videos SET duration = ?2 WHERE id = ?1", + params![video_id, seconds], + ) + .map_err(|e| e.to_string())?; + Ok(()) + } + + /// Videos whose length is still unknown, newest first. + pub fn videos_missing_duration(&self, limit: i64) -> Result, String> { + let mut stmt = self + .conn + .prepare( + "SELECT id FROM videos WHERE duration IS NULL + ORDER BY published DESC LIMIT ?1", + ) + .map_err(|e| e.to_string())?; + let rows = stmt + .query_map(params![limit], |r| r.get::<_, String>(0)) + .map_err(|e| e.to_string())?; + rows.collect::, _>>().map_err(|e| e.to_string()) + } + /// What a replacing import would destroy. Callers show this before asking /// the user to confirm, so nothing is deleted without being named first. pub fn preview_replace(&self, incoming: &[Channel]) -> Result { @@ -349,7 +382,8 @@ impl Db { let mut sql = String::from( "SELECT v.id, v.channel_id, COALESCE(c.title, ''), v.title, v.description, v.published, v.thumb_url, v.thumb_path, v.views, v.is_short, - d.state, d.path, d.pct, d.error, p.position, p.duration + d.state, d.path, d.pct, d.error, p.position, + COALESCE(v.duration, p.duration) FROM videos v LEFT JOIN channels c ON c.id = v.channel_id LEFT JOIN downloads d ON d.video_id = v.id @@ -759,6 +793,33 @@ mod tests { assert_eq!(feed[0].id, "b"); } + #[test] + fn a_recorded_duration_reaches_the_feed() { + let db = seeded(); + db.set_duration("a", 754).unwrap(); + let feed = db.list_feed(&FeedFilter::default()).unwrap(); + assert_eq!(feed.iter().find(|f| f.id == "a").unwrap().duration, Some(754.0)); + } + + #[test] + fn playback_duration_fills_in_when_the_video_length_is_unknown() { + let db = seeded(); + db.save_playback("b", 10.0, 300.0).unwrap(); + let feed = db.list_feed(&FeedFilter::default()).unwrap(); + assert_eq!(feed.iter().find(|f| f.id == "b").unwrap().duration, Some(300.0)); + } + + #[test] + fn videos_without_a_duration_are_reported_for_lookup() { + let db = seeded(); + assert_eq!(db.videos_missing_duration(10).unwrap().len(), 3); + db.set_duration("a", 100).unwrap(); + assert_eq!(db.videos_missing_duration(10).unwrap().len(), 2); + // A nonsense length is ignored rather than stored. + db.set_duration("b", 0).unwrap(); + assert_eq!(db.videos_missing_duration(10).unwrap().len(), 2); + } + #[test] fn downloads_in_progress_still_show_in_the_downloaded_filter() { let db = seeded(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 53073c5..3eadae5 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -160,6 +160,7 @@ fn build_menu(app: &tauri::AppHandle) -> tauri::Result> { commands::delete_download, commands::delete_all_downloads, commands::list_subtitles, + commands::fetch_durations, commands::get_connectivity, commands::set_library_path, commands::open_external, diff --git a/src/App.tsx b/src/App.tsx index df232b2..c40fcea 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,7 +1,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { cancelDownload, deleteAllDownloads, deleteDownload, downloadVideo, - onRefreshProgress, refreshFeeds, + fetchDurations, onRefreshProgress, refreshFeeds, } from "./api"; import Player from "./components/Player"; import Settings from "./components/Settings"; @@ -23,6 +23,12 @@ import { const TOAST_MS = 2400; /** How often to pull new videos while online, so the feed stays live. */ const AUTO_REFRESH_MS = 10 * 60 * 1000; +/** + * Video lengths trickle in. This was once every 12s and it got the whole IP + * challenged by YouTube, which broke playback and downloads too — the feed + * being fully annotated is not worth that. + */ +const DURATION_FILL_MS = 5 * 60 * 1000; function remembered(key: string): boolean { try { @@ -218,6 +224,30 @@ export default function App() { return () => clearInterval(id); }, [online, refreshing, playingIndex, doRefresh]); + // The Atom feed carries no duration, so lengths are looked up a batch at a + // time in the background and cached. Paused while the player is open. + useEffect(() => { + if (!online) return; + let stop = false; + // Set when the backend reports it is being refused, so we stop for the + // rest of the session rather than making the block worse. + let refused = false; + const tick = async () => { + if (stop || refused || playingIndex != null) return; + try { + if ((await fetchDurations()) > 0 && !stop) await reload(); + } catch { + refused = true; + } + }; + void tick(); + const id = setInterval(tick, DURATION_FILL_MS); + return () => { + stop = true; + clearInterval(id); + }; + }, [online, playingIndex, reload]); + // Refresh once on launch, as soon as there is a connection and something to // refresh, so the feed is current without anyone pressing anything. const launched = useRef(false); @@ -378,6 +408,7 @@ export default function App() { total={items.length} maxHeight={streamQuality === "best" ? null : Number(streamQuality)} subLang={subLang} + onSubLang={setSubLang} titleBarInset={titleBarInset} onPrev={ stepFrom(playingIndex, -1) != null diff --git a/src/api.ts b/src/api.ts index b9ba341..73cbbf0 100644 --- a/src/api.ts +++ b/src/api.ts @@ -29,6 +29,9 @@ export const downloadVideo = (videoId: string, quality: Quality, subLangs: strin export const deleteAllDownloads = () => invoke("delete_all_downloads"); +/** Fills in missing video lengths, a batch at a time. Returns how many. */ +export const fetchDurations = () => invoke("fetch_durations"); + /** WebVTT sidecars beside a downloaded video, as [language, path] pairs. */ export const listSubtitles = (videoId: string) => invoke>("list_subtitles", { videoId }); diff --git a/src/components/Player.tsx b/src/components/Player.tsx index c07b868..6ec66c6 100644 --- a/src/components/Player.tsx +++ b/src/components/Player.tsx @@ -20,6 +20,8 @@ interface Props { maxHeight: number | null; /** Preferred subtitle language, or "off". */ subLang: string; + /** Persists a subtitle choice made from the transport bar. */ + onSubLang: (l: string) => void; /** Position in the current feed, for the "3 of 180" readout. */ index: number; total: number; @@ -99,7 +101,7 @@ const RESUME_EDGE_S = 5; */ export default function Player({ item, path, onClose, onDelete, onPrev, onNext, onDownload, downloading, - maxHeight, subLang, index, total, titleBarInset, + maxHeight, subLang, onSubLang, index, total, titleBarInset, }: Props) { const streaming = path === null; const [src, setSrc] = useState(path ? fileUrl(path) : null); @@ -220,8 +222,14 @@ export default function Player({ return () => window.clearTimeout(hideTimer.current); }, [showChrome, src]); + const navIcon = + "grid h-[30px] w-[30px] shrink-0 place-items-center rounded-lg border border-slate-300 " + + "cursor-pointer hover:border-sky-500 hover:text-sky-600 disabled:opacity-30 " + + "disabled:cursor-not-allowed dark:border-slate-700 dark:hover:border-sky-500 " + + "dark:hover:text-sky-400"; + const navBtn = - "rounded-lg border border-slate-300 px-2 py-1.5 text-[11px] font-medium cursor-pointer " + + "inline-flex h-[30px] items-center rounded-lg border border-slate-300 px-2.5 text-[12px] font-medium cursor-pointer " + "hover:border-sky-500 hover:text-sky-600 disabled:opacity-30 disabled:cursor-not-allowed " + "disabled:hover:border-slate-300 disabled:hover:text-inherit " + "dark:border-slate-700 dark:hover:border-sky-500 dark:hover:text-sky-400"; @@ -234,7 +242,7 @@ export default function Player({ dark:border-slate-800 dark:bg-slate-900" > )} {!streaming && ( @@ -402,7 +419,7 @@ export default function Player({ onClick={() => openExternal(`https://www.youtube.com/watch?v=${item.id}`)} title="Open on YouTube" aria-label="Open on YouTube" - className={`${navBtn} w-[30px] p-0`} + className={navIcon} > void; /** Preferred subtitle language, or "off" to start with none. */ subLang: string; + /** Persists a choice made here, so the next video matches. */ + onSubLang: (l: string) => void; } const SKIP_S = 10; @@ -61,7 +63,9 @@ const btn = * Owning the bar is the only way to get every control into one strip along the * bottom, so the native ones are switched off entirely. */ -export default function PlayerControls({ videoRef, stageRef, onActivity, subLang }: Props) { +export default function PlayerControls({ + videoRef, stageRef, onActivity, subLang, onSubLang, +}: Props) { const [playing, setPlaying] = useState(false); const [time, setTime] = useState(0); const [duration, setDuration] = useState(0); @@ -148,6 +152,9 @@ export default function PlayerControls({ videoRef, stageRef, onActivity, subLang for (const t of Array.from(v.textTracks)) { t.mode = t === track ? "showing" : "disabled"; } + // The choice becomes the preference, so the next video matches without + // going back to Settings. + onSubLang(track ? (track.language || "off").split("-")[0] : "off"); bump((n) => n + 1); onActivity?.(); }; diff --git a/src/components/VideoRow.tsx b/src/components/VideoRow.tsx index 390316d..c2f36d7 100644 --- a/src/components/VideoRow.tsx +++ b/src/components/VideoRow.tsx @@ -3,7 +3,7 @@ import type { LiveDownload } from "../hooks/useDownloads"; import type { FeedItem } from "../types"; import DownloadButton from "./DownloadButton"; import WatchBar from "./WatchBar"; -import { compactViews, relativeTime } from "./format"; +import { clockDuration, compactViews, relativeTime } from "./format"; interface Props { item: FeedItem; @@ -49,12 +49,20 @@ export default function VideoRow({ )} {downloaded && ( Offline )} + {clockDuration(item.duration) && ( + + {clockDuration(item.duration)} + + )} diff --git a/src/components/VideoTile.tsx b/src/components/VideoTile.tsx index 414492a..cacdf71 100644 --- a/src/components/VideoTile.tsx +++ b/src/components/VideoTile.tsx @@ -3,7 +3,7 @@ import type { LiveDownload } from "../hooks/useDownloads"; import type { FeedItem } from "../types"; import DownloadButton from "./DownloadButton"; import WatchBar from "./WatchBar"; -import { compactViews, relativeTime } from "./format"; +import { clockDuration, compactViews, relativeTime } from "./format"; interface Props { item: FeedItem; @@ -46,12 +46,20 @@ export default function VideoTile({ )} {downloaded && ( Offline )} + {clockDuration(item.duration) && ( + + {clockDuration(item.duration)} + + )} diff --git a/src/components/format.ts b/src/components/format.ts index 5e0daf5..cd4971f 100644 --- a/src/components/format.ts +++ b/src/components/format.ts @@ -41,3 +41,13 @@ export function humanEta(seconds: number | null): string { const s = Math.floor(seconds % 60); return m > 0 ? `${m}m ${s}s left` : `${s}s left`; } + +/** Video length as YouTube shows it: 4:12, or 1:04:12 past an hour. */ +export function clockDuration(seconds: number | null): string { + if (seconds == null || !Number.isFinite(seconds) || seconds <= 0) return ""; + const h = Math.floor(seconds / 3600); + const m = Math.floor((seconds % 3600) / 60); + const s = Math.floor(seconds % 60); + const mm = h > 0 ? String(m).padStart(2, "0") : String(m); + return `${h > 0 ? `${h}:` : ""}${mm}:${String(s).padStart(2, "0")}`; +}