feat: durations, clean subtitles, icon buttons
Subtitles rendered pinned to the left edge and clipped, in half-grey karaoke text: YouTube's auto-captions carry align:start position:0% on every cue plus inline <00:00:12.480><c>word</c> timing tags. Both are now stripped after download, and existing downloads are tidied the first time they are listed. Thumbnails show video length. The Atom feed carries no duration, so it is read from the watch page and cached — but only a trickle. The first version fetched 24 pages every 12 seconds at six concurrent, roughly two requests a second sustained, and YouTube answered by challenging the whole IP: 'Sign in to confirm you are not a bot', which broke streaming and downloads too. It is now four pages every five minutes, one at a time, and stops for the session the moment a batch is refused. A subtitle chosen in the player becomes the stored preference, so the next video matches without going back to Settings. The back and download buttons are square icon buttons; the back glyph was off-centre because px-2 beat the p-0 meant to clear it.
This commit is contained in:
vincent
+191
-1
@@ -576,6 +576,85 @@ pub async fn save_playback(
|
||||
state.db.lock().await.save_playback(&video_id, position, duration)
|
||||
}
|
||||
|
||||
/// How much of a watch page to read. `lengthSeconds` sits in the player
|
||||
/// response near the top, so there is no need to pull a megabyte of HTML.
|
||||
const DURATION_PROBE_BYTES: &str = "bytes=0-262143";
|
||||
/// Deliberately small. An earlier version fetched 24 pages every 12 seconds at
|
||||
/// six concurrent — about two requests a second, sustained — and YouTube
|
||||
/// answered with "Sign in to confirm you're not a bot" for the whole IP,
|
||||
/// breaking playback and downloads as well. Filling every length matters far
|
||||
/// less than staying under the radar, so this trickles.
|
||||
const DURATION_BATCH: usize = 4;
|
||||
const DURATION_CONCURRENCY: usize = 1;
|
||||
|
||||
/// Fills in video lengths, which the Atom feed does not carry.
|
||||
///
|
||||
/// yt-dlp would cost seconds per video; a ranged GET of the watch page costs
|
||||
/// about one, and only ever runs for videos whose length is still unknown.
|
||||
#[tauri::command]
|
||||
pub async fn fetch_durations(state: State<'_, AppState>) -> Result<usize, String> {
|
||||
let ids = state
|
||||
.db
|
||||
.lock()
|
||||
.await
|
||||
.videos_missing_duration(DURATION_BATCH as i64)?;
|
||||
if ids.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let http = state.http.clone();
|
||||
let found = futures::stream::iter(ids.into_iter().map(|id| {
|
||||
let http = http.clone();
|
||||
async move {
|
||||
let secs = probe_duration(&http, &id).await;
|
||||
(id, secs)
|
||||
}
|
||||
}))
|
||||
.buffer_unordered(DURATION_CONCURRENCY)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
let attempted = found.len();
|
||||
let db = state.db.lock().await;
|
||||
let mut n = 0;
|
||||
for (id, secs) in found {
|
||||
if let Some(secs) = secs {
|
||||
db.set_duration(&id, secs)?;
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
drop(db);
|
||||
|
||||
// Every probe failing means YouTube is refusing us, not that these videos
|
||||
// have no length. Stop asking rather than hammering a closed door.
|
||||
if n == 0 && attempted > 0 {
|
||||
return Err("Duration lookup is being refused; backing off.".into());
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
async fn probe_duration(http: &reqwest::Client, video_id: &str) -> Option<i64> {
|
||||
let body = http
|
||||
.get(format!("https://www.youtube.com/watch?v={video_id}"))
|
||||
.header(reqwest::header::RANGE, DURATION_PROBE_BYTES)
|
||||
.send()
|
||||
.await
|
||||
.ok()?
|
||||
.text()
|
||||
.await
|
||||
.ok()?;
|
||||
parse_length_seconds(&body)
|
||||
}
|
||||
|
||||
/// Pulls `"lengthSeconds":"1315"` out of the watch page.
|
||||
pub fn parse_length_seconds(body: &str) -> Option<i64> {
|
||||
let needle = "\"lengthSeconds\":\"";
|
||||
let at = body.find(needle)? + needle.len();
|
||||
let rest = &body[at..];
|
||||
let end = rest.find('"')?;
|
||||
rest[..end].parse().ok().filter(|n| *n > 0)
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn list_channels(state: State<'_, AppState>) -> Result<Vec<ChannelWithCount>, String> {
|
||||
state.db.lock().await.list_channels()
|
||||
@@ -834,6 +913,10 @@ pub async fn download_video(
|
||||
.await
|
||||
.ok_or("Download finished but the file could not be located.")?,
|
||||
};
|
||||
// YouTube's captions arrive pinned to the left edge and full of
|
||||
// karaoke timing tags; clean them before they reach the player.
|
||||
tidy_subtitles(&library, &video_id).await;
|
||||
|
||||
let db = state.db.lock().await;
|
||||
db.set_download_state(&video_id, DownloadState::Done, None)?;
|
||||
db.set_download_path(&video_id, &path)?;
|
||||
@@ -897,6 +980,62 @@ pub async fn cancel_download(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rewrites downloaded WebVTT so it renders as ordinary subtitles.
|
||||
///
|
||||
/// YouTube's auto-captions carry `align:start position:0%` on every cue, which
|
||||
/// pins them to the left edge where long lines are clipped, plus inline
|
||||
/// `<00:00:12.480><c>word</c>` timing tags that render as half-grey karaoke
|
||||
/// text. Both are stripped; the timings themselves are untouched.
|
||||
async fn tidy_subtitles(library: &std::path::Path, video_id: &str) {
|
||||
let Ok(mut entries) = tokio::fs::read_dir(library).await else {
|
||||
return;
|
||||
};
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if !name.contains(video_id) || !name.ends_with(".vtt") {
|
||||
continue;
|
||||
}
|
||||
let path = entry.path();
|
||||
if let Ok(text) = tokio::fs::read_to_string(&path).await {
|
||||
let _ = tokio::fs::write(&path, tidy_vtt(&text)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn tidy_vtt(input: &str) -> String {
|
||||
let mut out = String::with_capacity(input.len());
|
||||
for line in input.lines() {
|
||||
if line.contains("-->") {
|
||||
// Keep the timing, drop every cue setting after it.
|
||||
let end = line.find("-->").map(|i| i + 3).unwrap_or(0);
|
||||
let rest = &line[end..];
|
||||
let stamp = rest.split_whitespace().next().unwrap_or("");
|
||||
out.push_str(&line[..end]);
|
||||
out.push(' ');
|
||||
out.push_str(stamp);
|
||||
} else {
|
||||
out.push_str(&strip_cue_tags(line));
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Removes `<...>` spans — both timestamps and `<c>` wrappers.
|
||||
fn strip_cue_tags(line: &str) -> String {
|
||||
let mut out = String::with_capacity(line.len());
|
||||
let mut depth = 0usize;
|
||||
for ch in line.chars() {
|
||||
match ch {
|
||||
'<' => depth += 1,
|
||||
'>' => depth = depth.saturating_sub(1),
|
||||
_ if depth == 0 => out.push(ch),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Locates a finished download by the `[<id>]` tag in its filename.
|
||||
async fn find_by_video_id(library: &std::path::Path, video_id: &str) -> Option<String> {
|
||||
let mut entries = tokio::fs::read_dir(library).await.ok()?;
|
||||
@@ -939,6 +1078,15 @@ pub async fn list_subtitles(
|
||||
if !name.contains(&video_id) || !name.ends_with(".vtt") {
|
||||
continue;
|
||||
}
|
||||
// Downloads made before the cleanup existed still carry YouTube's
|
||||
// edge-pinned cues, so tidy them the first time they are listed.
|
||||
let path = entry.path();
|
||||
if let Ok(text) = tokio::fs::read_to_string(&path).await {
|
||||
if text.contains("align:") || text.contains("position:") || text.contains("<c>") {
|
||||
let _ = tokio::fs::write(&path, tidy_vtt(&text)).await;
|
||||
}
|
||||
}
|
||||
|
||||
// yt-dlp names them "<base>.<lang>.vtt".
|
||||
let lang = name
|
||||
.trim_end_matches(".vtt")
|
||||
@@ -946,7 +1094,7 @@ pub async fn list_subtitles(
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
out.push((lang, entry.path().to_string_lossy().to_string()));
|
||||
out.push((lang, path.to_string_lossy().to_string()));
|
||||
}
|
||||
out.sort();
|
||||
Ok(out)
|
||||
@@ -1083,6 +1231,48 @@ mod tests {
|
||||
.to_string()
|
||||
}
|
||||
|
||||
use super::{parse_length_seconds, tidy_vtt};
|
||||
|
||||
/// Exactly the shape yt-dlp writes for YouTube auto-captions.
|
||||
const RAW_VTT: &str = concat!(
|
||||
"WEBVTT\nKind: captions\nLanguage: en\n\n",
|
||||
"00:00:12.400 --> 00:00:26.950 align:start position:0%\n",
|
||||
"Heat<00:00:12.480><c> up</c><00:00:12.480><c> here.</c>\n",
|
||||
);
|
||||
|
||||
#[test]
|
||||
fn cue_settings_that_pin_subtitles_to_the_edge_are_removed() {
|
||||
let out = tidy_vtt(RAW_VTT);
|
||||
assert!(!out.contains("align:start"));
|
||||
assert!(!out.contains("position:0%"));
|
||||
// The timing itself must survive intact.
|
||||
assert!(out.contains("00:00:12.400 --> 00:00:26.950"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn karaoke_timing_tags_are_stripped_leaving_plain_text() {
|
||||
let out = tidy_vtt(RAW_VTT);
|
||||
assert!(out.contains("Heat up here."));
|
||||
assert!(!out.contains("<c>"));
|
||||
assert!(!out.contains("00:00:12.480>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_header_survives_or_the_file_stops_being_webvtt() {
|
||||
assert!(tidy_vtt(RAW_VTT).starts_with("WEBVTT"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_the_length_out_of_a_watch_page() {
|
||||
assert_eq!(
|
||||
parse_length_seconds(r#"...,"lengthSeconds":"1315","isLive"..."#),
|
||||
Some(1315)
|
||||
);
|
||||
assert_eq!(parse_length_seconds("nothing here"), None);
|
||||
// A zero length is meaningless and must not be stored.
|
||||
assert_eq!(parse_length_seconds(r#""lengthSeconds":"0""#), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_original_language_becomes_the_default_track() {
|
||||
let out = rewrite_master(DUBBED, None, Some("en")).unwrap();
|
||||
|
||||
+62
-1
@@ -86,9 +86,42 @@ impl Db {
|
||||
fn init(conn: Connection) -> Result<Db, String> {
|
||||
conn.execute_batch(SCHEMA)
|
||||
.map_err(|e| format!("Cannot create schema: {e}"))?;
|
||||
// CREATE TABLE IF NOT EXISTS leaves existing databases alone, so new
|
||||
// columns need adding explicitly. The error when it already exists is
|
||||
// the expected case, not a failure.
|
||||
let _ = conn.execute("ALTER TABLE videos ADD COLUMN duration INTEGER", []);
|
||||
Ok(Db { conn })
|
||||
}
|
||||
|
||||
/// Records a video's length in seconds.
|
||||
pub fn set_duration(&self, video_id: &str, seconds: i64) -> Result<(), String> {
|
||||
if seconds <= 0 {
|
||||
return Ok(());
|
||||
}
|
||||
self.conn
|
||||
.execute(
|
||||
"UPDATE videos SET duration = ?2 WHERE id = ?1",
|
||||
params![video_id, seconds],
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Videos whose length is still unknown, newest first.
|
||||
pub fn videos_missing_duration(&self, limit: i64) -> Result<Vec<String>, String> {
|
||||
let mut stmt = self
|
||||
.conn
|
||||
.prepare(
|
||||
"SELECT id FROM videos WHERE duration IS NULL
|
||||
ORDER BY published DESC LIMIT ?1",
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let rows = stmt
|
||||
.query_map(params![limit], |r| r.get::<_, String>(0))
|
||||
.map_err(|e| e.to_string())?;
|
||||
rows.collect::<Result<Vec<_>, _>>().map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// What a replacing import would destroy. Callers show this before asking
|
||||
/// the user to confirm, so nothing is deleted without being named first.
|
||||
pub fn preview_replace(&self, incoming: &[Channel]) -> Result<ImportPreview, String> {
|
||||
@@ -349,7 +382,8 @@ impl Db {
|
||||
let mut sql = String::from(
|
||||
"SELECT v.id, v.channel_id, COALESCE(c.title, ''), v.title, v.description,
|
||||
v.published, v.thumb_url, v.thumb_path, v.views, v.is_short,
|
||||
d.state, d.path, d.pct, d.error, p.position, p.duration
|
||||
d.state, d.path, d.pct, d.error, p.position,
|
||||
COALESCE(v.duration, p.duration)
|
||||
FROM videos v
|
||||
LEFT JOIN channels c ON c.id = v.channel_id
|
||||
LEFT JOIN downloads d ON d.video_id = v.id
|
||||
@@ -759,6 +793,33 @@ mod tests {
|
||||
assert_eq!(feed[0].id, "b");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_recorded_duration_reaches_the_feed() {
|
||||
let db = seeded();
|
||||
db.set_duration("a", 754).unwrap();
|
||||
let feed = db.list_feed(&FeedFilter::default()).unwrap();
|
||||
assert_eq!(feed.iter().find(|f| f.id == "a").unwrap().duration, Some(754.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn playback_duration_fills_in_when_the_video_length_is_unknown() {
|
||||
let db = seeded();
|
||||
db.save_playback("b", 10.0, 300.0).unwrap();
|
||||
let feed = db.list_feed(&FeedFilter::default()).unwrap();
|
||||
assert_eq!(feed.iter().find(|f| f.id == "b").unwrap().duration, Some(300.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn videos_without_a_duration_are_reported_for_lookup() {
|
||||
let db = seeded();
|
||||
assert_eq!(db.videos_missing_duration(10).unwrap().len(), 3);
|
||||
db.set_duration("a", 100).unwrap();
|
||||
assert_eq!(db.videos_missing_duration(10).unwrap().len(), 2);
|
||||
// A nonsense length is ignored rather than stored.
|
||||
db.set_duration("b", 0).unwrap();
|
||||
assert_eq!(db.videos_missing_duration(10).unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downloads_in_progress_still_show_in_the_downloaded_filter() {
|
||||
let db = seeded();
|
||||
|
||||
@@ -160,6 +160,7 @@ fn build_menu(app: &tauri::AppHandle) -> tauri::Result<Menu<tauri::Wry>> {
|
||||
commands::delete_download,
|
||||
commands::delete_all_downloads,
|
||||
commands::list_subtitles,
|
||||
commands::fetch_durations,
|
||||
commands::get_connectivity,
|
||||
commands::set_library_path,
|
||||
commands::open_external,
|
||||
|
||||
Reference in New Issue
Block a user