feat: Takeout CSV, Atom feed, yt-dlp progress, and SQLite layers

37 unit tests covering the three parsers and the database, including
that a feed refresh preserves download state and that a truncated
feed response fails rather than storing partial results.
This commit is contained in:
vincent
2026-08-29 02:26:17 +02:00
parent 6fdad60190
commit b724e9d56f
11 changed files with 7139 additions and 6 deletions
+175
View File
@@ -0,0 +1,175 @@
//! Driving `yt-dlp` and interpreting its progress output.
//!
//! The format selector is deliberately narrow: H.264 video plus AAC audio in an
//! MP4 container. YouTube only serves H.264 up to 1080p — everything above that
//! is VP9 or AV1, which WKWebView cannot reliably play. Since FlightTube plays
//! downloads in its own window, a 4K file we cannot decode is worthless. Do not
//! widen this selector without also solving playback.
pub const FORMAT_SELECTOR: &str = "bv*[vcodec^=avc1]+ba[acodec^=mp4a]/bv*+ba/b";
/// Sentinel prefix so progress lines are distinguishable from yt-dlp's ordinary
/// chatter on the same stream.
pub const PROGRESS_TEMPLATE: &str = "FTPROG %(progress.downloaded_bytes)s %(progress.total_bytes)s %(progress.speed)s %(progress.eta)s";
#[derive(Debug, Clone, PartialEq)]
pub struct Progress {
pub downloaded: u64,
pub total: Option<u64>,
pub speed: Option<f64>,
pub eta: Option<u64>,
}
impl Progress {
pub fn pct(&self) -> Option<f64> {
match self.total {
Some(t) if t > 0 => Some((self.downloaded as f64 / t as f64) * 100.0),
_ => None,
}
}
}
/// yt-dlp writes `NA` for values it does not know yet (notably `total_bytes`
/// before the stream is resolved), and interleaves ordinary log lines on the
/// same stream. Anything that isn't a well-formed progress line yields `None`
/// rather than an error — a garbled line must never abort a running download.
fn opt_num<T: std::str::FromStr>(tok: &str) -> Option<T> {
if tok == "NA" || tok.is_empty() {
None
} else {
tok.parse::<T>().ok()
}
}
pub fn parse_progress_line(line: &str) -> Option<Progress> {
let rest = line.trim().strip_prefix("FTPROG")?;
let tokens: Vec<&str> = rest.split_whitespace().collect();
if tokens.len() != 4 {
return None;
}
// A progress line without a byte count tells us nothing; reject it.
let downloaded: u64 = opt_num(tokens[0])?;
Some(Progress {
downloaded,
total: opt_num::<u64>(tokens[1]),
// yt-dlp emits floats like `524288.0`; parse as f64 then keep as bytes.
speed: opt_num::<f64>(tokens[2]),
eta: opt_num::<f64>(tokens[3]).map(|v| v as u64),
})
}
/// Arguments for downloading one video. Kept separate from process spawning so
/// the argument construction is assertable in tests.
pub fn build_args(video_id: &str, out_template: &str) -> Vec<String> {
vec![
"-f".into(),
FORMAT_SELECTOR.into(),
"--merge-output-format".into(),
"mp4".into(),
"--no-playlist".into(),
"--newline".into(),
"--no-colors".into(),
"--progress".into(),
"--progress-template".into(),
PROGRESS_TEMPLATE.into(),
"--print".into(),
"after_move:FTPATH %(filepath)s".into(),
"-o".into(),
out_template.into(),
format!("https://www.youtube.com/watch?v={video_id}"),
]
}
/// yt-dlp reports the final path via `--print after_move:`, which is more
/// reliable than guessing the extension after a merge.
pub fn parse_final_path(line: &str) -> Option<String> {
line.trim()
.strip_prefix("FTPATH ")
.map(|p| p.trim().to_string())
.filter(|p| !p.is_empty())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_complete_line() {
let p = parse_progress_line("FTPROG 1048576 10485760 524288.0 18").unwrap();
assert_eq!(p.downloaded, 1_048_576);
assert_eq!(p.total, Some(10_485_760));
assert_eq!(p.speed, Some(524_288.0));
assert_eq!(p.eta, Some(18));
}
#[test]
fn handles_na_fields_before_stream_resolves() {
let p = parse_progress_line("FTPROG 4096 NA NA NA").unwrap();
assert_eq!(p.downloaded, 4096);
assert_eq!(p.total, None);
assert_eq!(p.speed, None);
assert_eq!(p.eta, None);
}
#[test]
fn tolerates_float_eta() {
let p = parse_progress_line("FTPROG 10 100 5.5 12.0").unwrap();
assert_eq!(p.eta, Some(12));
}
#[test]
fn ignores_ordinary_yt_dlp_output() {
assert!(parse_progress_line("[youtube] Extracting URL: https://x").is_none());
assert!(parse_progress_line("[Merger] Merging formats into \"x.mp4\"").is_none());
assert!(parse_progress_line("").is_none());
assert!(parse_progress_line("[download] 50% of 10MiB").is_none());
}
#[test]
fn ignores_garbage_without_panicking() {
assert!(parse_progress_line("FTPROG").is_none());
assert!(parse_progress_line("FTPROG a b c d").is_none());
assert!(parse_progress_line("FTPROG 1 2").is_none());
assert!(parse_progress_line("FTPROG 1 2 3 4 5").is_none());
assert!(parse_progress_line("FTPROG NA NA NA NA").is_none());
}
#[test]
fn computes_percent() {
let p = parse_progress_line("FTPROG 5000 10000 1.0 1").unwrap();
assert_eq!(p.pct(), Some(50.0));
let q = parse_progress_line("FTPROG 5000 NA NA NA").unwrap();
assert_eq!(q.pct(), None);
}
#[test]
fn percent_guards_against_zero_total() {
let p = Progress {
downloaded: 5,
total: Some(0),
speed: None,
eta: None,
};
assert_eq!(p.pct(), None);
}
#[test]
fn args_pin_h264_and_mp4() {
let args = build_args("abc123", "/tmp/%(id)s.%(ext)s");
assert!(args.contains(&FORMAT_SELECTOR.to_string()));
assert!(args.contains(&"mp4".to_string()));
assert!(args.contains(&"https://www.youtube.com/watch?v=abc123".to_string()));
assert!(args.contains(&"--no-playlist".to_string()));
}
#[test]
fn extracts_final_path() {
assert_eq!(
parse_final_path("FTPATH /Users/x/Movies/FlightTube/abc.mp4").unwrap(),
"/Users/x/Movies/FlightTube/abc.mp4"
);
assert!(parse_final_path("FTPATH ").is_none());
assert!(parse_final_path("[download] done").is_none());
}
}