feat: Takeout CSV, Atom feed, yt-dlp progress, and SQLite layers
37 unit tests covering the three parsers and the database, including that a feed refresh preserves download state and that a truncated feed response fails rather than storing partial results.
This commit is contained in:
@@ -0,0 +1,175 @@
|
||||
//! Driving `yt-dlp` and interpreting its progress output.
|
||||
//!
|
||||
//! The format selector is deliberately narrow: H.264 video plus AAC audio in an
|
||||
//! MP4 container. YouTube only serves H.264 up to 1080p — everything above that
|
||||
//! is VP9 or AV1, which WKWebView cannot reliably play. Since FlightTube plays
|
||||
//! downloads in its own window, a 4K file we cannot decode is worthless. Do not
|
||||
//! widen this selector without also solving playback.
|
||||
pub const FORMAT_SELECTOR: &str = "bv*[vcodec^=avc1]+ba[acodec^=mp4a]/bv*+ba/b";
|
||||
|
||||
/// Sentinel prefix so progress lines are distinguishable from yt-dlp's ordinary
|
||||
/// chatter on the same stream.
|
||||
pub const PROGRESS_TEMPLATE: &str = "FTPROG %(progress.downloaded_bytes)s %(progress.total_bytes)s %(progress.speed)s %(progress.eta)s";
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Progress {
|
||||
pub downloaded: u64,
|
||||
pub total: Option<u64>,
|
||||
pub speed: Option<f64>,
|
||||
pub eta: Option<u64>,
|
||||
}
|
||||
|
||||
impl Progress {
|
||||
pub fn pct(&self) -> Option<f64> {
|
||||
match self.total {
|
||||
Some(t) if t > 0 => Some((self.downloaded as f64 / t as f64) * 100.0),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// yt-dlp writes `NA` for values it does not know yet (notably `total_bytes`
|
||||
/// before the stream is resolved), and interleaves ordinary log lines on the
|
||||
/// same stream. Anything that isn't a well-formed progress line yields `None`
|
||||
/// rather than an error — a garbled line must never abort a running download.
|
||||
fn opt_num<T: std::str::FromStr>(tok: &str) -> Option<T> {
|
||||
if tok == "NA" || tok.is_empty() {
|
||||
None
|
||||
} else {
|
||||
tok.parse::<T>().ok()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_progress_line(line: &str) -> Option<Progress> {
|
||||
let rest = line.trim().strip_prefix("FTPROG")?;
|
||||
let tokens: Vec<&str> = rest.split_whitespace().collect();
|
||||
if tokens.len() != 4 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// A progress line without a byte count tells us nothing; reject it.
|
||||
let downloaded: u64 = opt_num(tokens[0])?;
|
||||
|
||||
Some(Progress {
|
||||
downloaded,
|
||||
total: opt_num::<u64>(tokens[1]),
|
||||
// yt-dlp emits floats like `524288.0`; parse as f64 then keep as bytes.
|
||||
speed: opt_num::<f64>(tokens[2]),
|
||||
eta: opt_num::<f64>(tokens[3]).map(|v| v as u64),
|
||||
})
|
||||
}
|
||||
|
||||
/// Arguments for downloading one video. Kept separate from process spawning so
|
||||
/// the argument construction is assertable in tests.
|
||||
pub fn build_args(video_id: &str, out_template: &str) -> Vec<String> {
|
||||
vec![
|
||||
"-f".into(),
|
||||
FORMAT_SELECTOR.into(),
|
||||
"--merge-output-format".into(),
|
||||
"mp4".into(),
|
||||
"--no-playlist".into(),
|
||||
"--newline".into(),
|
||||
"--no-colors".into(),
|
||||
"--progress".into(),
|
||||
"--progress-template".into(),
|
||||
PROGRESS_TEMPLATE.into(),
|
||||
"--print".into(),
|
||||
"after_move:FTPATH %(filepath)s".into(),
|
||||
"-o".into(),
|
||||
out_template.into(),
|
||||
format!("https://www.youtube.com/watch?v={video_id}"),
|
||||
]
|
||||
}
|
||||
|
||||
/// yt-dlp reports the final path via `--print after_move:`, which is more
|
||||
/// reliable than guessing the extension after a merge.
|
||||
pub fn parse_final_path(line: &str) -> Option<String> {
|
||||
line.trim()
|
||||
.strip_prefix("FTPATH ")
|
||||
.map(|p| p.trim().to_string())
|
||||
.filter(|p| !p.is_empty())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_complete_line() {
|
||||
let p = parse_progress_line("FTPROG 1048576 10485760 524288.0 18").unwrap();
|
||||
assert_eq!(p.downloaded, 1_048_576);
|
||||
assert_eq!(p.total, Some(10_485_760));
|
||||
assert_eq!(p.speed, Some(524_288.0));
|
||||
assert_eq!(p.eta, Some(18));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_na_fields_before_stream_resolves() {
|
||||
let p = parse_progress_line("FTPROG 4096 NA NA NA").unwrap();
|
||||
assert_eq!(p.downloaded, 4096);
|
||||
assert_eq!(p.total, None);
|
||||
assert_eq!(p.speed, None);
|
||||
assert_eq!(p.eta, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tolerates_float_eta() {
|
||||
let p = parse_progress_line("FTPROG 10 100 5.5 12.0").unwrap();
|
||||
assert_eq!(p.eta, Some(12));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_ordinary_yt_dlp_output() {
|
||||
assert!(parse_progress_line("[youtube] Extracting URL: https://x").is_none());
|
||||
assert!(parse_progress_line("[Merger] Merging formats into \"x.mp4\"").is_none());
|
||||
assert!(parse_progress_line("").is_none());
|
||||
assert!(parse_progress_line("[download] 50% of 10MiB").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_garbage_without_panicking() {
|
||||
assert!(parse_progress_line("FTPROG").is_none());
|
||||
assert!(parse_progress_line("FTPROG a b c d").is_none());
|
||||
assert!(parse_progress_line("FTPROG 1 2").is_none());
|
||||
assert!(parse_progress_line("FTPROG 1 2 3 4 5").is_none());
|
||||
assert!(parse_progress_line("FTPROG NA NA NA NA").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn computes_percent() {
|
||||
let p = parse_progress_line("FTPROG 5000 10000 1.0 1").unwrap();
|
||||
assert_eq!(p.pct(), Some(50.0));
|
||||
let q = parse_progress_line("FTPROG 5000 NA NA NA").unwrap();
|
||||
assert_eq!(q.pct(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percent_guards_against_zero_total() {
|
||||
let p = Progress {
|
||||
downloaded: 5,
|
||||
total: Some(0),
|
||||
speed: None,
|
||||
eta: None,
|
||||
};
|
||||
assert_eq!(p.pct(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn args_pin_h264_and_mp4() {
|
||||
let args = build_args("abc123", "/tmp/%(id)s.%(ext)s");
|
||||
assert!(args.contains(&FORMAT_SELECTOR.to_string()));
|
||||
assert!(args.contains(&"mp4".to_string()));
|
||||
assert!(args.contains(&"https://www.youtube.com/watch?v=abc123".to_string()));
|
||||
assert!(args.contains(&"--no-playlist".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_final_path() {
|
||||
assert_eq!(
|
||||
parse_final_path("FTPATH /Users/x/Movies/FlightTube/abc.mp4").unwrap(),
|
||||
"/Users/x/Movies/FlightTube/abc.mp4"
|
||||
);
|
||||
assert!(parse_final_path("FTPATH ").is_none());
|
||||
assert!(parse_final_path("[download] done").is_none());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user