feat: Takeout CSV, Atom feed, yt-dlp progress, and SQLite layers

37 unit tests covering the three parsers and the database, including
that a feed refresh preserves download state and that a truncated
feed response fails rather than storing partial results.
This commit is contained in:
vincent
2026-08-29 02:26:17 +02:00
parent 6fdad60190
commit b724e9d56f
11 changed files with 7139 additions and 6 deletions
+116
View File
@@ -0,0 +1,116 @@
//! Parsing of Google Takeout's `subscriptions.csv`.
//!
//! The export's real header is `Channel Id,Channel Url,Channel Title`, but the
//! column order has varied across Takeout versions and locales, so we resolve
//! columns by header name rather than by position.
use crate::models::Channel;
fn find_col(headers: &csv::StringRecord, name: &str) -> Option<usize> {
headers
.iter()
.position(|h| h.trim().eq_ignore_ascii_case(name))
}
pub fn parse_csv(input: &str) -> Result<Vec<Channel>, String> {
if input.trim().is_empty() {
return Err("The file is empty.".into());
}
let mut reader = csv::ReaderBuilder::new()
.flexible(true)
.from_reader(input.as_bytes());
let headers = reader
.headers()
.map_err(|e| format!("Could not read the CSV header: {e}"))?
.clone();
let id_col = find_col(&headers, "Channel Id")
.ok_or("This does not look like a Takeout subscriptions.csv: no 'Channel Id' column.")?;
let title_col = find_col(&headers, "Channel Title")
.ok_or("This does not look like a Takeout subscriptions.csv: no 'Channel Title' column.")?;
let url_col = find_col(&headers, "Channel Url");
let mut out = Vec::new();
for record in reader.records() {
let record = record.map_err(|e| format!("Malformed CSV row: {e}"))?;
let id = record.get(id_col).unwrap_or("").trim();
if id.is_empty() {
continue;
}
let title = record.get(title_col).unwrap_or("").trim();
let url = url_col
.and_then(|c| record.get(c))
.map(str::trim)
.filter(|u| !u.is_empty())
.map(str::to_string)
.unwrap_or_else(|| format!("https://www.youtube.com/channel/{id}"));
out.push(Channel {
id: id.to_string(),
title: if title.is_empty() {
id.to_string()
} else {
title.to_string()
},
url,
});
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_standard_export() {
let csv = "Channel Id,Channel Url,Channel Title\n\
UCabc,http://www.youtube.com/channel/UCabc,Linus Tech Tips\n";
let out = parse_csv(csv).unwrap();
assert_eq!(out.len(), 1);
assert_eq!(out[0].id, "UCabc");
assert_eq!(out[0].title, "Linus Tech Tips");
assert_eq!(out[0].url, "http://www.youtube.com/channel/UCabc");
}
#[test]
fn handles_commas_and_unicode_in_titles() {
let csv = "Channel Id,Channel Url,Channel Title\n\
UCx,http://y.com/UCx,\"Kurzgesagt In a Nutshell, Ltd\"\n";
let out = parse_csv(csv).unwrap();
assert_eq!(out[0].title, "Kurzgesagt In a Nutshell, Ltd");
}
#[test]
fn tolerates_reordered_columns() {
let csv = "Channel Title,Channel Id,Channel Url\nVeritasium,UCz,http://y.com/UCz\n";
let out = parse_csv(csv).unwrap();
assert_eq!(out[0].id, "UCz");
assert_eq!(out[0].title, "Veritasium");
}
#[test]
fn derives_url_when_column_absent() {
let csv = "Channel Id,Channel Title\nUCq,Some Channel\n";
let out = parse_csv(csv).unwrap();
assert_eq!(out[0].url, "https://www.youtube.com/channel/UCq");
}
#[test]
fn skips_blank_lines_and_empty_file() {
assert!(parse_csv("Channel Id,Channel Url,Channel Title\n\n")
.unwrap()
.is_empty());
assert!(parse_csv("").is_err());
}
#[test]
fn errors_on_missing_required_column() {
assert!(parse_csv("Foo,Bar\n1,2\n").is_err());
}
}