feat: Takeout CSV, Atom feed, yt-dlp progress, and SQLite layers
37 unit tests covering the three parsers and the database, including that a feed refresh preserves download state and that a truncated feed response fails rather than storing partial results.
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
//! Parsing of Google Takeout's `subscriptions.csv`.
|
||||
//!
|
||||
//! The export's real header is `Channel Id,Channel Url,Channel Title`, but the
|
||||
//! column order has varied across Takeout versions and locales, so we resolve
|
||||
//! columns by header name rather than by position.
|
||||
|
||||
use crate::models::Channel;
|
||||
|
||||
fn find_col(headers: &csv::StringRecord, name: &str) -> Option<usize> {
|
||||
headers
|
||||
.iter()
|
||||
.position(|h| h.trim().eq_ignore_ascii_case(name))
|
||||
}
|
||||
|
||||
pub fn parse_csv(input: &str) -> Result<Vec<Channel>, String> {
|
||||
if input.trim().is_empty() {
|
||||
return Err("The file is empty.".into());
|
||||
}
|
||||
|
||||
let mut reader = csv::ReaderBuilder::new()
|
||||
.flexible(true)
|
||||
.from_reader(input.as_bytes());
|
||||
|
||||
let headers = reader
|
||||
.headers()
|
||||
.map_err(|e| format!("Could not read the CSV header: {e}"))?
|
||||
.clone();
|
||||
|
||||
let id_col = find_col(&headers, "Channel Id")
|
||||
.ok_or("This does not look like a Takeout subscriptions.csv: no 'Channel Id' column.")?;
|
||||
let title_col = find_col(&headers, "Channel Title")
|
||||
.ok_or("This does not look like a Takeout subscriptions.csv: no 'Channel Title' column.")?;
|
||||
let url_col = find_col(&headers, "Channel Url");
|
||||
|
||||
let mut out = Vec::new();
|
||||
for record in reader.records() {
|
||||
let record = record.map_err(|e| format!("Malformed CSV row: {e}"))?;
|
||||
|
||||
let id = record.get(id_col).unwrap_or("").trim();
|
||||
if id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let title = record.get(title_col).unwrap_or("").trim();
|
||||
let url = url_col
|
||||
.and_then(|c| record.get(c))
|
||||
.map(str::trim)
|
||||
.filter(|u| !u.is_empty())
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(|| format!("https://www.youtube.com/channel/{id}"));
|
||||
|
||||
out.push(Channel {
|
||||
id: id.to_string(),
|
||||
title: if title.is_empty() {
|
||||
id.to_string()
|
||||
} else {
|
||||
title.to_string()
|
||||
},
|
||||
url,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_standard_export() {
|
||||
let csv = "Channel Id,Channel Url,Channel Title\n\
|
||||
UCabc,http://www.youtube.com/channel/UCabc,Linus Tech Tips\n";
|
||||
let out = parse_csv(csv).unwrap();
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].id, "UCabc");
|
||||
assert_eq!(out[0].title, "Linus Tech Tips");
|
||||
assert_eq!(out[0].url, "http://www.youtube.com/channel/UCabc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_commas_and_unicode_in_titles() {
|
||||
let csv = "Channel Id,Channel Url,Channel Title\n\
|
||||
UCx,http://y.com/UCx,\"Kurzgesagt – In a Nutshell, Ltd\"\n";
|
||||
let out = parse_csv(csv).unwrap();
|
||||
assert_eq!(out[0].title, "Kurzgesagt – In a Nutshell, Ltd");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tolerates_reordered_columns() {
|
||||
let csv = "Channel Title,Channel Id,Channel Url\nVeritasium,UCz,http://y.com/UCz\n";
|
||||
let out = parse_csv(csv).unwrap();
|
||||
assert_eq!(out[0].id, "UCz");
|
||||
assert_eq!(out[0].title, "Veritasium");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derives_url_when_column_absent() {
|
||||
let csv = "Channel Id,Channel Title\nUCq,Some Channel\n";
|
||||
let out = parse_csv(csv).unwrap();
|
||||
assert_eq!(out[0].url, "https://www.youtube.com/channel/UCq");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_blank_lines_and_empty_file() {
|
||||
assert!(parse_csv("Channel Id,Channel Url,Channel Title\n\n")
|
||||
.unwrap()
|
||||
.is_empty());
|
||||
assert!(parse_csv("").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn errors_on_missing_required_column() {
|
||||
assert!(parse_csv("Foo,Bar\n1,2\n").is_err());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user