vincent
b724e9d56f
37 unit tests covering the three parsers and the database, including that a feed refresh preserves download state and that a truncated feed response fails rather than storing partial results.
117 lines
3.6 KiB
Rust
117 lines
3.6 KiB
Rust
//! Parsing of Google Takeout's `subscriptions.csv`.
|
||
//!
|
||
//! The export's real header is `Channel Id,Channel Url,Channel Title`, but the
|
||
//! column order has varied across Takeout versions and locales, so we resolve
|
||
//! columns by header name rather than by position.
|
||
|
||
use crate::models::Channel;
|
||
|
||
fn find_col(headers: &csv::StringRecord, name: &str) -> Option<usize> {
|
||
headers
|
||
.iter()
|
||
.position(|h| h.trim().eq_ignore_ascii_case(name))
|
||
}
|
||
|
||
pub fn parse_csv(input: &str) -> Result<Vec<Channel>, String> {
|
||
if input.trim().is_empty() {
|
||
return Err("The file is empty.".into());
|
||
}
|
||
|
||
let mut reader = csv::ReaderBuilder::new()
|
||
.flexible(true)
|
||
.from_reader(input.as_bytes());
|
||
|
||
let headers = reader
|
||
.headers()
|
||
.map_err(|e| format!("Could not read the CSV header: {e}"))?
|
||
.clone();
|
||
|
||
let id_col = find_col(&headers, "Channel Id")
|
||
.ok_or("This does not look like a Takeout subscriptions.csv: no 'Channel Id' column.")?;
|
||
let title_col = find_col(&headers, "Channel Title")
|
||
.ok_or("This does not look like a Takeout subscriptions.csv: no 'Channel Title' column.")?;
|
||
let url_col = find_col(&headers, "Channel Url");
|
||
|
||
let mut out = Vec::new();
|
||
for record in reader.records() {
|
||
let record = record.map_err(|e| format!("Malformed CSV row: {e}"))?;
|
||
|
||
let id = record.get(id_col).unwrap_or("").trim();
|
||
if id.is_empty() {
|
||
continue;
|
||
}
|
||
|
||
let title = record.get(title_col).unwrap_or("").trim();
|
||
let url = url_col
|
||
.and_then(|c| record.get(c))
|
||
.map(str::trim)
|
||
.filter(|u| !u.is_empty())
|
||
.map(str::to_string)
|
||
.unwrap_or_else(|| format!("https://www.youtube.com/channel/{id}"));
|
||
|
||
out.push(Channel {
|
||
id: id.to_string(),
|
||
title: if title.is_empty() {
|
||
id.to_string()
|
||
} else {
|
||
title.to_string()
|
||
},
|
||
url,
|
||
});
|
||
}
|
||
|
||
Ok(out)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn parses_standard_export() {
|
||
let csv = "Channel Id,Channel Url,Channel Title\n\
|
||
UCabc,http://www.youtube.com/channel/UCabc,Linus Tech Tips\n";
|
||
let out = parse_csv(csv).unwrap();
|
||
assert_eq!(out.len(), 1);
|
||
assert_eq!(out[0].id, "UCabc");
|
||
assert_eq!(out[0].title, "Linus Tech Tips");
|
||
assert_eq!(out[0].url, "http://www.youtube.com/channel/UCabc");
|
||
}
|
||
|
||
#[test]
|
||
fn handles_commas_and_unicode_in_titles() {
|
||
let csv = "Channel Id,Channel Url,Channel Title\n\
|
||
UCx,http://y.com/UCx,\"Kurzgesagt – In a Nutshell, Ltd\"\n";
|
||
let out = parse_csv(csv).unwrap();
|
||
assert_eq!(out[0].title, "Kurzgesagt – In a Nutshell, Ltd");
|
||
}
|
||
|
||
#[test]
|
||
fn tolerates_reordered_columns() {
|
||
let csv = "Channel Title,Channel Id,Channel Url\nVeritasium,UCz,http://y.com/UCz\n";
|
||
let out = parse_csv(csv).unwrap();
|
||
assert_eq!(out[0].id, "UCz");
|
||
assert_eq!(out[0].title, "Veritasium");
|
||
}
|
||
|
||
#[test]
|
||
fn derives_url_when_column_absent() {
|
||
let csv = "Channel Id,Channel Title\nUCq,Some Channel\n";
|
||
let out = parse_csv(csv).unwrap();
|
||
assert_eq!(out[0].url, "https://www.youtube.com/channel/UCq");
|
||
}
|
||
|
||
#[test]
|
||
fn skips_blank_lines_and_empty_file() {
|
||
assert!(parse_csv("Channel Id,Channel Url,Channel Title\n\n")
|
||
.unwrap()
|
||
.is_empty());
|
||
assert!(parse_csv("").is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn errors_on_missing_required_column() {
|
||
assert!(parse_csv("Foo,Bar\n1,2\n").is_err());
|
||
}
|
||
}
|