use super::feeds; use crate::auth::extractor::AuthUser; use crate::error::AppError; use crate::json_serialization::user::JsonUser; use crate::models::feed::rss_feed::Feed; use crate::models::feed_item::new_feed_item::NewFeedItem; use crate::schema::feed_item::{feed_id, url}; use crate::{ database::establish_connection, schema::{ feed::{self, user_id}, feed_item, }, }; use actix_web::{web, HttpRequest, HttpResponse}; use chrono::{DateTime, Local, NaiveDateTime}; use dateparser::parse; use diesel::prelude::*; use rss::Item; use scraper::{Html, Selector}; use std::collections::{HashMap, HashSet}; fn get_date(date_str: &str) -> Result { if let Ok(result) = parse(date_str) { log::info!("Date: {:?}", result); return Ok(result.with_timezone(&Local).naive_local()); } DateTime::parse_from_rfc2822(date_str).map(|dt| dt.with_timezone(&Local).naive_local()) } // Some feeds (e.g. Deutsche Welle) embed responsive-image templates such as // `src="https://example.com/img_${formatId}.jpg"` that their own frontend // JavaScript fills in before loading — verbatim, they 404. Skip those and // pick the first with a real, directly loadable URL instead. fn image_src_is_resolvable(element: &scraper::ElementRef) -> bool { match element.value().attr("src") { Some(src) => !src.contains('{') && !src.to_lowercase().contains("%7b"), None => false, } } fn escape_html_attr(value: &str) -> String { value .replace('&', "&") .replace('"', """) .replace('<', "<") .replace('>', ">") } // Feed-supplied `` markup is rendered as-is (via `v-html`) in the frontend, // so strip everything except a harmless image tag before it's stored — in // particular event handlers like `onerror`/`onload` that a malicious feed // could use for XSS. fn sanitize_img_html(html: &str) -> String { let allowed_attributes = HashSet::from(["src", "alt", "title"]); ammonia::Builder::default() .tags(HashSet::from(["img"])) .tag_attributes(HashMap::from([("img", allowed_attributes)])) .clean(html) .to_string() } // Some feeds (e.g. Deutsche Welle) don't embed an in the item content at // all — they carry the article image as an RSS instead. Build an // tag from it so those feeds get a preview image too. fn enclosure_image_html(item: &Item) -> Option { let enclosure = item.enclosure()?; if !enclosure.mime_type().to_lowercase().starts_with("image/") { return None; } Some(format!( r#""#, escape_html_attr(enclosure.url()) )) } fn create_feed_item(item: Item, feed: &Feed, connection: &mut PgConnection) -> anyhow::Result<()> { // Items without a title or link are malformed/unusable — skip them rather // than failing the whole sync over one bad entry from an external feed. let Some(item_title) = item.title.clone() else { log::warn!("Skipping feed item without a title."); return Ok(()); }; if item.link.is_none() { log::warn!("Skipping feed item without a link: {}", item_title); return Ok(()); } log::info!("Create feed item: {}", item_title); // Items without a pub_date are treated as current (inserted unconditionally) // — feeds that don't publish dates are typically small/curated enough that // this is fine. let mut time: NaiveDateTime = Local::now().naive_local(); if let Some(pub_date) = item.pub_date() { time = match get_date(pub_date) { Ok(date) => date, Err(err) => { log::error!("could not parse pub date: {}", err); time } }; } let base_content: &str = item.content().or(item.description()).unwrap_or_default(); let frag = Html::parse_fragment(base_content); let mut content = "".to_string(); // Some feeds (e.g. Stuttgarter Nachrichten) embed a social-sharing widget // (WhatsApp/Email/Facebook/... links plus a "Link kopiert" tooltip) in the // article content. It's not part of the article and isn't present in the // scraped/readable edition either, so skip its text when flattening below. let selector_social_bar = Selector::parse("#article-social-bar").expect("\"#article-social-bar\" is a valid CSS selector"); let excluded_node_ids: std::collections::HashSet<_> = frag .select(&selector_social_bar) .flat_map(|el| el.descendants().map(|node| node.id())) .collect(); let selector_img = Selector::parse("img").expect("\"img\" is a valid CSS selector"); match frag.select(&selector_img).find(image_src_is_resolvable) { Some(image) => { content.push_str(&sanitize_img_html(&image.html())); content.push_str("
"); } None => { if let Some(image_html) = enclosure_image_html(&item) { content.push_str(&sanitize_img_html(&image_html)); content.push_str("
"); } } } for node in frag.tree.nodes() { if excluded_node_ids.contains(&node.id()) { continue; } if let scraper::node::Node::Text(text) = node.value() { content.push_str(&text.text); } } let new_feed_item = NewFeedItem::new( feed.id, content.clone(), item_title.clone(), item.link.expect("checked above"), Some(time), ); // `on_conflict` on the (feed_id, url) unique constraint makes this // insert idempotent, so two syncs for the same feed running concurrently // (e.g. a sync button press racing a page-reload sync) can't both pass a // check-then-insert race and create duplicate items. Keying on the // article's link rather than its title also handles feeds that edit a // headline after publishing while keeping the same link — the link is // the stable identity. let inserted_rows = diesel::insert_into(feed_item::table) .values(&new_feed_item) .on_conflict((feed_id, url)) .do_nothing() .execute(connection)?; if inserted_rows > 0 { log::info!("Inserted item: {}", item_title); } else { log::info!("Item {} already exists.", item_title); } Ok(()) } pub async fn sync( _req: HttpRequest, data: web::Json, auth_user: AuthUser, ) -> Result { let req_user_id: i32 = data.user_id; if auth_user.0 != req_user_id { return Ok(HttpResponse::Forbidden().finish()); } let mut connection: diesel::PgConnection = establish_connection(); let feeds: Vec = feed::table .filter(user_id.eq(req_user_id)) .load::(&mut connection)?; log::info!("Found {} feeds to sync.", feeds.len()); for feed in feeds { log::info!("Try to get url: {}", feed.url); let result = feeds::get_feed(&feed.url).await; match result { Ok(channel) => { for item in channel.into_items() { log::info!("{:?}", item); if let Err(e) = create_feed_item(item, &feed, &mut connection) { log::error!("Could not create feed item for {}: {:?}", feed.url, e); } } } Err(e) => log::error!("Could not get channel {}. Error: {}", feed.url, e), } } Ok(HttpResponse::Ok().finish()) } #[cfg(test)] mod tests { use crate::models::feed::new_feed::NewFeed; use crate::models::feed_item::rss_feed_item::FeedItem; use crate::models::user::new_user::NewUser; use crate::models::user::rss_user::User; use crate::schema::users; use crate::test_helpers::{delete_feed, delete_user, insert_feed, insert_user, unique_suffix}; use chrono::Duration; use super::*; #[test] fn get_date_parses_iso8601_dates() { assert!(get_date("2024-01-01T12:00:00Z").is_ok()); } #[test] fn get_date_parses_rfc2822_dates() { assert!(get_date("Tue, 03 Jun 2025 10:00:00 GMT").is_ok()); } #[test] fn get_date_returns_err_for_unparseable_dates() { assert!(get_date("not-a-date").is_err()); } #[test] fn create_feed_item_skips_template_placeholder_images() { let html = Html::parse_fragment( r#"

placeholder

real image

"#, ); let selector = Selector::parse("img").unwrap(); let chosen = html .select(&selector) .find(image_src_is_resolvable) .expect("should find a resolvable image"); assert_eq!( Some("https://example.test/real.jpg"), chosen.value().attr("src") ); } #[test] fn create_feed_item_finds_no_image_when_all_are_templated() { let html = Html::parse_fragment(r#""#); let selector = Selector::parse("img").unwrap(); assert!(html.select(&selector).find(image_src_is_resolvable).is_none()); } #[test] fn enclosure_image_html_builds_img_for_image_enclosures() { let mut item = Item::default(); item.set_enclosure(rss::Enclosure { url: "https://static.dw.com/image/73880499_302.jpg".to_string(), length: "2000".to_string(), mime_type: "image/jpeg".to_string(), }); assert_eq!( Some(r#""#.to_string()), enclosure_image_html(&item) ); } #[test] fn enclosure_image_html_ignores_non_image_enclosures() { let mut item = Item::default(); item.set_enclosure(rss::Enclosure { url: "https://example.test/episode.mp3".to_string(), length: "2000".to_string(), mime_type: "audio/mpeg".to_string(), }); assert_eq!(None, enclosure_image_html(&item)); } #[test] fn enclosure_image_html_escapes_url_attribute() { let mut item = Item::default(); item.set_enclosure(rss::Enclosure { url: "https://example.test/img.jpg?a=1&b=\"x\"".to_string(), length: "2000".to_string(), mime_type: "image/jpeg".to_string(), }); assert_eq!( Some(r#""#.to_string()), enclosure_image_html(&item) ); } #[test] fn sanitize_img_html_strips_event_handlers() { let sanitized = sanitize_img_html(r#""#); assert!(!sanitized.contains("onerror")); assert!(!sanitized.contains("alert")); assert!(sanitized.contains(r#"src="x""#)); } #[test] fn sanitize_img_html_keeps_only_allowed_attributes() { let sanitized = sanitize_img_html( r#""#, ); assert!(sanitized.contains(r#"src="https://example.test/img.jpg""#)); assert!(sanitized.contains(r#"alt="desc""#)); assert!(sanitized.contains(r#"title="t""#)); assert!(!sanitized.contains("style")); assert!(!sanitized.contains("class")); } #[actix_web::test] async fn create_feed_item_inserts_articles_older_than_two_weeks() { let mut connection = establish_connection(); let suffix = unique_suffix(); let new_user = NewUser::new( format!("age_test_{suffix}"), format!("age_{suffix}@example.test"), "secret".to_string(), ) .unwrap(); let user: User = diesel::insert_into(users::table) .values(&new_user) .get_result(&mut connection) .unwrap(); let new_feed = NewFeed::new( format!("Age test feed {suffix}"), format!("https://example.test/feed/{suffix}"), user.id, ); let feed: Feed = diesel::insert_into(feed::table) .values(&new_feed) .get_result(&mut connection) .unwrap(); // Item with a pub_date 20 days ago — should still be inserted, since // infrequently-updated feeds (or infrequent syncs) must not lose // articles the user hasn't seen yet. let old_date = (Local::now() - Duration::days(20)) .format("%a, %d %b %Y %H:%M:%S %z") .to_string(); let mut old_item = Item::default(); old_item.set_title(Some(format!("Old article {suffix}"))); old_item.set_link(Some(format!("https://example.test/old/{suffix}"))); old_item.set_pub_date(Some(old_date)); old_item.set_content(Some("

old

".to_string())); // Item without a pub_date — treated as current, should be inserted. let mut fresh_item = Item::default(); fresh_item.set_title(Some(format!("Fresh article {suffix}"))); fresh_item.set_link(Some(format!("https://example.test/fresh/{suffix}"))); fresh_item.set_content(Some("

fresh

".to_string())); create_feed_item(old_item, &feed, &mut connection).unwrap(); create_feed_item(fresh_item, &feed, &mut connection).unwrap(); let items: Vec = feed_item::table .filter(feed_id.eq(feed.id)) .load(&mut connection) .unwrap(); assert_eq!(2, items.len(), "both old and fresh items should be inserted"); diesel::delete(feed_item::table.filter(feed_id.eq(feed.id))) .execute(&mut connection) .ok(); diesel::delete(feed::table.filter(feed::id.eq(feed.id))) .execute(&mut connection) .ok(); diesel::delete(users::table.filter(users::id.eq(user.id))) .execute(&mut connection) .ok(); } #[actix_web::test] async fn create_feed_item_does_not_duplicate_existing_items() { let mut connection = establish_connection(); let suffix = unique_suffix(); let new_user = NewUser::new( format!("sync_test_{suffix}"), format!("sync_{suffix}@example.test"), "secret".to_string(), ) .unwrap(); let user: User = diesel::insert_into(users::table) .values(&new_user) .get_result(&mut connection) .unwrap(); let new_feed = NewFeed::new( format!("Sync test feed {suffix}"), format!("https://example.test/feed/{suffix}"), user.id, ); let feed: Feed = diesel::insert_into(feed::table) .values(&new_feed) .get_result(&mut connection) .unwrap(); let mut item = Item::default(); item.set_title(Some(format!("Sync test article {suffix}"))); item.set_link(Some(format!("https://example.test/article/{suffix}"))); item.set_content(Some("

Hello world

".to_string())); create_feed_item(item.clone(), &feed, &mut connection).unwrap(); create_feed_item(item, &feed, &mut connection).unwrap(); let items: Vec = feed_item::table .filter(feed_id.eq(feed.id)) .load(&mut connection) .unwrap(); assert_eq!(1, items.len(), "duplicate feed items should not be created"); diesel::delete(feed_item::table.filter(feed_id.eq(feed.id))) .execute(&mut connection) .ok(); diesel::delete(feed::table.filter(feed::id.eq(feed.id))) .execute(&mut connection) .ok(); diesel::delete(users::table.filter(users::id.eq(user.id))) .execute(&mut connection) .ok(); } #[actix_web::test] async fn create_feed_item_is_race_safe_under_concurrent_syncs() { let mut connection = establish_connection(); let suffix = unique_suffix(); let user = insert_user(&mut connection, "secret"); let feed = insert_feed(&mut connection, user.id); let mut item = Item::default(); item.set_title(Some(format!("Race test article {suffix}"))); item.set_link(Some(format!("https://example.test/race/{suffix}"))); item.set_content(Some("

Hello world

".to_string())); // Simulate two concurrent syncs for the same feed (e.g. a sync // button press racing a page-reload sync) — each on its own // connection/thread, released together so they genuinely overlap — // both inserting the same item. let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); let handles: Vec<_> = (0..2) .map(|_| { let feed = feed.clone(); let item = item.clone(); let barrier = barrier.clone(); std::thread::spawn(move || { let mut connection = establish_connection(); barrier.wait(); create_feed_item(item, &feed, &mut connection) }) }) .collect(); for handle in handles { handle.join().unwrap().unwrap(); } let items: Vec = feed_item::table .filter(feed_id.eq(feed.id)) .load(&mut connection) .unwrap(); assert_eq!( 1, items.len(), "concurrent syncs must not create duplicate feed items" ); diesel::delete(feed_item::table.filter(feed_id.eq(feed.id))) .execute(&mut connection) .ok(); delete_feed(&mut connection, feed.id); delete_user(&mut connection, user.id); } #[actix_web::test] async fn create_feed_item_strips_onerror_from_feed_image() { let mut connection = establish_connection(); let suffix = unique_suffix(); let new_user = NewUser::new( format!("xss_test_{suffix}"), format!("xss_{suffix}@example.test"), "secret".to_string(), ) .unwrap(); let user: User = diesel::insert_into(users::table) .values(&new_user) .get_result(&mut connection) .unwrap(); let new_feed = NewFeed::new( format!("XSS test feed {suffix}"), format!("https://example.test/feed/{suffix}"), user.id, ); let feed: Feed = diesel::insert_into(feed::table) .values(&new_feed) .get_result(&mut connection) .unwrap(); let mut item = Item::default(); item.set_title(Some(format!("XSS article {suffix}"))); item.set_link(Some(format!("https://example.test/xss/{suffix}"))); item.set_content(Some( r#"

text

"# .to_string(), )); create_feed_item(item, &feed, &mut connection).unwrap(); let stored: FeedItem = feed_item::table .filter(feed_id.eq(feed.id)) .first(&mut connection) .unwrap(); assert!(!stored.content.contains("onerror")); assert!(!stored.content.contains("alert")); assert!(stored.content.contains(r#"src="https://example.test/real.jpg""#)); diesel::delete(feed_item::table.filter(feed_id.eq(feed.id))) .execute(&mut connection) .ok(); diesel::delete(feed::table.filter(feed::id.eq(feed.id))) .execute(&mut connection) .ok(); diesel::delete(users::table.filter(users::id.eq(user.id))) .execute(&mut connection) .ok(); } #[actix_web::test] async fn create_feed_item_strips_social_sharing_widget() { let mut connection = establish_connection(); let suffix = unique_suffix(); let new_user = NewUser::new( format!("social_bar_test_{suffix}"), format!("social_bar_{suffix}@example.test"), "secret".to_string(), ) .unwrap(); let user: User = diesel::insert_into(users::table) .values(&new_user) .get_result(&mut connection) .unwrap(); let new_feed = NewFeed::new( format!("Social bar test feed {suffix}"), format!("https://example.test/feed/{suffix}"), user.id, ); let feed: Feed = diesel::insert_into(feed::table) .values(&new_feed) .get_result(&mut connection) .unwrap(); let mut item = Item::default(); item.set_title(Some(format!("Social bar article {suffix}"))); item.set_link(Some(format!("https://example.test/article/{suffix}"))); item.set_content(Some( r#"

Article text

  •  
  •  

    Link kopiert

"# .to_string(), )); create_feed_item(item, &feed, &mut connection).unwrap(); let items: Vec = feed_item::table .filter(feed_id.eq(feed.id)) .load(&mut connection) .unwrap(); assert_eq!(1, items.len()); assert!(items[0].content.contains("Article text")); assert!(!items[0].content.contains("Link kopiert")); diesel::delete(feed_item::table.filter(feed_id.eq(feed.id))) .execute(&mut connection) .ok(); diesel::delete(feed::table.filter(feed::id.eq(feed.id))) .execute(&mut connection) .ok(); diesel::delete(users::table.filter(users::id.eq(user.id))) .execute(&mut connection) .ok(); } }