rss-reader/src/reader/sync.rs

use super::feeds;
use crate::auth::extractor::AuthUser;
use crate::error::AppError;
use crate::json_serialization::user::JsonUser;
use crate::models::feed::rss_feed::Feed;
use crate::models::feed_item::new_feed_item::NewFeedItem;
use crate::schema::feed_item::{feed_id, url};
use crate::{
    database::establish_connection,
    schema::{
        feed::{self, user_id},
        feed_item,
    },
};
use actix_web::{web, HttpRequest, HttpResponse};
use chrono::{DateTime, Local, NaiveDateTime};
use dateparser::parse;
use diesel::prelude::*;
use rss::Item;
use scraper::{Html, Selector};
use std::collections::{HashMap, HashSet};

fn get_date(date_str: &str) -> Result<NaiveDateTime, chrono::ParseError> {
    if let Ok(result) = parse(date_str) {
        log::info!("Date: {:?}", result);
        return Ok(result.with_timezone(&Local).naive_local());
    }

    DateTime::parse_from_rfc2822(date_str).map(|dt| dt.with_timezone(&Local).naive_local())
}

// Some feeds (e.g. Deutsche Welle) embed responsive-image templates such as
// `src="https://example.com/img_${formatId}.jpg"` that their own frontend
// JavaScript fills in before loading — verbatim, they 404. Skip those and
// pick the first <img> with a real, directly loadable URL instead.
fn image_src_is_resolvable(element: &scraper::ElementRef) -> bool {
    match element.value().attr("src") {
        Some(src) => !src.contains('{') && !src.to_lowercase().contains("%7b"),
        None => false,
    }
}

fn escape_html_attr(value: &str) -> String {
    value
        .replace('&', "&amp;")
        .replace('"', "&quot;")
        .replace('<', "&lt;")
        .replace('>', "&gt;")
}

// Feed-supplied `<img>` markup is rendered as-is (via `v-html`) in the frontend,
// so strip everything except a harmless image tag before it's stored — in
// particular event handlers like `onerror`/`onload` that a malicious feed
// could use for XSS.
fn sanitize_img_html(html: &str) -> String {
    let allowed_attributes = HashSet::from(["src", "alt", "title"]);

    ammonia::Builder::default()
        .tags(HashSet::from(["img"]))
        .tag_attributes(HashMap::from([("img", allowed_attributes)]))
        .clean(html)
        .to_string()
}

// Some feeds (e.g. Deutsche Welle) don't embed an <img> in the item content at
// all — they carry the article image as an RSS <enclosure> instead. Build an
// <img> tag from it so those feeds get a preview image too.
fn enclosure_image_html(item: &Item) -> Option<String> {
    let enclosure = item.enclosure()?;
    if !enclosure.mime_type().to_lowercase().starts_with("image/") {
        return None;
    }
    Some(format!(
        r#"<img src="{}">"#,
        escape_html_attr(enclosure.url())
    ))
}

fn create_feed_item(item: Item, feed: &Feed, connection: &mut PgConnection) -> anyhow::Result<()> {
    // Items without a title or link are malformed/unusable — skip them rather
    // than failing the whole sync over one bad entry from an external feed.
    let Some(item_title) = item.title.clone() else {
        log::warn!("Skipping feed item without a title.");
        return Ok(());
    };
    if item.link.is_none() {
        log::warn!("Skipping feed item without a link: {}", item_title);
        return Ok(());
    }
    log::info!("Create feed item: {}", item_title);

    // Items without a pub_date are treated as current (inserted unconditionally)
    // — feeds that don't publish dates are typically small/curated enough that
    // this is fine.
    let mut time: NaiveDateTime = Local::now().naive_local();
    if let Some(pub_date) = item.pub_date() {
        time = match get_date(pub_date) {
            Ok(date) => date,
            Err(err) => {
                log::error!("could not parse pub date: {}", err);
                time
            }
        };
    }

    let base_content: &str = item.content().or(item.description()).unwrap_or_default();

    let frag = Html::parse_fragment(base_content);
    let mut content = "".to_string();

    // Some feeds (e.g. Stuttgarter Nachrichten) embed a social-sharing widget
    // (WhatsApp/Email/Facebook/... links plus a "Link kopiert" tooltip) in the
    // article content. It's not part of the article and isn't present in the
    // scraped/readable edition either, so skip its text when flattening below.
    let selector_social_bar =
        Selector::parse("#article-social-bar").expect("\"#article-social-bar\" is a valid CSS selector");
    let excluded_node_ids: std::collections::HashSet<_> = frag
        .select(&selector_social_bar)
        .flat_map(|el| el.descendants().map(|node| node.id()))
        .collect();

    let selector_img = Selector::parse("img").expect("\"img\" is a valid CSS selector");
    match frag.select(&selector_img).find(image_src_is_resolvable) {
        Some(image) => {
            content.push_str(&sanitize_img_html(&image.html()));
            content.push_str("<br>");
        }
        None => {
            if let Some(image_html) = enclosure_image_html(&item) {
                content.push_str(&sanitize_img_html(&image_html));
                content.push_str("<br>");
            }
        }
    }

    for node in frag.tree.nodes() {
        if excluded_node_ids.contains(&node.id()) {
            continue;
        }
        if let scraper::node::Node::Text(text) = node.value() {
            content.push_str(&text.text);
        }
    }

    let new_feed_item = NewFeedItem::new(
        feed.id,
        content.clone(),
        item_title.clone(),
        item.link.expect("checked above"),
        Some(time),
    );

    // `on_conflict` on the (feed_id, url) unique constraint makes this
    // insert idempotent, so two syncs for the same feed running concurrently
    // (e.g. a sync button press racing a page-reload sync) can't both pass a
    // check-then-insert race and create duplicate items. Keying on the
    // article's link rather than its title also handles feeds that edit a
    // headline after publishing while keeping the same link — the link is
    // the stable identity.
    let inserted_rows = diesel::insert_into(feed_item::table)
        .values(&new_feed_item)
        .on_conflict((feed_id, url))
        .do_nothing()
        .execute(connection)?;

    if inserted_rows > 0 {
        log::info!("Inserted item: {}", item_title);
    } else {
        log::info!("Item {} already exists.", item_title);
    }

    Ok(())
}

pub async fn sync(
    _req: HttpRequest,
    data: web::Json<JsonUser>,
    auth_user: AuthUser,
) -> Result<HttpResponse, AppError> {
    let req_user_id: i32 = data.user_id;

    if auth_user.0 != req_user_id {
        return Ok(HttpResponse::Forbidden().finish());
    }

    let mut connection: diesel::PgConnection = establish_connection();

    let feeds: Vec<Feed> = feed::table
        .filter(user_id.eq(req_user_id))
        .load::<Feed>(&mut connection)?;

    log::info!("Found {} feeds to sync.", feeds.len());

    for feed in feeds {
        log::info!("Try to get url: {}", feed.url);
        let result = feeds::get_feed(&feed.url).await;

        match result {
            Ok(channel) => {
                for item in channel.into_items() {
                    log::info!("{:?}", item);
                    if let Err(e) = create_feed_item(item, &feed, &mut connection) {
                        log::error!("Could not create feed item for {}: {:?}", feed.url, e);
                    }
                }
            }
            Err(e) => log::error!("Could not get channel {}. Error: {}", feed.url, e),
        }
    }

    Ok(HttpResponse::Ok().finish())
}

#[cfg(test)]
mod tests {
    use crate::models::feed::new_feed::NewFeed;
    use crate::models::feed_item::rss_feed_item::FeedItem;
    use crate::models::user::new_user::NewUser;
    use crate::models::user::rss_user::User;
    use crate::schema::users;
    use crate::test_helpers::{delete_feed, delete_user, insert_feed, insert_user, unique_suffix};
    use chrono::Duration;

    use super::*;

    #[test]
    fn get_date_parses_iso8601_dates() {
        assert!(get_date("2024-01-01T12:00:00Z").is_ok());
    }

    #[test]
    fn get_date_parses_rfc2822_dates() {
        assert!(get_date("Tue, 03 Jun 2025 10:00:00 GMT").is_ok());
    }

    #[test]
    fn get_date_returns_err_for_unparseable_dates() {
        assert!(get_date("not-a-date").is_err());
    }

    #[test]
    fn create_feed_item_skips_template_placeholder_images() {
        let html = Html::parse_fragment(
            r#"<img src="https://example.test/img_${formatId}.jpg"><p>placeholder</p>
               <img src="https://example.test/real.jpg"><p>real image</p>"#,
        );
        let selector = Selector::parse("img").unwrap();

        let chosen = html
            .select(&selector)
            .find(image_src_is_resolvable)
            .expect("should find a resolvable image");

        assert_eq!(
            Some("https://example.test/real.jpg"),
            chosen.value().attr("src")
        );
    }

    #[test]
    fn create_feed_item_finds_no_image_when_all_are_templated() {
        let html = Html::parse_fragment(r#"<img src="https://example.test/img_${formatId}.jpg">"#);
        let selector = Selector::parse("img").unwrap();

        assert!(html.select(&selector).find(image_src_is_resolvable).is_none());
    }

    #[test]
    fn enclosure_image_html_builds_img_for_image_enclosures() {
        let mut item = Item::default();
        item.set_enclosure(rss::Enclosure {
            url: "https://static.dw.com/image/73880499_302.jpg".to_string(),
            length: "2000".to_string(),
            mime_type: "image/jpeg".to_string(),
        });

        assert_eq!(
            Some(r#"<img src="https://static.dw.com/image/73880499_302.jpg">"#.to_string()),
            enclosure_image_html(&item)
        );
    }

    #[test]
    fn enclosure_image_html_ignores_non_image_enclosures() {
        let mut item = Item::default();
        item.set_enclosure(rss::Enclosure {
            url: "https://example.test/episode.mp3".to_string(),
            length: "2000".to_string(),
            mime_type: "audio/mpeg".to_string(),
        });

        assert_eq!(None, enclosure_image_html(&item));
    }

    #[test]
    fn enclosure_image_html_escapes_url_attribute() {
        let mut item = Item::default();
        item.set_enclosure(rss::Enclosure {
            url: "https://example.test/img.jpg?a=1&b=\"x\"".to_string(),
            length: "2000".to_string(),
            mime_type: "image/jpeg".to_string(),
        });

        assert_eq!(
            Some(r#"<img src="https://example.test/img.jpg?a=1&amp;b=&quot;x&quot;">"#.to_string()),
            enclosure_image_html(&item)
        );
    }

    #[test]
    fn sanitize_img_html_strips_event_handlers() {
        let sanitized = sanitize_img_html(r#"<img src="x" onerror="alert(1)">"#);

        assert!(!sanitized.contains("onerror"));
        assert!(!sanitized.contains("alert"));
        assert!(sanitized.contains(r#"src="x""#));
    }

    #[test]
    fn sanitize_img_html_keeps_only_allowed_attributes() {
        let sanitized = sanitize_img_html(
            r#"<img src="https://example.test/img.jpg" alt="desc" title="t" style="display:none" class="evil">"#,
        );

        assert!(sanitized.contains(r#"src="https://example.test/img.jpg""#));
        assert!(sanitized.contains(r#"alt="desc""#));
        assert!(sanitized.contains(r#"title="t""#));
        assert!(!sanitized.contains("style"));
        assert!(!sanitized.contains("class"));
    }

    #[actix_web::test]
    async fn create_feed_item_inserts_articles_older_than_two_weeks() {
        let mut connection = establish_connection();
        let suffix = unique_suffix();

        let new_user = NewUser::new(
            format!("age_test_{suffix}"),
            format!("age_{suffix}@example.test"),
            "secret".to_string(),
        )
        .unwrap();
        let user: User = diesel::insert_into(users::table)
            .values(&new_user)
            .get_result(&mut connection)
            .unwrap();

        let new_feed = NewFeed::new(
            format!("Age test feed {suffix}"),
            format!("https://example.test/feed/{suffix}"),
            user.id,
        );
        let feed: Feed = diesel::insert_into(feed::table)
            .values(&new_feed)
            .get_result(&mut connection)
            .unwrap();

        // Item with a pub_date 20 days ago — should still be inserted, since
        // infrequently-updated feeds (or infrequent syncs) must not lose
        // articles the user hasn't seen yet.
        let old_date = (Local::now() - Duration::days(20))
            .format("%a, %d %b %Y %H:%M:%S %z")
            .to_string();
        let mut old_item = Item::default();
        old_item.set_title(Some(format!("Old article {suffix}")));
        old_item.set_link(Some(format!("https://example.test/old/{suffix}")));
        old_item.set_pub_date(Some(old_date));
        old_item.set_content(Some("<p>old</p>".to_string()));

        // Item without a pub_date — treated as current, should be inserted.
        let mut fresh_item = Item::default();
        fresh_item.set_title(Some(format!("Fresh article {suffix}")));
        fresh_item.set_link(Some(format!("https://example.test/fresh/{suffix}")));
        fresh_item.set_content(Some("<p>fresh</p>".to_string()));

        create_feed_item(old_item, &feed, &mut connection).unwrap();
        create_feed_item(fresh_item, &feed, &mut connection).unwrap();

        let items: Vec<FeedItem> = feed_item::table
            .filter(feed_id.eq(feed.id))
            .load(&mut connection)
            .unwrap();

        assert_eq!(2, items.len(), "both old and fresh items should be inserted");

        diesel::delete(feed_item::table.filter(feed_id.eq(feed.id)))
            .execute(&mut connection)
            .ok();
        diesel::delete(feed::table.filter(feed::id.eq(feed.id)))
            .execute(&mut connection)
            .ok();
        diesel::delete(users::table.filter(users::id.eq(user.id)))
            .execute(&mut connection)
            .ok();
    }

    #[actix_web::test]
    async fn create_feed_item_does_not_duplicate_existing_items() {
        let mut connection = establish_connection();
        let suffix = unique_suffix();

        let new_user = NewUser::new(
            format!("sync_test_{suffix}"),
            format!("sync_{suffix}@example.test"),
            "secret".to_string(),
        )
        .unwrap();
        let user: User = diesel::insert_into(users::table)
            .values(&new_user)
            .get_result(&mut connection)
            .unwrap();

        let new_feed = NewFeed::new(
            format!("Sync test feed {suffix}"),
            format!("https://example.test/feed/{suffix}"),
            user.id,
        );
        let feed: Feed = diesel::insert_into(feed::table)
            .values(&new_feed)
            .get_result(&mut connection)
            .unwrap();

        let mut item = Item::default();
        item.set_title(Some(format!("Sync test article {suffix}")));
        item.set_link(Some(format!("https://example.test/article/{suffix}")));
        item.set_content(Some("<p>Hello world</p>".to_string()));

        create_feed_item(item.clone(), &feed, &mut connection).unwrap();
        create_feed_item(item, &feed, &mut connection).unwrap();

        let items: Vec<FeedItem> = feed_item::table
            .filter(feed_id.eq(feed.id))
            .load(&mut connection)
            .unwrap();
        assert_eq!(1, items.len(), "duplicate feed items should not be created");

        diesel::delete(feed_item::table.filter(feed_id.eq(feed.id)))
            .execute(&mut connection)
            .ok();
        diesel::delete(feed::table.filter(feed::id.eq(feed.id)))
            .execute(&mut connection)
            .ok();
        diesel::delete(users::table.filter(users::id.eq(user.id)))
            .execute(&mut connection)
            .ok();
    }

    #[actix_web::test]
    async fn create_feed_item_is_race_safe_under_concurrent_syncs() {
        let mut connection = establish_connection();
        let suffix = unique_suffix();

        let user = insert_user(&mut connection, "secret");
        let feed = insert_feed(&mut connection, user.id);

        let mut item = Item::default();
        item.set_title(Some(format!("Race test article {suffix}")));
        item.set_link(Some(format!("https://example.test/race/{suffix}")));
        item.set_content(Some("<p>Hello world</p>".to_string()));

        // Simulate two concurrent syncs for the same feed (e.g. a sync
        // button press racing a page-reload sync) — each on its own
        // connection/thread, released together so they genuinely overlap —
        // both inserting the same item.
        let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
        let handles: Vec<_> = (0..2)
            .map(|_| {
                let feed = feed.clone();
                let item = item.clone();
                let barrier = barrier.clone();
                std::thread::spawn(move || {
                    let mut connection = establish_connection();
                    barrier.wait();
                    create_feed_item(item, &feed, &mut connection)
                })
            })
            .collect();

        for handle in handles {
            handle.join().unwrap().unwrap();
        }

        let items: Vec<FeedItem> = feed_item::table
            .filter(feed_id.eq(feed.id))
            .load(&mut connection)
            .unwrap();
        assert_eq!(
            1,
            items.len(),
            "concurrent syncs must not create duplicate feed items"
        );

        diesel::delete(feed_item::table.filter(feed_id.eq(feed.id)))
            .execute(&mut connection)
            .ok();
        delete_feed(&mut connection, feed.id);
        delete_user(&mut connection, user.id);
    }

    #[actix_web::test]
    async fn create_feed_item_strips_onerror_from_feed_image() {
        let mut connection = establish_connection();
        let suffix = unique_suffix();

        let new_user = NewUser::new(
            format!("xss_test_{suffix}"),
            format!("xss_{suffix}@example.test"),
            "secret".to_string(),
        )
        .unwrap();
        let user: User = diesel::insert_into(users::table)
            .values(&new_user)
            .get_result(&mut connection)
            .unwrap();

        let new_feed = NewFeed::new(
            format!("XSS test feed {suffix}"),
            format!("https://example.test/feed/{suffix}"),
            user.id,
        );
        let feed: Feed = diesel::insert_into(feed::table)
            .values(&new_feed)
            .get_result(&mut connection)
            .unwrap();

        let mut item = Item::default();
        item.set_title(Some(format!("XSS article {suffix}")));
        item.set_link(Some(format!("https://example.test/xss/{suffix}")));
        item.set_content(Some(
            r#"<img src="https://example.test/real.jpg" onerror="alert(1)"><p>text</p>"#
                .to_string(),
        ));

        create_feed_item(item, &feed, &mut connection).unwrap();

        let stored: FeedItem = feed_item::table
            .filter(feed_id.eq(feed.id))
            .first(&mut connection)
            .unwrap();

        assert!(!stored.content.contains("onerror"));
        assert!(!stored.content.contains("alert"));
        assert!(stored.content.contains(r#"src="https://example.test/real.jpg""#));

        diesel::delete(feed_item::table.filter(feed_id.eq(feed.id)))
            .execute(&mut connection)
            .ok();
        diesel::delete(feed::table.filter(feed::id.eq(feed.id)))
            .execute(&mut connection)
            .ok();
        diesel::delete(users::table.filter(users::id.eq(user.id)))
            .execute(&mut connection)
            .ok();
    }

    #[actix_web::test]
    async fn create_feed_item_strips_social_sharing_widget() {
        let mut connection = establish_connection();
        let suffix = unique_suffix();

        let new_user = NewUser::new(
            format!("social_bar_test_{suffix}"),
            format!("social_bar_{suffix}@example.test"),
            "secret".to_string(),
        )
        .unwrap();
        let user: User = diesel::insert_into(users::table)
            .values(&new_user)
            .get_result(&mut connection)
            .unwrap();

        let new_feed = NewFeed::new(
            format!("Social bar test feed {suffix}"),
            format!("https://example.test/feed/{suffix}"),
            user.id,
        );
        let feed: Feed = diesel::insert_into(feed::table)
            .values(&new_feed)
            .get_result(&mut connection)
            .unwrap();

        let mut item = Item::default();
        item.set_title(Some(format!("Social bar article {suffix}")));
        item.set_link(Some(format!("https://example.test/article/{suffix}")));
        item.set_content(Some(
            r#"<p>Article text</p>
            <div id="article-social-bar" data-noprint="true">
              <ul>
                <li><a id="whatsapp" href="whatsapp://send?text=foo">&nbsp;</a></li>
                <li><a id="link_copy" onclick="copyToClipboard()">&nbsp;</a>
                  <p>Link kopiert</p>
                </li>
              </ul>
            </div>"#
                .to_string(),
        ));

        create_feed_item(item, &feed, &mut connection).unwrap();

        let items: Vec<FeedItem> = feed_item::table
            .filter(feed_id.eq(feed.id))
            .load(&mut connection)
            .unwrap();
        assert_eq!(1, items.len());
        assert!(items[0].content.contains("Article text"));
        assert!(!items[0].content.contains("Link kopiert"));

        diesel::delete(feed_item::table.filter(feed_id.eq(feed.id)))
            .execute(&mut connection)
            .ok();
        diesel::delete(feed::table.filter(feed::id.eq(feed.id)))
            .execute(&mut connection)
            .ok();
        diesel::delete(users::table.filter(users::id.eq(user.id)))
            .execute(&mut connection)
            .ok();
    }
}