diff --git a/migrations/2023-10-08-081246_update_feed_item/down.sql b/migrations/2023-10-08-081246_update_feed_item/down.sql new file mode 100644 index 0000000..7382f47 --- /dev/null +++ b/migrations/2023-10-08-081246_update_feed_item/down.sql @@ -0,0 +1,6 @@ +-- This file should undo anything in `up.sql` +ALTER TABLE feed_item +DROP COLUMN title; + +ALTER TABLE feed_item +DROP COLUMN url; diff --git a/migrations/2023-10-08-081246_update_feed_item/up.sql b/migrations/2023-10-08-081246_update_feed_item/up.sql new file mode 100644 index 0000000..a3a7518 --- /dev/null +++ b/migrations/2023-10-08-081246_update_feed_item/up.sql @@ -0,0 +1,6 @@ +-- Your SQL goes here +ALTER TABLE feed_item +ADD COLUMN title VARCHAR NOT NULL; + +ALTER TABLE feed_item +ADD COLUMN url VARCHAR NOT NULL; diff --git a/src/json_serialization/mod.rs b/src/json_serialization/mod.rs index ac4e56d..c19504e 100755 --- a/src/json_serialization/mod.rs +++ b/src/json_serialization/mod.rs @@ -1,4 +1,5 @@ pub mod articles; pub mod login; pub mod new_feed; +pub mod new_feed_item; pub mod new_user; diff --git a/src/json_serialization/new_feed_item.rs b/src/json_serialization/new_feed_item.rs new file mode 100644 index 0000000..a08fffe --- /dev/null +++ b/src/json_serialization/new_feed_item.rs @@ -0,0 +1,9 @@ +use serde::Deserialize; + +#[derive(Deserialize)] +pub struct NewFeedItemSchema { + pub content: String, + pub feed_id: i32, + pub url: String, + pub title: String, +} diff --git a/src/models/feed/rss_feed.rs b/src/models/feed/rss_feed.rs index fead698..dce0b62 100755 --- a/src/models/feed/rss_feed.rs +++ b/src/models/feed/rss_feed.rs @@ -2,7 +2,7 @@ use super::super::user::rss_user::User; use crate::schema::feed; use diesel::{Associations, Identifiable, Queryable}; -#[derive(Clone, Queryable, Identifiable, Associations)] +#[derive(Clone, Debug, Queryable, Identifiable, Associations)] #[diesel(belongs_to(User))] #[diesel(table_name=feed)] pub struct Feed { diff --git a/src/models/feed_item/mod.rs b/src/models/feed_item/mod.rs index d1698c1..37d1c31 100755 --- a/src/models/feed_item/mod.rs +++ b/src/models/feed_item/mod.rs @@ -1 +1,2 @@ +pub mod new_feed_item; mod rss_feed_item; diff --git a/src/models/feed_item/new_feed_item.rs b/src/models/feed_item/new_feed_item.rs new file mode 100755 index 0000000..35667e7 --- /dev/null +++ b/src/models/feed_item/new_feed_item.rs @@ -0,0 +1,40 @@ +// extern crate bcrypt; + +// use bcrypt::{hash, DEFAULT_COST}; +use diesel::Insertable; +// use uuid::Uuid; + +use crate::schema::feed_item; + +#[derive(Insertable, Clone)] +#[diesel(table_name=feed_item)] +pub struct NewFeedItem { + pub feed_id: i32, + pub content: String, + pub title: String, + pub url: String, +} + +impl NewFeedItem { + pub fn new(feed_id: i32, content: String, title: String, url: String) -> Self { + Self { + feed_id, + content, + title, + url, + } + } +} + +// impl NewUser { +// pub fn new(content: String, title: String, url: String) -> NewUser { +// let hashed_password: String = hash(password.as_str(), DEFAULT_COST).unwrap(); +// let uuid = Uuid::new_v4(); +// NewUser { +// username, +// email, +// password: hashed_password, +// unique_id: uuid.to_string(), +// } +// } +// } diff --git a/src/models/feed_item/rss_feed_item.rs b/src/models/feed_item/rss_feed_item.rs index 8b13789..fbf056e 100755 --- a/src/models/feed_item/rss_feed_item.rs +++ b/src/models/feed_item/rss_feed_item.rs @@ -1 +1,15 @@ +use diesel::{Associations, Identifiable, Queryable}; +use crate::schema::feed_item; + +#[derive(Clone, Queryable, Identifiable, Associations)] +#[diesel(belongs_to(Feed))] +#[diesel(table_name=feed_item)] +pub struct Feed { + pub id: i32, + pub feed_id: i32, + pub title: String, + pub url: String, + pub content: String, + pub read: bool, +} diff --git a/src/reader/feeds.rs b/src/reader/feeds.rs index c4d5b3b..cb77619 100755 --- a/src/reader/feeds.rs +++ b/src/reader/feeds.rs @@ -5,5 +5,6 @@ use rss::Channel; pub async fn get_feed(feed: &str) -> Result> { let content = reqwest::get(feed).await?.bytes().await?; let channel = Channel::read_from(&content[..])?; + log::info!("{:?}", channel); Ok(channel) } diff --git a/src/reader/structs/feed.rs b/src/reader/structs/feed.rs index 0aa323b..edbf681 100644 --- a/src/reader/structs/feed.rs +++ b/src/reader/structs/feed.rs @@ -7,9 +7,9 @@ pub struct Feed { pub title: String, pub items: Vec
, } - -impl Feed { - pub fn new(title: String, items: Vec
) -> Feed { - Feed { title, items } - } -} +// +// impl Feed { +// pub fn new(title: String, items: Vec
) -> Feed { +// Feed { title, items } +// } +// } diff --git a/src/reader/sync.rs b/src/reader/sync.rs index 0025a1c..616c865 100644 --- a/src/reader/sync.rs +++ b/src/reader/sync.rs @@ -1,12 +1,17 @@ use super::feeds; +use crate::models::feed::rss_feed::Feed; +use crate::models::feed_item::new_feed_item::NewFeedItem; use crate::{ database::establish_connection, - models::feed::rss_feed::Feed, - schema::feed::{self, user_id}, + schema::{ + feed::{self, user_id}, + feed_item, + }, }; use actix_web::{web, HttpRequest, HttpResponse, Responder}; use diesel::prelude::*; use futures::StreamExt; +use scraper::{Html, Selector}; use serde_derive::Deserialize; #[derive(Deserialize)] @@ -18,26 +23,63 @@ pub async fn sync(_req: HttpRequest, data: web::Json) -> impl Responde let mut connection: diesel::PgConnection = establish_connection(); let req_user_id = data.user_id.parse::().unwrap(); - log::info!("{:?}", req_user_id); let feed: Vec = feed::table .filter(user_id.eq(req_user_id)) .load::(&mut connection) .unwrap(); + log::info!("Found {} feeds to sync.", feed.len()); + // Create an asynchronous stream of Feed items let feed_stream = futures::stream::iter(feed.clone().into_iter()).map(|feed| { // Asynchronously fetch the feed_list for each feed + log::info!("processing feed: {:?}", feed); async move { - let _feed_list = feeds::get_feed(&feed.url).await.unwrap(); - // Process feed_list here + log::info!("start moved"); + let feed_list: rss::Channel = feeds::get_feed(&feed.url).await.unwrap(); + log::info!("{:?}", feed_list); + + feed_list.into_items().into_iter().for_each(|item| { + let title = item.title.unwrap(); + let frag = Html::parse_fragment(&item.content.unwrap()); + let mut content = "".to_string(); + let frag_clone = frag.clone(); + frag.tree.into_iter().for_each(|node| { + let selector_img = Selector::parse("img").unwrap(); + + for element in frag_clone.select(&selector_img) { + if !content.starts_with("") + } + } + if let scraper::node::Node::Text(text) = node { + content.push_str(&text.text); + } + }); + + let mut connection: diesel::PgConnection = establish_connection(); + let new_feed_item = + NewFeedItem::new(feed.id, content.clone(), title.clone(), feed.url.clone()); + let insert_result = diesel::insert_into(feed_item::table) + .values(&new_feed_item) + .execute(&mut connection); + + log::info!("{:?}", insert_result); + }); } }); // Execute the asynchronous stream - tokio::spawn(feed_stream.for_each(|_| async {})); + let result = tokio::spawn(feed_stream.for_each(|_| async {})).await; - HttpResponse::Ok() + if result.is_err() { + log::error!("{:?}", result); + HttpResponse::InternalServerError() + } else { + HttpResponse::Ok() + } } // pub async fn sync(req: HttpRequest) -> impl Responder { // let request = req.clone(); diff --git a/src/schema.rs b/src/schema.rs index 14d46e7..7ca84c8 100755 --- a/src/schema.rs +++ b/src/schema.rs @@ -15,6 +15,8 @@ diesel::table! { feed_id -> Int4, content -> Text, read -> Bool, + title -> Varchar, + url -> Varchar, } }