merge feed lib into project

Cargo.toml

@@ -11,12 +11,13 @@ rusqlite = {version=">=0.34",features=['bundled']}
 scraper = "0.23.1"
 directories = "6.0.0"
 chrono = "0.4.41"
-rss_content = { git = "https://code.gabe.rocks/gabriel/rss_content", version = "0.1.1" }
 url = "2.5.4"
 opml = "1.1.6"
 sha1 = "0.10.6"
 bytes = "1.11.1"
+ego-tree = "0.10.0"
 #rfd = "0.15.4" (for importing files)
+
 [profile.dev]
 debug=true
 incremental = true

src/main.rs

@@ -4,6 +4,7 @@ pub mod ui;
 pub mod db;
 pub mod files;
 pub mod widgets;
+mod rss_content;
 
 pub fn main() -> iced::Result {
     db::initialize();

src/net.rs

@@ -89,28 +89,20 @@ fn get_client(network: Network) -> Result<Client, Error> {
     }
 }
 
-pub fn get_content(url: &str) -> Option<String> {
+fn fetch(url: &str) -> Option<reqwest::blocking::Response> {
     let client = get_client(url_network(url)).unwrap();
-    let res = client.get(url).header(USER_AGENT, "RSS Reader").send();
+    client.get(url)
-    match res {
+        .header(USER_AGENT, "RSS Reader")
-        Ok(resp) => match resp.text() {
+        .send()
-            Ok(body) => return Some(body),
+        .ok()
-            Err(_) => return None,
+}
-        },
+
-        Err(_) => return None,
+pub fn get_content(url: &str) -> Option<String> {
-    }
+    fetch(url)?.text().ok()
 }
 
 pub fn get_bytes(url: &str) -> Option<bytes::Bytes> {
-    let client = get_client(url_network(url)).unwrap();
+    fetch(url)?.bytes().ok()
-    let res = client.get(url).header(USER_AGENT, "RSS Reader").send();
-    match res {
-        Ok(resp) => match resp.bytes() {
-            Ok(body) => return Some(body),
-            Err(_) => return None,
-        },
-        Err(_) => return None,
-    }
 }
 
 pub fn retrieve_opml(url: &str) -> Vec<Url> {

src/rss_content/mod.rs

@@ -0,0 +1,350 @@
+use scraper::{ElementRef, Html, Node};
+use iced::widget::markdown;
+
+/*
+    The goal here is to flatten the DOM as much as possible. 
+    paragraphs with fancy formatting are turned into markdown, same with 
+*/
+
+//Supported content
+#[derive(Debug,Clone)]
+pub enum Content {
+    Markdown(String),
+    MarkdownParsed(Vec<markdown::Item>),
+    Image(String),
+    Audio(String),
+    Video(String),
+    Ignore
+}
+
+pub fn parse_content(c: &str) -> Vec<Content>{
+    process_content(&itemize_content(c)).into_iter().map(|i| {
+        match i {
+            Content::Markdown(s) => {
+                Content::MarkdownParsed(markdown::parse(&s).collect())
+                //this is super lazy but it works....
+            }
+            _ => {i}
+        }
+    }).collect()
+}
+
+fn markdownify_child(item: &Item) -> String {
+    let mut result = "".to_owned();
+    match markdown_content(&item) {
+        Content::Markdown(s) => {
+            result = result + &s;
+        },
+        _ => {}
+    }
+    result
+
+}
+
+fn process_children(children: &Vec<Item>) -> String {
+    let mut result = "".to_owned();
+    for c in children{
+        result = result + &markdownify_child(c);
+    }
+    result
+}
+
+
+fn has_image(children: &Vec<Item>) -> bool {
+    for c in children {
+        match c {
+            Item::Image(_) => {
+                return true;
+            }
+            _ => {}
+        }
+    }
+    false
+}
+
+fn has_video(children: &Vec<Item>) -> bool{
+    for c in children {
+        match c {
+            Item::Video(_) => {
+            return true;
+            }
+            _ => {}
+        }
+        
+    }
+    false
+}
+
+fn markdown_content(item: &Item) -> Content {
+    let mut markdown = String::new();
+    match item {
+        Item::Title(n,children) => {
+            markdown = markdown + &"#".repeat(*n) + " " +&process_children(children);
+        },
+        Item::BoldedText(children) => {
+            markdown = format!("**{}**",process_children(children));
+        },
+        Item::EmphasisText(children) => {
+            markdown = format!("*{}*",process_children(children));
+        }
+        Item::Text(s) => {
+            markdown = markdown + s;
+        },
+        Item::Link(href, children) => {
+            markdown = markdown + &format!("[{}]({})",process_children(children),href);
+        }
+        Item::Paragraph(children) => {
+            markdown = markdown + &process_children(children);
+        }
+        Item::UnorderedList(children) => {
+            markdown = markdown + &process_children(children);
+        }
+        Item::OrderedList(children) => {
+            markdown = markdown + &process_children(children);
+        }
+        Item::ListItem(children) => {
+            markdown = "\n- ".to_owned() + &process_children(children);
+        }
+        Item::Code(children) => {
+            markdown = markdown + &format!("```{}```",&process_children(children));
+        }
+        Item::Blockquote(children) => {
+            markdown = markdown + "> " + &process_children(children);
+        }
+        _ => {}
+    }
+    Content::Markdown(markdown)
+}
+
+fn get_media_source(children: &Vec<Item>) -> Option<String> {
+    for c in children {
+        match c {
+            Item::Source(src) => {
+                return Some(src.to_owned());
+            }
+            _ => {}
+        }
+    }
+    None
+}
+
+fn media_content(item: &Item) -> Content{
+    match item {
+        Item::Link(_,children) => {
+            for c in children {
+                match c {
+                    Item::Video(_) => {return media_content(c);}
+                    Item::Audio(_) => {return media_content(c);}
+                    _ => {}
+                }
+            }
+            Content::Ignore
+        }
+        Item::Audio(children) => {
+            match get_media_source(children) {
+                Some(s) => {
+                    return Content::Audio(s);
+                }
+                None => {
+                    return Content::Markdown("<Audio Element with no source>".to_owned());
+                }
+            }
+        }
+        Item::Video(children) => {
+            match get_media_source(children) {
+                Some(source) => {
+                    return Content::Video(source)
+                }
+                None => {
+                    return Content::Markdown("<Video element with no source>".to_owned());
+                }
+            }
+        }
+        _ => {
+            Content::Markdown(format!("Incorrectly assigned element:{:#?}",item))
+        }
+    }
+}
+
+
+fn process_content(items: &Vec<Item>) -> Vec<Content> {
+    let mut result: Vec<Content> = Vec::new();
+    //println!("Converting {} items into Content",items.len());
+    for i in items {
+        match i {
+            Item::Title(_,_) => {
+                result.push(markdown_content(i));
+            }
+            Item::Paragraph(_) => {
+                result.push(markdown_content(i));
+            },
+            Item::Link(_,children) => {
+                if has_video(children){
+                    result.push(media_content(i));
+                }
+                else if has_image(children){
+                    result.push(media_content(i));
+
+                }
+                else {
+                    result.push(markdown_content(i))
+                }
+            }
+            Item::UnorderedList(_) => {
+                result.push(markdown_content(i));
+            }
+            Item::OrderedList(_) => {
+                result.push(markdown_content(i));
+            }
+            Item::ListItem(_) => {
+                result.push(markdown_content(i));
+            }
+            Item::Image(src) => {
+                result.push(Content::Image(src.to_owned()));
+            }
+            Item::Video(_) => {
+                result.push(media_content(i));
+            }
+            Item::Audio(_) => {
+                result.push(media_content(i));
+            }
+            Item::Container(children) => {
+                //should actually plan to contain things
+                for c in process_content(children){
+                    result.push(c);
+                }
+            }
+            Item::Code(_) => {
+                result.push(markdown_content(i));
+            }
+            Item::Blockquote(_) => {
+                result.push(markdown_content(i));
+            }
+            _ => {
+                result.push(Content::Ignore);
+            }
+        }
+    }
+
+    result
+}
+
+#[derive(Debug,Clone)]
+enum Item {
+    Ignore,
+    Title(usize,Vec<Item>),
+    Text(String),
+    //text, links, formatting are all markdown
+    //arguably, for better control it will be best to turn markdown into its own set of items
+    Image(String),
+    Video(Vec<Item>),
+    Audio(Vec<Item>),
+    Source(String),
+    Blockquote(Vec<Item>),
+    Code(Vec<Item>),
+    BoldedText(Vec<Item>),
+    EmphasisText(Vec<Item>),
+    UnorderedList(Vec<Item>),
+    OrderedList(Vec<Item>),
+    ListItem(Vec<Item>),
+    Paragraph(Vec<Item>),
+    Container(Vec<Item>),
+    Link(String,Vec<Item>),
+    Table(Vec<Item>)
+}
+
+
+fn itemize_content(content: &str) -> Vec<Item> {
+    let frag = Html::parse_fragment(content);
+    frag.root_element().children().map(|e|{
+        parse_items(e)
+    }).collect()
+}
+
+fn get_children(el: &ElementRef) -> Vec<Item>{
+    el.children().map(|c|{parse_items(c)}).collect()
+}
+
+fn parse_items(n: ego_tree::NodeRef<'_,Node>) -> Item{
+    if n.value().is_text(){
+        return Item::Text((&n.value().as_text().unwrap()).to_string())
+    }
+    if n.value().is_element(){
+        let el = ElementRef::wrap(n).unwrap();  
+        let tag_name = el.value().name();
+        match tag_name {
+            "h1" => {return Item::Title(1, get_children(&el))},
+            "h2" => {return Item::Title(2, get_children(&el))},
+            "h3" => {return Item::Title(3, get_children(&el))},
+            "h4" => {return Item::Title(4, get_children(&el))},
+            "h5" => {return Item::Title(5, get_children(&el))},
+            "h6" => {return Item::Title(6, get_children(&el))},
+            "strong" => {return Item::BoldedText(get_children(&el))},
+            "em" => {return Item::EmphasisText(get_children(&el))},
+            "br" => {return Item::Text("\n".to_owned())},
+            "hr" => {return Item::Text("---".to_owned())}
+            "p" => {
+                return Item::Paragraph(get_children(&el))
+            },
+            "a" => {
+                let href = match el.attr("href") {
+                    Some(link) => {link}
+                    None => {""}
+                };
+                return Item::Link(href.to_owned(),get_children(&el))
+                                            
+                
+            }
+            "img" => {
+                match el.attr("src") {
+                    Some(src) => {
+                        return Item::Image(src.to_owned())
+                    },
+                    None => {return Item::Ignore}
+                }
+            }
+            "source" => {
+                match el.attr("src") {
+                    Some(src) => {
+                        return Item::Source(src.to_owned())
+                    },
+                    None => {return Item::Ignore}
+                }
+            }
+            "video" => {
+                return Item::Video(get_children(&el))                
+            }
+            "audio" => {
+                return Item::Audio(get_children(&el))
+            }
+            "ol" => {
+                return Item::OrderedList(get_children(&el))
+            }
+            "ul" => {
+                return Item::UnorderedList(get_children(&el))
+            }
+            "li" => {
+                return Item::ListItem(get_children(&el))
+            }
+            "div" => {
+                return Item::Container(get_children(&el))
+            }
+            "code" => {
+                return Item::Code(get_children(&el))
+            }
+            "blockquote" => {
+                return Item::Blockquote(get_children(&el))
+            }
+
+            _ => {}
+        };  
+    }
+    
+    Item::Ignore
+
+
+
+}
+
+#[cfg(test)]
+ mod tests;

src/rss_content/tests/mod.rs

@@ -0,0 +1,67 @@
+mod example_data;
+use super::*;
+use example_data::*;
+
+fn get_feed(u: &str) -> rss::Channel {
+    rss::Channel::read_from(u.as_bytes()).unwrap()
+}
+
+#[test]
+fn fragments() {
+    for f in FRAGMENTS {
+        println!("Processing Fragment:{}",f);
+        let items = itemize_content(f);
+        let content = process_content(&items);
+        println!("Content:\n{:#?}",content);
+    }
+}
+
+#[test]
+fn content_test() {
+    let example_text = Item::Text("Example.com".to_owned());
+    let example_link = Item::Link("https://example.com".to_owned(), [example_text].to_vec());
+    let result = process_content(&[example_link].to_vec());
+    println!("Items to content parse result:\n{:#?}", result);
+}
+
+#[test]
+fn content_display() {
+    let feed = get_feed(example_data::GABE_ROCKS);
+    let content: &Vec<Content> = &process_content(&itemize_content(
+        feed.items.get(4).unwrap().content().unwrap(),
+    ));
+    println!("Content: {:#?}", content)
+}
+
+#[test]
+fn itemize_feeds() {
+    let _ = FEEDS.map(|u| {
+        let feed = get_feed(u);
+        let results: Vec<_> = feed
+            .items
+            .into_iter()
+            .map(|item| {
+                itemize_content(&item.content.unwrap());
+            })
+            .collect();
+        //let results: Vec<_> = itemize_content(u);
+        println!(
+            "Evaluated feed\nScanned {} items without errors",
+            results.len()
+        )
+    });
+}
+#[test]
+fn markdownify_feeds() {
+    let _ = FEEDS.map(|u| {
+        let feed = get_feed(u);
+        let results: Vec<_> = feed
+            .items
+            .into_iter()
+            .map(|item| {
+                process_content(&itemize_content(&item.content.unwrap()));
+            })
+            .collect();
+        println!("Processed {} items without errors", results.len())
+    });
+}

src/ui.rs

@@ -15,8 +15,8 @@ use iced::{
     Element,
     Length::Fill,
 };
-use rss_content::parse_content;
+use crate::rss_content::parse_content;
-use rss_content::Content;
+use crate::rss_content::Content;
 use url::Url;
 const ICON: &[u8] = include_bytes!("../assets/icon_placeholder.png");
 

src/widgets.rs

@@ -13,8 +13,7 @@ use iced::{
     widget::{button, column, container, text},
     Element,
 };
-use rss_content;
+use crate::rss_content::Content;
-use rss_content::Content;
 use ui::Message;
 
 pub fn list_feeds() -> iced::widget::Column<'static, Message> {