Add support for posts authored in markdown
This commit is contained in:
silverpill
parent
125595f940
commit
b26b2419ed
6 changed files with 283 additions and 9 deletions
111
Cargo.lock
generated
111
Cargo.lock
generated
|
|
@ -373,9 +373,9 @@ checksum = "d0d27fb6b6f1e43147af148af49d49329413ba781aa0d5e10979831c210173b5"
|
|||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.2.1"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitvec"
|
||||
|
|
@ -546,6 +546,21 @@ dependencies = [
|
|||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "comrak"
|
||||
version = "0.14.0"
|
||||
source = "git+https://github.com/kivikakk/comrak?rev=03238b81d0917acbea73a2d255603f8051e4dc04#03238b81d0917acbea73a2d255603f8051e4dc04"
|
||||
dependencies = [
|
||||
"entities",
|
||||
"memchr",
|
||||
"once_cell",
|
||||
"pest",
|
||||
"pest_derive",
|
||||
"regex",
|
||||
"typed-arena",
|
||||
"unicode_categories",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.6.0"
|
||||
|
|
@ -801,6 +816,12 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "entities"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca"
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.9.0"
|
||||
|
|
@ -1574,9 +1595,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.4.0"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
|
||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
|
|
@ -1656,6 +1677,7 @@ dependencies = [
|
|||
"base64",
|
||||
"chrono",
|
||||
"clap",
|
||||
"comrak",
|
||||
"deadpool",
|
||||
"deadpool-postgres",
|
||||
"dotenv",
|
||||
|
|
@ -2007,6 +2029,50 @@ version = "2.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||
|
||||
[[package]]
|
||||
name = "pest"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbc7bc69c062e492337d74d59b120c274fd3d261b6bf6d3207d499b4b379c41a"
|
||||
dependencies = [
|
||||
"thiserror",
|
||||
"ucd-trie",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pest_derive"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60b75706b9642ebcb34dab3bc7750f811609a0eb1dd8b88c2d15bf628c1c65b2"
|
||||
dependencies = [
|
||||
"pest",
|
||||
"pest_generator",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pest_generator"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4f9272122f5979a6511a749af9db9bfc810393f63119970d7085fed1c4ea0db"
|
||||
dependencies = [
|
||||
"pest",
|
||||
"pest_meta",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pest_meta"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c8717927f9b79515e565a64fe46c38b8cd0427e64c40680b14a7365ab09ac8d"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"pest",
|
||||
"sha1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.10.1"
|
||||
|
|
@ -2845,6 +2911,17 @@ dependencies = [
|
|||
"digest 0.10.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.2",
|
||||
"digest 0.10.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.9.5"
|
||||
|
|
@ -3068,18 +3145,18 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
|
|||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.24"
|
||||
version = "1.0.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
|
||||
checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.24"
|
||||
version = "1.0.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
|
||||
checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -3271,12 +3348,24 @@ version = "0.2.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
|
||||
|
||||
[[package]]
|
||||
name = "typed-arena"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0685c84d5d54d1c26f7d3eb96cd41550adb97baed141a761cf335d3d33bcd0ae"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
|
||||
|
||||
[[package]]
|
||||
name = "ucd-trie"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
|
||||
|
||||
[[package]]
|
||||
name = "uint"
|
||||
version = "0.9.1"
|
||||
|
|
@ -3338,6 +3427,12 @@ version = "0.2.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
|
||||
|
||||
[[package]]
|
||||
name = "unicode_categories"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "1.7.2"
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ base64 = "0.13.0"
|
|||
chrono = { version = "0.4.22", features = ["serde"] }
|
||||
# Used to build admin CLI tool
|
||||
clap = { version = "3.1.8", default-features = false, features = ["std", "derive"] }
|
||||
# Used for parsing markdown
|
||||
comrak = { git = "https://github.com/kivikakk/comrak", rev = "03238b81d0917acbea73a2d255603f8051e4dc04", default-features = false }
|
||||
# Used for pooling database connections
|
||||
deadpool = "0.9.2"
|
||||
deadpool-postgres = { version = "0.10.2", default-features = false }
|
||||
|
|
@ -67,7 +69,7 @@ sha2 = "0.9.5"
|
|||
# Used to verify EIP-4361 signatures
|
||||
siwe = "0.3.0"
|
||||
# Used for creating error types
|
||||
thiserror = "1.0.24"
|
||||
thiserror = "1.0.37"
|
||||
# Async runtime
|
||||
tokio = { version = "1.17.0", features = ["macros"] }
|
||||
# Used for working with Postgresql database
|
||||
|
|
|
|||
|
|
@ -510,6 +510,9 @@ paths:
|
|||
description: Media type of the post content.
|
||||
type: string
|
||||
default: text/html
|
||||
enum:
|
||||
- text/html
|
||||
- text/markdown
|
||||
'media_ids[]':
|
||||
description: Array of Attachment ids to be attached as media.
|
||||
type: array
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use crate::mastodon_api::accounts::types::Account;
|
|||
use crate::mastodon_api::media::types::Attachment;
|
||||
use crate::models::posts::types::{Post, PostCreateData, Visibility};
|
||||
use crate::models::profiles::types::DbActorProfile;
|
||||
use crate::utils::markdown::markdown_to_html;
|
||||
|
||||
/// https://docs.joinmastodon.org/entities/mention/
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -169,6 +170,10 @@ impl TryFrom<StatusData> for PostCreateData {
|
|||
};
|
||||
let content = match status_data.content_type.as_str() {
|
||||
"text/html" => status_data.status,
|
||||
"text/markdown" => {
|
||||
markdown_to_html(&status_data.status)
|
||||
.map_err(|_| ValidationError("invalid markdown"))?
|
||||
},
|
||||
_ => return Err(ValidationError("unsupported content type")),
|
||||
};
|
||||
let post_data = Self {
|
||||
|
|
|
|||
168
src/utils/markdown.rs
Normal file
168
src/utils/markdown.rs
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
use comrak::{
|
||||
format_commonmark,
|
||||
format_html,
|
||||
nodes::{Ast, AstNode, NodeValue},
|
||||
parse_document,
|
||||
Arena,
|
||||
ComrakOptions,
|
||||
ComrakExtensionOptions,
|
||||
ComrakParseOptions,
|
||||
ComrakRenderOptions,
|
||||
};
|
||||
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
pub enum MarkdownError {
|
||||
#[error(transparent)]
|
||||
IoError(#[from] std::io::Error),
|
||||
|
||||
#[error(transparent)]
|
||||
Utf8Error(#[from] std::string::FromUtf8Error),
|
||||
}
|
||||
|
||||
fn iter_nodes<'a, F>(
|
||||
node: &'a AstNode<'a>,
|
||||
func: &F,
|
||||
) -> Result<(), MarkdownError>
|
||||
where F: Fn(&'a AstNode<'a>) -> Result<(), MarkdownError>
|
||||
{
|
||||
func(node)?;
|
||||
for child in node.children() {
|
||||
iter_nodes(child, func)?;
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn node_to_markdown<'a>(
|
||||
node: &'a AstNode<'a>,
|
||||
options: &ComrakOptions,
|
||||
) -> Result<String, MarkdownError> {
|
||||
let mut output = vec![];
|
||||
format_commonmark(node, options, &mut output)?;
|
||||
let markdown = String::from_utf8(output)?
|
||||
.trim_end_matches('\n')
|
||||
.to_string();
|
||||
Ok(markdown)
|
||||
}
|
||||
|
||||
/// Supported markdown features:
|
||||
/// - bold and italic
|
||||
/// - links and autolinks
|
||||
/// - inline code and code blocks
|
||||
pub fn markdown_to_html(text: &str) -> Result<String, MarkdownError> {
|
||||
let options = ComrakOptions {
|
||||
extension: ComrakExtensionOptions {
|
||||
autolink: true,
|
||||
..Default::default()
|
||||
},
|
||||
parse: ComrakParseOptions::default(),
|
||||
render: ComrakRenderOptions {
|
||||
hardbreaks: true,
|
||||
escape: true,
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
let arena = Arena::new();
|
||||
let root = parse_document(
|
||||
&arena,
|
||||
text,
|
||||
&options,
|
||||
);
|
||||
|
||||
// Re-render blockquotes, headings, HRs, images and lists
|
||||
// TODO: disable parser rules https://github.com/kivikakk/comrak/issues/244
|
||||
iter_nodes(root, &|node| {
|
||||
let node_value = node.data.borrow().value.clone();
|
||||
match node_value {
|
||||
// Blocks
|
||||
NodeValue::BlockQuote | NodeValue::Heading(_) | NodeValue::ThematicBreak => {
|
||||
// Replace children with paragraph containing markdown
|
||||
let mut markdown = node_to_markdown(node, &options)?;
|
||||
if matches!(node_value, NodeValue::BlockQuote) {
|
||||
// Fix greentext
|
||||
markdown = markdown.replace("> ", ">");
|
||||
};
|
||||
for child in node.children() {
|
||||
child.detach();
|
||||
};
|
||||
let text = NodeValue::Text(markdown.as_bytes().to_vec());
|
||||
let text_node = arena.alloc(AstNode::from(text));
|
||||
node.append(text_node);
|
||||
let mut borrowed_node = node.data.borrow_mut();
|
||||
*borrowed_node = Ast::new(NodeValue::Paragraph);
|
||||
},
|
||||
// Inlines
|
||||
NodeValue::Image(_) => {
|
||||
// Replace node with text node containing markdown
|
||||
let markdown = node_to_markdown(node, &options)?;
|
||||
for child in node.children() {
|
||||
child.detach();
|
||||
};
|
||||
let text = NodeValue::Text(markdown.as_bytes().to_vec());
|
||||
let mut borrowed_node = node.data.borrow_mut();
|
||||
*borrowed_node = Ast::new(text);
|
||||
},
|
||||
NodeValue::List(_) => {
|
||||
// Replace list and list item nodes
|
||||
// while preserving their contents
|
||||
let mut replacements: Vec<&AstNode> = vec![];
|
||||
for list_item in node.children() {
|
||||
let mut contents = vec![];
|
||||
for paragraph in list_item.children() {
|
||||
for content_node in paragraph.children() {
|
||||
contents.push(content_node);
|
||||
};
|
||||
paragraph.detach();
|
||||
};
|
||||
let list_prefix_markdown =
|
||||
node_to_markdown(list_item, &options)?;
|
||||
let list_prefix =
|
||||
NodeValue::Text(list_prefix_markdown.as_bytes().to_vec());
|
||||
if !replacements.is_empty() {
|
||||
// Insert line break before next list item
|
||||
let linebreak = NodeValue::LineBreak;
|
||||
replacements.push(arena.alloc(AstNode::from(linebreak)));
|
||||
};
|
||||
replacements.push(arena.alloc(AstNode::from(list_prefix)));
|
||||
for content_node in contents {
|
||||
replacements.push(content_node);
|
||||
};
|
||||
list_item.detach();
|
||||
};
|
||||
for child_node in replacements {
|
||||
node.append(child_node);
|
||||
};
|
||||
let mut borrowed_node = node.data.borrow_mut();
|
||||
*borrowed_node = Ast::new(NodeValue::Paragraph);
|
||||
},
|
||||
_ => (),
|
||||
};
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
let mut output = vec![];
|
||||
format_html(root, &options, &mut output)?;
|
||||
let html = String::from_utf8(output)?
|
||||
// Fix hardbreaks
|
||||
.replace("<br />\n", "<br>")
|
||||
// Remove extra soft breaks
|
||||
.replace(">\n<", "><")
|
||||
.trim_end_matches('\n')
|
||||
.to_string();
|
||||
Ok(html)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_markdown_to_html() {
|
||||
let text = "# heading\n\ntest **bold** test *italic* test ~~strike~~ with `code`, <span>html</span> and https://example.com\nnew line\n\ntwo new lines and a list:\n- item 1\n- item 2\n\n>greentext\n\n---\n\nimage: \n\ncode block:\n```\nlet test\ntest = 1\n```";
|
||||
let html = markdown_to_html(text).unwrap();
|
||||
let expected_html = concat!(
|
||||
r#"<p># heading</p><p>test <strong>bold</strong> test <em>italic</em> test ~~strike~~ with <code>code</code>, <span>html</span> and <a href="https://example.com">https://example.com</a><br>new line</p><p>two new lines and a list:</p><p>- item 1<br>- item 2</p><p>>greentext</p><p>-----</p><p>image: </p><p>code block:</p>"#,
|
||||
"<pre><code>let test\ntest = 1\n</code></pre>",
|
||||
);
|
||||
assert_eq!(html, expected_html);
|
||||
}
|
||||
}
|
||||
|
|
@ -4,3 +4,4 @@ pub mod currencies;
|
|||
pub mod files;
|
||||
pub mod html;
|
||||
pub mod id;
|
||||
pub mod markdown;
|
||||
|
|
|
|||
Loading…
Reference in a new issue