From e6795946e47a67437db8c880e19c7ec1d1b5bca9 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 6 May 2024 13:25:42 +0200 Subject: [PATCH] Use middleware for plugin --- .gitignore | 2 +- .woodpecker.yml | 2 +- Cargo.lock | 6 +- Cargo.toml | 4 + api_tests/prepare-drone-federation-test.sh | 7 +- crates/api_crud/Cargo.toml | 2 - crates/api_crud/src/post/create.rs | 49 +-------- {example_plugin => plugins}/go.mod | 0 {example_plugin => plugins}/go.sum | 0 {example_plugin => plugins}/main.go | 4 +- src/api_routes_http.rs | 2 + src/lib.rs | 1 + src/plugin_middleware.rs | 120 +++++++++++++++++++++ 13 files changed, 140 insertions(+), 59 deletions(-) rename {example_plugin => plugins}/go.mod (100%) rename {example_plugin => plugins}/go.sum (100%) rename {example_plugin => plugins}/main.go (93%) create mode 100644 src/plugin_middleware.rs diff --git a/.gitignore b/.gitignore index c44fd85a8..f8cc6d979 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,4 @@ dev_pgdata/ *.sqldump # compiled example plugin -example_plugin/plugin.wasm +plugins/plugin.wasm diff --git a/.woodpecker.yml b/.woodpecker.yml index bd25574c4..66a45cde7 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -229,7 +229,7 @@ steps: DO_WRITE_HOSTS_FILE: "1" commands: - *install_pnpm - - apt update && apt install -y bash curl postgresql-client golang tinygo + - apt update && apt install -y bash curl postgresql-client golang - bash api_tests/prepare-drone-federation-test.sh - cd api_tests/ - pnpm i diff --git a/Cargo.lock b/Cargo.lock index 452b288db..f2cc71f55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3140,8 +3140,6 @@ dependencies = [ "actix-web", "anyhow", "bcrypt", - "extism", - "extism-convert", "futures", "lemmy_api_common", "lemmy_db_schema", @@ -3358,6 +3356,7 @@ version = "0.19.4-beta.6" dependencies = [ "activitypub_federation", "actix-cors", + "actix-http", "actix-web", "actix-web-prom", "chrono", @@ -3366,6 +3365,8 @@ dependencies = [ "console-subscriber 0.1.10", "diesel", "diesel-async", + "extism", + "extism-convert", "futures-util", "lemmy_api", "lemmy_api_common", @@ -3383,6 +3384,7 @@ dependencies = [ "reqwest", "reqwest-middleware", "reqwest-tracing", + "serde", "serde_json", "serial_test", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 77eed98bb..fc55ab236 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -204,7 +204,11 @@ chrono = { workspace = true } prometheus = { version = "0.13.3", features = ["process"] } serial_test = { workspace = true } clap = { workspace = true } +serde.workspace = true actix-web-prom = "0.7.0" +actix-http = "3.6.0" +extism = "1.2.0" +extism-convert = "1.2.0" [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/api_tests/prepare-drone-federation-test.sh b/api_tests/prepare-drone-federation-test.sh index 6470c110d..1a027d264 100755 --- a/api_tests/prepare-drone-federation-test.sh +++ b/api_tests/prepare-drone-federation-test.sh @@ -74,10 +74,9 @@ LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_delta.hjson \ target/lemmy_server >$LOG_DIR/lemmy_delta.out 2>&1 & # plugin setup -pushd example_plugin -# TODO: not in ubuntu repos, better to use only `go` -# TODO: prevent it from creating useless `go` folder in home dir -tinygo build -o plugin.wasm -target wasi main.go +pushd plugins +# need to use tinygo because apparently go has only experimental support for wasm target +GOPATH=$HOME/.local/share/go tinygo build -o plugin.wasm -target wasi main.go popd echo "start epsilon" diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index 34a463d17..e9053e318 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -31,8 +31,6 @@ anyhow.workspace = true serde.workspace = true webmention = "0.5.0" accept-language = "3.1.0" -extism = "1.2.0" -extism-convert = "1.2.0" [package.metadata.cargo-machete] ignored = ["futures"] diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index 643baac06..fcd274c03 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -1,6 +1,5 @@ use activitypub_federation::config::Data; use actix_web::web::Json; -use extism::*; use lemmy_api_common::{ build_response::build_post_response, context::LemmyContext, @@ -47,20 +46,16 @@ use lemmy_utils::{ }, }, }; -use serde::Serialize; -use std::{ffi::OsStr, fs::read_dir}; use tracing::Instrument; use url::Url; use webmention::{Webmention, WebmentionError}; #[tracing::instrument(skip(context))] pub async fn create_post( - mut data: Json, + data: Json, context: Data, local_user_view: LocalUserView, ) -> LemmyResult> { - plugin_hook("api_before_create_post", &mut (*data))?; - let local_site = LocalSite::read(&mut context.pool()).await?; honeypot_check(&data.honeypot)?; @@ -205,45 +200,5 @@ pub async fn create_post( } }; - let mut res = build_post_response(&context, community_id, &local_user_view.person, post_id) - .await? - .0; - - plugin_hook("api_after_create_post", &mut res)?; - Ok(Json(res)) -} - -fn load_plugins() -> LemmyResult { - // TODO: make dir configurable via env var - // TODO: should only read fs once at startup for performance - let plugin_paths = read_dir("example_plugin")?; - - let mut wasm_files = vec![]; - for path in plugin_paths { - let path = path?.path(); - if path.extension() == Some(OsStr::new("wasm")) { - wasm_files.push(path); - } - } - let manifest = Manifest::new(wasm_files); - let plugin = Plugin::new(manifest, [], true)?; - Ok(plugin) -} - -fn plugin_hook serde::Deserialize<'de> + Clone>( - name: &'static str, - data: &mut T, -) -> LemmyResult<()> { - let mut plugin = load_plugins()?; - if plugin.function_exists(name) { - *data = plugin - .call::, extism_convert::Json>(name, (*data).clone().into()) - .map_err(|e| { - dbg!(&e); - LemmyErrorType::PluginError(e.to_string()) - })? - .0 - .into(); - } - Ok(()) + build_post_response(&context, community_id, &local_user_view.person, post_id).await } diff --git a/example_plugin/go.mod b/plugins/go.mod similarity index 100% rename from example_plugin/go.mod rename to plugins/go.mod diff --git a/example_plugin/go.sum b/plugins/go.sum similarity index 100% rename from example_plugin/go.sum rename to plugins/go.sum diff --git a/example_plugin/main.go b/plugins/main.go similarity index 93% rename from example_plugin/main.go rename to plugins/main.go index 11df20e9c..63223082c 100644 --- a/example_plugin/main.go +++ b/plugins/main.go @@ -16,8 +16,8 @@ type CreatePost struct { Custom_thumbnail *string `json:"custom_thumbnail,omitempty"` } -//export api_before_create_post -func api_before_create_post() int32 { +//export api_before_post_post +func api_before_post_post() int32 { params := CreatePost{} // use json input helper, which automatically unmarshals the plugin input into your struct err := pdk.InputJSON(¶ms) diff --git a/src/api_routes_http.rs b/src/api_routes_http.rs index 013e2e092..68aeecea0 100644 --- a/src/api_routes_http.rs +++ b/src/api_routes_http.rs @@ -1,3 +1,4 @@ +use crate::plugin_middleware::PluginMiddleware; use actix_web::{guard, web}; use lemmy_api::{ comment::{ @@ -139,6 +140,7 @@ use lemmy_utils::rate_limit::RateLimitCell; pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { cfg.service( web::scope("/api/v3") + .wrap(PluginMiddleware::new()) .route("/image_proxy", web::get().to(image_proxy)) // Site .service( diff --git a/src/lib.rs b/src/lib.rs index 633fd5313..837191e9c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ pub mod api_routes_http; pub mod code_migrations; +pub mod plugin_middleware; pub mod prometheus_metrics; pub mod root_span_builder; pub mod scheduled_tasks; diff --git a/src/plugin_middleware.rs b/src/plugin_middleware.rs new file mode 100644 index 000000000..e464f61b7 --- /dev/null +++ b/src/plugin_middleware.rs @@ -0,0 +1,120 @@ +use actix_http::h1::Payload; +use actix_web::{ + body::MessageBody, + dev::{forward_ready, Service, ServiceRequest, ServiceResponse, Transform}, + web::Bytes, + Error, +}; +use core::future::Ready; +use extism::{Manifest, Plugin}; +use futures_util::future::LocalBoxFuture; +use lemmy_utils::{error::LemmyResult, LemmyErrorType}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{ffi::OsStr, fs::read_dir, future::ready, rc::Rc}; +use tracing::info; + +#[derive(Clone)] +pub struct PluginMiddleware {} + +impl PluginMiddleware { + pub fn new() -> Self { + PluginMiddleware {} + } +} +impl Transform for PluginMiddleware +where + S: Service, Error = Error> + 'static, + S::Future: 'static, + B: MessageBody + 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type Transform = SessionService; + type InitError = (); + type Future = Ready>; + + fn new_transform(&self, service: S) -> Self::Future { + ready(Ok(SessionService { + service: Rc::new(service), + })) + } +} + +pub struct SessionService { + service: Rc, +} + +impl Service for SessionService +where + S: Service, Error = Error> + 'static, + S::Future: 'static, + B: 'static, +{ + type Response = ServiceResponse; + type Error = Error; + type Future = LocalBoxFuture<'static, Result>; + + forward_ready!(service); + + fn call(&self, mut service_req: ServiceRequest) -> Self::Future { + let svc = self.service.clone(); + + Box::pin(async move { + let method = service_req.method(); + let path = service_req.path().replace("/api/v3/", "").replace("/", "_"); + // TODO: naming can be a bit silly, `POST /api/v3/post` becomes `api_before_post_post` + let plugin_hook = format!("api_before_{method}_{path}").to_lowercase(); + info!("Calling plugin hook {}", &plugin_hook); + if let Some(mut plugins) = load_plugins()? { + if plugins.function_exists(&plugin_hook) { + let payload = service_req.extract::().await?; + + let mut json: Value = serde_json::from_slice(&payload.to_vec())?; + call_plugin(plugins, &plugin_hook, &mut json)?; + + let (_, mut new_payload) = Payload::create(true); + new_payload.unread_data(Bytes::from(serde_json::to_vec_pretty(&json)?)); + service_req.set_payload(new_payload.into()); + } + } + let res = svc.call(service_req).await?; + Ok(res) + }) + } +} + +fn load_plugins() -> LemmyResult> { + // TODO: make dir configurable via env var + // TODO: should only read fs once at startup for performance + let plugin_paths = read_dir("plugins")?; + + let mut wasm_files = vec![]; + for path in plugin_paths { + let path = path?.path(); + if path.extension() == Some(OsStr::new("wasm")) { + wasm_files.push(path); + } + } + if !wasm_files.is_empty() { + // TODO: what if theres more than one plugin for the same hook? + let manifest = Manifest::new(wasm_files); + let plugin = Plugin::new(manifest, [], true)?; + Ok(Some(plugin)) + } else { + Ok(None) + } +} + +pub fn call_plugin Deserialize<'de> + Clone>( + mut plugins: Plugin, + name: &str, + data: &mut T, +) -> LemmyResult<()> { + *data = plugins + .call::, extism_convert::Json>(name, (*data).clone().into()) + .map_err(|e| LemmyErrorType::PluginError(e.to_string()))? + .0 + .into(); + Ok(()) +}