mirror of
https://github.com/LemmyNet/lemmy.git
synced 2024-06-26 09:10:32 +00:00
Merge 723045a32a
into 7c146272c3
This commit is contained in:
commit
01bc3df3c8
3
.gitignore
vendored
3
.gitignore
vendored
|
@ -34,3 +34,6 @@ dev_pgdata/
|
||||||
|
|
||||||
# database dumps
|
# database dumps
|
||||||
*.sqldump
|
*.sqldump
|
||||||
|
|
||||||
|
# compiled example plugin
|
||||||
|
plugins/plugin.wasm
|
||||||
|
|
|
@ -229,7 +229,7 @@ steps:
|
||||||
DO_WRITE_HOSTS_FILE: "1"
|
DO_WRITE_HOSTS_FILE: "1"
|
||||||
commands:
|
commands:
|
||||||
- *install_pnpm
|
- *install_pnpm
|
||||||
- apt update && apt install -y bash curl postgresql-client
|
- apt update && apt install -y bash curl postgresql-client golang
|
||||||
- bash api_tests/prepare-drone-federation-test.sh
|
- bash api_tests/prepare-drone-federation-test.sh
|
||||||
- cd api_tests/
|
- cd api_tests/
|
||||||
- pnpm i
|
- pnpm i
|
||||||
|
|
1268
Cargo.lock
generated
1268
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
@ -204,7 +204,9 @@ chrono = { workspace = true }
|
||||||
prometheus = { version = "0.13.3", features = ["process"] }
|
prometheus = { version = "0.13.3", features = ["process"] }
|
||||||
serial_test = { workspace = true }
|
serial_test = { workspace = true }
|
||||||
clap = { workspace = true }
|
clap = { workspace = true }
|
||||||
|
serde.workspace = true
|
||||||
actix-web-prom = "0.7.0"
|
actix-web-prom = "0.7.0"
|
||||||
|
actix-http = "3.6.0"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
pretty_assertions = { workspace = true }
|
pretty_assertions = { workspace = true }
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -49,8 +49,6 @@ else
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "$PWD"
|
|
||||||
|
|
||||||
LOG_DIR=target/log
|
LOG_DIR=target/log
|
||||||
mkdir -p $LOG_DIR
|
mkdir -p $LOG_DIR
|
||||||
|
|
||||||
|
@ -75,6 +73,12 @@ LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_delta.hjson \
|
||||||
LEMMY_DATABASE_URL="${LEMMY_DATABASE_URL}/lemmy_delta" \
|
LEMMY_DATABASE_URL="${LEMMY_DATABASE_URL}/lemmy_delta" \
|
||||||
target/lemmy_server >$LOG_DIR/lemmy_delta.out 2>&1 &
|
target/lemmy_server >$LOG_DIR/lemmy_delta.out 2>&1 &
|
||||||
|
|
||||||
|
# plugin setup
|
||||||
|
pushd plugins
|
||||||
|
# need to use tinygo because apparently go has only experimental support for wasm target
|
||||||
|
GOPATH=$HOME/.local/share/go tinygo build -o plugin.wasm -target wasi main.go
|
||||||
|
popd
|
||||||
|
|
||||||
echo "start epsilon"
|
echo "start epsilon"
|
||||||
# An instance who has a blocklist, with lemmy-alpha blocked
|
# An instance who has a blocklist, with lemmy-alpha blocked
|
||||||
LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_epsilon.hjson \
|
LEMMY_CONFIG_LOCATION=./docker/federation/lemmy_epsilon.hjson \
|
||||||
|
|
|
@ -11,7 +11,7 @@ killall -s1 lemmy_server || true
|
||||||
popd
|
popd
|
||||||
|
|
||||||
pnpm i
|
pnpm i
|
||||||
pnpm api-test || true
|
pnpm api-test-post || true
|
||||||
|
|
||||||
killall -s1 lemmy_server || true
|
killall -s1 lemmy_server || true
|
||||||
killall -s1 pict-rs || true
|
killall -s1 pict-rs || true
|
||||||
|
|
|
@ -763,3 +763,15 @@ test("Fetch post with redirect", async () => {
|
||||||
let gammaPost2 = await gamma.resolveObject(form);
|
let gammaPost2 = await gamma.resolveObject(form);
|
||||||
expect(gammaPost2.post).toBeDefined();
|
expect(gammaPost2.post).toBeDefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test.only("Plugin test", async () => {
|
||||||
|
let community = await createCommunity(epsilon);
|
||||||
|
let postRes = createPost(
|
||||||
|
epsilon,
|
||||||
|
community.community_view.community.id,
|
||||||
|
"https://example.com/",
|
||||||
|
"body",
|
||||||
|
"foobar",
|
||||||
|
);
|
||||||
|
expect((await postRes).post_view.post.name).toBe("Hello plugin!");
|
||||||
|
});
|
||||||
|
|
|
@ -47,6 +47,10 @@ html2md = "0.2.14"
|
||||||
html2text = "0.6.0"
|
html2text = "0.6.0"
|
||||||
stringreader = "0.1.1"
|
stringreader = "0.1.1"
|
||||||
enum_delegate = "0.2.0"
|
enum_delegate = "0.2.0"
|
||||||
|
extism = { version = "1.2.0", features = [
|
||||||
|
"register-filesystem",
|
||||||
|
], default-features = false }
|
||||||
|
extism-convert = { version = "1.2.0", default-features = false }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
serial_test = { workspace = true }
|
serial_test = { workspace = true }
|
||||||
|
|
|
@ -27,6 +27,7 @@ pub mod fetcher;
|
||||||
pub mod http;
|
pub mod http;
|
||||||
pub(crate) mod mentions;
|
pub(crate) mod mentions;
|
||||||
pub mod objects;
|
pub mod objects;
|
||||||
|
pub mod plugins;
|
||||||
pub mod protocol;
|
pub mod protocol;
|
||||||
|
|
||||||
pub const FEDERATION_HTTP_FETCH_LIMIT: u32 = 50;
|
pub const FEDERATION_HTTP_FETCH_LIMIT: u32 = 50;
|
||||||
|
|
|
@ -3,6 +3,7 @@ use crate::{
|
||||||
check_apub_id_valid_with_strictness,
|
check_apub_id_valid_with_strictness,
|
||||||
local_site_data_cached,
|
local_site_data_cached,
|
||||||
objects::{read_from_string_or_source_opt, verify_is_remote_object},
|
objects::{read_from_string_or_source_opt, verify_is_remote_object},
|
||||||
|
plugins::{call_plugin, load_plugins},
|
||||||
protocol::{
|
protocol::{
|
||||||
objects::{
|
objects::{
|
||||||
page::{Attachment, AttributedTo, Hashtag, HashtagType, Page, PageType},
|
page::{Attachment, AttributedTo, Hashtag, HashtagType, Page, PageType},
|
||||||
|
@ -50,6 +51,7 @@ use lemmy_utils::{
|
||||||
};
|
};
|
||||||
use std::ops::Deref;
|
use std::ops::Deref;
|
||||||
use stringreader::StringReader;
|
use stringreader::StringReader;
|
||||||
|
use tracing::info;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
const MAX_TITLE_LENGTH: usize = 200;
|
const MAX_TITLE_LENGTH: usize = 200;
|
||||||
|
@ -224,7 +226,7 @@ impl Object for ApubPost {
|
||||||
let first_attachment = page.attachment.first();
|
let first_attachment = page.attachment.first();
|
||||||
let local_site = LocalSite::read(&mut context.pool()).await.ok();
|
let local_site = LocalSite::read(&mut context.pool()).await.ok();
|
||||||
|
|
||||||
let form = if !page.is_mod_action(context).await? {
|
let mut form = if !page.is_mod_action(context).await? {
|
||||||
let url = if let Some(attachment) = first_attachment.cloned() {
|
let url = if let Some(attachment) = first_attachment.cloned() {
|
||||||
Some(attachment.url())
|
Some(attachment.url())
|
||||||
} else if page.kind == PageType::Video {
|
} else if page.kind == PageType::Video {
|
||||||
|
@ -275,8 +277,25 @@ impl Object for ApubPost {
|
||||||
.build()
|
.build()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// TODO: move this all into helper function
|
||||||
|
let before_plugin_hook = "federation_before_receive_post";
|
||||||
|
info!("Calling plugin hook {}", &before_plugin_hook);
|
||||||
|
if let Some(mut plugins) = load_plugins()? {
|
||||||
|
if plugins.function_exists(&before_plugin_hook) {
|
||||||
|
call_plugin(plugins, &before_plugin_hook, &mut form)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let timestamp = page.updated.or(page.published).unwrap_or_else(naive_now);
|
let timestamp = page.updated.or(page.published).unwrap_or_else(naive_now);
|
||||||
let post = Post::insert_apub(&mut context.pool(), timestamp, &form).await?;
|
let mut post = Post::insert_apub(&mut context.pool(), timestamp, &form).await?;
|
||||||
|
|
||||||
|
let after_plugin_hook = "federation_after_receive_post";
|
||||||
|
info!("Calling plugin hook {}", &after_plugin_hook);
|
||||||
|
if let Some(mut plugins) = load_plugins()? {
|
||||||
|
if plugins.function_exists(&after_plugin_hook) {
|
||||||
|
call_plugin(plugins, &after_plugin_hook, &mut post)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
generate_post_link_metadata(
|
generate_post_link_metadata(
|
||||||
post.clone(),
|
post.clone(),
|
||||||
|
|
39
crates/apub/src/plugins.rs
Normal file
39
crates/apub/src/plugins.rs
Normal file
|
@ -0,0 +1,39 @@
|
||||||
|
use extism::{Manifest, Plugin};
|
||||||
|
use lemmy_utils::{error::LemmyResult, LemmyErrorType};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::{ffi::OsStr, fs::read_dir};
|
||||||
|
|
||||||
|
pub fn load_plugins() -> LemmyResult<Option<Plugin>> {
|
||||||
|
// TODO: make dir configurable via env var
|
||||||
|
// TODO: should only read fs once at startup for performance
|
||||||
|
let plugin_paths = read_dir("plugins")?;
|
||||||
|
|
||||||
|
let mut wasm_files = vec![];
|
||||||
|
for path in plugin_paths {
|
||||||
|
let path = path?.path();
|
||||||
|
if path.extension() == Some(OsStr::new("wasm")) {
|
||||||
|
wasm_files.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !wasm_files.is_empty() {
|
||||||
|
// TODO: what if theres more than one plugin for the same hook?
|
||||||
|
let manifest = Manifest::new(wasm_files);
|
||||||
|
let plugin = Plugin::new(manifest, [], true)?;
|
||||||
|
Ok(Some(plugin))
|
||||||
|
} else {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn call_plugin<T: Serialize + for<'de> Deserialize<'de> + Clone>(
|
||||||
|
mut plugins: Plugin,
|
||||||
|
name: &str,
|
||||||
|
data: &mut T,
|
||||||
|
) -> LemmyResult<()> {
|
||||||
|
*data = plugins
|
||||||
|
.call::<extism_convert::Json<T>, extism_convert::Json<T>>(name, (*data).clone().into())
|
||||||
|
.map_err(|e| LemmyErrorType::PluginError(e.to_string()))?
|
||||||
|
.0
|
||||||
|
.into();
|
||||||
|
Ok(())
|
||||||
|
}
|
|
@ -60,7 +60,7 @@ pub struct Post {
|
||||||
pub alt_text: Option<String>,
|
pub alt_text: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, TypedBuilder)]
|
#[derive(Debug, Clone, TypedBuilder, Serialize, Deserialize)]
|
||||||
#[builder(field_defaults(default))]
|
#[builder(field_defaults(default))]
|
||||||
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
|
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
|
||||||
#[cfg_attr(feature = "full", diesel(table_name = post))]
|
#[cfg_attr(feature = "full", diesel(table_name = post))]
|
||||||
|
|
|
@ -175,6 +175,7 @@ pub enum LemmyErrorType {
|
||||||
InvalidBotAction,
|
InvalidBotAction,
|
||||||
CantBlockLocalInstance,
|
CantBlockLocalInstance,
|
||||||
UrlWithoutDomain,
|
UrlWithoutDomain,
|
||||||
|
PluginError(String),
|
||||||
Unknown(String),
|
Unknown(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
5
plugins/go.mod
Normal file
5
plugins/go.mod
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
module example_plugin
|
||||||
|
|
||||||
|
go 1.22.0
|
||||||
|
|
||||||
|
require github.com/extism/go-pdk v1.0.2 // indirect
|
2
plugins/go.sum
Normal file
2
plugins/go.sum
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
github.com/extism/go-pdk v1.0.2 h1:UB7oTW3tw2zoMlsUdBEDAAbhQg9OudzgNeyCwQYZ730=
|
||||||
|
github.com/extism/go-pdk v1.0.2/go.mod h1:Gz+LIU/YCKnKXhgge8yo5Yu1F/lbv7KtKFkiCSzW/P4=
|
40
plugins/main.go
Normal file
40
plugins/main.go
Normal file
|
@ -0,0 +1,40 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/extism/go-pdk"
|
||||||
|
)
|
||||||
|
|
||||||
|
type CreatePost struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Body *string `json:"body,omitempty"`
|
||||||
|
Community_id int32 `json:"community_id"`
|
||||||
|
Url *string `json:"url,omitempty"`
|
||||||
|
Alt_text *string `json:"alt_text,omitempty"`
|
||||||
|
Honeypot *string `json:"honeypot,omitempty"`
|
||||||
|
Nsfw *bool `json:"nsfw,omitempty"`
|
||||||
|
Language_id *int32 `json:"language_id,omitempty"`
|
||||||
|
Custom_thumbnail *string `json:"custom_thumbnail,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
//export api_before_post_post
|
||||||
|
func api_before_post_post() int32 {
|
||||||
|
params := CreatePost{}
|
||||||
|
// use json input helper, which automatically unmarshals the plugin input into your struct
|
||||||
|
err := pdk.InputJSON(¶ms)
|
||||||
|
if err != nil {
|
||||||
|
pdk.SetError(err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if params.Name == "foobar" {
|
||||||
|
params.Name = "Hello plugin!"
|
||||||
|
}
|
||||||
|
// use json output helper, which automatically marshals your struct to the plugin output
|
||||||
|
err = pdk.OutputJSON(params)
|
||||||
|
if err != nil {
|
||||||
|
pdk.SetError(err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {}
|
|
@ -1,3 +1,4 @@
|
||||||
|
use crate::plugin_middleware::PluginMiddleware;
|
||||||
use actix_web::{guard, web};
|
use actix_web::{guard, web};
|
||||||
use lemmy_api::{
|
use lemmy_api::{
|
||||||
comment::{
|
comment::{
|
||||||
|
@ -139,6 +140,7 @@ use lemmy_utils::rate_limit::RateLimitCell;
|
||||||
pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) {
|
pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) {
|
||||||
cfg.service(
|
cfg.service(
|
||||||
web::scope("/api/v3")
|
web::scope("/api/v3")
|
||||||
|
.wrap(PluginMiddleware::new())
|
||||||
.route("/image_proxy", web::get().to(image_proxy))
|
.route("/image_proxy", web::get().to(image_proxy))
|
||||||
// Site
|
// Site
|
||||||
.service(
|
.service(
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
pub mod api_routes_http;
|
pub mod api_routes_http;
|
||||||
pub mod code_migrations;
|
pub mod code_migrations;
|
||||||
|
pub mod plugin_middleware;
|
||||||
pub mod prometheus_metrics;
|
pub mod prometheus_metrics;
|
||||||
pub mod root_span_builder;
|
pub mod root_span_builder;
|
||||||
pub mod scheduled_tasks;
|
pub mod scheduled_tasks;
|
||||||
|
|
97
src/plugin_middleware.rs
Normal file
97
src/plugin_middleware.rs
Normal file
|
@ -0,0 +1,97 @@
|
||||||
|
use actix_http::{body::BoxBody, h1::Payload};
|
||||||
|
use actix_web::{
|
||||||
|
body::MessageBody,
|
||||||
|
dev::{forward_ready, Service, ServiceRequest, ServiceResponse, Transform},
|
||||||
|
web::Bytes,
|
||||||
|
Error,
|
||||||
|
};
|
||||||
|
use core::future::Ready;
|
||||||
|
use futures_util::future::LocalBoxFuture;
|
||||||
|
use lemmy_apub::plugins::{call_plugin, load_plugins};
|
||||||
|
use serde_json::Value;
|
||||||
|
use std::{future::ready, rc::Rc};
|
||||||
|
use tracing::info;
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct PluginMiddleware {}
|
||||||
|
|
||||||
|
impl PluginMiddleware {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
PluginMiddleware {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl<S> Transform<S, ServiceRequest> for PluginMiddleware
|
||||||
|
where
|
||||||
|
S: Service<ServiceRequest, Response = ServiceResponse<BoxBody>, Error = Error> + 'static,
|
||||||
|
S::Future: 'static,
|
||||||
|
{
|
||||||
|
type Response = ServiceResponse<BoxBody>;
|
||||||
|
type Error = Error;
|
||||||
|
type Transform = SessionService<S>;
|
||||||
|
type InitError = ();
|
||||||
|
type Future = Ready<Result<Self::Transform, Self::InitError>>;
|
||||||
|
|
||||||
|
fn new_transform(&self, service: S) -> Self::Future {
|
||||||
|
ready(Ok(SessionService {
|
||||||
|
service: Rc::new(service),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct SessionService<S> {
|
||||||
|
service: Rc<S>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<S> Service<ServiceRequest> for SessionService<S>
|
||||||
|
where
|
||||||
|
S: Service<ServiceRequest, Response = ServiceResponse<BoxBody>, Error = Error> + 'static,
|
||||||
|
S::Future: 'static,
|
||||||
|
{
|
||||||
|
type Response = ServiceResponse<BoxBody>;
|
||||||
|
type Error = Error;
|
||||||
|
type Future = LocalBoxFuture<'static, Result<Self::Response, Self::Error>>;
|
||||||
|
|
||||||
|
forward_ready!(service);
|
||||||
|
|
||||||
|
fn call(&self, mut service_req: ServiceRequest) -> Self::Future {
|
||||||
|
let svc = self.service.clone();
|
||||||
|
|
||||||
|
Box::pin(async move {
|
||||||
|
let method = service_req.method().clone();
|
||||||
|
let path = service_req.path().replace("/api/v3/", "").replace("/", "_");
|
||||||
|
// TODO: naming can be a bit silly, `POST /api/v3/post` becomes `api_before_post_post`
|
||||||
|
let before_plugin_hook = format!("api_before_{method}_{path}").to_lowercase();
|
||||||
|
|
||||||
|
info!("Calling plugin hook {}", &before_plugin_hook);
|
||||||
|
if let Some(mut plugins) = load_plugins()? {
|
||||||
|
if plugins.function_exists(&before_plugin_hook) {
|
||||||
|
let payload = service_req.extract::<Bytes>().await?;
|
||||||
|
|
||||||
|
let mut json: Value = serde_json::from_slice(&payload.to_vec())?;
|
||||||
|
call_plugin(plugins, &before_plugin_hook, &mut json)?;
|
||||||
|
|
||||||
|
let (_, mut new_payload) = Payload::create(true);
|
||||||
|
new_payload.unread_data(Bytes::from(serde_json::to_vec(&json)?));
|
||||||
|
service_req.set_payload(new_payload.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut res = svc.call(service_req).await?;
|
||||||
|
|
||||||
|
// TODO: add after hook
|
||||||
|
let after_plugin_hook = format!("api_after_{method}_{path}").to_lowercase();
|
||||||
|
info!("Calling plugin hook {}", &after_plugin_hook);
|
||||||
|
if let Some(mut plugins) = load_plugins()? {
|
||||||
|
if plugins.function_exists(&before_plugin_hook) {
|
||||||
|
res = res.map_body(|_, body| {
|
||||||
|
let mut json: Value =
|
||||||
|
serde_json::from_slice(&body.try_into_bytes().unwrap().to_vec()).unwrap();
|
||||||
|
call_plugin(plugins, &after_plugin_hook, &mut json).unwrap();
|
||||||
|
BoxBody::new(Bytes::from(serde_json::to_vec(&json).unwrap()))
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(res)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue