From b1fbe8c5afe7002f25e4d64109cae1ddbd20ede6 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sun, 17 Jul 2022 09:10:32 +0300 Subject: [PATCH 01/25] create sync mod --- Cargo.toml | 2 +- src/lib.rs | 35 +++++++++++++++++++---------------- src/{ => sync}/error.rs | 0 src/{ => sync}/executor.rs | 0 src/sync/mod.rs | 13 +++++++++++++ src/{ => sync}/queue.rs | 0 src/{ => sync}/scheduler.rs | 0 src/{ => sync}/schema.rs | 0 src/{ => sync}/worker_pool.rs | 0 9 files changed, 33 insertions(+), 17 deletions(-) rename src/{ => sync}/error.rs (100%) rename src/{ => sync}/executor.rs (100%) create mode 100644 src/sync/mod.rs rename src/{ => sync}/queue.rs (100%) rename src/{ => sync}/scheduler.rs (100%) rename src/{ => sync}/schema.rs (100%) rename src/{ => sync}/worker_pool.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 0e57e15..4b5bf86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ version = "0.6.0" authors = ["Ayrat Badykov "] description = "Background job processing library for Rust" repository = "https://github.com/ayrat555/fang" -edition = "2018" +edition = "2021" license = "MIT" readme = "README.md" rust-version = "1.62" diff --git a/src/lib.rs b/src/lib.rs index 34daa00..24f1a1f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,24 +1,27 @@ -#![allow(clippy::nonstandard_macro_braces)] -#![allow(clippy::extra_unused_lifetimes)] +// #![allow(clippy::nonstandard_macro_braces)] +// #![allow(clippy::extra_unused_lifetimes)] + +// pub mod error; +// pub mod executor; +// pub mod queue; +// pub mod scheduler; +// pub mod schema; +// pub mod worker_pool; + +// pub use error::FangError; +// pub use executor::*; +// pub use queue::*; +// pub use scheduler::*; +// pub use schema::*; +// pub use worker_pool::*; #[macro_use] extern crate diesel; -pub mod error; -pub mod executor; -pub mod queue; -pub mod scheduler; -pub mod schema; -pub mod worker_pool; - -pub use error::FangError; -pub use executor::*; -pub use queue::*; -pub use scheduler::*; -pub use schema::*; -pub use worker_pool::*; - #[doc(hidden)] pub use diesel::pg::PgConnection; #[doc(hidden)] pub use typetag; + +pub mod sync; +pub use sync::*; diff --git a/src/error.rs b/src/sync/error.rs similarity index 100% rename from src/error.rs rename to src/sync/error.rs diff --git a/src/executor.rs b/src/sync/executor.rs similarity index 100% rename from src/executor.rs rename to src/sync/executor.rs diff --git a/src/sync/mod.rs b/src/sync/mod.rs new file mode 100644 index 0000000..f2aaf7c --- /dev/null +++ b/src/sync/mod.rs @@ -0,0 +1,13 @@ +pub mod error; +pub mod executor; +pub mod queue; +pub mod scheduler; +pub mod schema; +pub mod worker_pool; + +pub use error::FangError; +pub use executor::*; +pub use queue::*; +pub use scheduler::*; +pub use schema::*; +pub use worker_pool::*; diff --git a/src/queue.rs b/src/sync/queue.rs similarity index 100% rename from src/queue.rs rename to src/sync/queue.rs diff --git a/src/scheduler.rs b/src/sync/scheduler.rs similarity index 100% rename from src/scheduler.rs rename to src/sync/scheduler.rs diff --git a/src/schema.rs b/src/sync/schema.rs similarity index 100% rename from src/schema.rs rename to src/sync/schema.rs diff --git a/src/worker_pool.rs b/src/sync/worker_pool.rs similarity index 100% rename from src/worker_pool.rs rename to src/sync/worker_pool.rs From f999e55fa49a21211cd5e20e6d32d09360f67508 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sun, 17 Jul 2022 09:16:54 +0300 Subject: [PATCH 02/25] fix clippy --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 24f1a1f..45caeb7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ // #![allow(clippy::nonstandard_macro_braces)] -// #![allow(clippy::extra_unused_lifetimes)] +#![allow(clippy::extra_unused_lifetimes)] // pub mod error; // pub mod executor; From a707ce2f5cc4e8c785d8d8d145f1b128d9c7c378 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sun, 17 Jul 2022 10:13:45 +0300 Subject: [PATCH 03/25] start working on async queries --- Cargo.toml | 2 ++ src/asynk/async_queue.rs | 11 +++++++++++ src/asynk/mod.rs | 1 + src/asynk/queries/insert_task.sql | 1 + src/lib.rs | 2 ++ 5 files changed, 17 insertions(+) create mode 100644 src/asynk/async_queue.rs create mode 100644 src/asynk/mod.rs create mode 100644 src/asynk/queries/insert_task.sql diff --git a/Cargo.toml b/Cargo.toml index 4b5bf86..7e34edd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,3 +22,5 @@ typetag = "0.2" log = "0.4" serde = { version = "1", features = ["derive"] } thiserror = "1.0" +bb8-postgres = "0.8" +tokio = { version = "1.20", features = ["full"] } diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs new file mode 100644 index 0000000..8a9393a --- /dev/null +++ b/src/asynk/async_queue.rs @@ -0,0 +1,11 @@ +use bb8_postgres::tokio_postgres::Client; + +pub struct AsyncQueue { + pg_client: Client, +} + +impl AsyncQueue { + pub fn new(pg_client: Client) -> Self { + Self { pg_client } + } +} diff --git a/src/asynk/mod.rs b/src/asynk/mod.rs new file mode 100644 index 0000000..3b8222c --- /dev/null +++ b/src/asynk/mod.rs @@ -0,0 +1 @@ +pub mod async_queue; diff --git a/src/asynk/queries/insert_task.sql b/src/asynk/queries/insert_task.sql new file mode 100644 index 0000000..312a460 --- /dev/null +++ b/src/asynk/queries/insert_task.sql @@ -0,0 +1 @@ +INSERT INTO "fang_tasks" ("metadata", "error_message", "state", task_type") VALUES ($1, $2, $3, $4); diff --git a/src/lib.rs b/src/lib.rs index 45caeb7..a6d3bf1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,3 +25,5 @@ pub use typetag; pub mod sync; pub use sync::*; + +pub mod asynk; From 5b61458b5fa86a4f7f3ffbacaa62cdcc0a665652 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Sun, 17 Jul 2022 12:06:23 +0000 Subject: [PATCH 04/25] sql queries (#22) --- src/asynk/queries/fail_task.sql | 1 + src/asynk/queries/insert_job.sql | 1 + src/asynk/queries/insert_periodic_job.sql | 1 + src/asynk/queries/insert_periodic_task.sql | 1 + src/asynk/queries/insert_task.sql | 2 +- src/asynk/queries/remove_all_periodic_tasks.sql | 1 + src/asynk/queries/remove_all_tasks.sql | 1 + src/asynk/queries/remove_task.sql | 1 + src/asynk/queries/remove_tasks_type.sql | 1 + src/asynk/queries/schedule_next_task.sql | 1 + src/asynk/queries/update_task_state.sql | 1 + 11 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 src/asynk/queries/fail_task.sql create mode 100644 src/asynk/queries/insert_job.sql create mode 100644 src/asynk/queries/insert_periodic_job.sql create mode 100644 src/asynk/queries/insert_periodic_task.sql create mode 100644 src/asynk/queries/remove_all_periodic_tasks.sql create mode 100644 src/asynk/queries/remove_all_tasks.sql create mode 100644 src/asynk/queries/remove_task.sql create mode 100644 src/asynk/queries/remove_tasks_type.sql create mode 100644 src/asynk/queries/schedule_next_task.sql create mode 100644 src/asynk/queries/update_task_state.sql diff --git a/src/asynk/queries/fail_task.sql b/src/asynk/queries/fail_task.sql new file mode 100644 index 0000000..91f7f46 --- /dev/null +++ b/src/asynk/queries/fail_task.sql @@ -0,0 +1 @@ +UPDATE "fang_tasks" SET "state" = $1 , "error_message" = $2 , "updated_at" = $3 WHERE id = $4 diff --git a/src/asynk/queries/insert_job.sql b/src/asynk/queries/insert_job.sql new file mode 100644 index 0000000..33aea7d --- /dev/null +++ b/src/asynk/queries/insert_job.sql @@ -0,0 +1 @@ +INSERT INTO "fang_tasks" ("metadata", "created_at") VALUES ($1, $2) diff --git a/src/asynk/queries/insert_periodic_job.sql b/src/asynk/queries/insert_periodic_job.sql new file mode 100644 index 0000000..881eab1 --- /dev/null +++ b/src/asynk/queries/insert_periodic_job.sql @@ -0,0 +1 @@ +INSERT INTO "fang_periodic_tasks" ("metadata", "scheduled_at" , "period_in_seconds") VALUES ($1, $2 , $3) diff --git a/src/asynk/queries/insert_periodic_task.sql b/src/asynk/queries/insert_periodic_task.sql new file mode 100644 index 0000000..d9b52b3 --- /dev/null +++ b/src/asynk/queries/insert_periodic_task.sql @@ -0,0 +1 @@ +INSERT INTO "fang_periodic_tasks" ("metadata", "period_in_seconds") VALUES ($1, $2) diff --git a/src/asynk/queries/insert_task.sql b/src/asynk/queries/insert_task.sql index 312a460..8981fcc 100644 --- a/src/asynk/queries/insert_task.sql +++ b/src/asynk/queries/insert_task.sql @@ -1 +1 @@ -INSERT INTO "fang_tasks" ("metadata", "error_message", "state", task_type") VALUES ($1, $2, $3, $4); +INSERT INTO "fang_tasks" ("metadata", "error_message", "state", "task_type") VALUES ($1, $2, $3, $4) diff --git a/src/asynk/queries/remove_all_periodic_tasks.sql b/src/asynk/queries/remove_all_periodic_tasks.sql new file mode 100644 index 0000000..75b5afe --- /dev/null +++ b/src/asynk/queries/remove_all_periodic_tasks.sql @@ -0,0 +1 @@ +DELETE FROM "fang_periodic_tasks" diff --git a/src/asynk/queries/remove_all_tasks.sql b/src/asynk/queries/remove_all_tasks.sql new file mode 100644 index 0000000..eaecbba --- /dev/null +++ b/src/asynk/queries/remove_all_tasks.sql @@ -0,0 +1 @@ +DELETE FROM "fang_tasks" diff --git a/src/asynk/queries/remove_task.sql b/src/asynk/queries/remove_task.sql new file mode 100644 index 0000000..b6da69f --- /dev/null +++ b/src/asynk/queries/remove_task.sql @@ -0,0 +1 @@ +DELETE FROM "fang_tasks" WHERE id = $1 diff --git a/src/asynk/queries/remove_tasks_type.sql b/src/asynk/queries/remove_tasks_type.sql new file mode 100644 index 0000000..e4de9c0 --- /dev/null +++ b/src/asynk/queries/remove_tasks_type.sql @@ -0,0 +1 @@ +DELETE FROM "fang_tasks" WHERE task_type = $1 diff --git a/src/asynk/queries/schedule_next_task.sql b/src/asynk/queries/schedule_next_task.sql new file mode 100644 index 0000000..3e0495c --- /dev/null +++ b/src/asynk/queries/schedule_next_task.sql @@ -0,0 +1 @@ +UPDATE "fang_periodic_tasks" SET "scheduled_at" = $1 , "updated_at" = $2 diff --git a/src/asynk/queries/update_task_state.sql b/src/asynk/queries/update_task_state.sql new file mode 100644 index 0000000..21b9e82 --- /dev/null +++ b/src/asynk/queries/update_task_state.sql @@ -0,0 +1 @@ +UPDATE "fang_tasks" SET "state" = $1 , "updated_at" = $2 WHERE id = $3 From c2c83d1031f8ae473e7dfbd2465cf6478d013e44 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sun, 17 Jul 2022 19:22:00 +0300 Subject: [PATCH 05/25] async insert task query (#23) * async insert task query * return the number of rows --- Cargo.toml | 4 +- src/asynk/async_queue.rs | 132 ++++++++++++++++++++++++++++-- src/asynk/async_runnable.rs | 19 +++++ src/asynk/mod.rs | 4 + src/asynk/queries/insert_job.sql | 2 +- src/asynk/queries/insert_task.sql | 2 +- 6 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 src/asynk/async_runnable.rs diff --git a/Cargo.toml b/Cargo.toml index 7e34edd..d5427dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,5 +22,7 @@ typetag = "0.2" log = "0.4" serde = { version = "1", features = ["derive"] } thiserror = "1.0" -bb8-postgres = "0.8" +bb8-postgres = {version = "0.8", features = ["with-serde_json-1"]} tokio = { version = "1.20", features = ["full"] } +async-trait = "0.1" +typed-builder = "0.10" diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 8a9393a..4f3fcec 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -1,11 +1,131 @@ -use bb8_postgres::tokio_postgres::Client; +use crate::asynk::AsyncRunnable; +use bb8_postgres::bb8::Pool; +use bb8_postgres::bb8::RunError; +use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; +use bb8_postgres::tokio_postgres::tls::TlsConnect; +use bb8_postgres::tokio_postgres::types::ToSql; +use bb8_postgres::tokio_postgres::Socket; +use bb8_postgres::PostgresConnectionManager; +use thiserror::Error; +use typed_builder::TypedBuilder; -pub struct AsyncQueue { - pg_client: Client, +#[derive(Error, Debug)] +pub enum AsyncQueueError { + #[error(transparent)] + PoolError(#[from] RunError), + #[error(transparent)] + PgError(#[from] bb8_postgres::tokio_postgres::Error), + #[error("returned invalid result (expected {expected:?}, found {found:?})")] + ResultError { expected: u64, found: u64 }, } -impl AsyncQueue { - pub fn new(pg_client: Client) -> Self { - Self { pg_client } +#[derive(Debug, TypedBuilder)] +pub struct AsyncQueue +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + pool: Pool>, + #[builder(default = false)] + test: bool, +} + +const INSERT_TASK_QUERY: &str = include_str!("queries/insert_task.sql"); + +impl AsyncQueue +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + pub fn new(pool: Pool>) -> Self { + AsyncQueue::builder().pool(pool).build() + } + + pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { + let json_task = serde_json::to_value(task).unwrap(); + let task_type = task.task_type(); + + self.execute_one(INSERT_TASK_QUERY, &[&json_task, &task_type]) + .await + } + + async fn execute_one( + &mut self, + query: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result { + let mut connection = self.pool.get().await?; + + let result = if self.test { + let transaction = connection.transaction().await?; + + let result = transaction.execute(query, params).await?; + + transaction.rollback().await?; + + result + } else { + connection.execute(query, params).await? + }; + + if result != 1 { + return Err(AsyncQueueError::ResultError { + expected: 1, + found: result, + }); + } + + Ok(result) + } +} + +#[cfg(test)] +mod async_queue_tests { + use super::AsyncQueue; + use crate::asynk::AsyncRunnable; + use crate::asynk::Error; + use async_trait::async_trait; + use bb8_postgres::bb8::Pool; + use bb8_postgres::tokio_postgres::Client; + use bb8_postgres::tokio_postgres::NoTls; + use bb8_postgres::PostgresConnectionManager; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize)] + struct Job { + pub number: u16, + } + + #[typetag::serde] + #[async_trait] + impl AsyncRunnable for Job { + async fn run(&self, _connection: &Client) -> Result<(), Error> { + Ok(()) + } + } + + #[tokio::test] + async fn insert_task_creates_new_task() { + let mut queue = queue().await; + + let result = queue.insert_task(&Job { number: 1 }).await.unwrap(); + + assert_eq!(1, result); + } + + async fn queue() -> AsyncQueue { + let pg_mgr = PostgresConnectionManager::new_from_stringlike( + "postgres://postgres:postgres@localhost/fang", + NoTls, + ) + .unwrap(); + + let pool = Pool::builder().build(pg_mgr).await.unwrap(); + + AsyncQueue::builder().pool(pool).test(true).build() } } diff --git a/src/asynk/async_runnable.rs b/src/asynk/async_runnable.rs new file mode 100644 index 0000000..a933a8d --- /dev/null +++ b/src/asynk/async_runnable.rs @@ -0,0 +1,19 @@ +use async_trait::async_trait; +use bb8_postgres::tokio_postgres::Client; + +const COMMON_TYPE: &str = "common"; + +#[derive(Debug)] +pub struct Error { + pub description: String, +} + +#[typetag::serde(tag = "type")] +#[async_trait] +pub trait AsyncRunnable { + async fn run(&self, client: &Client) -> Result<(), Error>; + + fn task_type(&self) -> String { + COMMON_TYPE.to_string() + } +} diff --git a/src/asynk/mod.rs b/src/asynk/mod.rs index 3b8222c..171d637 100644 --- a/src/asynk/mod.rs +++ b/src/asynk/mod.rs @@ -1 +1,5 @@ pub mod async_queue; +pub mod async_runnable; + +pub use async_runnable::AsyncRunnable; +pub use async_runnable::Error; diff --git a/src/asynk/queries/insert_job.sql b/src/asynk/queries/insert_job.sql index 33aea7d..2bef71d 100644 --- a/src/asynk/queries/insert_job.sql +++ b/src/asynk/queries/insert_job.sql @@ -1 +1 @@ -INSERT INTO "fang_tasks" ("metadata", "created_at") VALUES ($1, $2) +INSERT INTO "fang_tasks" ("metadata", "task_type") VALUES ($1, $2) diff --git a/src/asynk/queries/insert_task.sql b/src/asynk/queries/insert_task.sql index 8981fcc..2bef71d 100644 --- a/src/asynk/queries/insert_task.sql +++ b/src/asynk/queries/insert_task.sql @@ -1 +1 @@ -INSERT INTO "fang_tasks" ("metadata", "error_message", "state", "task_type") VALUES ($1, $2, $3, $4) +INSERT INTO "fang_tasks" ("metadata", "task_type") VALUES ($1, $2) From d186a5434c4274b5ce14a9de4f60f029acb7a633 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sun, 17 Jul 2022 22:34:55 +0300 Subject: [PATCH 06/25] allow to use transaction to be able to rollback in tests --- src/asynk/async_queue.rs | 49 ++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 4f3fcec..89d01f0 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -5,11 +5,12 @@ use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; use bb8_postgres::tokio_postgres::tls::TlsConnect; use bb8_postgres::tokio_postgres::types::ToSql; use bb8_postgres::tokio_postgres::Socket; +use bb8_postgres::tokio_postgres::Transaction; use bb8_postgres::PostgresConnectionManager; use thiserror::Error; use typed_builder::TypedBuilder; -#[derive(Error, Debug)] +#[derive(Debug, Error)] pub enum AsyncQueueError { #[error(transparent)] PoolError(#[from] RunError), @@ -17,24 +18,27 @@ pub enum AsyncQueueError { PgError(#[from] bb8_postgres::tokio_postgres::Error), #[error("returned invalid result (expected {expected:?}, found {found:?})")] ResultError { expected: u64, found: u64 }, + #[error("Queue doesn't have a connection")] + PoolAndTransactionEmpty, } -#[derive(Debug, TypedBuilder)] -pub struct AsyncQueue +#[derive(TypedBuilder)] +pub struct AsyncQueue<'a, Tls> where Tls: MakeTlsConnect + Clone + Send + Sync + 'static, >::Stream: Send + Sync, >::TlsConnect: Send, <>::TlsConnect as TlsConnect>::Future: Send, { - pool: Pool>, - #[builder(default = false)] - test: bool, + #[builder(default, setter(into))] + pool: Option>>, + #[builder(default, setter(into))] + transaction: Option>, } const INSERT_TASK_QUERY: &str = include_str!("queries/insert_task.sql"); -impl AsyncQueue +impl<'a, Tls> AsyncQueue<'a, Tls> where Tls: MakeTlsConnect + Clone + Send + Sync + 'static, >::Stream: Send + Sync, @@ -45,6 +49,10 @@ where AsyncQueue::builder().pool(pool).build() } + pub fn new_with_transaction(transaction: Transaction<'a>) -> Self { + AsyncQueue::builder().transaction(transaction).build() + } + pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { let json_task = serde_json::to_value(task).unwrap(); let task_type = task.task_type(); @@ -58,18 +66,14 @@ where query: &str, params: &[&(dyn ToSql + Sync)], ) -> Result { - let mut connection = self.pool.get().await?; + let result = if let Some(pool) = &self.pool { + let connection = pool.get().await?; - let result = if self.test { - let transaction = connection.transaction().await?; - - let result = transaction.execute(query, params).await?; - - transaction.rollback().await?; - - result - } else { connection.execute(query, params).await? + } else if let Some(transaction) = &self.transaction { + transaction.execute(query, params).await? + } else { + return Err(AsyncQueueError::PoolAndTransactionEmpty); }; if result != 1 { @@ -110,22 +114,23 @@ mod async_queue_tests { #[tokio::test] async fn insert_task_creates_new_task() { - let mut queue = queue().await; + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); + let mut queue = AsyncQueue::::new_with_transaction(transaction); let result = queue.insert_task(&Job { number: 1 }).await.unwrap(); assert_eq!(1, result); } - async fn queue() -> AsyncQueue { + async fn pool() -> Pool> { let pg_mgr = PostgresConnectionManager::new_from_stringlike( "postgres://postgres:postgres@localhost/fang", NoTls, ) .unwrap(); - let pool = Pool::builder().build(pg_mgr).await.unwrap(); - - AsyncQueue::builder().pool(pool).test(true).build() + Pool::builder().build(pg_mgr).await.unwrap() } } From a60eb08fa63fbdaec05e2a458d595ac5098a4cd4 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Mon, 18 Jul 2022 10:44:05 +0000 Subject: [PATCH 07/25] fang_tasks queries done (#24) * fang_tasks queries done * queries with transaction rollback * deleting unused imports * some tests * deleting schema , deleting feature section , updated_at corrected * rollback and commit methods and execute modified * transaction empty message * fix cargo clippy --- Cargo.toml | 2 +- src/asynk/async_queue.rs | 117 +++++++++++++++++++++++++++++++++++---- src/lib.rs | 40 +++++++++++++ src/sync/executor.rs | 4 +- src/sync/queue.rs | 38 +------------ src/sync/scheduler.rs | 4 +- src/sync/worker_pool.rs | 2 +- 7 files changed, 153 insertions(+), 54 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d5427dc..919566c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ typetag = "0.2" log = "0.4" serde = { version = "1", features = ["derive"] } thiserror = "1.0" -bb8-postgres = {version = "0.8", features = ["with-serde_json-1"]} +bb8-postgres = {version = "0.8", features = ["with-serde_json-1" , "with-uuid-0_8" , "with-chrono-0_4"]} tokio = { version = "1.20", features = ["full"] } async-trait = "0.1" typed-builder = "0.10" diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 89d01f0..99e4fef 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -1,4 +1,5 @@ use crate::asynk::AsyncRunnable; +use crate::Task; use bb8_postgres::bb8::Pool; use bb8_postgres::bb8::RunError; use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; @@ -7,6 +8,7 @@ use bb8_postgres::tokio_postgres::types::ToSql; use bb8_postgres::tokio_postgres::Socket; use bb8_postgres::tokio_postgres::Transaction; use bb8_postgres::PostgresConnectionManager; +use chrono::Utc; use thiserror::Error; use typed_builder::TypedBuilder; @@ -20,6 +22,8 @@ pub enum AsyncQueueError { ResultError { expected: u64, found: u64 }, #[error("Queue doesn't have a connection")] PoolAndTransactionEmpty, + #[error("Need to create a transaction to perform this operation")] + TransactionEmpty, } #[derive(TypedBuilder)] @@ -37,6 +41,11 @@ where } const INSERT_TASK_QUERY: &str = include_str!("queries/insert_task.sql"); +const UPDATE_TASK_STATE_QUERY: &str = include_str!("queries/update_task_state.sql"); +const FAIL_TASK_QUERY: &str = include_str!("queries/fail_task.sql"); +const REMOVE_ALL_TASK_QUERY: &str = include_str!("queries/remove_all_tasks.sql"); +const REMOVE_TASK_QUERY: &str = include_str!("queries/remove_task.sql"); +const REMOVE_TASKS_TYPE_QUERY: &str = include_str!("queries/remove_tasks_type.sql"); impl<'a, Tls> AsyncQueue<'a, Tls> where @@ -52,19 +61,73 @@ where pub fn new_with_transaction(transaction: Transaction<'a>) -> Self { AsyncQueue::builder().transaction(transaction).build() } - + pub async fn rollback(mut self) -> Result, AsyncQueueError> { + let transaction = self.transaction; + self.transaction = None; + match transaction { + Some(tr) => { + tr.rollback().await?; + Ok(self) + } + None => Err(AsyncQueueError::TransactionEmpty), + } + } + pub async fn commit(mut self) -> Result, AsyncQueueError> { + let transaction = self.transaction; + self.transaction = None; + match transaction { + Some(tr) => { + tr.commit().await?; + Ok(self) + } + None => Err(AsyncQueueError::TransactionEmpty), + } + } pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { - let json_task = serde_json::to_value(task).unwrap(); + let metadata = serde_json::to_value(task).unwrap(); let task_type = task.task_type(); - self.execute_one(INSERT_TASK_QUERY, &[&json_task, &task_type]) + self.execute(INSERT_TASK_QUERY, &[&metadata, &task_type], Some(1)) .await } + pub async fn update_task_state( + &mut self, + task: &Task, + state: &str, + ) -> Result { + let updated_at = Utc::now(); + self.execute( + UPDATE_TASK_STATE_QUERY, + &[&state, &updated_at, &task.id], + Some(1), + ) + .await + } + pub async fn remove_all_tasks(&mut self) -> Result { + self.execute(REMOVE_ALL_TASK_QUERY, &[], None).await + } + pub async fn remove_task(&mut self, task: &Task) -> Result { + self.execute(REMOVE_TASK_QUERY, &[&task.id], Some(1)).await + } + pub async fn remove_tasks_type(&mut self, task_type: &str) -> Result { + self.execute(REMOVE_TASKS_TYPE_QUERY, &[&task_type], None) + .await + } + pub async fn fail_task(&mut self, task: &Task) -> Result { + let updated_at = Utc::now(); + self.execute( + FAIL_TASK_QUERY, + &[&"failed", &task.error_message, &updated_at, &task.id], + Some(1), + ) + .await + } - async fn execute_one( + async fn execute( &mut self, query: &str, params: &[&(dyn ToSql + Sync)], + expected_result_count: Option, ) -> Result { let result = if let Some(pool) = &self.pool { let connection = pool.get().await?; @@ -75,14 +138,14 @@ where } else { return Err(AsyncQueueError::PoolAndTransactionEmpty); }; - - if result != 1 { - return Err(AsyncQueueError::ResultError { - expected: 1, - found: result, - }); + if let Some(expected_result) = expected_result_count { + if result != expected_result { + return Err(AsyncQueueError::ResultError { + expected: expected_result, + found: result, + }); + } } - Ok(result) } } @@ -122,6 +185,38 @@ mod async_queue_tests { let result = queue.insert_task(&Job { number: 1 }).await.unwrap(); assert_eq!(1, result); + queue.rollback().await.unwrap(); + } + + #[tokio::test] + async fn remove_all_tasks_test() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); + let mut queue = AsyncQueue::::new_with_transaction(transaction); + + let result = queue.insert_task(&Job { number: 1 }).await.unwrap(); + assert_eq!(1, result); + let result = queue.insert_task(&Job { number: 2 }).await.unwrap(); + assert_eq!(1, result); + let result = queue.remove_all_tasks().await.unwrap(); + assert_eq!(2, result); + queue.rollback().await.unwrap(); + } + #[tokio::test] + async fn remove_tasks_type_test() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); + let mut queue = AsyncQueue::::new_with_transaction(transaction); + + let result = queue.insert_task(&Job { number: 1 }).await.unwrap(); + assert_eq!(1, result); + let result = queue.insert_task(&Job { number: 2 }).await.unwrap(); + assert_eq!(1, result); + let result = queue.remove_tasks_type("common").await.unwrap(); + assert_eq!(2, result); + queue.rollback().await.unwrap(); } async fn pool() -> Pool> { diff --git a/src/lib.rs b/src/lib.rs index a6d3bf1..14694f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,46 @@ // pub use scheduler::*; // pub use schema::*; // pub use worker_pool::*; +use chrono::DateTime; +use chrono::Utc; +use uuid::Uuid; + +#[derive(Queryable, Identifiable, Debug, Eq, PartialEq, Clone)] +#[table_name = "fang_tasks"] +pub struct Task { + pub id: Uuid, + pub metadata: serde_json::Value, + pub error_message: Option, + pub state: FangTaskState, + pub task_type: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Queryable, Identifiable, Debug, Eq, PartialEq, Clone)] +#[table_name = "fang_periodic_tasks"] +pub struct PeriodicTask { + pub id: Uuid, + pub metadata: serde_json::Value, + pub period_in_seconds: i32, + pub scheduled_at: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Insertable)] +#[table_name = "fang_tasks"] +pub struct NewTask { + pub metadata: serde_json::Value, + pub task_type: String, +} + +#[derive(Insertable)] +#[table_name = "fang_periodic_tasks"] +pub struct NewPeriodicTask { + pub metadata: serde_json::Value, + pub period_in_seconds: i32, +} #[macro_use] extern crate diesel; diff --git a/src/sync/executor.rs b/src/sync/executor.rs index 1255b7b..ba0d3d9 100644 --- a/src/sync/executor.rs +++ b/src/sync/executor.rs @@ -1,7 +1,7 @@ use crate::error::FangError; use crate::queue::Queue; -use crate::queue::Task; use crate::worker_pool::{SharedState, WorkerState}; +use crate::Task; use diesel::pg::PgConnection; use diesel::r2d2::{ConnectionManager, PooledConnection}; use log::error; @@ -185,10 +185,10 @@ mod executor_tests { use super::Executor; use super::RetentionMode; use super::Runnable; - use crate::queue::NewTask; use crate::queue::Queue; use crate::schema::FangTaskState; use crate::typetag; + use crate::NewTask; use diesel::connection::Connection; use diesel::pg::PgConnection; use diesel::r2d2::{ConnectionManager, PooledConnection}; diff --git a/src/sync/queue.rs b/src/sync/queue.rs index c5afd77..4232e64 100644 --- a/src/sync/queue.rs +++ b/src/sync/queue.rs @@ -2,6 +2,7 @@ use crate::executor::Runnable; use crate::schema::fang_periodic_tasks; use crate::schema::fang_tasks; use crate::schema::FangTaskState; +use crate::{NewPeriodicTask, NewTask, PeriodicTask, Task}; use chrono::DateTime; use chrono::Duration; use chrono::Utc; @@ -13,43 +14,6 @@ use dotenv::dotenv; use std::env; use uuid::Uuid; -#[derive(Queryable, Identifiable, Debug, Eq, PartialEq, Clone)] -#[table_name = "fang_tasks"] -pub struct Task { - pub id: Uuid, - pub metadata: serde_json::Value, - pub error_message: Option, - pub state: FangTaskState, - pub task_type: String, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Queryable, Identifiable, Debug, Eq, PartialEq, Clone)] -#[table_name = "fang_periodic_tasks"] -pub struct PeriodicTask { - pub id: Uuid, - pub metadata: serde_json::Value, - pub period_in_seconds: i32, - pub scheduled_at: Option>, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Insertable)] -#[table_name = "fang_tasks"] -pub struct NewTask { - pub metadata: serde_json::Value, - pub task_type: String, -} - -#[derive(Insertable)] -#[table_name = "fang_periodic_tasks"] -pub struct NewPeriodicTask { - pub metadata: serde_json::Value, - pub period_in_seconds: i32, -} - pub struct Queue { pub connection: PgConnection, } diff --git a/src/sync/scheduler.rs b/src/sync/scheduler.rs index 70224ae..25812d3 100644 --- a/src/sync/scheduler.rs +++ b/src/sync/scheduler.rs @@ -1,6 +1,6 @@ use crate::executor::Runnable; -use crate::queue::PeriodicTask; use crate::queue::Queue; +use crate::PeriodicTask; use std::thread; use std::time::Duration; @@ -82,9 +82,9 @@ mod job_scheduler_tests { use crate::executor::Error; use crate::executor::Runnable; use crate::queue::Queue; - use crate::queue::Task; use crate::schema::fang_tasks; use crate::typetag; + use crate::Task; use diesel::pg::PgConnection; use diesel::prelude::*; use serde::{Deserialize, Serialize}; diff --git a/src/sync/worker_pool.rs b/src/sync/worker_pool.rs index bb4d394..0380148 100644 --- a/src/sync/worker_pool.rs +++ b/src/sync/worker_pool.rs @@ -226,9 +226,9 @@ mod job_pool_tests { use crate::executor::RetentionMode; use crate::executor::Runnable; use crate::queue::Queue; - use crate::queue::Task; use crate::schema::{fang_tasks, FangTaskState}; use crate::typetag; + use crate::Task; use diesel::pg::PgConnection; use diesel::prelude::*; use serde::{Deserialize, Serialize}; From 9c478e66a4eed65df100759c901af2b95d519feb Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:47:55 +0000 Subject: [PATCH 08/25] Renaming task to job (#29) (#31) --- src/sync/executor.rs | 30 ++++++------- src/sync/queue.rs | 98 ++++++++++++++++++++--------------------- src/sync/scheduler.rs | 10 ++--- src/sync/worker_pool.rs | 52 +++++++++++----------- 4 files changed, 95 insertions(+), 95 deletions(-) diff --git a/src/sync/executor.rs b/src/sync/executor.rs index ba0d3d9..69f9bcf 100644 --- a/src/sync/executor.rs +++ b/src/sync/executor.rs @@ -195,12 +195,12 @@ mod executor_tests { use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] - struct ExecutorJobTest { + struct ExecutorTaskTest { pub number: u16, } #[typetag::serde] - impl Runnable for ExecutorJobTest { + impl Runnable for ExecutorTaskTest { fn run(&self, _connection: &PgConnection) -> Result<(), Error> { println!("the number is {}", self.number); @@ -209,12 +209,12 @@ mod executor_tests { } #[derive(Serialize, Deserialize)] - struct FailedJob { + struct FailedTask { pub number: u16, } #[typetag::serde] - impl Runnable for FailedJob { + impl Runnable for FailedTask { fn run(&self, _connection: &PgConnection) -> Result<(), Error> { let message = format!("the number is {}", self.number); @@ -225,10 +225,10 @@ mod executor_tests { } #[derive(Serialize, Deserialize)] - struct JobType1 {} + struct TaskType1 {} #[typetag::serde] - impl Runnable for JobType1 { + impl Runnable for TaskType1 { fn run(&self, _connection: &PgConnection) -> Result<(), Error> { Ok(()) } @@ -239,10 +239,10 @@ mod executor_tests { } #[derive(Serialize, Deserialize)] - struct JobType2 {} + struct TaskType2 {} #[typetag::serde] - impl Runnable for JobType2 { + impl Runnable for TaskType2 { fn run(&self, _connection: &PgConnection) -> Result<(), Error> { Ok(()) } @@ -258,7 +258,7 @@ mod executor_tests { #[test] fn executes_and_finishes_task() { - let job = ExecutorJobTest { number: 10 }; + let job = ExecutorTaskTest { number: 10 }; let new_task = NewTask { metadata: serialize(&job), @@ -289,16 +289,16 @@ mod executor_tests { #[test] #[ignore] fn executes_task_only_of_specific_type() { - let job1 = JobType1 {}; - let job2 = JobType2 {}; + let task1 = TaskType1 {}; + let task2 = TaskType2 {}; let new_task1 = NewTask { - metadata: serialize(&job1), + metadata: serialize(&task1), task_type: "type1".to_string(), }; let new_task2 = NewTask { - metadata: serialize(&job2), + metadata: serialize(&task2), task_type: "type2".to_string(), }; @@ -331,10 +331,10 @@ mod executor_tests { #[test] fn saves_error_for_failed_task() { - let job = FailedJob { number: 10 }; + let task = FailedTask { number: 10 }; let new_task = NewTask { - metadata: serialize(&job), + metadata: serialize(&task), task_type: "common".to_string(), }; diff --git a/src/sync/queue.rs b/src/sync/queue.rs index 4232e64..62a93d9 100644 --- a/src/sync/queue.rs +++ b/src/sync/queue.rs @@ -41,19 +41,19 @@ impl Queue { Self { connection } } - pub fn push_task(&self, job: &dyn Runnable) -> Result { - Self::push_task_query(&self.connection, job) + pub fn push_task(&self, task: &dyn Runnable) -> Result { + Self::push_task_query(&self.connection, task) } - pub fn push_task_query(connection: &PgConnection, job: &dyn Runnable) -> Result { - let json_job = serde_json::to_value(job).unwrap(); + pub fn push_task_query(connection: &PgConnection, task: &dyn Runnable) -> Result { + let json_task = serde_json::to_value(task).unwrap(); - match Self::find_task_by_metadata_query(connection, &json_job) { + match Self::find_task_by_metadata_query(connection, &json_task) { Some(task) => Ok(task), None => { let new_task = NewTask { - metadata: json_job.clone(), - task_type: job.task_type(), + metadata: json_task.clone(), + task_type: task.task_type(), }; Self::insert_query(connection, &new_task) } @@ -62,24 +62,24 @@ impl Queue { pub fn push_periodic_task( &self, - job: &dyn Runnable, + task: &dyn Runnable, period: i32, ) -> Result { - Self::push_periodic_task_query(&self.connection, job, period) + Self::push_periodic_task_query(&self.connection, task, period) } pub fn push_periodic_task_query( connection: &PgConnection, - job: &dyn Runnable, + task: &dyn Runnable, period: i32, ) -> Result { - let json_job = serde_json::to_value(job).unwrap(); + let json_task = serde_json::to_value(task).unwrap(); - match Self::find_periodic_task_by_metadata_query(connection, &json_job) { + match Self::find_periodic_task_by_metadata_query(connection, &json_task) { Some(task) => Ok(task), None => { let new_task = NewPeriodicTask { - metadata: json_job, + metadata: json_task, period_in_seconds: period, }; @@ -90,8 +90,8 @@ impl Queue { } } - pub fn enqueue_task(job: &dyn Runnable) -> Result { - Self::new().push_task(job) + pub fn enqueue_task(task: &dyn Runnable) -> Result { + Self::new().push_task(task) } pub fn insert(&self, params: &NewTask) -> Result { @@ -404,11 +404,11 @@ mod queue_tests { queue.connection.test_transaction::<(), Error, _>(|| { let timestamp1 = Utc::now() - Duration::hours(40); - let task1 = insert_job(serde_json::json!(true), timestamp1, &queue.connection); + let task1 = insert_task(serde_json::json!(true), timestamp1, &queue.connection); let timestamp2 = Utc::now() - Duration::hours(20); - insert_job(serde_json::json!(false), timestamp2, &queue.connection); + insert_task(serde_json::json!(false), timestamp2, &queue.connection); let found_task = queue.fetch_task(&None).unwrap(); @@ -423,7 +423,7 @@ mod queue_tests { let queue = Queue::new(); queue.connection.test_transaction::<(), Error, _>(|| { - let task = insert_new_job(&queue.connection); + let task = insert_new_task(&queue.connection); let updated_task = queue.finish_task(&task).unwrap(); @@ -438,7 +438,7 @@ mod queue_tests { let queue = Queue::new(); queue.connection.test_transaction::<(), Error, _>(|| { - let task = insert_new_job(&queue.connection); + let task = insert_new_task(&queue.connection); let error = "Failed".to_string(); let updated_task = queue.fail_task(&task, error.clone()).unwrap(); @@ -455,7 +455,7 @@ mod queue_tests { let queue = Queue::new(); queue.connection.test_transaction::<(), Error, _>(|| { - let _task = insert_new_job(&queue.connection); + let _task = insert_new_task(&queue.connection); let updated_task = queue.fetch_and_touch(&None).unwrap().unwrap(); @@ -483,8 +483,8 @@ mod queue_tests { let queue = Queue::new(); queue.connection.test_transaction::<(), Error, _>(|| { - let job = Job { number: 10 }; - let task = queue.push_task(&job).unwrap(); + let task = PepeTask { number: 10 }; + let task = queue.push_task(&task).unwrap(); let mut m = serde_json::value::Map::new(); m.insert( @@ -493,7 +493,7 @@ mod queue_tests { ); m.insert( "type".to_string(), - serde_json::value::Value::String("Job".to_string()), + serde_json::value::Value::String("PepeTask".to_string()), ); assert_eq!(task.metadata, serde_json::value::Value::Object(m)); @@ -507,10 +507,10 @@ mod queue_tests { let queue = Queue::new(); queue.connection.test_transaction::<(), Error, _>(|| { - let job = Job { number: 10 }; - let task2 = queue.push_task(&job).unwrap(); + let task = PepeTask { number: 10 }; + let task2 = queue.push_task(&task).unwrap(); - let task1 = queue.push_task(&job).unwrap(); + let task1 = queue.push_task(&task).unwrap(); assert_eq!(task1.id, task2.id); @@ -523,8 +523,8 @@ mod queue_tests { let queue = Queue::new(); queue.connection.test_transaction::<(), Error, _>(|| { - let job = Job { number: 10 }; - let task = queue.push_periodic_task(&job, 60).unwrap(); + let task = PepeTask { number: 10 }; + let task = queue.push_periodic_task(&task, 60).unwrap(); assert_eq!(task.period_in_seconds, 60); assert!(queue.find_periodic_task_by_id(task.id).is_some()); @@ -534,14 +534,14 @@ mod queue_tests { } #[test] - fn push_periodic_task_returns_existing_job() { + fn push_periodic_task_returns_existing_task() { let queue = Queue::new(); queue.connection.test_transaction::<(), Error, _>(|| { - let job = Job { number: 10 }; - let task1 = queue.push_periodic_task(&job, 60).unwrap(); + let task = PepeTask { number: 10 }; + let task1 = queue.push_periodic_task(&task, 60).unwrap(); - let task2 = queue.push_periodic_task(&job, 60).unwrap(); + let task2 = queue.push_periodic_task(&task, 60).unwrap(); assert_eq!(task1.id, task2.id); @@ -554,12 +554,12 @@ mod queue_tests { let queue = Queue::new(); queue.connection.test_transaction::<(), Error, _>(|| { - let job = Job { number: 10 }; - let task = queue.push_periodic_task(&job, 60).unwrap(); + let task = PepeTask { number: 10 }; + let task = queue.push_periodic_task(&task, 60).unwrap(); let schedule_in_future = Utc::now() + Duration::hours(100); - insert_periodic_job( + insert_periodic_task( serde_json::json!(true), schedule_in_future, 100, @@ -581,7 +581,7 @@ mod queue_tests { queue.connection.test_transaction::<(), Error, _>(|| { let task = - insert_periodic_job(serde_json::json!(true), Utc::now(), 100, &queue.connection); + insert_periodic_task(serde_json::json!(true), Utc::now(), 100, &queue.connection); let updated_task = queue.schedule_next_task_execution(&task).unwrap(); @@ -604,7 +604,7 @@ mod queue_tests { queue.connection.test_transaction::<(), Error, _>(|| { let task = - insert_periodic_job(serde_json::json!(true), Utc::now(), 100, &queue.connection); + insert_periodic_task(serde_json::json!(true), Utc::now(), 100, &queue.connection); let result = queue.remove_all_periodic_tasks().unwrap(); @@ -621,7 +621,7 @@ mod queue_tests { let queue = Queue::new(); queue.connection.test_transaction::<(), Error, _>(|| { - let task = insert_job(serde_json::json!(true), Utc::now(), &queue.connection); + let task = insert_task(serde_json::json!(true), Utc::now(), &queue.connection); let result = queue.remove_all_tasks().unwrap(); assert_eq!(1, result); @@ -639,7 +639,7 @@ mod queue_tests { queue.connection.test_transaction::<(), Error, _>(|| { let schedule_in_future = Utc::now() + Duration::hours(100); - insert_periodic_job( + insert_periodic_task( serde_json::json!(true), schedule_in_future, 100, @@ -647,7 +647,7 @@ mod queue_tests { ); let task = - insert_periodic_job(serde_json::json!(true), Utc::now(), 100, &queue.connection); + insert_periodic_task(serde_json::json!(true), Utc::now(), 100, &queue.connection); let tasks = queue.fetch_periodic_tasks(100).unwrap(); @@ -726,8 +726,8 @@ mod queue_tests { let queue = Queue::new(); let timestamp1 = Utc::now() - Duration::hours(40); - let task1 = insert_job( - serde_json::json!(Job { number: 12 }), + let task1 = insert_task( + serde_json::json!(PepeTask { number: 12 }), timestamp1, &queue.connection, ); @@ -736,8 +736,8 @@ mod queue_tests { let timestamp2 = Utc::now() - Duration::hours(20); - let task2 = insert_job( - serde_json::json!(Job { number: 11 }), + let task2 = insert_task( + serde_json::json!(PepeTask { number: 11 }), timestamp2, &queue.connection, ); @@ -772,12 +772,12 @@ mod queue_tests { } #[derive(Serialize, Deserialize)] - struct Job { + struct PepeTask { pub number: u16, } #[typetag::serde] - impl Runnable for Job { + impl Runnable for PepeTask { fn run(&self, _connection: &PgConnection) -> Result<(), ExecutorError> { println!("the number is {}", self.number); @@ -785,7 +785,7 @@ mod queue_tests { } } - fn insert_job( + fn insert_task( metadata: serde_json::Value, timestamp: DateTime, connection: &PgConnection, @@ -799,7 +799,7 @@ mod queue_tests { .unwrap() } - fn insert_periodic_job( + fn insert_periodic_task( metadata: serde_json::Value, timestamp: DateTime, period_in_seconds: i32, @@ -815,7 +815,7 @@ mod queue_tests { .unwrap() } - fn insert_new_job(connection: &PgConnection) -> Task { + fn insert_new_task(connection: &PgConnection) -> Task { diesel::insert_into(fang_tasks::table) .values(&vec![(fang_tasks::metadata.eq(serde_json::json!(true)),)]) .get_result::(connection) diff --git a/src/sync/scheduler.rs b/src/sync/scheduler.rs index 25812d3..ef52b20 100644 --- a/src/sync/scheduler.rs +++ b/src/sync/scheduler.rs @@ -77,7 +77,7 @@ impl Scheduler { } #[cfg(test)] -mod job_scheduler_tests { +mod task_scheduler_tests { use super::Scheduler; use crate::executor::Error; use crate::executor::Runnable; @@ -92,10 +92,10 @@ mod job_scheduler_tests { use std::time::Duration; #[derive(Serialize, Deserialize)] - struct ScheduledJob {} + struct ScheduledTask {} #[typetag::serde] - impl Runnable for ScheduledJob { + impl Runnable for ScheduledTask { fn run(&self, _connection: &PgConnection) -> Result<(), Error> { Ok(()) } @@ -107,10 +107,10 @@ mod job_scheduler_tests { #[test] #[ignore] - fn schedules_jobs() { + fn schedules_tasks() { let queue = Queue::new(); - queue.push_periodic_task(&ScheduledJob {}, 10).unwrap(); + queue.push_periodic_task(&ScheduledTask {}, 10).unwrap(); Scheduler::start(1, 2); let sleep_duration = Duration::from_secs(15); diff --git a/src/sync/worker_pool.rs b/src/sync/worker_pool.rs index 0380148..97f0ec5 100644 --- a/src/sync/worker_pool.rs +++ b/src/sync/worker_pool.rs @@ -219,7 +219,7 @@ impl Drop for WorkerThread { } #[cfg(test)] -mod job_pool_tests { +mod task_pool_tests { use super::WorkerParams; use super::WorkerPool; use crate::executor::Error; @@ -236,12 +236,12 @@ mod job_pool_tests { use std::time::Duration; #[derive(Serialize, Deserialize)] - struct MyJob { + struct MyTask { pub number: u16, pub current_thread_name: String, } - impl MyJob { + impl MyTask { pub fn new(number: u16) -> Self { let handle = thread::current(); let current_thread_name = handle.name().unwrap().to_string(); @@ -254,13 +254,13 @@ mod job_pool_tests { } #[typetag::serde] - impl Runnable for MyJob { + impl Runnable for MyTask { fn run(&self, connection: &PgConnection) -> Result<(), Error> { thread::sleep(Duration::from_secs(3)); - let new_job = MyJob::new(self.number + 1); + let new_task = MyTask::new(self.number + 1); - Queue::push_task_query(connection, &new_job).unwrap(); + Queue::push_task_query(connection, &new_task).unwrap(); Ok(()) } @@ -271,12 +271,12 @@ mod job_pool_tests { } #[derive(Serialize, Deserialize)] - struct ShutdownJob { + struct ShutdownTask { pub number: u16, pub current_thread_name: String, } - impl ShutdownJob { + impl ShutdownTask { pub fn new(number: u16) -> Self { let handle = thread::current(); let current_thread_name = handle.name().unwrap().to_string(); @@ -289,13 +289,13 @@ mod job_pool_tests { } #[typetag::serde] - impl Runnable for ShutdownJob { + impl Runnable for ShutdownTask { fn run(&self, connection: &PgConnection) -> Result<(), Error> { thread::sleep(Duration::from_secs(3)); - let new_job = MyJob::new(self.number + 1); + let new_task = MyTask::new(self.number + 1); - Queue::push_task_query(connection, &new_job).unwrap(); + Queue::push_task_query(connection, &new_task).unwrap(); Ok(()) } @@ -320,14 +320,14 @@ mod job_pool_tests { let mut worker_params = WorkerParams::new(); worker_params.set_retention_mode(RetentionMode::KeepAll); - let mut job_pool = WorkerPool::new_with_params(2, worker_params); + let mut task_pool = WorkerPool::new_with_params(2, worker_params); - queue.push_task(&ShutdownJob::new(100)).unwrap(); - queue.push_task(&ShutdownJob::new(200)).unwrap(); + queue.push_task(&ShutdownTask::new(100)).unwrap(); + queue.push_task(&ShutdownTask::new(200)).unwrap(); - job_pool.start().unwrap(); + task_pool.start().unwrap(); thread::sleep(Duration::from_secs(1)); - job_pool.shutdown().unwrap(); + task_pool.shutdown().unwrap(); thread::sleep(Duration::from_secs(5)); let tasks = get_all_tasks(&queue.connection, "shutdown_test"); @@ -351,12 +351,12 @@ mod job_pool_tests { let mut worker_params = WorkerParams::new(); worker_params.set_retention_mode(RetentionMode::KeepAll); - let mut job_pool = WorkerPool::new_with_params(2, worker_params); + let mut task_pool = WorkerPool::new_with_params(2, worker_params); - queue.push_task(&MyJob::new(100)).unwrap(); - queue.push_task(&MyJob::new(200)).unwrap(); + queue.push_task(&MyTask::new(100)).unwrap(); + queue.push_task(&MyTask::new(200)).unwrap(); - job_pool.start().unwrap(); + task_pool.start().unwrap(); thread::sleep(Duration::from_secs(100)); @@ -364,19 +364,19 @@ mod job_pool_tests { assert!(tasks.len() > 40); - let test_worker1_jobs = tasks.clone().into_iter().filter(|job| { - serde_json::to_string(&job.metadata) + let test_worker1_tasks = tasks.clone().into_iter().filter(|task| { + serde_json::to_string(&task.metadata) .unwrap() .contains("worker_1") }); - let test_worker2_jobs = tasks.into_iter().filter(|job| { - serde_json::to_string(&job.metadata) + let test_worker2_tasks = tasks.into_iter().filter(|task| { + serde_json::to_string(&task.metadata) .unwrap() .contains("worker_2") }); - assert!(test_worker1_jobs.count() > 20); - assert!(test_worker2_jobs.count() > 20); + assert!(test_worker1_tasks.count() > 20); + assert!(test_worker2_tasks.count() > 20); } } From abe324fc57886b5f077d5cf8b7b14bd240cce1ce Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Tue, 19 Jul 2022 13:49:43 +0000 Subject: [PATCH 09/25] Fetch task (#26) * fetch task async and renaming every Job/job to Task/task * fixing cargo clippy * Task instead Job in readme * change one task because if not test will fail, redefined all tasks structs * derive feature * deleting schema * changing query * task builder * fix bug enum fang task state * fetch_test based on metadata, good improve testing fetch_task * deleting toSql FromSql derives that are useless for Task * builders and fail_task change --- Cargo.toml | 3 +- README.md | 32 ++--- src/asynk/async_queue.rs | 174 ++++++++++++++++++++++++-- src/asynk/queries/fetch_task_type.sql | 1 + src/lib.rs | 40 ------ src/sync/queue.rs | 38 +++++- 6 files changed, 219 insertions(+), 69 deletions(-) create mode 100644 src/asynk/queries/fetch_task_type.sql diff --git a/Cargo.toml b/Cargo.toml index 919566c..6741d8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,8 @@ typetag = "0.2" log = "0.4" serde = { version = "1", features = ["derive"] } thiserror = "1.0" -bb8-postgres = {version = "0.8", features = ["with-serde_json-1" , "with-uuid-0_8" , "with-chrono-0_4"]} +bb8-postgres = {version = "0.8", features = ["with-serde_json-1" , "with-uuid-0_8" , "with-chrono-0_4" ]} +postgres-types = { version = "0.X.X", features = ["derive"] } tokio = { version = "1.20", features = ["full"] } async-trait = "0.1" typed-builder = "0.10" diff --git a/README.md b/README.md index e836446..f7c2d2d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # Fang -Background job processing library for Rust. It uses Postgres DB as a task queue. +Background task processing library for Rust. It uses Postgres DB as a task queue. ## Installation @@ -24,9 +24,9 @@ serde = { version = "1.0", features = ["derive"] } ## Usage -### Defining a job +### Defining a task -Every job should implement `fang::Runnable` trait which is used by `fang` to execute it. +Every task should implement `fang::Runnable` trait which is used by `fang` to execute it. ```rust use fang::Error; @@ -36,12 +36,12 @@ use fang::PgConnection; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] -struct Job { +struct MyTask { pub number: u16, } #[typetag::serde] -impl Runnable for Job { +impl Runnable for MyTask { fn run(&self, _connection: &PgConnection) -> Result<(), Error> { println!("the number is {}", self.number); @@ -50,13 +50,13 @@ impl Runnable for Job { } ``` -As you can see from the example above, the trait implementation has `#[typetag::serde]` attribute which is used to deserialize the job. +As you can see from the example above, the trait implementation has `#[typetag::serde]` attribute which is used to deserialize the task. -The second parameter of the `run` function is diesel's PgConnection, You can re-use it to manipulate the job queue, for example, to add a new job during the current job's execution. Or you can just re-use it in your own queries if you're using diesel. If you don't need it, just ignore it. +The second parameter of the `run` function is diesel's PgConnection, You can re-use it to manipulate the task queue, for example, to add a new job during the current job's execution. Or you can just re-use it in your own queries if you're using diesel. If you don't need it, just ignore it. -### Enqueuing a job +### Enqueuing a task -To enqueue a job use `Queue::enqueue_task` +To enqueue a task use `Queue::enqueue_task` ```rust @@ -64,17 +64,17 @@ use fang::Queue; ... -Queue::enqueue_task(&Job { number: 10 }).unwrap(); +Queue::enqueue_task(&MyTask { number: 10 }).unwrap(); ``` -The example above creates a new postgres connection on every call. If you want to reuse the same postgres connection to enqueue several jobs use Postgres struct instance: +The example above creates a new postgres connection on every call. If you want to reuse the same postgres connection to enqueue several tasks use Postgres struct instance: ```rust let queue = Queue::new(); for id in &unsynced_feed_ids { - queue.push_task(&SyncFeedJob { feed_id: *id }).unwrap(); + queue.push_task(&SyncFeedMyTask { feed_id: *id }).unwrap(); } ``` @@ -82,7 +82,7 @@ for id in &unsynced_feed_ids { Or you can use `PgConnection` struct: ```rust -Queue::push_task_query(pg_connection, &new_job).unwrap(); +Queue::push_task_query(pg_connection, &new_task).unwrap(); ``` ### Starting workers @@ -131,7 +131,7 @@ Add `task_type` method to the `Runnable` trait implementation: ... #[typetag::serde] -impl Runnable for Job { +impl Runnable for MyTask { fn run(&self) -> Result<(), Error> { println!("the number is {}", self.number); @@ -222,11 +222,11 @@ use fang::Queue; let queue = Queue::new(); queue - .push_periodic_task(&SyncJob::default(), 120) + .push_periodic_task(&SyncMyTask::default(), 120) .unwrap(); queue - .push_periodic_task(&DeliverJob::default(), 60) + .push_periodic_task(&DeliverMyTask::default(), 60) .unwrap(); Scheduler::start(10, 5); diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 99e4fef..1bb4386 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -1,17 +1,85 @@ use crate::asynk::AsyncRunnable; -use crate::Task; use bb8_postgres::bb8::Pool; use bb8_postgres::bb8::RunError; +use bb8_postgres::tokio_postgres::row::Row; use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; use bb8_postgres::tokio_postgres::tls::TlsConnect; -use bb8_postgres::tokio_postgres::types::ToSql; use bb8_postgres::tokio_postgres::Socket; use bb8_postgres::tokio_postgres::Transaction; use bb8_postgres::PostgresConnectionManager; +use chrono::DateTime; use chrono::Utc; +use postgres_types::{FromSql, ToSql}; use thiserror::Error; use typed_builder::TypedBuilder; +use uuid::Uuid; +#[derive(Debug, Eq, PartialEq, Clone, ToSql, FromSql)] +#[postgres(name = "fang_task_state")] +pub enum FangTaskState { + #[postgres(name = "new")] + New, + #[postgres(name = "in_progress")] + InProgress, + #[postgres(name = "failed")] + Failed, + #[postgres(name = "finished")] + Finished, +} +impl Default for FangTaskState { + fn default() -> Self { + FangTaskState::New + } +} +#[derive(TypedBuilder, Debug, Eq, PartialEq, Clone)] +pub struct Task { + #[builder(setter(into))] + pub id: Uuid, + #[builder(setter(into))] + pub metadata: serde_json::Value, + #[builder(setter(into))] + pub error_message: Option, + #[builder(default, setter(into))] + pub state: FangTaskState, + #[builder(setter(into))] + pub task_type: String, + #[builder(setter(into))] + pub created_at: DateTime, + #[builder(setter(into))] + pub updated_at: DateTime, +} + +#[derive(TypedBuilder, Debug, Eq, PartialEq, Clone)] +pub struct PeriodicTask { + #[builder(setter(into))] + pub id: Uuid, + #[builder(setter(into))] + pub metadata: serde_json::Value, + #[builder(setter(into))] + pub period_in_seconds: i32, + #[builder(setter(into))] + pub scheduled_at: Option>, + #[builder(setter(into))] + pub created_at: DateTime, + #[builder(setter(into))] + pub updated_at: DateTime, +} + +#[derive(TypedBuilder, Debug, Eq, PartialEq, Clone)] +pub struct NewTask { + #[builder(setter(into))] + pub metadata: serde_json::Value, + #[builder(setter(into))] + pub task_type: String, +} + +#[derive(TypedBuilder, Debug, Eq, PartialEq, Clone)] +pub struct NewPeriodicTask { + #[builder(setter(into))] + pub metadata: serde_json::Value, + #[builder(setter(into))] + pub period_in_seconds: i32, +} #[derive(Debug, Error)] pub enum AsyncQueueError { #[error(transparent)] @@ -46,6 +114,7 @@ const FAIL_TASK_QUERY: &str = include_str!("queries/fail_task.sql"); const REMOVE_ALL_TASK_QUERY: &str = include_str!("queries/remove_all_tasks.sql"); const REMOVE_TASK_QUERY: &str = include_str!("queries/remove_task.sql"); const REMOVE_TASKS_TYPE_QUERY: &str = include_str!("queries/remove_tasks_type.sql"); +const FETCH_TASK_TYPE_QUERY: &str = include_str!("queries/fetch_task_type.sql"); impl<'a, Tls> AsyncQueue<'a, Tls> where @@ -83,6 +152,58 @@ where None => Err(AsyncQueueError::TransactionEmpty), } } + pub async fn fetch_task( + &mut self, + task_type: &Option, + ) -> Result { + let mut task = match task_type { + None => self.get_task_type("common").await?, + Some(task_type_str) => self.get_task_type(task_type_str).await?, + }; + self.update_task_state(&task, FangTaskState::InProgress) + .await?; + task.state = FangTaskState::InProgress; + Ok(task) + } + pub async fn get_task_type(&mut self, task_type: &str) -> Result { + let row: Row = self.get_row(FETCH_TASK_TYPE_QUERY, &[&task_type]).await?; + let id: Uuid = row.get("id"); + let metadata: serde_json::Value = row.get("metadata"); + let error_message: Option = match row.try_get("error_message") { + Ok(error_message) => Some(error_message), + Err(_) => None, + }; + let state: FangTaskState = FangTaskState::New; + let task_type: String = row.get("task_type"); + let created_at: DateTime = row.get("created_at"); + let updated_at: DateTime = row.get("updated_at"); + let task = Task::builder() + .id(id) + .metadata(metadata) + .error_message(error_message) + .state(state) + .task_type(task_type) + .created_at(created_at) + .updated_at(updated_at) + .build(); + Ok(task) + } + pub async fn get_row( + &mut self, + query: &str, + params: &[&(dyn ToSql + Sync)], + ) -> Result { + let row: Row = if let Some(pool) = &self.pool { + let connection = pool.get().await?; + + connection.query_one(query, params).await? + } else if let Some(transaction) = &self.transaction { + transaction.query_one(query, params).await? + } else { + return Err(AsyncQueueError::PoolAndTransactionEmpty); + }; + Ok(row) + } pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { let metadata = serde_json::to_value(task).unwrap(); let task_type = task.task_type(); @@ -93,7 +214,7 @@ where pub async fn update_task_state( &mut self, task: &Task, - state: &str, + state: FangTaskState, ) -> Result { let updated_at = Utc::now(); self.execute( @@ -117,7 +238,12 @@ where let updated_at = Utc::now(); self.execute( FAIL_TASK_QUERY, - &[&"failed", &task.error_message, &updated_at, &task.id], + &[ + &FangTaskState::Failed, + &task.error_message, + &updated_at, + &task.id, + ], Some(1), ) .await @@ -163,13 +289,13 @@ mod async_queue_tests { use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] - struct Job { + struct AsyncTask { pub number: u16, } #[typetag::serde] #[async_trait] - impl AsyncRunnable for Job { + impl AsyncRunnable for AsyncTask { async fn run(&self, _connection: &Client) -> Result<(), Error> { Ok(()) } @@ -182,7 +308,7 @@ mod async_queue_tests { let transaction = connection.transaction().await.unwrap(); let mut queue = AsyncQueue::::new_with_transaction(transaction); - let result = queue.insert_task(&Job { number: 1 }).await.unwrap(); + let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); assert_eq!(1, result); queue.rollback().await.unwrap(); @@ -195,14 +321,40 @@ mod async_queue_tests { let transaction = connection.transaction().await.unwrap(); let mut queue = AsyncQueue::::new_with_transaction(transaction); - let result = queue.insert_task(&Job { number: 1 }).await.unwrap(); + let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); assert_eq!(1, result); - let result = queue.insert_task(&Job { number: 2 }).await.unwrap(); + let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); assert_eq!(1, result); let result = queue.remove_all_tasks().await.unwrap(); assert_eq!(2, result); queue.rollback().await.unwrap(); } + + #[tokio::test] + async fn fetch_test() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); + let mut queue = AsyncQueue::::new_with_transaction(transaction); + + let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); + assert_eq!(1, result); + let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); + assert_eq!(1, result); + let task = queue.fetch_task(&None).await.unwrap(); + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); + assert_eq!(Some(1), number); + assert_eq!(Some("AsyncTask"), type_task); + let task = queue.fetch_task(&None).await.unwrap(); + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); + assert_eq!(Some(2), number); + assert_eq!(Some("AsyncTask"), type_task); + queue.rollback().await.unwrap(); + } #[tokio::test] async fn remove_tasks_type_test() { let pool = pool().await; @@ -210,9 +362,9 @@ mod async_queue_tests { let transaction = connection.transaction().await.unwrap(); let mut queue = AsyncQueue::::new_with_transaction(transaction); - let result = queue.insert_task(&Job { number: 1 }).await.unwrap(); + let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); assert_eq!(1, result); - let result = queue.insert_task(&Job { number: 2 }).await.unwrap(); + let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); assert_eq!(1, result); let result = queue.remove_tasks_type("common").await.unwrap(); assert_eq!(2, result); diff --git a/src/asynk/queries/fetch_task_type.sql b/src/asynk/queries/fetch_task_type.sql new file mode 100644 index 0000000..360a5fa --- /dev/null +++ b/src/asynk/queries/fetch_task_type.sql @@ -0,0 +1 @@ +SELECT * FROM fang_tasks WHERE state = 'new' AND task_type = $1 ORDER BY created_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED diff --git a/src/lib.rs b/src/lib.rs index 14694f9..a6d3bf1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,46 +14,6 @@ // pub use scheduler::*; // pub use schema::*; // pub use worker_pool::*; -use chrono::DateTime; -use chrono::Utc; -use uuid::Uuid; - -#[derive(Queryable, Identifiable, Debug, Eq, PartialEq, Clone)] -#[table_name = "fang_tasks"] -pub struct Task { - pub id: Uuid, - pub metadata: serde_json::Value, - pub error_message: Option, - pub state: FangTaskState, - pub task_type: String, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Queryable, Identifiable, Debug, Eq, PartialEq, Clone)] -#[table_name = "fang_periodic_tasks"] -pub struct PeriodicTask { - pub id: Uuid, - pub metadata: serde_json::Value, - pub period_in_seconds: i32, - pub scheduled_at: Option>, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Insertable)] -#[table_name = "fang_tasks"] -pub struct NewTask { - pub metadata: serde_json::Value, - pub task_type: String, -} - -#[derive(Insertable)] -#[table_name = "fang_periodic_tasks"] -pub struct NewPeriodicTask { - pub metadata: serde_json::Value, - pub period_in_seconds: i32, -} #[macro_use] extern crate diesel; diff --git a/src/sync/queue.rs b/src/sync/queue.rs index 62a93d9..8302078 100644 --- a/src/sync/queue.rs +++ b/src/sync/queue.rs @@ -2,7 +2,6 @@ use crate::executor::Runnable; use crate::schema::fang_periodic_tasks; use crate::schema::fang_tasks; use crate::schema::FangTaskState; -use crate::{NewPeriodicTask, NewTask, PeriodicTask, Task}; use chrono::DateTime; use chrono::Duration; use chrono::Utc; @@ -14,6 +13,43 @@ use dotenv::dotenv; use std::env; use uuid::Uuid; +#[derive(Queryable, Identifiable, Debug, Eq, PartialEq, Clone)] +#[table_name = "fang_tasks"] +pub struct Task { + pub id: Uuid, + pub metadata: serde_json::Value, + pub error_message: Option, + pub state: FangTaskState, + pub task_type: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Queryable, Identifiable, Debug, Eq, PartialEq, Clone)] +#[table_name = "fang_periodic_tasks"] +pub struct PeriodicTask { + pub id: Uuid, + pub metadata: serde_json::Value, + pub period_in_seconds: i32, + pub scheduled_at: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Insertable)] +#[table_name = "fang_tasks"] +pub struct NewTask { + pub metadata: serde_json::Value, + pub task_type: String, +} + +#[derive(Insertable)] +#[table_name = "fang_periodic_tasks"] +pub struct NewPeriodicTask { + pub metadata: serde_json::Value, + pub period_in_seconds: i32, +} + pub struct Queue { pub connection: PgConnection, } From 3602097fb6b61fb208785e0176c87108593ebc38 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Tue, 19 Jul 2022 23:05:09 +0300 Subject: [PATCH 10/25] fetch_task static and AsyncQueue method (#32) * fetch and touch a task inside of db transaction * remove transaction from AsyncQueue * fix `fetch_and_touch` test * uncomment one more test --- src/asynk/async_queue.rs | 353 ++++++++++++++++++++++----------------- 1 file changed, 203 insertions(+), 150 deletions(-) diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 1bb4386..5543c67 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -14,6 +14,16 @@ use thiserror::Error; use typed_builder::TypedBuilder; use uuid::Uuid; +const INSERT_TASK_QUERY: &str = include_str!("queries/insert_task.sql"); +const UPDATE_TASK_STATE_QUERY: &str = include_str!("queries/update_task_state.sql"); +const FAIL_TASK_QUERY: &str = include_str!("queries/fail_task.sql"); +const REMOVE_ALL_TASK_QUERY: &str = include_str!("queries/remove_all_tasks.sql"); +const REMOVE_TASK_QUERY: &str = include_str!("queries/remove_task.sql"); +const REMOVE_TASKS_TYPE_QUERY: &str = include_str!("queries/remove_tasks_type.sql"); +const FETCH_TASK_TYPE_QUERY: &str = include_str!("queries/fetch_task_type.sql"); + +const DEFAULT_TASK_TYPE: &str = "common"; + #[derive(Debug, Eq, PartialEq, Clone, ToSql, FromSql)] #[postgres(name = "fang_task_state")] pub enum FangTaskState { @@ -88,35 +98,19 @@ pub enum AsyncQueueError { PgError(#[from] bb8_postgres::tokio_postgres::Error), #[error("returned invalid result (expected {expected:?}, found {found:?})")] ResultError { expected: u64, found: u64 }, - #[error("Queue doesn't have a connection")] - PoolAndTransactionEmpty, - #[error("Need to create a transaction to perform this operation")] - TransactionEmpty, } -#[derive(TypedBuilder)] -pub struct AsyncQueue<'a, Tls> +pub struct AsyncQueue where Tls: MakeTlsConnect + Clone + Send + Sync + 'static, >::Stream: Send + Sync, >::TlsConnect: Send, <>::TlsConnect as TlsConnect>::Future: Send, { - #[builder(default, setter(into))] - pool: Option>>, - #[builder(default, setter(into))] - transaction: Option>, + pool: Pool>, } -const INSERT_TASK_QUERY: &str = include_str!("queries/insert_task.sql"); -const UPDATE_TASK_STATE_QUERY: &str = include_str!("queries/update_task_state.sql"); -const FAIL_TASK_QUERY: &str = include_str!("queries/fail_task.sql"); -const REMOVE_ALL_TASK_QUERY: &str = include_str!("queries/remove_all_tasks.sql"); -const REMOVE_TASK_QUERY: &str = include_str!("queries/remove_task.sql"); -const REMOVE_TASKS_TYPE_QUERY: &str = include_str!("queries/remove_tasks_type.sql"); -const FETCH_TASK_TYPE_QUERY: &str = include_str!("queries/fetch_task_type.sql"); - -impl<'a, Tls> AsyncQueue<'a, Tls> +impl AsyncQueue where Tls: MakeTlsConnect + Clone + Send + Sync + 'static, >::Stream: Send + Sync, @@ -124,116 +118,55 @@ where <>::TlsConnect as TlsConnect>::Future: Send, { pub fn new(pool: Pool>) -> Self { - AsyncQueue::builder().pool(pool).build() + AsyncQueue { pool } } - pub fn new_with_transaction(transaction: Transaction<'a>) -> Self { - AsyncQueue::builder().transaction(transaction).build() - } - pub async fn rollback(mut self) -> Result, AsyncQueueError> { - let transaction = self.transaction; - self.transaction = None; - match transaction { - Some(tr) => { - tr.rollback().await?; - Ok(self) - } - None => Err(AsyncQueueError::TransactionEmpty), - } - } - pub async fn commit(mut self) -> Result, AsyncQueueError> { - let transaction = self.transaction; - self.transaction = None; - match transaction { - Some(tr) => { - tr.commit().await?; - Ok(self) - } - None => Err(AsyncQueueError::TransactionEmpty), - } - } - pub async fn fetch_task( + pub async fn fetch_and_touch_task( &mut self, task_type: &Option, ) -> Result { - let mut task = match task_type { - None => self.get_task_type("common").await?, - Some(task_type_str) => self.get_task_type(task_type_str).await?, - }; - self.update_task_state(&task, FangTaskState::InProgress) - .await?; - task.state = FangTaskState::InProgress; - Ok(task) - } - pub async fn get_task_type(&mut self, task_type: &str) -> Result { - let row: Row = self.get_row(FETCH_TASK_TYPE_QUERY, &[&task_type]).await?; - let id: Uuid = row.get("id"); - let metadata: serde_json::Value = row.get("metadata"); - let error_message: Option = match row.try_get("error_message") { - Ok(error_message) => Some(error_message), - Err(_) => None, - }; - let state: FangTaskState = FangTaskState::New; - let task_type: String = row.get("task_type"); - let created_at: DateTime = row.get("created_at"); - let updated_at: DateTime = row.get("updated_at"); - let task = Task::builder() - .id(id) - .metadata(metadata) - .error_message(error_message) - .state(state) - .task_type(task_type) - .created_at(created_at) - .updated_at(updated_at) - .build(); + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let task = Self::fetch_and_touch_task_query(&mut transaction, task_type).await?; + + transaction.commit().await?; + Ok(task) } + pub async fn get_row( &mut self, query: &str, params: &[&(dyn ToSql + Sync)], ) -> Result { - let row: Row = if let Some(pool) = &self.pool { - let connection = pool.get().await?; + let connection = self.pool.get().await?; + + let row = connection.query_one(query, params).await?; - connection.query_one(query, params).await? - } else if let Some(transaction) = &self.transaction { - transaction.query_one(query, params).await? - } else { - return Err(AsyncQueueError::PoolAndTransactionEmpty); - }; Ok(row) } - pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { - let metadata = serde_json::to_value(task).unwrap(); - let task_type = task.task_type(); - self.execute(INSERT_TASK_QUERY, &[&metadata, &task_type], Some(1)) - .await - } - pub async fn update_task_state( - &mut self, - task: &Task, - state: FangTaskState, - ) -> Result { - let updated_at = Utc::now(); - self.execute( - UPDATE_TASK_STATE_QUERY, - &[&state, &updated_at, &task.id], - Some(1), - ) - .await + pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + Self::insert_task_query(&mut transaction, task).await } + pub async fn remove_all_tasks(&mut self) -> Result { self.execute(REMOVE_ALL_TASK_QUERY, &[], None).await } + pub async fn remove_task(&mut self, task: &Task) -> Result { self.execute(REMOVE_TASK_QUERY, &[&task.id], Some(1)).await } + pub async fn remove_tasks_type(&mut self, task_type: &str) -> Result { self.execute(REMOVE_TASKS_TYPE_QUERY, &[&task_type], None) .await } + pub async fn fail_task(&mut self, task: &Task) -> Result { let updated_at = Utc::now(); self.execute( @@ -255,15 +188,9 @@ where params: &[&(dyn ToSql + Sync)], expected_result_count: Option, ) -> Result { - let result = if let Some(pool) = &self.pool { - let connection = pool.get().await?; + let connection = self.pool.get().await?; - connection.execute(query, params).await? - } else if let Some(transaction) = &self.transaction { - transaction.execute(query, params).await? - } else { - return Err(AsyncQueueError::PoolAndTransactionEmpty); - }; + let result = connection.execute(query, params).await?; if let Some(expected_result) = expected_result_count { if result != expected_result { return Err(AsyncQueueError::ResultError { @@ -274,6 +201,109 @@ where } Ok(result) } + + pub async fn fetch_and_touch_task_query( + transaction: &mut Transaction<'_>, + task_type: &Option, + ) -> Result { + let mut task = match task_type { + None => Self::get_task_type(transaction, DEFAULT_TASK_TYPE).await?, + Some(task_type_str) => Self::get_task_type(transaction, task_type_str).await?, + }; + + Self::update_task_state(transaction, &task, FangTaskState::InProgress).await?; + + task.state = FangTaskState::InProgress; + + Ok(task) + } + + pub async fn get_task_type( + transaction: &mut Transaction<'_>, + task_type: &str, + ) -> Result { + let row: Row = transaction + .query_one(FETCH_TASK_TYPE_QUERY, &[&task_type]) + .await?; + + let task = Self::row_to_task(row); + + Ok(task) + } + + pub async fn update_task_state( + transaction: &mut Transaction<'_>, + task: &Task, + state: FangTaskState, + ) -> Result { + let updated_at = Utc::now(); + + Self::execute_query( + transaction, + UPDATE_TASK_STATE_QUERY, + &[&state, &updated_at, &task.id], + Some(1), + ) + .await + } + + pub async fn insert_task_query( + transaction: &mut Transaction<'_>, + task: &dyn AsyncRunnable, + ) -> Result { + let metadata = serde_json::to_value(task).unwrap(); + let task_type = task.task_type(); + + Self::execute_query( + transaction, + INSERT_TASK_QUERY, + &[&metadata, &task_type], + Some(1), + ) + .await + } + + pub async fn execute_query( + transaction: &mut Transaction<'_>, + query: &str, + params: &[&(dyn ToSql + Sync)], + expected_result_count: Option, + ) -> Result { + let result = transaction.execute(query, params).await?; + + if let Some(expected_result) = expected_result_count { + if result != expected_result { + return Err(AsyncQueueError::ResultError { + expected: expected_result, + found: result, + }); + } + } + Ok(result) + } + + fn row_to_task(row: Row) -> Task { + let id: Uuid = row.get("id"); + let metadata: serde_json::Value = row.get("metadata"); + let error_message: Option = match row.try_get("error_message") { + Ok(error_message) => Some(error_message), + Err(_) => None, + }; + let state: FangTaskState = FangTaskState::New; + let task_type: String = row.get("task_type"); + let created_at: DateTime = row.get("created_at"); + let updated_at: DateTime = row.get("updated_at"); + + Task::builder() + .id(id) + .metadata(metadata) + .error_message(error_message) + .state(state) + .task_type(task_type) + .created_at(created_at) + .updated_at(updated_at) + .build() + } } #[cfg(test)] @@ -305,71 +335,94 @@ mod async_queue_tests { async fn insert_task_creates_new_task() { let pool = pool().await; let mut connection = pool.get().await.unwrap(); - let transaction = connection.transaction().await.unwrap(); - let mut queue = AsyncQueue::::new_with_transaction(transaction); + let mut transaction = connection.transaction().await.unwrap(); - let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); + let result = + AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) + .await + .unwrap(); assert_eq!(1, result); - queue.rollback().await.unwrap(); + transaction.rollback().await.unwrap(); } - #[tokio::test] - async fn remove_all_tasks_test() { - let pool = pool().await; - let mut connection = pool.get().await.unwrap(); - let transaction = connection.transaction().await.unwrap(); - let mut queue = AsyncQueue::::new_with_transaction(transaction); + // #[tokio::test] + // async fn remove_all_tasks_test() { + // let pool = pool().await; + // let mut connection = pool.get().await.unwrap(); + // let transaction = connection.transaction().await.unwrap(); + // let mut queue = AsyncQueue::::new_with_transaction(transaction); - let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); - assert_eq!(1, result); - let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); - assert_eq!(1, result); - let result = queue.remove_all_tasks().await.unwrap(); - assert_eq!(2, result); - queue.rollback().await.unwrap(); - } + // let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); + // assert_eq!(1, result); + + // let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); + // assert_eq!(1, result); + + // let result = queue.remove_all_tasks().await.unwrap(); + // assert_eq!(2, result); + + // queue.rollback().await.unwrap(); + // } #[tokio::test] - async fn fetch_test() { + async fn fetch_and_touch_test() { let pool = pool().await; let mut connection = pool.get().await.unwrap(); - let transaction = connection.transaction().await.unwrap(); - let mut queue = AsyncQueue::::new_with_transaction(transaction); + let mut transaction = connection.transaction().await.unwrap(); - let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); + let result = + AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) + .await + .unwrap(); assert_eq!(1, result); - let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); + + let result = + AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 2 }) + .await + .unwrap(); assert_eq!(1, result); - let task = queue.fetch_task(&None).await.unwrap(); + + let task = AsyncQueue::::fetch_and_touch_task_query(&mut transaction, &None) + .await + .unwrap(); let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); + assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let task = queue.fetch_task(&None).await.unwrap(); + + let task = AsyncQueue::::fetch_and_touch_task_query(&mut transaction, &None) + .await + .unwrap(); let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); + assert_eq!(Some(2), number); assert_eq!(Some("AsyncTask"), type_task); - queue.rollback().await.unwrap(); - } - #[tokio::test] - async fn remove_tasks_type_test() { - let pool = pool().await; - let mut connection = pool.get().await.unwrap(); - let transaction = connection.transaction().await.unwrap(); - let mut queue = AsyncQueue::::new_with_transaction(transaction); - let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); - assert_eq!(1, result); - let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); - assert_eq!(1, result); - let result = queue.remove_tasks_type("common").await.unwrap(); - assert_eq!(2, result); - queue.rollback().await.unwrap(); + transaction.rollback().await.unwrap(); } + // #[tokio::test] + // async fn remove_tasks_type_test() { + // let pool = pool().await; + // let mut connection = pool.get().await.unwrap(); + // let transaction = connection.transaction().await.unwrap(); + // let mut queue = AsyncQueue::::new_with_transaction(transaction); + + // let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); + // assert_eq!(1, result); + + // let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); + // assert_eq!(1, result); + + // let result = queue.remove_tasks_type("common").await.unwrap(); + // assert_eq!(2, result); + + // queue.rollback().await.unwrap(); + // } async fn pool() -> Pool> { let pg_mgr = PostgresConnectionManager::new_from_stringlike( From 8f6f905fe93c8e3c35b3ff0722c789568cf8cd9d Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 20 Jul 2022 13:01:33 +0300 Subject: [PATCH 11/25] Use transaction in all queries (#33) * Use transaction in all queries * fix clippy --- src/asynk/async_queue.rs | 202 +++++++++++++++++++++++++-------------- 1 file changed, 128 insertions(+), 74 deletions(-) diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 5543c67..e5f70f7 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -135,45 +135,97 @@ where Ok(task) } - pub async fn get_row( - &mut self, - query: &str, - params: &[&(dyn ToSql + Sync)], - ) -> Result { - let connection = self.pool.get().await?; - - let row = connection.query_one(query, params).await?; - - Ok(row) - } - pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { let mut connection = self.pool.get().await?; let mut transaction = connection.transaction().await?; - Self::insert_task_query(&mut transaction, task).await + let result = Self::insert_task_query(&mut transaction, task).await?; + + transaction.commit().await?; + + Ok(result) } pub async fn remove_all_tasks(&mut self) -> Result { - self.execute(REMOVE_ALL_TASK_QUERY, &[], None).await + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let result = Self::remove_all_tasks_query(&mut transaction).await?; + + transaction.commit().await?; + + Ok(result) } pub async fn remove_task(&mut self, task: &Task) -> Result { - self.execute(REMOVE_TASK_QUERY, &[&task.id], Some(1)).await + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let result = Self::remove_task_query(&mut transaction, task).await?; + + transaction.commit().await?; + + Ok(result) } pub async fn remove_tasks_type(&mut self, task_type: &str) -> Result { - self.execute(REMOVE_TASKS_TYPE_QUERY, &[&task_type], None) - .await + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let result = Self::remove_tasks_type_query(&mut transaction, task_type).await?; + + transaction.commit().await?; + + Ok(result) } - pub async fn fail_task(&mut self, task: &Task) -> Result { + pub async fn fail_task( + &mut self, + task: &Task, + error_message: &str, + ) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let result = Self::fail_task_query(&mut transaction, task, error_message).await?; + transaction.commit().await?; + + Ok(result) + } + + pub async fn remove_all_tasks_query( + transaction: &mut Transaction<'_>, + ) -> Result { + Self::execute_query(transaction, REMOVE_ALL_TASK_QUERY, &[], None).await + } + + pub async fn remove_task_query( + transaction: &mut Transaction<'_>, + task: &Task, + ) -> Result { + Self::execute_query(transaction, REMOVE_TASK_QUERY, &[&task.id], Some(1)).await + } + + pub async fn remove_tasks_type_query( + transaction: &mut Transaction<'_>, + task_type: &str, + ) -> Result { + Self::execute_query(transaction, REMOVE_TASKS_TYPE_QUERY, &[&task_type], None).await + } + + pub async fn fail_task_query( + transaction: &mut Transaction<'_>, + task: &Task, + error_message: &str, + ) -> Result { let updated_at = Utc::now(); - self.execute( + + Self::execute_query( + transaction, FAIL_TASK_QUERY, &[ &FangTaskState::Failed, - &task.error_message, + &error_message, &updated_at, &task.id, ], @@ -182,43 +234,25 @@ where .await } - async fn execute( - &mut self, - query: &str, - params: &[&(dyn ToSql + Sync)], - expected_result_count: Option, - ) -> Result { - let connection = self.pool.get().await?; - - let result = connection.execute(query, params).await?; - if let Some(expected_result) = expected_result_count { - if result != expected_result { - return Err(AsyncQueueError::ResultError { - expected: expected_result, - found: result, - }); - } - } - Ok(result) - } - pub async fn fetch_and_touch_task_query( transaction: &mut Transaction<'_>, task_type: &Option, ) -> Result { - let mut task = match task_type { - None => Self::get_task_type(transaction, DEFAULT_TASK_TYPE).await?, - Some(task_type_str) => Self::get_task_type(transaction, task_type_str).await?, + let task_type = match task_type { + Some(passed_task_type) => passed_task_type, + None => DEFAULT_TASK_TYPE, }; - Self::update_task_state(transaction, &task, FangTaskState::InProgress).await?; + let mut task = Self::get_task_type_query(transaction, task_type).await?; + + Self::update_task_state_query(transaction, &task, FangTaskState::InProgress).await?; task.state = FangTaskState::InProgress; Ok(task) } - pub async fn get_task_type( + pub async fn get_task_type_query( transaction: &mut Transaction<'_>, task_type: &str, ) -> Result { @@ -231,7 +265,7 @@ where Ok(task) } - pub async fn update_task_state( + pub async fn update_task_state_query( transaction: &mut Transaction<'_>, task: &Task, state: FangTaskState, @@ -346,24 +380,31 @@ mod async_queue_tests { transaction.rollback().await.unwrap(); } - // #[tokio::test] - // async fn remove_all_tasks_test() { - // let pool = pool().await; - // let mut connection = pool.get().await.unwrap(); - // let transaction = connection.transaction().await.unwrap(); - // let mut queue = AsyncQueue::::new_with_transaction(transaction); + #[tokio::test] + async fn remove_all_tasks_test() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let mut transaction = connection.transaction().await.unwrap(); - // let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); - // assert_eq!(1, result); + let result = + AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) + .await + .unwrap(); + assert_eq!(1, result); - // let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); - // assert_eq!(1, result); + let result = + AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 2 }) + .await + .unwrap(); + assert_eq!(1, result); - // let result = queue.remove_all_tasks().await.unwrap(); - // assert_eq!(2, result); + let result = AsyncQueue::::remove_all_tasks_query(&mut transaction) + .await + .unwrap(); + assert_eq!(2, result); - // queue.rollback().await.unwrap(); - // } + transaction.rollback().await.unwrap(); + } #[tokio::test] async fn fetch_and_touch_test() { @@ -405,24 +446,37 @@ mod async_queue_tests { transaction.rollback().await.unwrap(); } - // #[tokio::test] - // async fn remove_tasks_type_test() { - // let pool = pool().await; - // let mut connection = pool.get().await.unwrap(); - // let transaction = connection.transaction().await.unwrap(); - // let mut queue = AsyncQueue::::new_with_transaction(transaction); - // let result = queue.insert_task(&AsyncTask { number: 1 }).await.unwrap(); - // assert_eq!(1, result); + #[tokio::test] + async fn remove_tasks_type_test() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let mut transaction = connection.transaction().await.unwrap(); - // let result = queue.insert_task(&AsyncTask { number: 2 }).await.unwrap(); - // assert_eq!(1, result); + let result = + AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) + .await + .unwrap(); + assert_eq!(1, result); - // let result = queue.remove_tasks_type("common").await.unwrap(); - // assert_eq!(2, result); + let result = + AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 2 }) + .await + .unwrap(); + assert_eq!(1, result); - // queue.rollback().await.unwrap(); - // } + let result = AsyncQueue::::remove_tasks_type_query(&mut transaction, "mytype") + .await + .unwrap(); + assert_eq!(0, result); + + let result = AsyncQueue::::remove_tasks_type_query(&mut transaction, "common") + .await + .unwrap(); + assert_eq!(2, result); + + transaction.rollback().await.unwrap(); + } async fn pool() -> Pool> { let pg_mgr = PostgresConnectionManager::new_from_stringlike( From f56792a5580aa58216348ac8afbe72fd7fbea8e8 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Wed, 20 Jul 2022 14:27:01 +0000 Subject: [PATCH 12/25] Returning Insert query (#34) * insert query return task * deleting schema --- src/asynk/async_queue.rs | 83 ++++++++++++++++------- src/asynk/queries/insert_job.sql | 1 - src/asynk/queries/insert_periodic_job.sql | 1 - src/asynk/queries/insert_task.sql | 2 +- 4 files changed, 59 insertions(+), 28 deletions(-) delete mode 100644 src/asynk/queries/insert_job.sql delete mode 100644 src/asynk/queries/insert_periodic_job.sql diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index e5f70f7..f6d6542 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -135,15 +135,15 @@ where Ok(task) } - pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { + pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { let mut connection = self.pool.get().await?; let mut transaction = connection.transaction().await?; - let result = Self::insert_task_query(&mut transaction, task).await?; + let task = Self::insert_task_query(&mut transaction, task).await?; transaction.commit().await?; - Ok(result) + Ok(task) } pub async fn remove_all_tasks(&mut self) -> Result { @@ -284,17 +284,15 @@ where pub async fn insert_task_query( transaction: &mut Transaction<'_>, task: &dyn AsyncRunnable, - ) -> Result { + ) -> Result { let metadata = serde_json::to_value(task).unwrap(); let task_type = task.task_type(); - Self::execute_query( - transaction, - INSERT_TASK_QUERY, - &[&metadata, &task_type], - Some(1), - ) - .await + let row: Row = transaction + .query_one(INSERT_TASK_QUERY, &[&metadata, &task_type]) + .await?; + let task = Self::row_to_task(row); + Ok(task) } pub async fn execute_query( @@ -371,12 +369,16 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let mut transaction = connection.transaction().await.unwrap(); - let result = + let task = AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) .await .unwrap(); + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); - assert_eq!(1, result); + assert_eq!(Some(1), number); + assert_eq!(Some("AsyncTask"), type_task); transaction.rollback().await.unwrap(); } @@ -386,17 +388,28 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let mut transaction = connection.transaction().await.unwrap(); - let result = + let task = AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) .await .unwrap(); - assert_eq!(1, result); - let result = + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); + + assert_eq!(Some(1), number); + assert_eq!(Some("AsyncTask"), type_task); + + let task = AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 2 }) .await .unwrap(); - assert_eq!(1, result); + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); + + assert_eq!(Some(2), number); + assert_eq!(Some("AsyncTask"), type_task); let result = AsyncQueue::::remove_all_tasks_query(&mut transaction) .await @@ -412,17 +425,27 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let mut transaction = connection.transaction().await.unwrap(); - let result = + let task = AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) .await .unwrap(); - assert_eq!(1, result); + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); - let result = + assert_eq!(Some(1), number); + assert_eq!(Some("AsyncTask"), type_task); + + let task = AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 2 }) .await .unwrap(); - assert_eq!(1, result); + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); + + assert_eq!(Some(2), number); + assert_eq!(Some("AsyncTask"), type_task); let task = AsyncQueue::::fetch_and_touch_task_query(&mut transaction, &None) .await @@ -453,17 +476,27 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let mut transaction = connection.transaction().await.unwrap(); - let result = + let task = AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) .await .unwrap(); - assert_eq!(1, result); + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); - let result = + assert_eq!(Some(1), number); + assert_eq!(Some("AsyncTask"), type_task); + + let task = AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 2 }) .await .unwrap(); - assert_eq!(1, result); + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); + + assert_eq!(Some(2), number); + assert_eq!(Some("AsyncTask"), type_task); let result = AsyncQueue::::remove_tasks_type_query(&mut transaction, "mytype") .await diff --git a/src/asynk/queries/insert_job.sql b/src/asynk/queries/insert_job.sql deleted file mode 100644 index 2bef71d..0000000 --- a/src/asynk/queries/insert_job.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO "fang_tasks" ("metadata", "task_type") VALUES ($1, $2) diff --git a/src/asynk/queries/insert_periodic_job.sql b/src/asynk/queries/insert_periodic_job.sql deleted file mode 100644 index 881eab1..0000000 --- a/src/asynk/queries/insert_periodic_job.sql +++ /dev/null @@ -1 +0,0 @@ -INSERT INTO "fang_periodic_tasks" ("metadata", "scheduled_at" , "period_in_seconds") VALUES ($1, $2 , $3) diff --git a/src/asynk/queries/insert_task.sql b/src/asynk/queries/insert_task.sql index 2bef71d..c76a4b8 100644 --- a/src/asynk/queries/insert_task.sql +++ b/src/asynk/queries/insert_task.sql @@ -1 +1 @@ -INSERT INTO "fang_tasks" ("metadata", "task_type") VALUES ($1, $2) +INSERT INTO "fang_tasks" ("metadata", "task_type") VALUES ($1, $2) RETURNING id , metadata , error_message , task_type , created_at , updated_at From d3805cd562b6bc0d9435a21acbd058d88a1a28f3 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:26:11 +0000 Subject: [PATCH 13/25] Returning fail task (#35) * fail task returning * fix clippy * state check --- src/asynk/async_queue.rs | 65 ++++++++++++++++++++++--------- src/asynk/queries/fail_task.sql | 2 +- src/asynk/queries/insert_task.sql | 2 +- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index f6d6542..e404272 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -181,16 +181,16 @@ where pub async fn fail_task( &mut self, - task: &Task, + task: Task, error_message: &str, - ) -> Result { + ) -> Result { let mut connection = self.pool.get().await?; let mut transaction = connection.transaction().await?; - let result = Self::fail_task_query(&mut transaction, task, error_message).await?; + let task = Self::fail_task_query(&mut transaction, task, error_message).await?; transaction.commit().await?; - Ok(result) + Ok(task) } pub async fn remove_all_tasks_query( @@ -215,23 +215,24 @@ where pub async fn fail_task_query( transaction: &mut Transaction<'_>, - task: &Task, + task: Task, error_message: &str, - ) -> Result { + ) -> Result { let updated_at = Utc::now(); - Self::execute_query( - transaction, - FAIL_TASK_QUERY, - &[ - &FangTaskState::Failed, - &error_message, - &updated_at, - &task.id, - ], - Some(1), - ) - .await + let row: Row = transaction + .query_one( + FAIL_TASK_QUERY, + &[ + &FangTaskState::Failed, + &error_message, + &updated_at, + &task.id, + ], + ) + .await?; + let failed_task = Self::row_to_task(row); + Ok(failed_task) } pub async fn fetch_and_touch_task_query( @@ -321,7 +322,7 @@ where Ok(error_message) => Some(error_message), Err(_) => None, }; - let state: FangTaskState = FangTaskState::New; + let state: FangTaskState = row.get("state"); let task_type: String = row.get("task_type"); let created_at: DateTime = row.get("created_at"); let updated_at: DateTime = row.get("updated_at"); @@ -341,6 +342,7 @@ where #[cfg(test)] mod async_queue_tests { use super::AsyncQueue; + use super::FangTaskState; use crate::asynk::AsyncRunnable; use crate::asynk::Error; use async_trait::async_trait; @@ -381,6 +383,31 @@ mod async_queue_tests { assert_eq!(Some("AsyncTask"), type_task); transaction.rollback().await.unwrap(); } + #[tokio::test] + async fn failed_task_query_test() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let mut transaction = connection.transaction().await.unwrap(); + + let task = + AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) + .await + .unwrap(); + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); + let id = task.id; + assert_eq!(Some(1), number); + assert_eq!(Some("AsyncTask"), type_task); + let failed_task = + AsyncQueue::::fail_task_query(&mut transaction, task, "Some error") + .await + .unwrap(); + assert_eq!(id, failed_task.id); + assert_eq!(Some("Some error"), failed_task.error_message.as_deref()); + assert_eq!(FangTaskState::Failed, failed_task.state); + transaction.rollback().await.unwrap(); + } #[tokio::test] async fn remove_all_tasks_test() { diff --git a/src/asynk/queries/fail_task.sql b/src/asynk/queries/fail_task.sql index 91f7f46..416d91f 100644 --- a/src/asynk/queries/fail_task.sql +++ b/src/asynk/queries/fail_task.sql @@ -1 +1 @@ -UPDATE "fang_tasks" SET "state" = $1 , "error_message" = $2 , "updated_at" = $3 WHERE id = $4 +UPDATE "fang_tasks" SET "state" = $1 , "error_message" = $2 , "updated_at" = $3 WHERE id = $4 RETURNING id , state , metadata , error_message , task_type , created_at , updated_at diff --git a/src/asynk/queries/insert_task.sql b/src/asynk/queries/insert_task.sql index c76a4b8..b6ec160 100644 --- a/src/asynk/queries/insert_task.sql +++ b/src/asynk/queries/insert_task.sql @@ -1 +1 @@ -INSERT INTO "fang_tasks" ("metadata", "task_type") VALUES ($1, $2) RETURNING id , metadata , error_message , task_type , created_at , updated_at +INSERT INTO "fang_tasks" ("metadata", "task_type") VALUES ($1, $2) RETURNING id , state , metadata , error_message , task_type , created_at , updated_at From d585bde8704dc13106bdeaaf7c6049b53b4b658f Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Wed, 20 Jul 2022 19:08:02 +0300 Subject: [PATCH 14/25] split into async and blocking features (#36) * split into async and blocking features * update Cargo.toml file * fix example --- Cargo.toml | 56 +++++++++++++++++++++----- fang_examples/simple_worker/Cargo.toml | 2 +- src/{sync => blocking}/error.rs | 0 src/{sync => blocking}/executor.rs | 0 src/{sync => blocking}/mod.rs | 0 src/{sync => blocking}/queue.rs | 0 src/{sync => blocking}/scheduler.rs | 0 src/{sync => blocking}/schema.rs | 0 src/{sync => blocking}/worker_pool.rs | 0 src/lib.rs | 25 ++++-------- 10 files changed, 54 insertions(+), 29 deletions(-) rename src/{sync => blocking}/error.rs (100%) rename src/{sync => blocking}/executor.rs (100%) rename src/{sync => blocking}/mod.rs (100%) rename src/{sync => blocking}/queue.rs (100%) rename src/{sync => blocking}/scheduler.rs (100%) rename src/{sync => blocking}/schema.rs (100%) rename src/{sync => blocking}/worker_pool.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 6741d8f..0f2b123 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,19 +11,53 @@ rust-version = "1.62" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +blocking = ["diesel", "diesel-derive-enum", "dotenv"] +asynk = ["bb8-postgres", "postgres-types", "tokio", "async-trait", "typed-builder"] + [dependencies] -diesel = { version = "1.4", features = ["postgres", "serde_json", "chrono", "uuidv07", "r2d2"] } -diesel-derive-enum = { version = "1", features = ["postgres"] } -dotenv = "0.15" -uuid = { version = "0.8", features = ["v4"] } chrono = "0.4" -serde_json = "1" -typetag = "0.2" log = "0.4" serde = { version = "1", features = ["derive"] } +serde_json = "1" thiserror = "1.0" -bb8-postgres = {version = "0.8", features = ["with-serde_json-1" , "with-uuid-0_8" , "with-chrono-0_4" ]} -postgres-types = { version = "0.X.X", features = ["derive"] } -tokio = { version = "1.20", features = ["full"] } -async-trait = "0.1" -typed-builder = "0.10" +typetag = "0.2" +uuid = { version = "0.8", features = ["v4"] } + + +[dependencies.diesel] +version = "1.4" +features = ["postgres", "serde_json", "chrono", "uuidv07", "r2d2"] +optional = true + +[dependencies.diesel-derive-enum] +version = "1" +features = ["postgres"] +optional = true + +[dependencies.dotenv] +version = "0.15" +optional = true + +[dependencies.bb8-postgres] +version = "0.8" +features = ["with-serde_json-1" , "with-uuid-0_8" , "with-chrono-0_4"] +optional = true + +[dependencies.postgres-types] +version = "0.X.X" +features = ["derive"] +optional = true + +[dependencies.tokio] +version = "1.20" +features = ["full"] +optional = true + +[dependencies.async-trait] +version = "0.1" +optional = true + +[dependencies.typed-builder] +version = "0.10" +optional = true diff --git a/fang_examples/simple_worker/Cargo.toml b/fang_examples/simple_worker/Cargo.toml index f1c3328..287c552 100644 --- a/fang_examples/simple_worker/Cargo.toml +++ b/fang_examples/simple_worker/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -fang = { path = "../../" } +fang = { path = "../../" , features = ["blocking"]} serde = { version = "1.0", features = ["derive"] } signal-hook = "0.3.10" dotenv = "0.15.0" diff --git a/src/sync/error.rs b/src/blocking/error.rs similarity index 100% rename from src/sync/error.rs rename to src/blocking/error.rs diff --git a/src/sync/executor.rs b/src/blocking/executor.rs similarity index 100% rename from src/sync/executor.rs rename to src/blocking/executor.rs diff --git a/src/sync/mod.rs b/src/blocking/mod.rs similarity index 100% rename from src/sync/mod.rs rename to src/blocking/mod.rs diff --git a/src/sync/queue.rs b/src/blocking/queue.rs similarity index 100% rename from src/sync/queue.rs rename to src/blocking/queue.rs diff --git a/src/sync/scheduler.rs b/src/blocking/scheduler.rs similarity index 100% rename from src/sync/scheduler.rs rename to src/blocking/scheduler.rs diff --git a/src/sync/schema.rs b/src/blocking/schema.rs similarity index 100% rename from src/sync/schema.rs rename to src/blocking/schema.rs diff --git a/src/sync/worker_pool.rs b/src/blocking/worker_pool.rs similarity index 100% rename from src/sync/worker_pool.rs rename to src/blocking/worker_pool.rs diff --git a/src/lib.rs b/src/lib.rs index a6d3bf1..cf12c64 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,29 +1,20 @@ -// #![allow(clippy::nonstandard_macro_braces)] #![allow(clippy::extra_unused_lifetimes)] -// pub mod error; -// pub mod executor; -// pub mod queue; -// pub mod scheduler; -// pub mod schema; -// pub mod worker_pool; - -// pub use error::FangError; -// pub use executor::*; -// pub use queue::*; -// pub use scheduler::*; -// pub use schema::*; -// pub use worker_pool::*; - #[macro_use] +#[cfg(feature = "blocking")] extern crate diesel; #[doc(hidden)] +#[cfg(feature = "blocking")] pub use diesel::pg::PgConnection; + #[doc(hidden)] pub use typetag; -pub mod sync; -pub use sync::*; +#[cfg(feature = "blocking")] +pub mod blocking; +#[cfg(feature = "blocking")] +pub use blocking::*; +#[cfg(feature = "asynk")] pub mod asynk; From 8d0a23e2f9ebc9d81107c51442f79f46ed690cea Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Sat, 23 Jul 2022 14:24:22 +0000 Subject: [PATCH 15/25] Async Worker (#44) * stating with async worker * async sleep solve and update state test * uncomenting line * retention mode and sleep params in lib.rs * fixing code style * trying to solve * dont like changes * Add AsyncQueueable trait (#45) * add trait * update runnable trait * fix test Co-authored-by: Ayrat Badykov --- Cargo.toml | 1 + src/asynk/async_queue.rs | 356 +++++++++++++++--------- src/asynk/async_runnable.rs | 6 +- src/asynk/async_worker.rs | 109 ++++++++ src/asynk/mod.rs | 2 +- src/asynk/queries/update_task_state.sql | 2 +- src/blocking/executor.rs | 44 +-- src/blocking/worker_pool.rs | 5 +- src/lib.rs | 36 +++ 9 files changed, 383 insertions(+), 178 deletions(-) create mode 100644 src/asynk/async_worker.rs diff --git a/Cargo.toml b/Cargo.toml index 0f2b123..fb739d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ rust-version = "1.62" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] +default = ["blocking", "asynk"] blocking = ["diesel", "diesel-derive-enum", "dotenv"] asynk = ["bb8-postgres", "postgres-types", "tokio", "async-trait", "typed-builder"] diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index e404272..628ffd8 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -1,4 +1,4 @@ -use crate::asynk::AsyncRunnable; +use async_trait::async_trait; use bb8_postgres::bb8::Pool; use bb8_postgres::bb8::RunError; use bb8_postgres::tokio_postgres::row::Row; @@ -90,6 +90,7 @@ pub struct NewPeriodicTask { #[builder(setter(into))] pub period_in_seconds: i32, } + #[derive(Debug, Error)] pub enum AsyncQueueError { #[error(transparent)] @@ -100,6 +101,36 @@ pub enum AsyncQueueError { ResultError { expected: u64, found: u64 }, } +#[async_trait] +pub trait AsyncQueueable { + async fn fetch_and_touch_task( + &mut self, + task_type: &Option, + ) -> Result, AsyncQueueError>; + + async fn insert_task( + &mut self, + task: serde_json::Value, + task_type: &str, + ) -> Result; + + async fn remove_all_tasks(&mut self) -> Result; + + async fn remove_task(&mut self, task: Task) -> Result; + + async fn remove_tasks_type(&mut self, task_type: &str) -> Result; + + async fn update_task_state( + &mut self, + task: Task, + state: FangTaskState, + ) -> Result; + + async fn fail_task(&mut self, task: Task, error_message: &str) + -> Result; +} + +#[derive(Debug, Clone)] pub struct AsyncQueue where Tls: MakeTlsConnect + Clone + Send + Sync + 'static, @@ -117,80 +148,11 @@ where >::TlsConnect: Send, <>::TlsConnect as TlsConnect>::Future: Send, { - pub fn new(pool: Pool>) -> Self { - AsyncQueue { pool } - } + pub async fn connect(uri: impl ToString, tls: Tls) -> Result { + let manager = PostgresConnectionManager::new_from_stringlike(uri, tls)?; + let pool = Pool::builder().build(manager).await?; - pub async fn fetch_and_touch_task( - &mut self, - task_type: &Option, - ) -> Result { - let mut connection = self.pool.get().await?; - let mut transaction = connection.transaction().await?; - - let task = Self::fetch_and_touch_task_query(&mut transaction, task_type).await?; - - transaction.commit().await?; - - Ok(task) - } - - pub async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { - let mut connection = self.pool.get().await?; - let mut transaction = connection.transaction().await?; - - let task = Self::insert_task_query(&mut transaction, task).await?; - - transaction.commit().await?; - - Ok(task) - } - - pub async fn remove_all_tasks(&mut self) -> Result { - let mut connection = self.pool.get().await?; - let mut transaction = connection.transaction().await?; - - let result = Self::remove_all_tasks_query(&mut transaction).await?; - - transaction.commit().await?; - - Ok(result) - } - - pub async fn remove_task(&mut self, task: &Task) -> Result { - let mut connection = self.pool.get().await?; - let mut transaction = connection.transaction().await?; - - let result = Self::remove_task_query(&mut transaction, task).await?; - - transaction.commit().await?; - - Ok(result) - } - - pub async fn remove_tasks_type(&mut self, task_type: &str) -> Result { - let mut connection = self.pool.get().await?; - let mut transaction = connection.transaction().await?; - - let result = Self::remove_tasks_type_query(&mut transaction, task_type).await?; - - transaction.commit().await?; - - Ok(result) - } - - pub async fn fail_task( - &mut self, - task: Task, - error_message: &str, - ) -> Result { - let mut connection = self.pool.get().await?; - let mut transaction = connection.transaction().await?; - - let task = Self::fail_task_query(&mut transaction, task, error_message).await?; - transaction.commit().await?; - - Ok(task) + Ok(Self { pool }) } pub async fn remove_all_tasks_query( @@ -201,7 +163,7 @@ where pub async fn remove_task_query( transaction: &mut Transaction<'_>, - task: &Task, + task: Task, ) -> Result { Self::execute_query(transaction, REMOVE_TASK_QUERY, &[&task.id], Some(1)).await } @@ -238,19 +200,25 @@ where pub async fn fetch_and_touch_task_query( transaction: &mut Transaction<'_>, task_type: &Option, - ) -> Result { + ) -> Result, AsyncQueueError> { let task_type = match task_type { Some(passed_task_type) => passed_task_type, None => DEFAULT_TASK_TYPE, }; - let mut task = Self::get_task_type_query(transaction, task_type).await?; - - Self::update_task_state_query(transaction, &task, FangTaskState::InProgress).await?; - - task.state = FangTaskState::InProgress; - - Ok(task) + let task = match Self::get_task_type_query(transaction, task_type).await { + Ok(some_task) => Some(some_task), + Err(_) => None, + }; + let result_task = if let Some(some_task) = task { + Some( + Self::update_task_state_query(transaction, some_task, FangTaskState::InProgress) + .await?, + ) + } else { + None + }; + Ok(result_task) } pub async fn get_task_type_query( @@ -268,27 +236,23 @@ where pub async fn update_task_state_query( transaction: &mut Transaction<'_>, - task: &Task, + task: Task, state: FangTaskState, - ) -> Result { + ) -> Result { let updated_at = Utc::now(); - Self::execute_query( - transaction, - UPDATE_TASK_STATE_QUERY, - &[&state, &updated_at, &task.id], - Some(1), - ) - .await + let row: Row = transaction + .query_one(UPDATE_TASK_STATE_QUERY, &[&state, &updated_at, &task.id]) + .await?; + let task = Self::row_to_task(row); + Ok(task) } pub async fn insert_task_query( transaction: &mut Transaction<'_>, - task: &dyn AsyncRunnable, + metadata: serde_json::Value, + task_type: &str, ) -> Result { - let metadata = serde_json::to_value(task).unwrap(); - let task_type = task.task_type(); - let row: Row = transaction .query_one(INSERT_TASK_QUERY, &[&metadata, &task_type]) .await?; @@ -339,16 +303,117 @@ where } } +#[async_trait] +impl AsyncQueueable for AsyncQueue +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + async fn fetch_and_touch_task( + &mut self, + task_type: &Option, + ) -> Result, AsyncQueueError> { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let task = Self::fetch_and_touch_task_query(&mut transaction, task_type).await?; + + transaction.commit().await?; + + Ok(task) + } + + async fn insert_task( + &mut self, + metadata: serde_json::Value, + task_type: &str, + ) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let task = Self::insert_task_query(&mut transaction, metadata, task_type).await?; + + transaction.commit().await?; + + Ok(task) + } + + async fn remove_all_tasks(&mut self) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let result = Self::remove_all_tasks_query(&mut transaction).await?; + + transaction.commit().await?; + + Ok(result) + } + + async fn remove_task(&mut self, task: Task) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let result = Self::remove_task_query(&mut transaction, task).await?; + + transaction.commit().await?; + + Ok(result) + } + + async fn remove_tasks_type(&mut self, task_type: &str) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let result = Self::remove_tasks_type_query(&mut transaction, task_type).await?; + + transaction.commit().await?; + + Ok(result) + } + + async fn update_task_state( + &mut self, + task: Task, + state: FangTaskState, + ) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let task = Self::update_task_state_query(&mut transaction, task, state).await?; + transaction.commit().await?; + + Ok(task) + } + + async fn fail_task( + &mut self, + task: Task, + error_message: &str, + ) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let task = Self::fail_task_query(&mut transaction, task, error_message).await?; + transaction.commit().await?; + + Ok(task) + } +} + #[cfg(test)] mod async_queue_tests { use super::AsyncQueue; + use super::AsyncQueueable; use super::FangTaskState; + use super::Task; use crate::asynk::AsyncRunnable; use crate::asynk::Error; use async_trait::async_trait; use bb8_postgres::bb8::Pool; - use bb8_postgres::tokio_postgres::Client; use bb8_postgres::tokio_postgres::NoTls; + use bb8_postgres::tokio_postgres::Transaction; use bb8_postgres::PostgresConnectionManager; use serde::{Deserialize, Serialize}; @@ -358,9 +423,9 @@ mod async_queue_tests { } #[typetag::serde] - #[async_trait] + #[async_trait(?Send)] impl AsyncRunnable for AsyncTask { - async fn run(&self, _connection: &Client) -> Result<(), Error> { + async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { Ok(()) } } @@ -371,10 +436,14 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let mut transaction = connection.transaction().await.unwrap(); + let task = AsyncTask { number: 1 }; + let metadata = serde_json::to_value(&task as &dyn AsyncRunnable).unwrap(); + let task = - AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) + AsyncQueue::::insert_task_query(&mut transaction, metadata, &task.task_type()) .await .unwrap(); + let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); @@ -383,29 +452,62 @@ mod async_queue_tests { assert_eq!(Some("AsyncTask"), type_task); transaction.rollback().await.unwrap(); } + + #[tokio::test] + async fn update_task_state_test() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let mut transaction = connection.transaction().await.unwrap(); + + let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; + + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); + let id = task.id; + + assert_eq!(Some(1), number); + assert_eq!(Some("AsyncTask"), type_task); + + let finished_task = AsyncQueue::::update_task_state_query( + &mut transaction, + task, + FangTaskState::Finished, + ) + .await + .unwrap(); + + assert_eq!(id, finished_task.id); + assert_eq!(FangTaskState::Finished, finished_task.state); + + transaction.rollback().await.unwrap(); + } + #[tokio::test] async fn failed_task_query_test() { let pool = pool().await; let mut connection = pool.get().await.unwrap(); let mut transaction = connection.transaction().await.unwrap(); - let task = - AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) - .await - .unwrap(); + let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; + let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); let id = task.id; + assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); + let failed_task = AsyncQueue::::fail_task_query(&mut transaction, task, "Some error") .await .unwrap(); + assert_eq!(id, failed_task.id); assert_eq!(Some("Some error"), failed_task.error_message.as_deref()); assert_eq!(FangTaskState::Failed, failed_task.state); + transaction.rollback().await.unwrap(); } @@ -415,10 +517,7 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let mut transaction = connection.transaction().await.unwrap(); - let task = - AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) - .await - .unwrap(); + let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -427,10 +526,8 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let task = - AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 2 }) - .await - .unwrap(); + let task = insert_task(&mut transaction, &AsyncTask { number: 2 }).await; + let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); @@ -452,10 +549,8 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let mut transaction = connection.transaction().await.unwrap(); - let task = - AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) - .await - .unwrap(); + let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; + let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); @@ -463,10 +558,8 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let task = - AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 2 }) - .await - .unwrap(); + let task = insert_task(&mut transaction, &AsyncTask { number: 2 }).await; + let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); @@ -476,7 +569,9 @@ mod async_queue_tests { let task = AsyncQueue::::fetch_and_touch_task_query(&mut transaction, &None) .await + .unwrap() .unwrap(); + let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); @@ -486,6 +581,7 @@ mod async_queue_tests { let task = AsyncQueue::::fetch_and_touch_task_query(&mut transaction, &None) .await + .unwrap() .unwrap(); let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -503,10 +599,8 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let mut transaction = connection.transaction().await.unwrap(); - let task = - AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 1 }) - .await - .unwrap(); + let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; + let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); @@ -514,10 +608,8 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let task = - AsyncQueue::::insert_task_query(&mut transaction, &AsyncTask { number: 2 }) - .await - .unwrap(); + let task = insert_task(&mut transaction, &AsyncTask { number: 2 }).await; + let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); @@ -547,4 +639,12 @@ mod async_queue_tests { Pool::builder().build(pg_mgr).await.unwrap() } + + async fn insert_task(transaction: &mut Transaction<'_>, task: &dyn AsyncRunnable) -> Task { + let metadata = serde_json::to_value(task).unwrap(); + + AsyncQueue::::insert_task_query(transaction, metadata, &task.task_type()) + .await + .unwrap() + } } diff --git a/src/asynk/async_runnable.rs b/src/asynk/async_runnable.rs index a933a8d..048b3bd 100644 --- a/src/asynk/async_runnable.rs +++ b/src/asynk/async_runnable.rs @@ -1,5 +1,5 @@ +use crate::asynk::async_queue::AsyncQueueable; use async_trait::async_trait; -use bb8_postgres::tokio_postgres::Client; const COMMON_TYPE: &str = "common"; @@ -9,9 +9,9 @@ pub struct Error { } #[typetag::serde(tag = "type")] -#[async_trait] +#[async_trait(?Send)] pub trait AsyncRunnable { - async fn run(&self, client: &Client) -> Result<(), Error>; + async fn run(&self, client: &mut dyn AsyncQueueable) -> Result<(), Error>; fn task_type(&self) -> String { COMMON_TYPE.to_string() diff --git a/src/asynk/async_worker.rs b/src/asynk/async_worker.rs new file mode 100644 index 0000000..fa4c1cc --- /dev/null +++ b/src/asynk/async_worker.rs @@ -0,0 +1,109 @@ +use crate::asynk::async_queue::AsyncQueue; +use crate::asynk::async_queue::AsyncQueueable; +use crate::asynk::async_queue::FangTaskState; +use crate::asynk::async_queue::Task; +use crate::asynk::async_runnable::AsyncRunnable; +use crate::asynk::Error; +use crate::{RetentionMode, SleepParams}; +use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; +use bb8_postgres::tokio_postgres::tls::TlsConnect; +use bb8_postgres::tokio_postgres::Socket; +use log::error; +use std::time::Duration; +use typed_builder::TypedBuilder; + +#[derive(TypedBuilder, Debug)] +pub struct AsyncWorker +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + #[builder(setter(into))] + pub queue: AsyncQueue, + #[builder(setter(into))] + pub task_type: Option, + #[builder(setter(into))] + pub sleep_params: SleepParams, + #[builder(setter(into))] + pub retention_mode: RetentionMode, +} +impl AsyncWorker +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + pub async fn run(&mut self, task: Task) { + let result = self.execute_task(task).await; + self.finalize_task(result).await + } + async fn execute_task(&mut self, task: Task) -> Result { + let actual_task: Box = + serde_json::from_value(task.metadata.clone()).unwrap(); + + let task_result = actual_task.run(&mut self.queue).await; + match task_result { + Ok(()) => Ok(task), + Err(error) => Err((task, error.description)), + } + } + async fn finalize_task(&mut self, result: Result) { + match self.retention_mode { + RetentionMode::KeepAll => { + match result { + Ok(task) => self + .queue + .update_task_state(task, FangTaskState::Finished) + .await + .unwrap(), + Err((task, error)) => self.queue.fail_task(task, &error).await.unwrap(), + }; + } + RetentionMode::RemoveAll => { + match result { + Ok(task) => self.queue.remove_task(task).await.unwrap(), + Err((task, _error)) => self.queue.remove_task(task).await.unwrap(), + }; + } + RetentionMode::RemoveFinished => match result { + Ok(task) => { + self.queue.remove_task(task).await.unwrap(); + } + Err((task, error)) => { + self.queue.fail_task(task, &error).await.unwrap(); + } + }, + } + } + pub async fn sleep(&mut self) { + self.sleep_params.maybe_increase_sleep_period(); + + tokio::time::sleep(Duration::from_secs(self.sleep_params.sleep_period)).await; + } + pub async fn run_tasks(&mut self) -> Result<(), Error> { + loop { + match self + .queue + .fetch_and_touch_task(&self.task_type.clone()) + .await + { + Ok(Some(task)) => { + self.sleep_params.maybe_reset_sleep_period(); + self.run(task).await; + } + Ok(None) => { + self.sleep().await; + } + + Err(error) => { + error!("Failed to fetch a task {:?}", error); + + self.sleep().await; + } + }; + } + } +} diff --git a/src/asynk/mod.rs b/src/asynk/mod.rs index 171d637..bf6e0cf 100644 --- a/src/asynk/mod.rs +++ b/src/asynk/mod.rs @@ -1,5 +1,5 @@ pub mod async_queue; pub mod async_runnable; - +pub mod async_worker; pub use async_runnable::AsyncRunnable; pub use async_runnable::Error; diff --git a/src/asynk/queries/update_task_state.sql b/src/asynk/queries/update_task_state.sql index 21b9e82..afca5c0 100644 --- a/src/asynk/queries/update_task_state.sql +++ b/src/asynk/queries/update_task_state.sql @@ -1 +1 @@ -UPDATE "fang_tasks" SET "state" = $1 , "updated_at" = $2 WHERE id = $3 +UPDATE "fang_tasks" SET "state" = $1 , "updated_at" = $2 WHERE id = $3 RETURNING id , state , metadata , error_message , task_type , created_at , updated_at diff --git a/src/blocking/executor.rs b/src/blocking/executor.rs index 69f9bcf..353bedc 100644 --- a/src/blocking/executor.rs +++ b/src/blocking/executor.rs @@ -1,7 +1,8 @@ use crate::error::FangError; use crate::queue::Queue; +use crate::queue::Task; use crate::worker_pool::{SharedState, WorkerState}; -use crate::Task; +use crate::{RetentionMode, SleepParams}; use diesel::pg::PgConnection; use diesel::r2d2::{ConnectionManager, PooledConnection}; use log::error; @@ -15,47 +16,6 @@ pub struct Executor { pub retention_mode: RetentionMode, shared_state: Option, } - -#[derive(Clone)] -pub enum RetentionMode { - KeepAll, - RemoveAll, - RemoveFinished, -} - -#[derive(Clone)] -pub struct SleepParams { - pub sleep_period: u64, - pub max_sleep_period: u64, - pub min_sleep_period: u64, - pub sleep_step: u64, -} - -impl SleepParams { - pub fn maybe_reset_sleep_period(&mut self) { - if self.sleep_period != self.min_sleep_period { - self.sleep_period = self.min_sleep_period; - } - } - - pub fn maybe_increase_sleep_period(&mut self) { - if self.sleep_period < self.max_sleep_period { - self.sleep_period += self.sleep_step; - } - } -} - -impl Default for SleepParams { - fn default() -> Self { - SleepParams { - sleep_period: 5, - max_sleep_period: 15, - min_sleep_period: 5, - sleep_step: 5, - } - } -} - #[derive(Debug)] pub struct Error { pub description: String, diff --git a/src/blocking/worker_pool.rs b/src/blocking/worker_pool.rs index 97f0ec5..651c787 100644 --- a/src/blocking/worker_pool.rs +++ b/src/blocking/worker_pool.rs @@ -2,9 +2,8 @@ use crate::diesel::r2d2; use crate::diesel::PgConnection; use crate::error::FangError; use crate::executor::Executor; -use crate::executor::RetentionMode; -use crate::executor::SleepParams; use crate::queue::Queue; +use crate::{RetentionMode, SleepParams}; use log::error; use log::info; use std::collections::HashMap; @@ -223,11 +222,11 @@ mod task_pool_tests { use super::WorkerParams; use super::WorkerPool; use crate::executor::Error; - use crate::executor::RetentionMode; use crate::executor::Runnable; use crate::queue::Queue; use crate::schema::{fang_tasks, FangTaskState}; use crate::typetag; + use crate::RetentionMode; use crate::Task; use diesel::pg::PgConnection; use diesel::prelude::*; diff --git a/src/lib.rs b/src/lib.rs index cf12c64..d4fc169 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,40 @@ #![allow(clippy::extra_unused_lifetimes)] +#[derive(Clone, Debug)] +pub enum RetentionMode { + KeepAll, + RemoveAll, + RemoveFinished, +} +#[derive(Clone, Debug)] +pub struct SleepParams { + pub sleep_period: u64, + pub max_sleep_period: u64, + pub min_sleep_period: u64, + pub sleep_step: u64, +} +impl SleepParams { + pub fn maybe_reset_sleep_period(&mut self) { + if self.sleep_period != self.min_sleep_period { + self.sleep_period = self.min_sleep_period; + } + } + + pub fn maybe_increase_sleep_period(&mut self) { + if self.sleep_period < self.max_sleep_period { + self.sleep_period += self.sleep_step; + } + } +} +impl Default for SleepParams { + fn default() -> Self { + SleepParams { + sleep_period: 5, + max_sleep_period: 15, + min_sleep_period: 5, + sleep_step: 5, + } + } +} #[macro_use] #[cfg(feature = "blocking")] From 2d724c377628a66f29edaa66002482980e80e978 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Wed, 27 Jul 2022 17:05:05 +0000 Subject: [PATCH 16/25] Async worker test (#46) * preparing code to tests * changing queue tests and starting worker tests with new struct AsyncQueueTest * fix clippy and delete Debug trait * fix warnings * Get task by id in worker tests and AsyncQueueTest #[cfg(test)] * Defaults and AsyncWorker Builder * Testing :D * Execute task specific type * deleting Options and comment * insert task is back git !! * Test remove tasks and changing insert task auxiliar func --- src/asynk/async_queue.rs | 223 +++++++++++++------ src/asynk/async_worker.rs | 322 +++++++++++++++++++++++---- src/asynk/queries/get_task_by_id.sql | 1 + src/lib.rs | 5 + 4 files changed, 435 insertions(+), 116 deletions(-) create mode 100644 src/asynk/queries/get_task_by_id.sql diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 628ffd8..5b90ca9 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -1,9 +1,11 @@ +use crate::asynk::async_runnable::Error as FangError; use async_trait::async_trait; use bb8_postgres::bb8::Pool; use bb8_postgres::bb8::RunError; use bb8_postgres::tokio_postgres::row::Row; -use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; -use bb8_postgres::tokio_postgres::tls::TlsConnect; +#[cfg(test)] +use bb8_postgres::tokio_postgres::tls::NoTls; +use bb8_postgres::tokio_postgres::tls::{MakeTlsConnect, TlsConnect}; use bb8_postgres::tokio_postgres::Socket; use bb8_postgres::tokio_postgres::Transaction; use bb8_postgres::PostgresConnectionManager; @@ -21,8 +23,10 @@ const REMOVE_ALL_TASK_QUERY: &str = include_str!("queries/remove_all_tasks.sql") const REMOVE_TASK_QUERY: &str = include_str!("queries/remove_task.sql"); const REMOVE_TASKS_TYPE_QUERY: &str = include_str!("queries/remove_tasks_type.sql"); const FETCH_TASK_TYPE_QUERY: &str = include_str!("queries/fetch_task_type.sql"); +#[cfg(test)] +const GET_TASK_BY_ID_QUERY: &str = include_str!("queries/get_task_by_id.sql"); -const DEFAULT_TASK_TYPE: &str = "common"; +pub const DEFAULT_TASK_TYPE: &str = "common"; #[derive(Debug, Eq, PartialEq, Clone, ToSql, FromSql)] #[postgres(name = "fang_task_state")] @@ -100,7 +104,14 @@ pub enum AsyncQueueError { #[error("returned invalid result (expected {expected:?}, found {found:?})")] ResultError { expected: u64, found: u64 }, } - +impl From for FangError { + fn from(error: AsyncQueueError) -> Self { + let message = format!("{:?}", error); + FangError { + description: message, + } + } +} #[async_trait] pub trait AsyncQueueable { async fn fetch_and_touch_task( @@ -141,6 +152,98 @@ where pool: Pool>, } +#[cfg(test)] +pub struct AsyncQueueTest<'a> { + pub transaction: Transaction<'a>, +} + +#[cfg(test)] +impl<'a> AsyncQueueTest<'a> { + pub async fn get_task_by_id(&mut self, id: Uuid) -> Result { + let row: Row = self + .transaction + .query_one(GET_TASK_BY_ID_QUERY, &[&id]) + .await?; + + let task = AsyncQueue::::row_to_task(row); + Ok(task) + } +} + +#[cfg(test)] +#[async_trait] +impl AsyncQueueable for AsyncQueueTest<'_> { + async fn fetch_and_touch_task( + &mut self, + task_type: &Option, + ) -> Result, AsyncQueueError> { + let transaction = &mut self.transaction; + + let task = AsyncQueue::::fetch_and_touch_task_query(transaction, task_type).await?; + + Ok(task) + } + + async fn insert_task( + &mut self, + metadata: serde_json::Value, + task_type: &str, + ) -> Result { + let transaction = &mut self.transaction; + + let task = AsyncQueue::::insert_task_query(transaction, metadata, task_type).await?; + + Ok(task) + } + + async fn remove_all_tasks(&mut self) -> Result { + let transaction = &mut self.transaction; + + let result = AsyncQueue::::remove_all_tasks_query(transaction).await?; + + Ok(result) + } + + async fn remove_task(&mut self, task: Task) -> Result { + let transaction = &mut self.transaction; + + let result = AsyncQueue::::remove_task_query(transaction, task).await?; + + Ok(result) + } + + async fn remove_tasks_type(&mut self, task_type: &str) -> Result { + let transaction = &mut self.transaction; + + let result = AsyncQueue::::remove_tasks_type_query(transaction, task_type).await?; + + Ok(result) + } + + async fn update_task_state( + &mut self, + task: Task, + state: FangTaskState, + ) -> Result { + let transaction = &mut self.transaction; + + let task = AsyncQueue::::update_task_state_query(transaction, task, state).await?; + + Ok(task) + } + + async fn fail_task( + &mut self, + task: Task, + error_message: &str, + ) -> Result { + let transaction = &mut self.transaction; + + let task = AsyncQueue::::fail_task_query(transaction, task, error_message).await?; + + Ok(task) + } +} impl AsyncQueue where Tls: MakeTlsConnect + Clone + Send + Sync + 'static, @@ -404,7 +507,7 @@ where #[cfg(test)] mod async_queue_tests { - use super::AsyncQueue; + use super::AsyncQueueTest; use super::AsyncQueueable; use super::FangTaskState; use super::Task; @@ -413,7 +516,6 @@ mod async_queue_tests { use async_trait::async_trait; use bb8_postgres::bb8::Pool; use bb8_postgres::tokio_postgres::NoTls; - use bb8_postgres::tokio_postgres::Transaction; use bb8_postgres::PostgresConnectionManager; use serde::{Deserialize, Serialize}; @@ -434,15 +536,11 @@ mod async_queue_tests { async fn insert_task_creates_new_task() { let pool = pool().await; let mut connection = pool.get().await.unwrap(); - let mut transaction = connection.transaction().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); - let task = AsyncTask { number: 1 }; - let metadata = serde_json::to_value(&task as &dyn AsyncRunnable).unwrap(); + let mut test = AsyncQueueTest { transaction }; - let task = - AsyncQueue::::insert_task_query(&mut transaction, metadata, &task.task_type()) - .await - .unwrap(); + let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -450,16 +548,18 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - transaction.rollback().await.unwrap(); + test.transaction.rollback().await.unwrap(); } #[tokio::test] async fn update_task_state_test() { let pool = pool().await; let mut connection = pool.get().await.unwrap(); - let mut transaction = connection.transaction().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); - let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; + let mut test = AsyncQueueTest { transaction }; + + let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -469,27 +569,26 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let finished_task = AsyncQueue::::update_task_state_query( - &mut transaction, - task, - FangTaskState::Finished, - ) - .await - .unwrap(); + let finished_task = test + .update_task_state(task, FangTaskState::Finished) + .await + .unwrap(); assert_eq!(id, finished_task.id); assert_eq!(FangTaskState::Finished, finished_task.state); - transaction.rollback().await.unwrap(); + test.transaction.rollback().await.unwrap(); } #[tokio::test] async fn failed_task_query_test() { let pool = pool().await; let mut connection = pool.get().await.unwrap(); - let mut transaction = connection.transaction().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); - let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; + let mut test = AsyncQueueTest { transaction }; + + let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -499,25 +598,24 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let failed_task = - AsyncQueue::::fail_task_query(&mut transaction, task, "Some error") - .await - .unwrap(); + let failed_task = test.fail_task(task, "Some error").await.unwrap(); assert_eq!(id, failed_task.id); assert_eq!(Some("Some error"), failed_task.error_message.as_deref()); assert_eq!(FangTaskState::Failed, failed_task.state); - transaction.rollback().await.unwrap(); + test.transaction.rollback().await.unwrap(); } #[tokio::test] async fn remove_all_tasks_test() { let pool = pool().await; let mut connection = pool.get().await.unwrap(); - let mut transaction = connection.transaction().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); - let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; + let mut test = AsyncQueueTest { transaction }; + + let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -526,7 +624,7 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let task = insert_task(&mut transaction, &AsyncTask { number: 2 }).await; + let task = insert_task(&mut test, &AsyncTask { number: 2 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -535,21 +633,21 @@ mod async_queue_tests { assert_eq!(Some(2), number); assert_eq!(Some("AsyncTask"), type_task); - let result = AsyncQueue::::remove_all_tasks_query(&mut transaction) - .await - .unwrap(); + let result = test.remove_all_tasks().await.unwrap(); assert_eq!(2, result); - transaction.rollback().await.unwrap(); + test.transaction.rollback().await.unwrap(); } #[tokio::test] async fn fetch_and_touch_test() { let pool = pool().await; let mut connection = pool.get().await.unwrap(); - let mut transaction = connection.transaction().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); - let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; + let mut test = AsyncQueueTest { transaction }; + + let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -558,7 +656,7 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let task = insert_task(&mut transaction, &AsyncTask { number: 2 }).await; + let task = insert_task(&mut test, &AsyncTask { number: 2 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -567,10 +665,7 @@ mod async_queue_tests { assert_eq!(Some(2), number); assert_eq!(Some("AsyncTask"), type_task); - let task = AsyncQueue::::fetch_and_touch_task_query(&mut transaction, &None) - .await - .unwrap() - .unwrap(); + let task = test.fetch_and_touch_task(&None).await.unwrap().unwrap(); let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -579,10 +674,7 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let task = AsyncQueue::::fetch_and_touch_task_query(&mut transaction, &None) - .await - .unwrap() - .unwrap(); + let task = test.fetch_and_touch_task(&None).await.unwrap().unwrap(); let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); @@ -590,16 +682,18 @@ mod async_queue_tests { assert_eq!(Some(2), number); assert_eq!(Some("AsyncTask"), type_task); - transaction.rollback().await.unwrap(); + test.transaction.rollback().await.unwrap(); } #[tokio::test] async fn remove_tasks_type_test() { let pool = pool().await; let mut connection = pool.get().await.unwrap(); - let mut transaction = connection.transaction().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); - let task = insert_task(&mut transaction, &AsyncTask { number: 1 }).await; + let mut test = AsyncQueueTest { transaction }; + + let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -608,7 +702,7 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let task = insert_task(&mut transaction, &AsyncTask { number: 2 }).await; + let task = insert_task(&mut test, &AsyncTask { number: 2 }).await; let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -617,17 +711,18 @@ mod async_queue_tests { assert_eq!(Some(2), number); assert_eq!(Some("AsyncTask"), type_task); - let result = AsyncQueue::::remove_tasks_type_query(&mut transaction, "mytype") - .await - .unwrap(); + let result = test.remove_tasks_type("mytype").await.unwrap(); assert_eq!(0, result); - let result = AsyncQueue::::remove_tasks_type_query(&mut transaction, "common") - .await - .unwrap(); + let result = test.remove_tasks_type("common").await.unwrap(); assert_eq!(2, result); - transaction.rollback().await.unwrap(); + test.transaction.rollback().await.unwrap(); + } + + async fn insert_task(test: &mut AsyncQueueTest<'_>, task: &dyn AsyncRunnable) -> Task { + let metadata = serde_json::to_value(task).unwrap(); + test.insert_task(metadata, &task.task_type()).await.unwrap() } async fn pool() -> Pool> { @@ -639,12 +734,4 @@ mod async_queue_tests { Pool::builder().build(pg_mgr).await.unwrap() } - - async fn insert_task(transaction: &mut Transaction<'_>, task: &dyn AsyncRunnable) -> Task { - let metadata = serde_json::to_value(task).unwrap(); - - AsyncQueue::::insert_task_query(transaction, metadata, &task.task_type()) - .await - .unwrap() - } } diff --git a/src/asynk/async_worker.rs b/src/asynk/async_worker.rs index fa4c1cc..d151b55 100644 --- a/src/asynk/async_worker.rs +++ b/src/asynk/async_worker.rs @@ -1,42 +1,27 @@ -use crate::asynk::async_queue::AsyncQueue; use crate::asynk::async_queue::AsyncQueueable; use crate::asynk::async_queue::FangTaskState; use crate::asynk::async_queue::Task; +use crate::asynk::async_queue::DEFAULT_TASK_TYPE; use crate::asynk::async_runnable::AsyncRunnable; use crate::asynk::Error; use crate::{RetentionMode, SleepParams}; -use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; -use bb8_postgres::tokio_postgres::tls::TlsConnect; -use bb8_postgres::tokio_postgres::Socket; use log::error; use std::time::Duration; use typed_builder::TypedBuilder; -#[derive(TypedBuilder, Debug)] -pub struct AsyncWorker -where - Tls: MakeTlsConnect + Clone + Send + Sync + 'static, - >::Stream: Send + Sync, - >::TlsConnect: Send, - <>::TlsConnect as TlsConnect>::Future: Send, -{ - #[builder(setter(into))] - pub queue: AsyncQueue, - #[builder(setter(into))] - pub task_type: Option, +#[derive(TypedBuilder)] +pub struct AsyncWorker<'a> { #[builder(setter(into))] + pub queue: &'a mut dyn AsyncQueueable, + #[builder(default=DEFAULT_TASK_TYPE.to_string() , setter(into))] + pub task_type: String, + #[builder(default, setter(into))] pub sleep_params: SleepParams, - #[builder(setter(into))] + #[builder(default, setter(into))] pub retention_mode: RetentionMode, } -impl AsyncWorker -where - Tls: MakeTlsConnect + Clone + Send + Sync + 'static, - >::Stream: Send + Sync, - >::TlsConnect: Send, - <>::TlsConnect as TlsConnect>::Future: Send, -{ - pub async fn run(&mut self, task: Task) { +impl<'a> AsyncWorker<'a> { + pub async fn run(&mut self, task: Task) -> Result<(), Error> { let result = self.execute_task(task).await; self.finalize_task(result).await } @@ -44,36 +29,44 @@ where let actual_task: Box = serde_json::from_value(task.metadata.clone()).unwrap(); - let task_result = actual_task.run(&mut self.queue).await; + let task_result = actual_task.run(self.queue).await; match task_result { Ok(()) => Ok(task), Err(error) => Err((task, error.description)), } } - async fn finalize_task(&mut self, result: Result) { + async fn finalize_task(&mut self, result: Result) -> Result<(), Error> { match self.retention_mode { - RetentionMode::KeepAll => { - match result { - Ok(task) => self - .queue - .update_task_state(task, FangTaskState::Finished) - .await - .unwrap(), - Err((task, error)) => self.queue.fail_task(task, &error).await.unwrap(), - }; - } - RetentionMode::RemoveAll => { - match result { - Ok(task) => self.queue.remove_task(task).await.unwrap(), - Err((task, _error)) => self.queue.remove_task(task).await.unwrap(), - }; - } - RetentionMode::RemoveFinished => match result { + RetentionMode::KeepAll => match result { Ok(task) => { - self.queue.remove_task(task).await.unwrap(); + self.queue + .update_task_state(task, FangTaskState::Finished) + .await?; + Ok(()) } Err((task, error)) => { - self.queue.fail_task(task, &error).await.unwrap(); + self.queue.fail_task(task, &error).await?; + Ok(()) + } + }, + RetentionMode::RemoveAll => match result { + Ok(task) => { + self.queue.remove_task(task).await?; + Ok(()) + } + Err((task, _error)) => { + self.queue.remove_task(task).await?; + Ok(()) + } + }, + RetentionMode::RemoveFinished => match result { + Ok(task) => { + self.queue.remove_task(task).await?; + Ok(()) + } + Err((task, error)) => { + self.queue.fail_task(task, &error).await?; + Ok(()) } }, } @@ -87,12 +80,12 @@ where loop { match self .queue - .fetch_and_touch_task(&self.task_type.clone()) + .fetch_and_touch_task(&Some(self.task_type.clone())) .await { Ok(Some(task)) => { self.sleep_params.maybe_reset_sleep_period(); - self.run(task).await; + self.run(task).await? } Ok(None) => { self.sleep().await; @@ -106,4 +99,237 @@ where }; } } + #[cfg(test)] + pub async fn run_tasks_until_none(&mut self) -> Result<(), Error> { + loop { + match self + .queue + .fetch_and_touch_task(&Some(self.task_type.clone())) + .await + { + Ok(Some(task)) => { + self.sleep_params.maybe_reset_sleep_period(); + self.run(task).await? + } + Ok(None) => { + return Ok(()); + } + Err(error) => { + error!("Failed to fetch a task {:?}", error); + + self.sleep().await; + } + }; + } + } +} + +#[cfg(test)] +mod async_worker_tests { + use super::AsyncWorker; + use crate::asynk::async_queue::AsyncQueueTest; + use crate::asynk::async_queue::AsyncQueueable; + use crate::asynk::async_queue::FangTaskState; + use crate::asynk::async_worker::Task; + use crate::asynk::AsyncRunnable; + use crate::asynk::Error; + use crate::RetentionMode; + use async_trait::async_trait; + use bb8_postgres::bb8::Pool; + use bb8_postgres::tokio_postgres::NoTls; + use bb8_postgres::PostgresConnectionManager; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize)] + struct WorkerAsyncTask { + pub number: u16, + } + + #[typetag::serde] + #[async_trait(?Send)] + impl AsyncRunnable for WorkerAsyncTask { + async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { + Ok(()) + } + } + #[derive(Serialize, Deserialize)] + struct AsyncFailedTask { + pub number: u16, + } + + #[typetag::serde] + #[async_trait(?Send)] + impl AsyncRunnable for AsyncFailedTask { + async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { + let message = format!("number {} is wrong :(", self.number); + + Err(Error { + description: message, + }) + } + } + + #[derive(Serialize, Deserialize)] + struct AsyncTaskType1 {} + + #[typetag::serde] + #[async_trait(?Send)] + impl AsyncRunnable for AsyncTaskType1 { + async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { + Ok(()) + } + + fn task_type(&self) -> String { + "type1".to_string() + } + } + + #[derive(Serialize, Deserialize)] + struct AsyncTaskType2 {} + + #[typetag::serde] + #[async_trait(?Send)] + impl AsyncRunnable for AsyncTaskType2 { + async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { + Ok(()) + } + + fn task_type(&self) -> String { + "type2".to_string() + } + } + #[tokio::test] + async fn execute_and_finishes_task() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); + + let mut test = AsyncQueueTest { transaction }; + + let task = insert_task(&mut test, &WorkerAsyncTask { number: 1 }).await; + let id = task.id; + + let mut worker = AsyncWorker::builder() + .queue(&mut test as &mut dyn AsyncQueueable) + .retention_mode(RetentionMode::KeepAll) + .build(); + + worker.run(task).await.unwrap(); + let task_finished = test.get_task_by_id(id).await.unwrap(); + assert_eq!(id, task_finished.id); + assert_eq!(FangTaskState::Finished, task_finished.state); + test.transaction.rollback().await.unwrap(); + } + #[tokio::test] + async fn saves_error_for_failed_task() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); + + let mut test = AsyncQueueTest { transaction }; + + let task = insert_task(&mut test, &AsyncFailedTask { number: 1 }).await; + let id = task.id; + + let mut worker = AsyncWorker::builder() + .queue(&mut test as &mut dyn AsyncQueueable) + .retention_mode(RetentionMode::KeepAll) + .build(); + + worker.run(task).await.unwrap(); + let task_finished = test.get_task_by_id(id).await.unwrap(); + + assert_eq!(id, task_finished.id); + assert_eq!(FangTaskState::Failed, task_finished.state); + assert_eq!( + "number 1 is wrong :(".to_string(), + task_finished.error_message.unwrap() + ); + test.transaction.rollback().await.unwrap(); + } + #[tokio::test] + async fn executes_task_only_of_specific_type() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); + + let mut test = AsyncQueueTest { transaction }; + + let task1 = insert_task(&mut test, &AsyncTaskType1 {}).await; + let task12 = insert_task(&mut test, &AsyncTaskType1 {}).await; + let task2 = insert_task(&mut test, &AsyncTaskType2 {}).await; + + let id1 = task1.id; + let id12 = task12.id; + let id2 = task2.id; + + let mut worker = AsyncWorker::builder() + .queue(&mut test as &mut dyn AsyncQueueable) + .task_type("type1".to_string()) + .retention_mode(RetentionMode::KeepAll) + .build(); + + worker.run_tasks_until_none().await.unwrap(); + let task1 = test.get_task_by_id(id1).await.unwrap(); + let task12 = test.get_task_by_id(id12).await.unwrap(); + let task2 = test.get_task_by_id(id2).await.unwrap(); + + assert_eq!(id1, task1.id); + assert_eq!(id12, task12.id); + assert_eq!(id2, task2.id); + assert_eq!(FangTaskState::Finished, task1.state); + assert_eq!(FangTaskState::Finished, task12.state); + assert_eq!(FangTaskState::New, task2.state); + test.transaction.rollback().await.unwrap(); + } + #[tokio::test] + async fn remove_when_finished() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); + + let mut test = AsyncQueueTest { transaction }; + + let task1 = insert_task(&mut test, &AsyncTaskType1 {}).await; + let task12 = insert_task(&mut test, &AsyncTaskType1 {}).await; + let task2 = insert_task(&mut test, &AsyncTaskType2 {}).await; + + let _id1 = task1.id; + let _id12 = task12.id; + let id2 = task2.id; + + let mut worker = AsyncWorker::builder() + .queue(&mut test as &mut dyn AsyncQueueable) + .task_type("type1".to_string()) + .build(); + + worker.run_tasks_until_none().await.unwrap(); + let task = test + .fetch_and_touch_task(&Some("type1".to_string())) + .await + .unwrap(); + assert_eq!(None, task); + + let task2 = test + .fetch_and_touch_task(&Some("type2".to_string())) + .await + .unwrap() + .unwrap(); + assert_eq!(id2, task2.id); + + test.transaction.rollback().await.unwrap(); + } + async fn insert_task(test: &mut AsyncQueueTest<'_>, task: &dyn AsyncRunnable) -> Task { + let metadata = serde_json::to_value(task).unwrap(); + test.insert_task(metadata, &task.task_type()).await.unwrap() + } + async fn pool() -> Pool> { + let pg_mgr = PostgresConnectionManager::new_from_stringlike( + "postgres://postgres:postgres@localhost/fang", + NoTls, + ) + .unwrap(); + + Pool::builder().build(pg_mgr).await.unwrap() + } } diff --git a/src/asynk/queries/get_task_by_id.sql b/src/asynk/queries/get_task_by_id.sql new file mode 100644 index 0000000..608166f --- /dev/null +++ b/src/asynk/queries/get_task_by_id.sql @@ -0,0 +1 @@ +SELECT * FROM fang_tasks WHERE id = $1 diff --git a/src/lib.rs b/src/lib.rs index d4fc169..c9d36f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,11 @@ pub enum RetentionMode { RemoveAll, RemoveFinished, } +impl Default for RetentionMode { + fn default() -> Self { + RetentionMode::RemoveAll + } +} #[derive(Clone, Debug)] pub struct SleepParams { pub sleep_period: u64, From 6222c15d997543bb7ded2783d08cea729d7f00c7 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Sun, 31 Jul 2022 11:40:24 +0000 Subject: [PATCH 17/25] Async scheduled tasks (#47) * preparing queries for periodic tasks * fixing clippy and some other things * delete src/schema * ignoring src/schema.rs * fix compile error * fix clippy * Scheduler * Scheduler done ?? * improve scheduler (#48) * improve scheduler * add number of restarts * fix clippy * add into attribute * make number of restarts pub * fixing bugs and start async scheduler tests * fixing insert periodic task bug and scheduler tests bug * fix clippy * fix clippy tests * fix clippy tests again * fix task type :D * Builder AsyncQueueTest and sleep in start Co-authored-by: Ayrat Badykov --- .gitignore | 1 + Cargo.toml | 8 +- src/asynk/async_queue.rs | 308 +++++++++++++++--- src/asynk/async_scheduler.rs | 222 +++++++++++++ src/asynk/async_worker.rs | 34 +- src/asynk/mod.rs | 1 + src/asynk/queries/fetch_periodic_tasks.sql | 1 + .../queries/find_periodic_task_by_id.sql | 1 + ...get_task_by_id.sql => find_task_by_id.sql} | 0 src/asynk/queries/find_task_by_metadata.sql | 1 + src/asynk/queries/insert_periodic_task.sql | 2 +- src/asynk/queries/schedule_next_task.sql | 2 +- src/blocking/executor.rs | 1 + 13 files changed, 525 insertions(+), 57 deletions(-) create mode 100644 src/asynk/async_scheduler.rs create mode 100644 src/asynk/queries/fetch_periodic_tasks.sql create mode 100644 src/asynk/queries/find_periodic_task_by_id.sql rename src/asynk/queries/{get_task_by_id.sql => find_task_by_id.sql} (100%) create mode 100644 src/asynk/queries/find_task_by_metadata.sql diff --git a/.gitignore b/.gitignore index 96ef6c0..626b0a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target Cargo.lock +src/schema.rs diff --git a/Cargo.toml b/Cargo.toml index fb739d7..d8d67aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ rust-version = "1.62" [features] default = ["blocking", "asynk"] blocking = ["diesel", "diesel-derive-enum", "dotenv"] -asynk = ["bb8-postgres", "postgres-types", "tokio", "async-trait", "typed-builder"] +asynk = ["bb8-postgres", "postgres-types", "tokio", "async-trait", "typed-builder", "async-recursion"] [dependencies] chrono = "0.4" @@ -45,6 +45,7 @@ version = "0.8" features = ["with-serde_json-1" , "with-uuid-0_8" , "with-chrono-0_4"] optional = true + [dependencies.postgres-types] version = "0.X.X" features = ["derive"] @@ -62,3 +63,8 @@ optional = true [dependencies.typed-builder] version = "0.10" optional = true + + +[dependencies.async-recursion] +version = "1" +optional = true \ No newline at end of file diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 5b90ca9..2fb3e9f 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -10,6 +10,7 @@ use bb8_postgres::tokio_postgres::Socket; use bb8_postgres::tokio_postgres::Transaction; use bb8_postgres::PostgresConnectionManager; use chrono::DateTime; +use chrono::Duration; use chrono::Utc; use postgres_types::{FromSql, ToSql}; use thiserror::Error; @@ -17,14 +18,21 @@ use typed_builder::TypedBuilder; use uuid::Uuid; const INSERT_TASK_QUERY: &str = include_str!("queries/insert_task.sql"); +const INSERT_PERIODIC_TASK_QUERY: &str = include_str!("queries/insert_periodic_task.sql"); +const SCHEDULE_NEXT_TASK_QUERY: &str = include_str!("queries/schedule_next_task.sql"); const UPDATE_TASK_STATE_QUERY: &str = include_str!("queries/update_task_state.sql"); const FAIL_TASK_QUERY: &str = include_str!("queries/fail_task.sql"); const REMOVE_ALL_TASK_QUERY: &str = include_str!("queries/remove_all_tasks.sql"); const REMOVE_TASK_QUERY: &str = include_str!("queries/remove_task.sql"); const REMOVE_TASKS_TYPE_QUERY: &str = include_str!("queries/remove_tasks_type.sql"); const FETCH_TASK_TYPE_QUERY: &str = include_str!("queries/fetch_task_type.sql"); +const FETCH_PERIODIC_TASKS_QUERY: &str = include_str!("queries/fetch_periodic_tasks.sql"); +const FIND_TASK_BY_METADATA_QUERY: &str = include_str!("queries/find_task_by_metadata.sql"); + #[cfg(test)] -const GET_TASK_BY_ID_QUERY: &str = include_str!("queries/get_task_by_id.sql"); +const FIND_TASK_BY_ID_QUERY: &str = include_str!("queries/find_task_by_id.sql"); +#[cfg(test)] +const FIND_PERIODIC_TASK_BY_ID_QUERY: &str = include_str!("queries/find_periodic_task_by_id.sql"); pub const DEFAULT_TASK_TYPE: &str = "common"; @@ -40,11 +48,13 @@ pub enum FangTaskState { #[postgres(name = "finished")] Finished, } + impl Default for FangTaskState { fn default() -> Self { FangTaskState::New } } + #[derive(TypedBuilder, Debug, Eq, PartialEq, Clone)] pub struct Task { #[builder(setter(into))] @@ -79,22 +89,6 @@ pub struct PeriodicTask { pub updated_at: DateTime, } -#[derive(TypedBuilder, Debug, Eq, PartialEq, Clone)] -pub struct NewTask { - #[builder(setter(into))] - pub metadata: serde_json::Value, - #[builder(setter(into))] - pub task_type: String, -} - -#[derive(TypedBuilder, Debug, Eq, PartialEq, Clone)] -pub struct NewPeriodicTask { - #[builder(setter(into))] - pub metadata: serde_json::Value, - #[builder(setter(into))] - pub period_in_seconds: i32, -} - #[derive(Debug, Error)] pub enum AsyncQueueError { #[error(transparent)] @@ -104,6 +98,7 @@ pub enum AsyncQueueError { #[error("returned invalid result (expected {expected:?}, found {found:?})")] ResultError { expected: u64, found: u64 }, } + impl From for FangError { fn from(error: AsyncQueueError) -> Self { let message = format!("{:?}", error); @@ -112,11 +107,12 @@ impl From for FangError { } } } + #[async_trait] pub trait AsyncQueueable { async fn fetch_and_touch_task( &mut self, - task_type: &Option, + task_type: Option, ) -> Result, AsyncQueueError>; async fn insert_task( @@ -124,7 +120,6 @@ pub trait AsyncQueueable { task: serde_json::Value, task_type: &str, ) -> Result; - async fn remove_all_tasks(&mut self) -> Result; async fn remove_task(&mut self, task: Task) -> Result; @@ -139,9 +134,25 @@ pub trait AsyncQueueable { async fn fail_task(&mut self, task: Task, error_message: &str) -> Result; + + async fn fetch_periodic_tasks( + &mut self, + error_margin_seconds: i64, + ) -> Result>, AsyncQueueError>; + + async fn insert_periodic_task( + &mut self, + metadata: serde_json::Value, + timestamp: DateTime, + period: i32, + ) -> Result; + async fn schedule_next_task( + &mut self, + periodic_task: PeriodicTask, + ) -> Result; } -#[derive(Debug, Clone)] +#[derive(TypedBuilder, Debug, Clone)] pub struct AsyncQueue where Tls: MakeTlsConnect + Clone + Send + Sync + 'static, @@ -149,25 +160,44 @@ where >::TlsConnect: Send, <>::TlsConnect as TlsConnect>::Future: Send, { + #[builder(setter(into))] pool: Pool>, + #[builder(default = false, setter(into))] + duplicated_tasks: bool, } #[cfg(test)] +#[derive(TypedBuilder)] pub struct AsyncQueueTest<'a> { + #[builder(setter(into))] pub transaction: Transaction<'a>, + #[builder(default = false, setter(into))] + pub duplicated_tasks: bool, } #[cfg(test)] impl<'a> AsyncQueueTest<'a> { - pub async fn get_task_by_id(&mut self, id: Uuid) -> Result { + pub async fn find_task_by_id(&mut self, id: Uuid) -> Result { let row: Row = self .transaction - .query_one(GET_TASK_BY_ID_QUERY, &[&id]) + .query_one(FIND_TASK_BY_ID_QUERY, &[&id]) .await?; let task = AsyncQueue::::row_to_task(row); Ok(task) } + pub async fn find_periodic_task_by_id( + &mut self, + id: Uuid, + ) -> Result { + let row: Row = self + .transaction + .query_one(FIND_PERIODIC_TASK_BY_ID_QUERY, &[&id]) + .await?; + + let task = AsyncQueue::::row_to_periodic_task(row); + Ok(task) + } } #[cfg(test)] @@ -175,7 +205,7 @@ impl<'a> AsyncQueueTest<'a> { impl AsyncQueueable for AsyncQueueTest<'_> { async fn fetch_and_touch_task( &mut self, - task_type: &Option, + task_type: Option, ) -> Result, AsyncQueueError> { let transaction = &mut self.transaction; @@ -191,11 +221,57 @@ impl AsyncQueueable for AsyncQueueTest<'_> { ) -> Result { let transaction = &mut self.transaction; - let task = AsyncQueue::::insert_task_query(transaction, metadata, task_type).await?; - + let task: Task = if self.duplicated_tasks { + AsyncQueue::::insert_task_query(transaction, metadata, task_type).await? + } else { + AsyncQueue::::insert_task_if_not_exist_query(transaction, metadata, task_type) + .await? + }; Ok(task) } + async fn schedule_next_task( + &mut self, + periodic_task: PeriodicTask, + ) -> Result { + let transaction = &mut self.transaction; + + let periodic_task = + AsyncQueue::::schedule_next_task_query(transaction, periodic_task).await?; + + Ok(periodic_task) + } + async fn insert_periodic_task( + &mut self, + metadata: serde_json::Value, + timestamp: DateTime, + period: i32, + ) -> Result { + let transaction = &mut self.transaction; + + let periodic_task = AsyncQueue::::insert_periodic_task_query( + transaction, + metadata, + timestamp, + period, + ) + .await?; + + Ok(periodic_task) + } + + async fn fetch_periodic_tasks( + &mut self, + error_margin_seconds: i64, + ) -> Result>, AsyncQueueError> { + let transaction = &mut self.transaction; + + let periodic_task = + AsyncQueue::::fetch_periodic_tasks_query(transaction, error_margin_seconds) + .await?; + + Ok(periodic_task) + } async fn remove_all_tasks(&mut self) -> Result { let transaction = &mut self.transaction; @@ -251,11 +327,18 @@ where >::TlsConnect: Send, <>::TlsConnect as TlsConnect>::Future: Send, { - pub async fn connect(uri: impl ToString, tls: Tls) -> Result { + pub async fn connect( + uri: impl ToString, + tls: Tls, + duplicated_tasks: bool, + ) -> Result { let manager = PostgresConnectionManager::new_from_stringlike(uri, tls)?; let pool = Pool::builder().build(manager).await?; - Ok(Self { pool }) + Ok(Self { + pool, + duplicated_tasks, + }) } pub async fn remove_all_tasks_query( @@ -302,14 +385,14 @@ where pub async fn fetch_and_touch_task_query( transaction: &mut Transaction<'_>, - task_type: &Option, + task_type: Option, ) -> Result, AsyncQueueError> { let task_type = match task_type { Some(passed_task_type) => passed_task_type, - None => DEFAULT_TASK_TYPE, + None => DEFAULT_TASK_TYPE.to_string(), }; - let task = match Self::get_task_type_query(transaction, task_type).await { + let task = match Self::get_task_type_query(transaction, &task_type).await { Ok(some_task) => Some(some_task), Err(_) => None, }; @@ -362,7 +445,59 @@ where let task = Self::row_to_task(row); Ok(task) } + pub async fn schedule_next_task_query( + transaction: &mut Transaction<'_>, + periodic_task: PeriodicTask, + ) -> Result { + let updated_at = Utc::now(); + let scheduled_at = updated_at + Duration::seconds(periodic_task.period_in_seconds.into()); + let row: Row = transaction + .query_one(SCHEDULE_NEXT_TASK_QUERY, &[&scheduled_at, &updated_at]) + .await?; + + let periodic_task = Self::row_to_periodic_task(row); + Ok(periodic_task) + } + pub async fn insert_periodic_task_query( + transaction: &mut Transaction<'_>, + metadata: serde_json::Value, + timestamp: DateTime, + period: i32, + ) -> Result { + let row: Row = transaction + .query_one( + INSERT_PERIODIC_TASK_QUERY, + &[&metadata, ×tamp, &period], + ) + .await?; + let periodic_task = Self::row_to_periodic_task(row); + Ok(periodic_task) + } + + pub async fn fetch_periodic_tasks_query( + transaction: &mut Transaction<'_>, + error_margin_seconds: i64, + ) -> Result>, AsyncQueueError> { + let current_time = Utc::now(); + + let low_limit = current_time - Duration::seconds(error_margin_seconds); + let high_limit = current_time + Duration::seconds(error_margin_seconds); + let rows: Vec = transaction + .query(FETCH_PERIODIC_TASKS_QUERY, &[&low_limit, &high_limit]) + .await?; + + let periodic_tasks: Vec = rows + .into_iter() + .map(|row| Self::row_to_periodic_task(row)) + .collect(); + + if periodic_tasks.is_empty() { + Ok(None) + } else { + Ok(Some(periodic_tasks)) + } + } pub async fn execute_query( transaction: &mut Transaction<'_>, query: &str, @@ -382,6 +517,49 @@ where Ok(result) } + pub async fn insert_task_if_not_exist_query( + transaction: &mut Transaction<'_>, + metadata: serde_json::Value, + task_type: &str, + ) -> Result { + match Self::find_task_by_metadata_query(transaction, &metadata).await { + Some(task) => Ok(task), + None => Self::insert_task_query(transaction, metadata, task_type).await, + } + } + pub async fn find_task_by_metadata_query( + transaction: &mut Transaction<'_>, + metadata: &serde_json::Value, + ) -> Option { + let result = transaction + .query_one(FIND_TASK_BY_METADATA_QUERY, &[metadata]) + .await; + + match result { + Ok(row) => Some(Self::row_to_task(row)), + Err(_) => None, + } + } + fn row_to_periodic_task(row: Row) -> PeriodicTask { + let id: Uuid = row.get("id"); + let metadata: serde_json::Value = row.get("metadata"); + let period_in_seconds: i32 = row.get("period_in_seconds"); + let scheduled_at: Option> = match row.try_get("scheduled_at") { + Ok(datetime) => Some(datetime), + Err(_) => None, + }; + let created_at: DateTime = row.get("created_at"); + let updated_at: DateTime = row.get("updated_at"); + + PeriodicTask::builder() + .id(id) + .metadata(metadata) + .period_in_seconds(period_in_seconds) + .scheduled_at(scheduled_at) + .created_at(created_at) + .updated_at(updated_at) + .build() + } fn row_to_task(row: Row) -> Task { let id: Uuid = row.get("id"); let metadata: serde_json::Value = row.get("metadata"); @@ -416,7 +594,7 @@ where { async fn fetch_and_touch_task( &mut self, - task_type: &Option, + task_type: Option, ) -> Result, AsyncQueueError> { let mut connection = self.pool.get().await?; let mut transaction = connection.transaction().await?; @@ -436,13 +614,63 @@ where let mut connection = self.pool.get().await?; let mut transaction = connection.transaction().await?; - let task = Self::insert_task_query(&mut transaction, metadata, task_type).await?; + let task: Task = if self.duplicated_tasks { + Self::insert_task_query(&mut transaction, metadata, task_type).await? + } else { + Self::insert_task_if_not_exist_query(&mut transaction, metadata, task_type).await? + }; transaction.commit().await?; Ok(task) } + async fn insert_periodic_task( + &mut self, + metadata: serde_json::Value, + timestamp: DateTime, + period: i32, + ) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let periodic_task = + Self::insert_periodic_task_query(&mut transaction, metadata, timestamp, period).await?; + + transaction.commit().await?; + + Ok(periodic_task) + } + + async fn schedule_next_task( + &mut self, + periodic_task: PeriodicTask, + ) -> Result { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let periodic_task = Self::schedule_next_task_query(&mut transaction, periodic_task).await?; + + transaction.commit().await?; + + Ok(periodic_task) + } + + async fn fetch_periodic_tasks( + &mut self, + error_margin_seconds: i64, + ) -> Result>, AsyncQueueError> { + let mut connection = self.pool.get().await?; + let mut transaction = connection.transaction().await?; + + let periodic_task = + Self::fetch_periodic_tasks_query(&mut transaction, error_margin_seconds).await?; + + transaction.commit().await?; + + Ok(periodic_task) + } + async fn remove_all_tasks(&mut self) -> Result { let mut connection = self.pool.get().await?; let mut transaction = connection.transaction().await?; @@ -538,7 +766,7 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; @@ -557,7 +785,7 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; @@ -586,7 +814,7 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; @@ -613,7 +841,7 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; @@ -645,7 +873,7 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; @@ -665,7 +893,7 @@ mod async_queue_tests { assert_eq!(Some(2), number); assert_eq!(Some("AsyncTask"), type_task); - let task = test.fetch_and_touch_task(&None).await.unwrap().unwrap(); + let task = test.fetch_and_touch_task(None).await.unwrap().unwrap(); let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); @@ -674,7 +902,7 @@ mod async_queue_tests { assert_eq!(Some(1), number); assert_eq!(Some("AsyncTask"), type_task); - let task = test.fetch_and_touch_task(&None).await.unwrap().unwrap(); + let task = test.fetch_and_touch_task(None).await.unwrap().unwrap(); let metadata = task.metadata.as_object().unwrap(); let number = metadata["number"].as_u64(); let type_task = metadata["type"].as_str(); @@ -691,7 +919,7 @@ mod async_queue_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task = insert_task(&mut test, &AsyncTask { number: 1 }).await; diff --git a/src/asynk/async_scheduler.rs b/src/asynk/async_scheduler.rs new file mode 100644 index 0000000..e369f27 --- /dev/null +++ b/src/asynk/async_scheduler.rs @@ -0,0 +1,222 @@ +use crate::asynk::async_queue::AsyncQueueable; +use crate::asynk::async_queue::PeriodicTask; +use crate::asynk::AsyncRunnable; +use crate::asynk::Error; +use async_recursion::async_recursion; +use log::error; +use std::time::Duration; +use tokio::time::sleep; +use typed_builder::TypedBuilder; + +#[derive(TypedBuilder)] +pub struct Scheduler<'a> { + #[builder(setter(into))] + pub check_period: u64, + #[builder(setter(into))] + pub error_margin_seconds: u64, + #[builder(setter(into))] + pub queue: &'a mut dyn AsyncQueueable, + #[builder(default = 0, setter(into))] + pub number_of_restarts: u32, +} + +impl<'a> Scheduler<'a> { + #[async_recursion(?Send)] + pub async fn start(&mut self) -> Result<(), Error> { + let task_res = self.schedule_loop().await; + + sleep(Duration::from_secs(1)).await; + + match task_res { + Err(err) => { + error!( + "Scheduler failed, restarting {:?}. Number of restarts {}", + err, self.number_of_restarts + ); + self.number_of_restarts += 1; + self.start().await + } + Ok(_) => { + error!( + "Scheduler stopped. restarting. Number of restarts {}", + self.number_of_restarts + ); + self.number_of_restarts += 1; + self.start().await + } + } + } + + pub async fn schedule_loop(&mut self) -> Result<(), Error> { + let sleep_duration = Duration::from_secs(self.check_period); + + loop { + self.schedule().await?; + + sleep(sleep_duration).await; + } + } + + pub async fn schedule(&mut self) -> Result<(), Error> { + if let Some(tasks) = self + .queue + .fetch_periodic_tasks(self.error_margin_seconds as i64) + .await? + { + for task in tasks { + self.process_task(task).await?; + } + }; + Ok(()) + } + + async fn process_task(&mut self, task: PeriodicTask) -> Result<(), Error> { + match task.scheduled_at { + None => { + self.queue.schedule_next_task(task).await?; + } + Some(_) => { + let metadata = task.metadata.clone(); + + let actual_task: Box = + serde_json::from_value(task.metadata.clone()).unwrap(); + + self.queue + .insert_task(metadata, &(*actual_task).task_type()) + .await?; + + self.queue.schedule_next_task(task).await?; + } + } + Ok(()) + } + + #[cfg(test)] + async fn schedule_test(&mut self) -> Result<(), Error> { + let sleep_duration = Duration::from_secs(self.check_period); + + loop { + match self + .queue + .fetch_periodic_tasks(self.error_margin_seconds as i64) + .await? + { + Some(tasks) => { + for task in tasks { + self.process_task(task).await?; + } + + return Ok(()); + } + None => { + sleep(sleep_duration).await; + } + }; + } + } +} + +#[cfg(test)] +mod async_scheduler_tests { + use super::Scheduler; + use crate::asynk::async_queue::AsyncQueueTest; + use crate::asynk::async_queue::AsyncQueueable; + use crate::asynk::async_queue::PeriodicTask; + use crate::asynk::AsyncRunnable; + use crate::asynk::Error; + use async_trait::async_trait; + use bb8_postgres::bb8::Pool; + use bb8_postgres::tokio_postgres::NoTls; + use bb8_postgres::PostgresConnectionManager; + use chrono::DateTime; + use chrono::Duration as OtherDuration; + use chrono::Utc; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize)] + struct AsyncScheduledTask { + pub number: u16, + } + + #[typetag::serde] + #[async_trait(?Send)] + impl AsyncRunnable for AsyncScheduledTask { + async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { + Ok(()) + } + fn task_type(&self) -> String { + "schedule".to_string() + } + } + + #[tokio::test] + async fn schedules_tasks() { + let pool = pool().await; + let mut connection = pool.get().await.unwrap(); + let transaction = connection.transaction().await.unwrap(); + + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); + + let schedule_in_future = Utc::now() + OtherDuration::seconds(5); + + let _periodic_task = insert_periodic_task( + &mut test, + &AsyncScheduledTask { number: 1 }, + schedule_in_future, + 10, + ) + .await; + + let check_period: u64 = 1; + let error_margin_seconds: u64 = 2; + + let mut scheduler = Scheduler::builder() + .check_period(check_period) + .error_margin_seconds(error_margin_seconds) + .queue(&mut test as &mut dyn AsyncQueueable) + .build(); + // Scheduler start tricky not loop :) + scheduler.schedule_test().await.unwrap(); + + let task = scheduler + .queue + .fetch_and_touch_task(Some("schedule".to_string())) + .await + .unwrap() + .unwrap(); + + let metadata = task.metadata.as_object().unwrap(); + let number = metadata["number"].as_u64(); + let type_task = metadata["type"].as_str(); + + let runnable_task: Box = + serde_json::from_value(task.metadata.clone()).unwrap(); + + assert_eq!("schedule", runnable_task.task_type()); + assert_eq!(Some("AsyncScheduledTask"), type_task); + assert_eq!(Some(1), number); + } + + async fn insert_periodic_task( + test: &mut AsyncQueueTest<'_>, + task: &dyn AsyncRunnable, + timestamp: DateTime, + period_in_seconds: i32, + ) -> PeriodicTask { + let metadata = serde_json::to_value(task).unwrap(); + + test.insert_periodic_task(metadata, timestamp, period_in_seconds) + .await + .unwrap() + } + + async fn pool() -> Pool> { + let pg_mgr = PostgresConnectionManager::new_from_stringlike( + "postgres://postgres:postgres@localhost/fang", + NoTls, + ) + .unwrap(); + + Pool::builder().build(pg_mgr).await.unwrap() + } +} diff --git a/src/asynk/async_worker.rs b/src/asynk/async_worker.rs index d151b55..71ec481 100644 --- a/src/asynk/async_worker.rs +++ b/src/asynk/async_worker.rs @@ -13,18 +13,20 @@ use typed_builder::TypedBuilder; pub struct AsyncWorker<'a> { #[builder(setter(into))] pub queue: &'a mut dyn AsyncQueueable, - #[builder(default=DEFAULT_TASK_TYPE.to_string() , setter(into))] + #[builder(default=DEFAULT_TASK_TYPE.to_string(), setter(into))] pub task_type: String, #[builder(default, setter(into))] pub sleep_params: SleepParams, #[builder(default, setter(into))] pub retention_mode: RetentionMode, } + impl<'a> AsyncWorker<'a> { pub async fn run(&mut self, task: Task) -> Result<(), Error> { let result = self.execute_task(task).await; self.finalize_task(result).await } + async fn execute_task(&mut self, task: Task) -> Result { let actual_task: Box = serde_json::from_value(task.metadata.clone()).unwrap(); @@ -35,6 +37,7 @@ impl<'a> AsyncWorker<'a> { Err(error) => Err((task, error.description)), } } + async fn finalize_task(&mut self, result: Result) -> Result<(), Error> { match self.retention_mode { RetentionMode::KeepAll => match result { @@ -71,16 +74,18 @@ impl<'a> AsyncWorker<'a> { }, } } + pub async fn sleep(&mut self) { self.sleep_params.maybe_increase_sleep_period(); tokio::time::sleep(Duration::from_secs(self.sleep_params.sleep_period)).await; } + pub async fn run_tasks(&mut self) -> Result<(), Error> { loop { match self .queue - .fetch_and_touch_task(&Some(self.task_type.clone())) + .fetch_and_touch_task(Some(self.task_type.clone())) .await { Ok(Some(task)) => { @@ -99,12 +104,13 @@ impl<'a> AsyncWorker<'a> { }; } } + #[cfg(test)] pub async fn run_tasks_until_none(&mut self) -> Result<(), Error> { loop { match self .queue - .fetch_and_touch_task(&Some(self.task_type.clone())) + .fetch_and_touch_task(Some(self.task_type.clone())) .await { Ok(Some(task)) => { @@ -204,7 +210,7 @@ mod async_worker_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task = insert_task(&mut test, &WorkerAsyncTask { number: 1 }).await; let id = task.id; @@ -215,7 +221,7 @@ mod async_worker_tests { .build(); worker.run(task).await.unwrap(); - let task_finished = test.get_task_by_id(id).await.unwrap(); + let task_finished = test.find_task_by_id(id).await.unwrap(); assert_eq!(id, task_finished.id); assert_eq!(FangTaskState::Finished, task_finished.state); test.transaction.rollback().await.unwrap(); @@ -226,7 +232,7 @@ mod async_worker_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task = insert_task(&mut test, &AsyncFailedTask { number: 1 }).await; let id = task.id; @@ -237,7 +243,7 @@ mod async_worker_tests { .build(); worker.run(task).await.unwrap(); - let task_finished = test.get_task_by_id(id).await.unwrap(); + let task_finished = test.find_task_by_id(id).await.unwrap(); assert_eq!(id, task_finished.id); assert_eq!(FangTaskState::Failed, task_finished.state); @@ -253,7 +259,7 @@ mod async_worker_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task1 = insert_task(&mut test, &AsyncTaskType1 {}).await; let task12 = insert_task(&mut test, &AsyncTaskType1 {}).await; @@ -270,9 +276,9 @@ mod async_worker_tests { .build(); worker.run_tasks_until_none().await.unwrap(); - let task1 = test.get_task_by_id(id1).await.unwrap(); - let task12 = test.get_task_by_id(id12).await.unwrap(); - let task2 = test.get_task_by_id(id2).await.unwrap(); + let task1 = test.find_task_by_id(id1).await.unwrap(); + let task12 = test.find_task_by_id(id12).await.unwrap(); + let task2 = test.find_task_by_id(id2).await.unwrap(); assert_eq!(id1, task1.id); assert_eq!(id12, task12.id); @@ -288,7 +294,7 @@ mod async_worker_tests { let mut connection = pool.get().await.unwrap(); let transaction = connection.transaction().await.unwrap(); - let mut test = AsyncQueueTest { transaction }; + let mut test = AsyncQueueTest::builder().transaction(transaction).build(); let task1 = insert_task(&mut test, &AsyncTaskType1 {}).await; let task12 = insert_task(&mut test, &AsyncTaskType1 {}).await; @@ -305,13 +311,13 @@ mod async_worker_tests { worker.run_tasks_until_none().await.unwrap(); let task = test - .fetch_and_touch_task(&Some("type1".to_string())) + .fetch_and_touch_task(Some("type1".to_string())) .await .unwrap(); assert_eq!(None, task); let task2 = test - .fetch_and_touch_task(&Some("type2".to_string())) + .fetch_and_touch_task(Some("type2".to_string())) .await .unwrap() .unwrap(); diff --git a/src/asynk/mod.rs b/src/asynk/mod.rs index bf6e0cf..45de86c 100644 --- a/src/asynk/mod.rs +++ b/src/asynk/mod.rs @@ -1,5 +1,6 @@ pub mod async_queue; pub mod async_runnable; +pub mod async_scheduler; pub mod async_worker; pub use async_runnable::AsyncRunnable; pub use async_runnable::Error; diff --git a/src/asynk/queries/fetch_periodic_tasks.sql b/src/asynk/queries/fetch_periodic_tasks.sql new file mode 100644 index 0000000..5d7529e --- /dev/null +++ b/src/asynk/queries/fetch_periodic_tasks.sql @@ -0,0 +1 @@ +SELECT * FROM fang_periodic_tasks WHERE scheduled_at BETWEEN $1 AND $2 OR scheduled_at IS NULL diff --git a/src/asynk/queries/find_periodic_task_by_id.sql b/src/asynk/queries/find_periodic_task_by_id.sql new file mode 100644 index 0000000..67c0316 --- /dev/null +++ b/src/asynk/queries/find_periodic_task_by_id.sql @@ -0,0 +1 @@ +SELECT * FROM fang_periodic_tasks WHERE id = $1 diff --git a/src/asynk/queries/get_task_by_id.sql b/src/asynk/queries/find_task_by_id.sql similarity index 100% rename from src/asynk/queries/get_task_by_id.sql rename to src/asynk/queries/find_task_by_id.sql diff --git a/src/asynk/queries/find_task_by_metadata.sql b/src/asynk/queries/find_task_by_metadata.sql new file mode 100644 index 0000000..0e38bfa --- /dev/null +++ b/src/asynk/queries/find_task_by_metadata.sql @@ -0,0 +1 @@ +SELECT * FROM fang_tasks WHERE metadata = $1 LIMIT 1 diff --git a/src/asynk/queries/insert_periodic_task.sql b/src/asynk/queries/insert_periodic_task.sql index d9b52b3..6f2eaaf 100644 --- a/src/asynk/queries/insert_periodic_task.sql +++ b/src/asynk/queries/insert_periodic_task.sql @@ -1 +1 @@ -INSERT INTO "fang_periodic_tasks" ("metadata", "period_in_seconds") VALUES ($1, $2) +INSERT INTO "fang_periodic_tasks" ("metadata", "scheduled_at", "period_in_seconds") VALUES ($1, $2, $3) RETURNING id , metadata , period_in_seconds , scheduled_at , created_at , updated_at diff --git a/src/asynk/queries/schedule_next_task.sql b/src/asynk/queries/schedule_next_task.sql index 3e0495c..95b8138 100644 --- a/src/asynk/queries/schedule_next_task.sql +++ b/src/asynk/queries/schedule_next_task.sql @@ -1 +1 @@ -UPDATE "fang_periodic_tasks" SET "scheduled_at" = $1 , "updated_at" = $2 +UPDATE "fang_periodic_tasks" SET "scheduled_at" = $1 , "updated_at" = $2 RETURNING id , metadata , period_in_seconds , scheduled_at , created_at , updated_at diff --git a/src/blocking/executor.rs b/src/blocking/executor.rs index 353bedc..1e7d17b 100644 --- a/src/blocking/executor.rs +++ b/src/blocking/executor.rs @@ -16,6 +16,7 @@ pub struct Executor { pub retention_mode: RetentionMode, shared_state: Option, } + #[derive(Debug)] pub struct Error { pub description: String, From 133d1427611b6c26a69b545d31f3515a4743fcfb Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Sun, 31 Jul 2022 16:32:37 +0300 Subject: [PATCH 18/25] Async worker pool draft (#49) * Async worker pool draft * ignore all targets * simple worker example * add debug logs * fix * fix tests * remove log * remove debug logs * fix clippy --- .gitignore | 2 +- fang_examples/simple_async_worker/Cargo.toml | 16 ++++ fang_examples/simple_async_worker/src/lib.rs | 34 ++++++++ fang_examples/simple_async_worker/src/main.rs | 41 ++++++++++ src/asynk/async_queue.rs | 4 +- src/asynk/async_runnable.rs | 4 +- src/asynk/async_scheduler.rs | 2 +- src/asynk/async_worker.rs | 8 +- src/asynk/async_worker_pool.rs | 78 +++++++++++++++++++ src/asynk/mod.rs | 2 + src/lib.rs | 3 + 11 files changed, 184 insertions(+), 10 deletions(-) create mode 100644 fang_examples/simple_async_worker/Cargo.toml create mode 100644 fang_examples/simple_async_worker/src/lib.rs create mode 100644 fang_examples/simple_async_worker/src/main.rs create mode 100644 src/asynk/async_worker_pool.rs diff --git a/.gitignore b/.gitignore index 626b0a1..63b2ddb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -/target +**/target Cargo.lock src/schema.rs diff --git a/fang_examples/simple_async_worker/Cargo.toml b/fang_examples/simple_async_worker/Cargo.toml new file mode 100644 index 0000000..0f138f5 --- /dev/null +++ b/fang_examples/simple_async_worker/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "simple_async_worker" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +fang = { path = "../../" , features = ["asynk"]} +env_logger = "0.9.0" +log = "0.4.0" +serde = { version = "1.0", features = ["derive"] } +tokio = { version = "1", features = ["full"] } +serde_json = "1" +async-trait = "0.1" +tokio-postgres = "0.7" diff --git a/fang_examples/simple_async_worker/src/lib.rs b/fang_examples/simple_async_worker/src/lib.rs new file mode 100644 index 0000000..58c206e --- /dev/null +++ b/fang_examples/simple_async_worker/src/lib.rs @@ -0,0 +1,34 @@ +use async_trait::async_trait; +use fang::asynk::async_queue::AsyncQueueable; +use fang::asynk::async_runnable::Error; +use fang::typetag; +use fang::AsyncRunnable; +use serde::Deserialize; +use serde::Serialize; +use std::time::Duration; + +#[derive(Serialize, Deserialize)] +pub struct MyTask { + pub number: u16, +} + +impl MyTask { + pub fn new(number: u16) -> Self { + Self { number } + } +} + +#[async_trait] +#[typetag::serde] +impl AsyncRunnable for MyTask { + async fn run(&self, queue: &mut dyn AsyncQueueable) -> Result<(), Error> { + log::info!("the curreny number is {}", self.number); + tokio::time::sleep(Duration::from_secs(3)).await; + + let new_task = MyTask::new(self.number + 1); + let metadata = serde_json::to_value(&new_task as &dyn AsyncRunnable).unwrap(); + queue.insert_task(metadata, "common").await.unwrap(); + + Ok(()) + } +} diff --git a/fang_examples/simple_async_worker/src/main.rs b/fang_examples/simple_async_worker/src/main.rs new file mode 100644 index 0000000..c5012b0 --- /dev/null +++ b/fang_examples/simple_async_worker/src/main.rs @@ -0,0 +1,41 @@ +use fang::asynk::async_queue::AsyncQueue; +use fang::asynk::async_queue::AsyncQueueable; +use fang::asynk::async_worker_pool::AsyncWorkerPool; +use fang::AsyncRunnable; +use simple_async_worker::MyTask; +use std::time::Duration; +use tokio_postgres::NoTls; + +#[tokio::main] +async fn main() { + env_logger::init(); + + log::info!("Starting..."); + + let mut queue = AsyncQueue::connect("postgres://postgres:postgres@localhost/fang", NoTls, true) + .await + .unwrap(); + + log::info!("Queue connected..."); + + let mut pool = AsyncWorkerPool::builder() + .queue(queue.clone()) + .number_of_workers(2 as u16) + .build(); + + log::info!("Pool created ..."); + + pool.start().await; + log::info!("Workers started ..."); + + let task1 = MyTask::new(0); + let task2 = MyTask::new(20_000); + + let metadata1 = serde_json::to_value(&task1 as &dyn AsyncRunnable).unwrap(); + let metadata2 = serde_json::to_value(&task2 as &dyn AsyncRunnable).unwrap(); + + queue.insert_task(metadata1, "common").await.unwrap(); + queue.insert_task(metadata2, "common").await.unwrap(); + + tokio::time::sleep(Duration::from_secs(100)).await; +} diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 2fb3e9f..24cdd47 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -109,7 +109,7 @@ impl From for FangError { } #[async_trait] -pub trait AsyncQueueable { +pub trait AsyncQueueable: Send { async fn fetch_and_touch_task( &mut self, task_type: Option, @@ -753,7 +753,7 @@ mod async_queue_tests { } #[typetag::serde] - #[async_trait(?Send)] + #[async_trait] impl AsyncRunnable for AsyncTask { async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { Ok(()) diff --git a/src/asynk/async_runnable.rs b/src/asynk/async_runnable.rs index 048b3bd..e14f698 100644 --- a/src/asynk/async_runnable.rs +++ b/src/asynk/async_runnable.rs @@ -9,8 +9,8 @@ pub struct Error { } #[typetag::serde(tag = "type")] -#[async_trait(?Send)] -pub trait AsyncRunnable { +#[async_trait] +pub trait AsyncRunnable: Send + Sync { async fn run(&self, client: &mut dyn AsyncQueueable) -> Result<(), Error>; fn task_type(&self) -> String { diff --git a/src/asynk/async_scheduler.rs b/src/asynk/async_scheduler.rs index e369f27..fa53a63 100644 --- a/src/asynk/async_scheduler.rs +++ b/src/asynk/async_scheduler.rs @@ -139,7 +139,7 @@ mod async_scheduler_tests { } #[typetag::serde] - #[async_trait(?Send)] + #[async_trait] impl AsyncRunnable for AsyncScheduledTask { async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { Ok(()) diff --git a/src/asynk/async_worker.rs b/src/asynk/async_worker.rs index 71ec481..63425fd 100644 --- a/src/asynk/async_worker.rs +++ b/src/asynk/async_worker.rs @@ -152,7 +152,7 @@ mod async_worker_tests { } #[typetag::serde] - #[async_trait(?Send)] + #[async_trait] impl AsyncRunnable for WorkerAsyncTask { async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { Ok(()) @@ -164,7 +164,7 @@ mod async_worker_tests { } #[typetag::serde] - #[async_trait(?Send)] + #[async_trait] impl AsyncRunnable for AsyncFailedTask { async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { let message = format!("number {} is wrong :(", self.number); @@ -179,7 +179,7 @@ mod async_worker_tests { struct AsyncTaskType1 {} #[typetag::serde] - #[async_trait(?Send)] + #[async_trait] impl AsyncRunnable for AsyncTaskType1 { async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { Ok(()) @@ -194,7 +194,7 @@ mod async_worker_tests { struct AsyncTaskType2 {} #[typetag::serde] - #[async_trait(?Send)] + #[async_trait] impl AsyncRunnable for AsyncTaskType2 { async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { Ok(()) diff --git a/src/asynk/async_worker_pool.rs b/src/asynk/async_worker_pool.rs new file mode 100644 index 0000000..876bcce --- /dev/null +++ b/src/asynk/async_worker_pool.rs @@ -0,0 +1,78 @@ +use crate::asynk::async_queue::AsyncQueue; +use crate::asynk::async_queue::AsyncQueueable; +use crate::asynk::async_worker::AsyncWorker; +use crate::asynk::Error; +use crate::RetentionMode; +use crate::SleepParams; +use async_recursion::async_recursion; +use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; +use bb8_postgres::tokio_postgres::tls::TlsConnect; +use bb8_postgres::tokio_postgres::Socket; +use log::error; +use std::time::Duration; +use typed_builder::TypedBuilder; + +#[derive(TypedBuilder, Clone)] +pub struct AsyncWorkerPool +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + #[builder(setter(into))] + pub queue: AsyncQueue, + #[builder(setter(into))] + pub number_of_workers: u16, +} + +#[derive(TypedBuilder, Clone)] +pub struct WorkerParams { + #[builder(setter(into, strip_option), default)] + pub retention_mode: Option, + #[builder(setter(into, strip_option), default)] + pub sleep_params: Option, + #[builder(setter(into, strip_option), default)] + pub task_type: Option, +} + +impl AsyncWorkerPool +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + pub async fn start(&mut self) { + for _idx in 1..self.number_of_workers + 1 { + let queue = self.queue.clone(); + tokio::spawn(async move { Self::supervise_worker(queue).await }); + } + } + + #[async_recursion] + pub async fn supervise_worker(queue: AsyncQueue) -> Result<(), Error> { + let result = Self::run_worker(queue.clone()).await; + + tokio::time::sleep(Duration::from_secs(1)).await; + + match result { + Err(err) => { + error!("Worker failed. Restarting. {:?}", err); + Self::supervise_worker(queue).await + } + Ok(_) => { + error!("Worker stopped. Restarting"); + Self::supervise_worker(queue).await + } + } + } + + pub async fn run_worker(mut queue: AsyncQueue) -> Result<(), Error> { + let mut worker = AsyncWorker::builder() + .queue(&mut queue as &mut dyn AsyncQueueable) + .build(); + + worker.run_tasks().await + } +} diff --git a/src/asynk/mod.rs b/src/asynk/mod.rs index 45de86c..ab9fb02 100644 --- a/src/asynk/mod.rs +++ b/src/asynk/mod.rs @@ -2,5 +2,7 @@ pub mod async_queue; pub mod async_runnable; pub mod async_scheduler; pub mod async_worker; +pub mod async_worker_pool; + pub use async_runnable::AsyncRunnable; pub use async_runnable::Error; diff --git a/src/lib.rs b/src/lib.rs index c9d36f7..f0cec22 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,3 +59,6 @@ pub use blocking::*; #[cfg(feature = "asynk")] pub mod asynk; + +#[cfg(feature = "asynk")] +pub use asynk::*; From 7e6619637387d9bcf795d5d44e980efb9124ad0b Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Sun, 31 Jul 2022 15:15:00 +0000 Subject: [PATCH 19/25] Async Improvements AsyncRunnable param (#51) * AsyncRunnable param in insert_task and insert_periodic_task * fix example --- fang_examples/simple_async_worker/src/lib.rs | 6 +- fang_examples/simple_async_worker/src/main.rs | 13 +-- src/asynk/async_queue.rs | 81 ++++++++++--------- src/asynk/async_scheduler.rs | 10 +-- src/asynk/async_worker.rs | 3 +- 5 files changed, 57 insertions(+), 56 deletions(-) diff --git a/fang_examples/simple_async_worker/src/lib.rs b/fang_examples/simple_async_worker/src/lib.rs index 58c206e..bbef181 100644 --- a/fang_examples/simple_async_worker/src/lib.rs +++ b/fang_examples/simple_async_worker/src/lib.rs @@ -26,8 +26,10 @@ impl AsyncRunnable for MyTask { tokio::time::sleep(Duration::from_secs(3)).await; let new_task = MyTask::new(self.number + 1); - let metadata = serde_json::to_value(&new_task as &dyn AsyncRunnable).unwrap(); - queue.insert_task(metadata, "common").await.unwrap(); + queue + .insert_task(&new_task as &dyn AsyncRunnable) + .await + .unwrap(); Ok(()) } diff --git a/fang_examples/simple_async_worker/src/main.rs b/fang_examples/simple_async_worker/src/main.rs index c5012b0..7adb9da 100644 --- a/fang_examples/simple_async_worker/src/main.rs +++ b/fang_examples/simple_async_worker/src/main.rs @@ -31,11 +31,14 @@ async fn main() { let task1 = MyTask::new(0); let task2 = MyTask::new(20_000); - let metadata1 = serde_json::to_value(&task1 as &dyn AsyncRunnable).unwrap(); - let metadata2 = serde_json::to_value(&task2 as &dyn AsyncRunnable).unwrap(); - - queue.insert_task(metadata1, "common").await.unwrap(); - queue.insert_task(metadata2, "common").await.unwrap(); + queue + .insert_task(&task1 as &dyn AsyncRunnable) + .await + .unwrap(); + queue + .insert_task(&task2 as &dyn AsyncRunnable) + .await + .unwrap(); tokio::time::sleep(Duration::from_secs(100)).await; } diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index 24cdd47..a6b4791 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -1,4 +1,5 @@ use crate::asynk::async_runnable::Error as FangError; +use crate::AsyncRunnable; use async_trait::async_trait; use bb8_postgres::bb8::Pool; use bb8_postgres::bb8::RunError; @@ -95,6 +96,8 @@ pub enum AsyncQueueError { PoolError(#[from] RunError), #[error(transparent)] PgError(#[from] bb8_postgres::tokio_postgres::Error), + #[error(transparent)] + SerdeError(#[from] serde_json::Error), #[error("returned invalid result (expected {expected:?}, found {found:?})")] ResultError { expected: u64, found: u64 }, } @@ -115,11 +118,7 @@ pub trait AsyncQueueable: Send { task_type: Option, ) -> Result, AsyncQueueError>; - async fn insert_task( - &mut self, - task: serde_json::Value, - task_type: &str, - ) -> Result; + async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result; async fn remove_all_tasks(&mut self) -> Result; async fn remove_task(&mut self, task: Task) -> Result; @@ -142,7 +141,7 @@ pub trait AsyncQueueable: Send { async fn insert_periodic_task( &mut self, - metadata: serde_json::Value, + task: &dyn AsyncRunnable, timestamp: DateTime, period: i32, ) -> Result; @@ -214,18 +213,20 @@ impl AsyncQueueable for AsyncQueueTest<'_> { Ok(task) } - async fn insert_task( - &mut self, - metadata: serde_json::Value, - task_type: &str, - ) -> Result { + async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { let transaction = &mut self.transaction; + let metadata = serde_json::to_value(task)?; + let task: Task = if self.duplicated_tasks { - AsyncQueue::::insert_task_query(transaction, metadata, task_type).await? + AsyncQueue::::insert_task_query(transaction, metadata, &task.task_type()).await? } else { - AsyncQueue::::insert_task_if_not_exist_query(transaction, metadata, task_type) - .await? + AsyncQueue::::insert_task_if_not_exist_query( + transaction, + metadata, + &task.task_type(), + ) + .await? }; Ok(task) } @@ -243,12 +244,14 @@ impl AsyncQueueable for AsyncQueueTest<'_> { } async fn insert_periodic_task( &mut self, - metadata: serde_json::Value, + task: &dyn AsyncRunnable, timestamp: DateTime, period: i32, ) -> Result { let transaction = &mut self.transaction; + let metadata = serde_json::to_value(task)?; + let periodic_task = AsyncQueue::::insert_periodic_task_query( transaction, metadata, @@ -341,27 +344,27 @@ where }) } - pub async fn remove_all_tasks_query( + async fn remove_all_tasks_query( transaction: &mut Transaction<'_>, ) -> Result { Self::execute_query(transaction, REMOVE_ALL_TASK_QUERY, &[], None).await } - pub async fn remove_task_query( + async fn remove_task_query( transaction: &mut Transaction<'_>, task: Task, ) -> Result { Self::execute_query(transaction, REMOVE_TASK_QUERY, &[&task.id], Some(1)).await } - pub async fn remove_tasks_type_query( + async fn remove_tasks_type_query( transaction: &mut Transaction<'_>, task_type: &str, ) -> Result { Self::execute_query(transaction, REMOVE_TASKS_TYPE_QUERY, &[&task_type], None).await } - pub async fn fail_task_query( + async fn fail_task_query( transaction: &mut Transaction<'_>, task: Task, error_message: &str, @@ -383,7 +386,7 @@ where Ok(failed_task) } - pub async fn fetch_and_touch_task_query( + async fn fetch_and_touch_task_query( transaction: &mut Transaction<'_>, task_type: Option, ) -> Result, AsyncQueueError> { @@ -407,7 +410,7 @@ where Ok(result_task) } - pub async fn get_task_type_query( + async fn get_task_type_query( transaction: &mut Transaction<'_>, task_type: &str, ) -> Result { @@ -420,7 +423,7 @@ where Ok(task) } - pub async fn update_task_state_query( + async fn update_task_state_query( transaction: &mut Transaction<'_>, task: Task, state: FangTaskState, @@ -434,7 +437,7 @@ where Ok(task) } - pub async fn insert_task_query( + async fn insert_task_query( transaction: &mut Transaction<'_>, metadata: serde_json::Value, task_type: &str, @@ -445,7 +448,7 @@ where let task = Self::row_to_task(row); Ok(task) } - pub async fn schedule_next_task_query( + async fn schedule_next_task_query( transaction: &mut Transaction<'_>, periodic_task: PeriodicTask, ) -> Result { @@ -459,7 +462,7 @@ where let periodic_task = Self::row_to_periodic_task(row); Ok(periodic_task) } - pub async fn insert_periodic_task_query( + async fn insert_periodic_task_query( transaction: &mut Transaction<'_>, metadata: serde_json::Value, timestamp: DateTime, @@ -475,7 +478,7 @@ where Ok(periodic_task) } - pub async fn fetch_periodic_tasks_query( + async fn fetch_periodic_tasks_query( transaction: &mut Transaction<'_>, error_margin_seconds: i64, ) -> Result>, AsyncQueueError> { @@ -498,7 +501,7 @@ where Ok(Some(periodic_tasks)) } } - pub async fn execute_query( + async fn execute_query( transaction: &mut Transaction<'_>, query: &str, params: &[&(dyn ToSql + Sync)], @@ -517,7 +520,7 @@ where Ok(result) } - pub async fn insert_task_if_not_exist_query( + async fn insert_task_if_not_exist_query( transaction: &mut Transaction<'_>, metadata: serde_json::Value, task_type: &str, @@ -527,7 +530,7 @@ where None => Self::insert_task_query(transaction, metadata, task_type).await, } } - pub async fn find_task_by_metadata_query( + async fn find_task_by_metadata_query( transaction: &mut Transaction<'_>, metadata: &serde_json::Value, ) -> Option { @@ -606,18 +609,17 @@ where Ok(task) } - async fn insert_task( - &mut self, - metadata: serde_json::Value, - task_type: &str, - ) -> Result { + async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { let mut connection = self.pool.get().await?; let mut transaction = connection.transaction().await?; + let metadata = serde_json::to_value(task)?; + let task: Task = if self.duplicated_tasks { - Self::insert_task_query(&mut transaction, metadata, task_type).await? + Self::insert_task_query(&mut transaction, metadata, &task.task_type()).await? } else { - Self::insert_task_if_not_exist_query(&mut transaction, metadata, task_type).await? + Self::insert_task_if_not_exist_query(&mut transaction, metadata, &task.task_type()) + .await? }; transaction.commit().await?; @@ -627,13 +629,15 @@ where async fn insert_periodic_task( &mut self, - metadata: serde_json::Value, + task: &dyn AsyncRunnable, timestamp: DateTime, period: i32, ) -> Result { let mut connection = self.pool.get().await?; let mut transaction = connection.transaction().await?; + let metadata = serde_json::to_value(task)?; + let periodic_task = Self::insert_periodic_task_query(&mut transaction, metadata, timestamp, period).await?; @@ -949,8 +953,7 @@ mod async_queue_tests { } async fn insert_task(test: &mut AsyncQueueTest<'_>, task: &dyn AsyncRunnable) -> Task { - let metadata = serde_json::to_value(task).unwrap(); - test.insert_task(metadata, &task.task_type()).await.unwrap() + test.insert_task(task).await.unwrap() } async fn pool() -> Pool> { diff --git a/src/asynk/async_scheduler.rs b/src/asynk/async_scheduler.rs index fa53a63..aec1561 100644 --- a/src/asynk/async_scheduler.rs +++ b/src/asynk/async_scheduler.rs @@ -76,14 +76,10 @@ impl<'a> Scheduler<'a> { self.queue.schedule_next_task(task).await?; } Some(_) => { - let metadata = task.metadata.clone(); - let actual_task: Box = serde_json::from_value(task.metadata.clone()).unwrap(); - self.queue - .insert_task(metadata, &(*actual_task).task_type()) - .await?; + self.queue.insert_task(&*actual_task).await?; self.queue.schedule_next_task(task).await?; } @@ -203,9 +199,7 @@ mod async_scheduler_tests { timestamp: DateTime, period_in_seconds: i32, ) -> PeriodicTask { - let metadata = serde_json::to_value(task).unwrap(); - - test.insert_periodic_task(metadata, timestamp, period_in_seconds) + test.insert_periodic_task(task, timestamp, period_in_seconds) .await .unwrap() } diff --git a/src/asynk/async_worker.rs b/src/asynk/async_worker.rs index 63425fd..16f8e08 100644 --- a/src/asynk/async_worker.rs +++ b/src/asynk/async_worker.rs @@ -326,8 +326,7 @@ mod async_worker_tests { test.transaction.rollback().await.unwrap(); } async fn insert_task(test: &mut AsyncQueueTest<'_>, task: &dyn AsyncRunnable) -> Task { - let metadata = serde_json::to_value(task).unwrap(); - test.insert_task(metadata, &task.task_type()).await.unwrap() + test.insert_task(task).await.unwrap() } async fn pool() -> Pool> { let pg_mgr = PostgresConnectionManager::new_from_stringlike( From 785dc88b8f7913187edd8b14906661a0bf028928 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Mon, 1 Aug 2022 16:55:36 +0000 Subject: [PATCH 20/25] configure workers (#52) --- src/asynk/async_worker_pool.rs | 35 ++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/src/asynk/async_worker_pool.rs b/src/asynk/async_worker_pool.rs index 876bcce..9ee1139 100644 --- a/src/asynk/async_worker_pool.rs +++ b/src/asynk/async_worker_pool.rs @@ -2,8 +2,7 @@ use crate::asynk::async_queue::AsyncQueue; use crate::asynk::async_queue::AsyncQueueable; use crate::asynk::async_worker::AsyncWorker; use crate::asynk::Error; -use crate::RetentionMode; -use crate::SleepParams; +use crate::{RetentionMode, SleepParams}; use async_recursion::async_recursion; use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; use bb8_postgres::tokio_postgres::tls::TlsConnect; @@ -22,6 +21,10 @@ where { #[builder(setter(into))] pub queue: AsyncQueue, + #[builder(default, setter(into))] + pub sleep_params: SleepParams, + #[builder(default, setter(into))] + pub retention_mode: RetentionMode, #[builder(setter(into))] pub number_of_workers: u16, } @@ -46,31 +49,47 @@ where pub async fn start(&mut self) { for _idx in 1..self.number_of_workers + 1 { let queue = self.queue.clone(); - tokio::spawn(async move { Self::supervise_worker(queue).await }); + let sleep_params = self.sleep_params.clone(); + let retention_mode = self.retention_mode.clone(); + + tokio::spawn(async move { + Self::supervise_worker(queue, sleep_params, retention_mode).await + }); } } #[async_recursion] - pub async fn supervise_worker(queue: AsyncQueue) -> Result<(), Error> { - let result = Self::run_worker(queue.clone()).await; + pub async fn supervise_worker( + queue: AsyncQueue, + sleep_params: SleepParams, + retention_mode: RetentionMode, + ) -> Result<(), Error> { + let result = + Self::run_worker(queue.clone(), sleep_params.clone(), retention_mode.clone()).await; tokio::time::sleep(Duration::from_secs(1)).await; match result { Err(err) => { error!("Worker failed. Restarting. {:?}", err); - Self::supervise_worker(queue).await + Self::supervise_worker(queue, sleep_params, retention_mode).await } Ok(_) => { error!("Worker stopped. Restarting"); - Self::supervise_worker(queue).await + Self::supervise_worker(queue, sleep_params, retention_mode).await } } } - pub async fn run_worker(mut queue: AsyncQueue) -> Result<(), Error> { + pub async fn run_worker( + mut queue: AsyncQueue, + sleep_params: SleepParams, + retention_mode: RetentionMode, + ) -> Result<(), Error> { let mut worker = AsyncWorker::builder() .queue(&mut queue as &mut dyn AsyncQueueable) + .sleep_params(sleep_params) + .retention_mode(retention_mode) .build(); worker.run_tasks().await From 36393676e70e362c7f2f2d317aede87d200ae9e8 Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Mon, 1 Aug 2022 19:57:17 +0300 Subject: [PATCH 21/25] limit the number of features for tokio (#53) --- Cargo.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d8d67aa..ebffffd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ optional = true [dependencies.tokio] version = "1.20" -features = ["full"] +features = ["rt", "time"] optional = true [dependencies.async-trait] @@ -64,7 +64,9 @@ optional = true version = "0.10" optional = true - [dependencies.async-recursion] version = "1" -optional = true \ No newline at end of file +optional = true + +[dev-dependencies] +tokio = { version = "1.20", features = ["macros"] } \ No newline at end of file From 549f5a1c4bdc444cf067d248c4cdf2d1b78f7902 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Mon, 1 Aug 2022 19:37:32 +0000 Subject: [PATCH 22/25] Max number of conns (#54) * Max number of conns * fixing builder , connect and example * fix comments * fix comments * Update src/asynk/async_queue.rs Co-authored-by: Ayrat Badykov --- fang_examples/simple_async_worker/src/main.rs | 13 ++-- src/asynk/async_queue.rs | 71 +++++++++++++------ src/asynk/async_worker_pool.rs | 4 +- 3 files changed, 58 insertions(+), 30 deletions(-) diff --git a/fang_examples/simple_async_worker/src/main.rs b/fang_examples/simple_async_worker/src/main.rs index 7adb9da..b4cd5f1 100644 --- a/fang_examples/simple_async_worker/src/main.rs +++ b/fang_examples/simple_async_worker/src/main.rs @@ -11,16 +11,19 @@ async fn main() { env_logger::init(); log::info!("Starting..."); + let max_pool_size: u32 = 2; + let mut queue = AsyncQueue::builder() + .uri("postgres://postgres:postgres@localhost/fang") + .max_pool_size(max_pool_size) + .duplicated_tasks(true) + .build(); - let mut queue = AsyncQueue::connect("postgres://postgres:postgres@localhost/fang", NoTls, true) - .await - .unwrap(); - + queue.connect(NoTls).await.unwrap(); log::info!("Queue connected..."); let mut pool = AsyncWorkerPool::builder() + .number_of_workers(max_pool_size) .queue(queue.clone()) - .number_of_workers(2 as u16) .build(); log::info!("Pool created ..."); diff --git a/src/asynk/async_queue.rs b/src/asynk/async_queue.rs index a6b4791..6c59704 100644 --- a/src/asynk/async_queue.rs +++ b/src/asynk/async_queue.rs @@ -100,6 +100,10 @@ pub enum AsyncQueueError { SerdeError(#[from] serde_json::Error), #[error("returned invalid result (expected {expected:?}, found {found:?})")] ResultError { expected: u64, found: u64 }, + #[error( + "AsyncQueue is not connected :( , call connect() method first and then perform operations" + )] + NotConnectedError, } impl From for FangError { @@ -159,10 +163,16 @@ where >::TlsConnect: Send, <>::TlsConnect as TlsConnect>::Future: Send, { + #[builder(default=None, setter(skip))] + pool: Option>>, #[builder(setter(into))] - pool: Pool>, + uri: String, + #[builder(setter(into))] + max_pool_size: u32, #[builder(default = false, setter(into))] duplicated_tasks: bool, + #[builder(default = false, setter(skip))] + connected: bool, } #[cfg(test)] @@ -330,20 +340,25 @@ where >::TlsConnect: Send, <>::TlsConnect as TlsConnect>::Future: Send, { - pub async fn connect( - uri: impl ToString, - tls: Tls, - duplicated_tasks: bool, - ) -> Result { - let manager = PostgresConnectionManager::new_from_stringlike(uri, tls)?; - let pool = Pool::builder().build(manager).await?; - - Ok(Self { - pool, - duplicated_tasks, - }) + pub fn check_if_connection(&self) -> Result<(), AsyncQueueError> { + if self.connected { + Ok(()) + } else { + Err(AsyncQueueError::NotConnectedError) + } } + pub async fn connect(&mut self, tls: Tls) -> Result<(), AsyncQueueError> { + let manager = PostgresConnectionManager::new_from_stringlike(self.uri.clone(), tls)?; + let pool = Pool::builder() + .max_size(self.max_pool_size) + .build(manager) + .await?; + + self.pool = Some(pool); + self.connected = true; + Ok(()) + } async fn remove_all_tasks_query( transaction: &mut Transaction<'_>, ) -> Result { @@ -599,7 +614,8 @@ where &mut self, task_type: Option, ) -> Result, AsyncQueueError> { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let task = Self::fetch_and_touch_task_query(&mut transaction, task_type).await?; @@ -610,7 +626,8 @@ where } async fn insert_task(&mut self, task: &dyn AsyncRunnable) -> Result { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let metadata = serde_json::to_value(task)?; @@ -633,7 +650,8 @@ where timestamp: DateTime, period: i32, ) -> Result { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let metadata = serde_json::to_value(task)?; @@ -650,7 +668,8 @@ where &mut self, periodic_task: PeriodicTask, ) -> Result { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let periodic_task = Self::schedule_next_task_query(&mut transaction, periodic_task).await?; @@ -664,7 +683,8 @@ where &mut self, error_margin_seconds: i64, ) -> Result>, AsyncQueueError> { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let periodic_task = @@ -676,7 +696,8 @@ where } async fn remove_all_tasks(&mut self) -> Result { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let result = Self::remove_all_tasks_query(&mut transaction).await?; @@ -687,7 +708,8 @@ where } async fn remove_task(&mut self, task: Task) -> Result { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let result = Self::remove_task_query(&mut transaction, task).await?; @@ -698,7 +720,8 @@ where } async fn remove_tasks_type(&mut self, task_type: &str) -> Result { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let result = Self::remove_tasks_type_query(&mut transaction, task_type).await?; @@ -713,7 +736,8 @@ where task: Task, state: FangTaskState, ) -> Result { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let task = Self::update_task_state_query(&mut transaction, task, state).await?; @@ -727,7 +751,8 @@ where task: Task, error_message: &str, ) -> Result { - let mut connection = self.pool.get().await?; + self.check_if_connection()?; + let mut connection = self.pool.as_ref().unwrap().get().await?; let mut transaction = connection.transaction().await?; let task = Self::fail_task_query(&mut transaction, task, error_message).await?; diff --git a/src/asynk/async_worker_pool.rs b/src/asynk/async_worker_pool.rs index 9ee1139..df7c201 100644 --- a/src/asynk/async_worker_pool.rs +++ b/src/asynk/async_worker_pool.rs @@ -26,7 +26,7 @@ where #[builder(default, setter(into))] pub retention_mode: RetentionMode, #[builder(setter(into))] - pub number_of_workers: u16, + pub number_of_workers: u32, } #[derive(TypedBuilder, Clone)] @@ -47,7 +47,7 @@ where <>::TlsConnect as TlsConnect>::Future: Send, { pub async fn start(&mut self) { - for _idx in 1..self.number_of_workers + 1 { + for _idx in 0..self.number_of_workers { let queue = self.queue.clone(); let sleep_params = self.sleep_params.clone(); let retention_mode = self.retention_mode.clone(); From 140b19e6e4b8a78eac112c6a6ea89f06b87d2a04 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Tue, 2 Aug 2022 14:32:58 +0000 Subject: [PATCH 23/25] Generic Worker Pool (#55) * generic async worker pool !! * cfg tests --- fang_examples/simple_async_worker/src/main.rs | 2 +- src/asynk/async_worker.rs | 99 +++++++++++++++++-- src/asynk/async_worker_pool.rs | 28 ++---- 3 files changed, 99 insertions(+), 30 deletions(-) diff --git a/fang_examples/simple_async_worker/src/main.rs b/fang_examples/simple_async_worker/src/main.rs index b4cd5f1..a228219 100644 --- a/fang_examples/simple_async_worker/src/main.rs +++ b/fang_examples/simple_async_worker/src/main.rs @@ -21,7 +21,7 @@ async fn main() { queue.connect(NoTls).await.unwrap(); log::info!("Queue connected..."); - let mut pool = AsyncWorkerPool::builder() + let mut pool: AsyncWorkerPool> = AsyncWorkerPool::builder() .number_of_workers(max_pool_size) .queue(queue.clone()) .build(); diff --git a/src/asynk/async_worker.rs b/src/asynk/async_worker.rs index 16f8e08..5c7a89f 100644 --- a/src/asynk/async_worker.rs +++ b/src/asynk/async_worker.rs @@ -10,9 +10,12 @@ use std::time::Duration; use typed_builder::TypedBuilder; #[derive(TypedBuilder)] -pub struct AsyncWorker<'a> { +pub struct AsyncWorker +where + AQueue: AsyncQueueable + Clone + Sync + 'static, +{ #[builder(setter(into))] - pub queue: &'a mut dyn AsyncQueueable, + pub queue: AQueue, #[builder(default=DEFAULT_TASK_TYPE.to_string(), setter(into))] pub task_type: String, #[builder(default, setter(into))] @@ -21,7 +24,10 @@ pub struct AsyncWorker<'a> { pub retention_mode: RetentionMode, } -impl<'a> AsyncWorker<'a> { +impl AsyncWorker +where + AQueue: AsyncQueueable + Clone + Sync + 'static, +{ pub async fn run(&mut self, task: Task) -> Result<(), Error> { let result = self.execute_task(task).await; self.finalize_task(result).await @@ -31,7 +37,7 @@ impl<'a> AsyncWorker<'a> { let actual_task: Box = serde_json::from_value(task.metadata.clone()).unwrap(); - let task_result = actual_task.run(self.queue).await; + let task_result = actual_task.run(&mut self.queue).await; match task_result { Ok(()) => Ok(task), Err(error) => Err((task, error.description)), @@ -104,8 +110,81 @@ impl<'a> AsyncWorker<'a> { }; } } +} - #[cfg(test)] +#[cfg(test)] +#[derive(TypedBuilder)] +pub struct AsyncWorkerTest<'a> { + #[builder(setter(into))] + pub queue: &'a mut dyn AsyncQueueable, + #[builder(default=DEFAULT_TASK_TYPE.to_string(), setter(into))] + pub task_type: String, + #[builder(default, setter(into))] + pub sleep_params: SleepParams, + #[builder(default, setter(into))] + pub retention_mode: RetentionMode, +} + +#[cfg(test)] +impl<'a> AsyncWorkerTest<'a> { + pub async fn run(&mut self, task: Task) -> Result<(), Error> { + let result = self.execute_task(task).await; + self.finalize_task(result).await + } + + async fn execute_task(&mut self, task: Task) -> Result { + let actual_task: Box = + serde_json::from_value(task.metadata.clone()).unwrap(); + + let task_result = actual_task.run(self.queue).await; + match task_result { + Ok(()) => Ok(task), + Err(error) => Err((task, error.description)), + } + } + + async fn finalize_task(&mut self, result: Result) -> Result<(), Error> { + match self.retention_mode { + RetentionMode::KeepAll => match result { + Ok(task) => { + self.queue + .update_task_state(task, FangTaskState::Finished) + .await?; + Ok(()) + } + Err((task, error)) => { + self.queue.fail_task(task, &error).await?; + Ok(()) + } + }, + RetentionMode::RemoveAll => match result { + Ok(task) => { + self.queue.remove_task(task).await?; + Ok(()) + } + Err((task, _error)) => { + self.queue.remove_task(task).await?; + Ok(()) + } + }, + RetentionMode::RemoveFinished => match result { + Ok(task) => { + self.queue.remove_task(task).await?; + Ok(()) + } + Err((task, error)) => { + self.queue.fail_task(task, &error).await?; + Ok(()) + } + }, + } + } + + pub async fn sleep(&mut self) { + self.sleep_params.maybe_increase_sleep_period(); + + tokio::time::sleep(Duration::from_secs(self.sleep_params.sleep_period)).await; + } pub async fn run_tasks_until_none(&mut self) -> Result<(), Error> { loop { match self @@ -132,7 +211,7 @@ impl<'a> AsyncWorker<'a> { #[cfg(test)] mod async_worker_tests { - use super::AsyncWorker; + use super::AsyncWorkerTest; use crate::asynk::async_queue::AsyncQueueTest; use crate::asynk::async_queue::AsyncQueueable; use crate::asynk::async_queue::FangTaskState; @@ -215,7 +294,7 @@ mod async_worker_tests { let task = insert_task(&mut test, &WorkerAsyncTask { number: 1 }).await; let id = task.id; - let mut worker = AsyncWorker::builder() + let mut worker = AsyncWorkerTest::builder() .queue(&mut test as &mut dyn AsyncQueueable) .retention_mode(RetentionMode::KeepAll) .build(); @@ -237,7 +316,7 @@ mod async_worker_tests { let task = insert_task(&mut test, &AsyncFailedTask { number: 1 }).await; let id = task.id; - let mut worker = AsyncWorker::builder() + let mut worker = AsyncWorkerTest::builder() .queue(&mut test as &mut dyn AsyncQueueable) .retention_mode(RetentionMode::KeepAll) .build(); @@ -269,7 +348,7 @@ mod async_worker_tests { let id12 = task12.id; let id2 = task2.id; - let mut worker = AsyncWorker::builder() + let mut worker = AsyncWorkerTest::builder() .queue(&mut test as &mut dyn AsyncQueueable) .task_type("type1".to_string()) .retention_mode(RetentionMode::KeepAll) @@ -304,7 +383,7 @@ mod async_worker_tests { let _id12 = task12.id; let id2 = task2.id; - let mut worker = AsyncWorker::builder() + let mut worker = AsyncWorkerTest::builder() .queue(&mut test as &mut dyn AsyncQueueable) .task_type("type1".to_string()) .build(); diff --git a/src/asynk/async_worker_pool.rs b/src/asynk/async_worker_pool.rs index df7c201..8a60622 100644 --- a/src/asynk/async_worker_pool.rs +++ b/src/asynk/async_worker_pool.rs @@ -1,26 +1,19 @@ -use crate::asynk::async_queue::AsyncQueue; use crate::asynk::async_queue::AsyncQueueable; use crate::asynk::async_worker::AsyncWorker; use crate::asynk::Error; use crate::{RetentionMode, SleepParams}; use async_recursion::async_recursion; -use bb8_postgres::tokio_postgres::tls::MakeTlsConnect; -use bb8_postgres::tokio_postgres::tls::TlsConnect; -use bb8_postgres::tokio_postgres::Socket; use log::error; use std::time::Duration; use typed_builder::TypedBuilder; #[derive(TypedBuilder, Clone)] -pub struct AsyncWorkerPool +pub struct AsyncWorkerPool where - Tls: MakeTlsConnect + Clone + Send + Sync + 'static, - >::Stream: Send + Sync, - >::TlsConnect: Send, - <>::TlsConnect as TlsConnect>::Future: Send, + AQueue: AsyncQueueable + Clone + Sync + 'static, { #[builder(setter(into))] - pub queue: AsyncQueue, + pub queue: AQueue, #[builder(default, setter(into))] pub sleep_params: SleepParams, #[builder(default, setter(into))] @@ -39,12 +32,9 @@ pub struct WorkerParams { pub task_type: Option, } -impl AsyncWorkerPool +impl AsyncWorkerPool where - Tls: MakeTlsConnect + Clone + Send + Sync + 'static, - >::Stream: Send + Sync, - >::TlsConnect: Send, - <>::TlsConnect as TlsConnect>::Future: Send, + AQueue: AsyncQueueable + Clone + Sync + 'static, { pub async fn start(&mut self) { for _idx in 0..self.number_of_workers { @@ -60,7 +50,7 @@ where #[async_recursion] pub async fn supervise_worker( - queue: AsyncQueue, + queue: AQueue, sleep_params: SleepParams, retention_mode: RetentionMode, ) -> Result<(), Error> { @@ -82,12 +72,12 @@ where } pub async fn run_worker( - mut queue: AsyncQueue, + queue: AQueue, sleep_params: SleepParams, retention_mode: RetentionMode, ) -> Result<(), Error> { - let mut worker = AsyncWorker::builder() - .queue(&mut queue as &mut dyn AsyncQueueable) + let mut worker: AsyncWorker = AsyncWorker::builder() + .queue(queue) .sleep_params(sleep_params) .retention_mode(retention_mode) .build(); From cf2ce19c970e93d71e387526e882c67db53cea18 Mon Sep 17 00:00:00 2001 From: Pmarquez <48651252+pxp9@users.noreply.github.com> Date: Wed, 3 Aug 2022 08:37:53 +0000 Subject: [PATCH 24/25] Handle imports and dependencies (#56) * imports, exports and README * serde_json not necessay * deleting tokio_postgres * authors * blocking example * Update fang_examples/simple_async_worker/src/lib.rs Co-authored-by: Ayrat Badykov --- Cargo.toml | 7 +- README.md | 177 +++++++++++++++++- fang_examples/simple_async_worker/Cargo.toml | 4 - fang_examples/simple_async_worker/src/lib.rs | 8 +- fang_examples/simple_async_worker/src/main.rs | 2 +- fang_examples/simple_worker/Cargo.toml | 1 - fang_examples/simple_worker/src/lib.rs | 4 +- src/lib.rs | 14 ++ 8 files changed, 193 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ebffffd..be5fed3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "fang" version = "0.6.0" -authors = ["Ayrat Badykov "] +authors = ["Ayrat Badykov " , "Pepe Márquez "] description = "Background job processing library for Rust" repository = "https://github.com/ayrat555/fang" edition = "2021" @@ -19,7 +19,8 @@ asynk = ["bb8-postgres", "postgres-types", "tokio", "async-trait", "typed-build [dependencies] chrono = "0.4" log = "0.4" -serde = { version = "1", features = ["derive"] } +serde = "1" +serde_derive = "1.0.141" serde_json = "1" thiserror = "1.0" typetag = "0.2" @@ -69,4 +70,4 @@ version = "1" optional = true [dev-dependencies] -tokio = { version = "1.20", features = ["macros"] } \ No newline at end of file +tokio = { version = "1.20", features = ["macros"] } diff --git a/README.md b/README.md index f7c2d2d..e4b5093 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,17 @@ Background task processing library for Rust. It uses Postgres DB as a task queue 1. Add this to your Cargo.toml +### Blocking ```toml [dependencies] -fang = "0.6" -serde = { version = "1.0", features = ["derive"] } +fang = { version = "0.6" , features = ["blocking"]} ``` +### Async +```toml +[dependencies] +fang = { version = "0.6" , features = ["asynk"]} +``` *Supports rustc 1.62+* 2. Create `fang_tasks` table in the Postgres database. The migration can be found in [the migrations directory](https://github.com/ayrat555/fang/blob/master/migrations/2021-06-05-112912_create_fang_tasks/up.sql). @@ -26,16 +31,18 @@ serde = { version = "1.0", features = ["derive"] } ### Defining a task -Every task should implement `fang::Runnable` trait which is used by `fang` to execute it. +#### Blocking +Every task in blocking should implement `fang::Runnable` trait which is used by `fang` to execute it. ```rust use fang::Error; use fang::Runnable; use fang::typetag; use fang::PgConnection; -use serde::{Deserialize, Serialize}; +use fang::serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] +#[serde(crate = "fang::serde")] struct MyTask { pub number: u16, } @@ -54,9 +61,40 @@ As you can see from the example above, the trait implementation has `#[typetag:: The second parameter of the `run` function is diesel's PgConnection, You can re-use it to manipulate the task queue, for example, to add a new job during the current job's execution. Or you can just re-use it in your own queries if you're using diesel. If you don't need it, just ignore it. + +#### Async +Every task in async should implement `fang::AsyncRunnable` trait which is used by `fang` to execute it. + +Also be careful to not to call with the same name two impl of AsyncRunnable, because will cause a fail with serde. +```rust +use fang::AsyncRunnable; +use fang::asynk::async_queue::AsyncQueueable; +use fang::serde::{Deserialize, Serialize}; +use fang::async_trait; + +#[derive(Serialize, Deserialize)] +#[serde(crate = "fang::serde")] +struct AsyncTask { + pub number: u16, +} + +#[typetag::serde] +#[async_trait] +impl AsyncRunnable for AsyncTask { + async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), Error> { + Ok(()) + } + // this func is optional to impl + // Default task-type it is common + fn task_type(&self) -> String { + "my-task-type".to_string() + } +} +``` ### Enqueuing a task -To enqueue a task use `Queue::enqueue_task` +#### Blocking +To enqueue a task in blocking use `Queue::enqueue_task` ```rust @@ -85,11 +123,53 @@ Or you can use `PgConnection` struct: Queue::push_task_query(pg_connection, &new_task).unwrap(); ``` +#### Async +To enqueue a task in async use `AsyncQueueable::insert_task` +depending of the backend that you prefer you will need to do it with a specific queue. + +For Postgres backend. +```rust +use fang::asynk::async_queue::AsyncQueue; +use fang::NoTls; +use fang::AsyncRunnable; + +// Create a AsyncQueue +let max_pool_size: u32 = 2; + +let mut queue = AsyncQueue::builder() + // Postgres database url + .uri("postgres://postgres:postgres@localhost/fang") + // Max number of connections that are allowed + .max_pool_size(max_pool_size) + // false if would like Uniqueness in tasks + .duplicated_tasks(true) + .build(); + +// Always connect first in order to perform any operation +queue.connect(NoTls).await.unwrap(); + +``` +For easy example we are using NoTls type, if for some reason you would like to encrypt postgres traffic. + +You can implement a Tls type. + +It is well documented for [openssl](https://docs.rs/postgres-openssl/latest/postgres_openssl/) and [native-tls](https://docs.rs/postgres-native-tls/latest/postgres_native_tls/) + +```rust +// AsyncTask from first example +let task = AsyncTask { 8 }; +let task_returned = queue + .insert_task(&task as &dyn AsyncRunnable) + .await + .unwrap(); +``` + ### Starting workers +#### Blocking Every worker runs in a separate thread. In case of panic, they are always restarted. -Use `WorkerPool` to start workers. `WorkerPool::new` accepts one parameter - the number of workers. +Use `WorkerPool` for blocking to start workers. `WorkerPool::new` accepts one parameter - the number of workers. ```rust @@ -112,17 +192,45 @@ worker_pool.shutdown() Using a library like [signal-hook][signal-hook], it's possible to gracefully shutdown a worker. See the Simple Worker for an example implementation. +#### Async +Every worker runs in a separate tokio thread. In case of panic, they are always restarted. +Use `AsyncWorkerPool` for async to start workers. + +```rust +use fang::asynk::async_worker_pool::AsyncWorkerPool; + +// Need to create a queue like before +// Also insert some tasks + +let mut pool: AsyncWorkerPool> = AsyncWorkerPool::builder() + .number_of_workers(max_pool_size) + .queue(queue.clone()) + .build(); + +pool.start().await; +``` + + Check out: - [Simple Worker Example](https://github.com/ayrat555/fang/tree/master/fang_examples/simple_worker) - simple worker example +- [Simple Async Worker Example](https://github.com/ayrat555/fang/tree/master/fang_examples/simple_async_worker) - simple async worker example - [El Monitorro](https://github.com/ayrat555/el_monitorro) - telegram feed reader. It uses Fang to synchronize feeds and deliver updates to users. ### Configuration +#### Blocking + To configure workers, instead of `WorkerPool::new` which uses default values, use `WorkerPool.new_with_params`. It accepts two parameters - the number of workers and `WorkerParams` struct. +#### Async + +Just use `TypeBuilder` done for `AsyncWorkerPool`. + ### Configuring the type of workers +#### Blocking + You can start workers for a specific types of tasks. These workers will be executing only tasks of the specified type. Add `task_type` method to the `Runnable` trait implementation: @@ -156,6 +264,12 @@ WorkerPool::new_with_params(10, worker_params).start(); Without setting `task_type` workers will be executing any type of task. +#### Async + +Same as Blocking. + +Use `TypeBuilder` for `AsyncWorker`. + ### Configuring retention mode By default, all successfully finished tasks are removed from the DB, failed tasks aren't. @@ -172,12 +286,17 @@ pub enum RetentionMode { Set retention mode with `set_retention_mode`: +#### Blocking + ```rust let mut worker_params = WorkerParams::new(); worker_params.set_retention_mode(RetentionMode::RemoveAll); WorkerPool::new_with_params(10, worker_params).start(); ``` +#### Async + +Set it in `AsyncWorker` `TypeBuilder`. ### Configuring sleep values @@ -189,7 +308,7 @@ pub struct SleepParams { pub max_sleep_period: u64, \\ default value is 15 pub min_sleep_period: u64, \\ default value is 5 pub sleep_step: u64, \\ default value is 5 -}p +} ``` If there are no tasks in the DB, a worker sleeps for `sleep_period` and each time this value increases by `sleep_step` until it reaches `max_sleep_period`. `min_sleep_period` is the initial value for `sleep_period`. All values are in seconds. @@ -208,6 +327,9 @@ worker_params.set_sleep_params(sleep_params); WorkerPool::new_with_params(10, worker_params).start(); ``` +#### Async + +Set it in `AsyncWorker` `TypeBuilder`. ## Periodic Tasks @@ -215,6 +337,8 @@ Fang can add tasks to `fang_tasks` periodically. To use this feature first run [ Usage example: +#### Blocking + ```rust use fang::Scheduler; use fang::Queue; @@ -238,6 +362,39 @@ In the example above, `push_periodic_task` is used to save the specified task to - Db check period in seconds - Acceptable error limit in seconds - |current_time - scheduled_time| < error +#### Async +```rust +use fang::asynk::async_scheduler::Scheduler; +use fang::asynk::async_queue::AsyncQueueable; +use fang::asynk::async_queue::AsyncQueue; + +// Build a AsyncQueue as before + +let schedule_in_future = Utc::now() + OtherDuration::seconds(5); + +let _periodic_task = queue.insert_periodic_task( + &AsyncTask { number: 1 }, + schedule_in_future, + 10, +) +.await; + +let check_period: u64 = 1; +let error_margin_seconds: u64 = 2; + +let mut scheduler = Scheduler::builder() + .check_period(check_period) + .error_margin_seconds(error_margin_seconds) + .queue(&mut queue as &mut dyn AsyncQueueable) + .build(); + +// Add some more task in other thread or before loop + +// Scheduler Loop +scheduler.start().await.unwrap(); +``` + + ## Contributing 1. [Fork it!](https://github.com/ayrat555/fang/fork) @@ -268,9 +425,11 @@ cargo test --all-features -- --ignored --test-threads=1 docker kill postgres ``` -## Author +## Authors -Ayrat Badykov (@ayrat555) +- Ayrat Badykov (@ayrat555) + +- Pepe Márquez (@pxp9) [s1]: https://img.shields.io/crates/v/fang.svg diff --git a/fang_examples/simple_async_worker/Cargo.toml b/fang_examples/simple_async_worker/Cargo.toml index 0f138f5..cf56b40 100644 --- a/fang_examples/simple_async_worker/Cargo.toml +++ b/fang_examples/simple_async_worker/Cargo.toml @@ -9,8 +9,4 @@ edition = "2021" fang = { path = "../../" , features = ["asynk"]} env_logger = "0.9.0" log = "0.4.0" -serde = { version = "1.0", features = ["derive"] } tokio = { version = "1", features = ["full"] } -serde_json = "1" -async-trait = "0.1" -tokio-postgres = "0.7" diff --git a/fang_examples/simple_async_worker/src/lib.rs b/fang_examples/simple_async_worker/src/lib.rs index bbef181..28207d7 100644 --- a/fang_examples/simple_async_worker/src/lib.rs +++ b/fang_examples/simple_async_worker/src/lib.rs @@ -1,13 +1,13 @@ -use async_trait::async_trait; +use fang::async_trait; use fang::asynk::async_queue::AsyncQueueable; use fang::asynk::async_runnable::Error; +use fang::serde::{Deserialize, Serialize}; use fang::typetag; use fang::AsyncRunnable; -use serde::Deserialize; -use serde::Serialize; use std::time::Duration; #[derive(Serialize, Deserialize)] +#[serde(crate = "fang::serde")] pub struct MyTask { pub number: u16, } @@ -22,7 +22,7 @@ impl MyTask { #[typetag::serde] impl AsyncRunnable for MyTask { async fn run(&self, queue: &mut dyn AsyncQueueable) -> Result<(), Error> { - log::info!("the curreny number is {}", self.number); + log::info!("the current number is {}", self.number); tokio::time::sleep(Duration::from_secs(3)).await; let new_task = MyTask::new(self.number + 1); diff --git a/fang_examples/simple_async_worker/src/main.rs b/fang_examples/simple_async_worker/src/main.rs index a228219..61fb09e 100644 --- a/fang_examples/simple_async_worker/src/main.rs +++ b/fang_examples/simple_async_worker/src/main.rs @@ -2,9 +2,9 @@ use fang::asynk::async_queue::AsyncQueue; use fang::asynk::async_queue::AsyncQueueable; use fang::asynk::async_worker_pool::AsyncWorkerPool; use fang::AsyncRunnable; +use fang::NoTls; use simple_async_worker::MyTask; use std::time::Duration; -use tokio_postgres::NoTls; #[tokio::main] async fn main() { diff --git a/fang_examples/simple_worker/Cargo.toml b/fang_examples/simple_worker/Cargo.toml index 287c552..b833463 100644 --- a/fang_examples/simple_worker/Cargo.toml +++ b/fang_examples/simple_worker/Cargo.toml @@ -7,7 +7,6 @@ edition = "2018" [dependencies] fang = { path = "../../" , features = ["blocking"]} -serde = { version = "1.0", features = ["derive"] } signal-hook = "0.3.10" dotenv = "0.15.0" env_logger = "0.9.0" diff --git a/fang_examples/simple_worker/src/lib.rs b/fang_examples/simple_worker/src/lib.rs index e4554a9..f79c02d 100644 --- a/fang_examples/simple_worker/src/lib.rs +++ b/fang_examples/simple_worker/src/lib.rs @@ -1,14 +1,14 @@ +use fang::serde::{Deserialize, Serialize}; use fang::typetag; use fang::Error; use fang::PgConnection; use fang::Queue; use fang::Runnable; -use serde::Deserialize; -use serde::Serialize; use std::thread; use std::time::Duration; #[derive(Serialize, Deserialize)] +#[serde(crate = "fang::serde")] pub struct MyJob { pub number: u16, pub current_thread_name: String, diff --git a/src/lib.rs b/src/lib.rs index f0cec22..217f9ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -52,6 +52,12 @@ pub use diesel::pg::PgConnection; #[doc(hidden)] pub use typetag; +#[doc(hidden)] +pub extern crate serde; + +#[doc(hidden)] +pub use serde_derive::{Deserialize, Serialize}; + #[cfg(feature = "blocking")] pub mod blocking; #[cfg(feature = "blocking")] @@ -62,3 +68,11 @@ pub mod asynk; #[cfg(feature = "asynk")] pub use asynk::*; + +#[cfg(feature = "asynk")] +#[doc(hidden)] +pub use bb8_postgres::tokio_postgres::tls::NoTls; + +#[cfg(feature = "asynk")] +#[doc(hidden)] +pub use async_trait::async_trait; From ead367d33f1dd6048c9cec460f28fa43de39ea6d Mon Sep 17 00:00:00 2001 From: Ayrat Badykov Date: Thu, 4 Aug 2022 18:22:53 +0300 Subject: [PATCH 25/25] Recover from panics draft (#57) * Recover from panics draft * Handling worker pool panics and Scheduler panics , fix clippy also * Update src/asynk/async_scheduler.rs Co-authored-by: Ayrat Badykov * add new task Co-authored-by: pxp9 <48651252+pxp9@users.noreply.github.com> --- Cargo.toml | 5 +- fang_examples/simple_async_worker/src/lib.rs | 41 ++++++- fang_examples/simple_async_worker/src/main.rs | 9 +- src/asynk/async_scheduler.rs | 103 +++++++++++++----- src/asynk/async_worker_pool.rs | 62 +++++------ 5 files changed, 149 insertions(+), 71 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index be5fed3..e2b9287 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ optional = true [dependencies.tokio] version = "1.20" -features = ["rt", "time"] +features = ["rt", "time", "macros"] optional = true [dependencies.async-trait] @@ -68,6 +68,3 @@ optional = true [dependencies.async-recursion] version = "1" optional = true - -[dev-dependencies] -tokio = { version = "1.20", features = ["macros"] } diff --git a/fang_examples/simple_async_worker/src/lib.rs b/fang_examples/simple_async_worker/src/lib.rs index 28207d7..25951b3 100644 --- a/fang_examples/simple_async_worker/src/lib.rs +++ b/fang_examples/simple_async_worker/src/lib.rs @@ -18,19 +18,54 @@ impl MyTask { } } +#[derive(Serialize, Deserialize)] +#[serde(crate = "fang::serde")] +pub struct MyFailingTask { + pub number: u16, +} + +impl MyFailingTask { + pub fn new(number: u16) -> Self { + Self { number } + } +} + #[async_trait] #[typetag::serde] impl AsyncRunnable for MyTask { async fn run(&self, queue: &mut dyn AsyncQueueable) -> Result<(), Error> { - log::info!("the current number is {}", self.number); - tokio::time::sleep(Duration::from_secs(3)).await; - let new_task = MyTask::new(self.number + 1); queue .insert_task(&new_task as &dyn AsyncRunnable) .await .unwrap(); + log::info!("the current number is {}", self.number); + tokio::time::sleep(Duration::from_secs(3)).await; + Ok(()) } } + +#[async_trait] +#[typetag::serde] +impl AsyncRunnable for MyFailingTask { + async fn run(&self, queue: &mut dyn AsyncQueueable) -> Result<(), Error> { + let new_task = MyFailingTask::new(self.number + 1); + queue + .insert_task(&new_task as &dyn AsyncRunnable) + .await + .unwrap(); + + log::info!("the current number is {}", self.number); + tokio::time::sleep(Duration::from_secs(3)).await; + + let b = true; + + if b { + panic!("Hello!"); + } else { + Ok(()) + } + } +} diff --git a/fang_examples/simple_async_worker/src/main.rs b/fang_examples/simple_async_worker/src/main.rs index 61fb09e..ff58e8c 100644 --- a/fang_examples/simple_async_worker/src/main.rs +++ b/fang_examples/simple_async_worker/src/main.rs @@ -3,6 +3,7 @@ use fang::asynk::async_queue::AsyncQueueable; use fang::asynk::async_worker_pool::AsyncWorkerPool; use fang::AsyncRunnable; use fang::NoTls; +use simple_async_worker::MyFailingTask; use simple_async_worker::MyTask; use std::time::Duration; @@ -22,7 +23,7 @@ async fn main() { log::info!("Queue connected..."); let mut pool: AsyncWorkerPool> = AsyncWorkerPool::builder() - .number_of_workers(max_pool_size) + .number_of_workers(10_u32) .queue(queue.clone()) .build(); @@ -33,6 +34,7 @@ async fn main() { let task1 = MyTask::new(0); let task2 = MyTask::new(20_000); + let task3 = MyFailingTask::new(50_000); queue .insert_task(&task1 as &dyn AsyncRunnable) @@ -43,5 +45,10 @@ async fn main() { .await .unwrap(); + queue + .insert_task(&task3 as &dyn AsyncRunnable) + .await + .unwrap(); + tokio::time::sleep(Duration::from_secs(100)).await; } diff --git a/src/asynk/async_scheduler.rs b/src/asynk/async_scheduler.rs index aec1561..eed582a 100644 --- a/src/asynk/async_scheduler.rs +++ b/src/asynk/async_scheduler.rs @@ -5,56 +5,75 @@ use crate::asynk::Error; use async_recursion::async_recursion; use log::error; use std::time::Duration; +use tokio::task::JoinHandle; use tokio::time::sleep; use typed_builder::TypedBuilder; -#[derive(TypedBuilder)] -pub struct Scheduler<'a> { +#[derive(TypedBuilder, Clone)] +pub struct Scheduler +where + AQueue: AsyncQueueable + Clone + Sync + 'static, +{ #[builder(setter(into))] pub check_period: u64, #[builder(setter(into))] pub error_margin_seconds: u64, #[builder(setter(into))] - pub queue: &'a mut dyn AsyncQueueable, + pub queue: AQueue, #[builder(default = 0, setter(into))] pub number_of_restarts: u32, } -impl<'a> Scheduler<'a> { +impl Scheduler +where + AQueue: AsyncQueueable + Clone + Sync + 'static, +{ #[async_recursion(?Send)] pub async fn start(&mut self) -> Result<(), Error> { - let task_res = self.schedule_loop().await; + let join_handle: JoinHandle> = self.schedule_loop().await; - sleep(Duration::from_secs(1)).await; - - match task_res { + match join_handle.await { Err(err) => { error!( - "Scheduler failed, restarting {:?}. Number of restarts {}", + "Scheduler panicked, restarting {:?}. Number of restarts {}", err, self.number_of_restarts ); self.number_of_restarts += 1; + sleep(Duration::from_secs(1)).await; self.start().await } - Ok(_) => { - error!( - "Scheduler stopped. restarting. Number of restarts {}", - self.number_of_restarts - ); - self.number_of_restarts += 1; - self.start().await - } + Ok(task_res) => match task_res { + Err(err) => { + error!( + "Scheduler failed, restarting {:?}. Number of restarts {}", + err, self.number_of_restarts + ); + self.number_of_restarts += 1; + self.start().await + } + Ok(_) => { + error!( + "Scheduler stopped. restarting. Number of restarts {}", + self.number_of_restarts + ); + self.number_of_restarts += 1; + self.start().await + } + }, } } - pub async fn schedule_loop(&mut self) -> Result<(), Error> { - let sleep_duration = Duration::from_secs(self.check_period); + pub async fn schedule_loop(&mut self) -> JoinHandle> { + let mut scheduler = self.clone(); + tokio::spawn(async move { + let sleep_duration = Duration::from_secs(scheduler.check_period); - loop { - self.schedule().await?; + loop { + scheduler.schedule().await?; - sleep(sleep_duration).await; - } + sleep(sleep_duration).await; + } + }) } pub async fn schedule(&mut self) -> Result<(), Error> { @@ -86,8 +105,23 @@ impl<'a> Scheduler<'a> { } Ok(()) } +} - #[cfg(test)] +#[cfg(test)] +#[derive(TypedBuilder)] +pub struct SchedulerTest<'a> { + #[builder(setter(into))] + pub check_period: u64, + #[builder(setter(into))] + pub error_margin_seconds: u64, + #[builder(setter(into))] + pub queue: &'a mut dyn AsyncQueueable, + #[builder(default = 0, setter(into))] + pub number_of_restarts: u32, +} + +#[cfg(test)] +impl<'a> SchedulerTest<'a> { async fn schedule_test(&mut self) -> Result<(), Error> { let sleep_duration = Duration::from_secs(self.check_period); @@ -110,11 +144,28 @@ impl<'a> Scheduler<'a> { }; } } + + async fn process_task(&mut self, task: PeriodicTask) -> Result<(), Error> { + match task.scheduled_at { + None => { + self.queue.schedule_next_task(task).await?; + } + Some(_) => { + let actual_task: Box = + serde_json::from_value(task.metadata.clone()).unwrap(); + + self.queue.insert_task(&*actual_task).await?; + + self.queue.schedule_next_task(task).await?; + } + } + Ok(()) + } } #[cfg(test)] mod async_scheduler_tests { - use super::Scheduler; + use super::SchedulerTest; use crate::asynk::async_queue::AsyncQueueTest; use crate::asynk::async_queue::AsyncQueueable; use crate::asynk::async_queue::PeriodicTask; @@ -166,7 +217,7 @@ mod async_scheduler_tests { let check_period: u64 = 1; let error_margin_seconds: u64 = 2; - let mut scheduler = Scheduler::builder() + let mut scheduler = SchedulerTest::builder() .check_period(check_period) .error_margin_seconds(error_margin_seconds) .queue(&mut test as &mut dyn AsyncQueueable) diff --git a/src/asynk/async_worker_pool.rs b/src/asynk/async_worker_pool.rs index 8a60622..d84754c 100644 --- a/src/asynk/async_worker_pool.rs +++ b/src/asynk/async_worker_pool.rs @@ -4,7 +4,7 @@ use crate::asynk::Error; use crate::{RetentionMode, SleepParams}; use async_recursion::async_recursion; use log::error; -use std::time::Duration; +use tokio::task::JoinHandle; use typed_builder::TypedBuilder; #[derive(TypedBuilder, Clone)] @@ -22,55 +22,43 @@ where pub number_of_workers: u32, } -#[derive(TypedBuilder, Clone)] -pub struct WorkerParams { - #[builder(setter(into, strip_option), default)] - pub retention_mode: Option, - #[builder(setter(into, strip_option), default)] - pub sleep_params: Option, - #[builder(setter(into, strip_option), default)] - pub task_type: Option, -} - impl AsyncWorkerPool where AQueue: AsyncQueueable + Clone + Sync + 'static, { pub async fn start(&mut self) { - for _idx in 0..self.number_of_workers { - let queue = self.queue.clone(); - let sleep_params = self.sleep_params.clone(); - let retention_mode = self.retention_mode.clone(); - - tokio::spawn(async move { - Self::supervise_worker(queue, sleep_params, retention_mode).await - }); + for idx in 0..self.number_of_workers { + let pool = self.clone(); + tokio::spawn(Self::supervise_task(pool, 0, idx)); } } #[async_recursion] - pub async fn supervise_worker( - queue: AQueue, - sleep_params: SleepParams, - retention_mode: RetentionMode, - ) -> Result<(), Error> { - let result = - Self::run_worker(queue.clone(), sleep_params.clone(), retention_mode.clone()).await; + pub async fn supervise_task(pool: AsyncWorkerPool, restarts: u64, worker_number: u32) { + let restarts = restarts + 1; + let join_handle = Self::spawn_worker( + pool.queue.clone(), + pool.sleep_params.clone(), + pool.retention_mode.clone(), + ) + .await; - tokio::time::sleep(Duration::from_secs(1)).await; - - match result { - Err(err) => { - error!("Worker failed. Restarting. {:?}", err); - Self::supervise_worker(queue, sleep_params, retention_mode).await - } - Ok(_) => { - error!("Worker stopped. Restarting"); - Self::supervise_worker(queue, sleep_params, retention_mode).await - } + if (join_handle.await).is_err() { + error!( + "Worker {} stopped. Restarting. the number of restarts {}", + worker_number, restarts, + ); + Self::supervise_task(pool, restarts, worker_number).await; } } + pub async fn spawn_worker( + queue: AQueue, + sleep_params: SleepParams, + retention_mode: RetentionMode, + ) -> JoinHandle> { + tokio::spawn(async move { Self::run_worker(queue, sleep_params, retention_mode).await }) + } pub async fn run_worker( queue: AQueue, sleep_params: SleepParams,