Re-org project

This commit is contained in:
Rafael Caricio 2023-03-05 01:19:35 +01:00
parent ada981865d
commit cb1ee8e854
Signed by: rafaelcaricio
GPG key ID: 3C86DBCE8E93C947
13 changed files with 76 additions and 212 deletions

View file

@ -1,113 +0,0 @@
# Changelog
## 0.10.2 (2023-02-23)
### Improved
- update `diesel-derive-enum` from `2.0.0-rc.0` to `2.0.1` - [#111](https://github.com/ayrat555/fang/pull/111)
- use `dotenvy` instead of `dotenv` - [#111](https://github.com/ayrat555/fang/pull/111)
- update `tokio` from `1.20` to `1.25` - [#111](https://github.com/ayrat555/fang/pull/111)
## 0.10.1 (2023-02-03)
### Improved
- Update typed-builder requirement from 0.11 to 0.12 - [#108](https://github.com/ayrat555/fang/pull/108)
- Add dodcumentation - [#96](https://github.com/ayrat555/fang/pull/96), [#104](https://github.com/ayrat555/fang/pull/104)
- Fix clippy warnings - [#109](https://github.com/ayrat555/fang/pull/109)
## 0.10.0 (2022-09-27)
### Added
- Add retries for tasks - [#92](https://github.com/ayrat555/fang/pull/92)
### Improved
- Update the cron crate from 0.11 to 0.12 - [#97](https://github.com/ayrat555/fang/pull/97)
## 0.9.1 (2022-09-14)
### Added
- Add methods to remove a task by uniq hash or id - [#90](https://github.com/ayrat555/fang/pull/90)
## 0.9.0 (2022-09-02)
### [#69](https://github.com/ayrat555/fang/pull/69)
### Added
- Added cron support for scheduled tasks.
### Improved
- Major refactoring of the blocking module.
- Delete the graceful shutdown in blocking module.
- Simplify database schema.
- Re-write scheduled tasks in asynk module.
- Re-write the errors in both modules.
- Update diesel crate to 2.0 version.
- Update uuid crate to 1.1 version.
## 0.8.0 (2022-08-18)
- Use Duration in SleepParams and schedulers - [#67](https://github.com/ayrat555/fang/pull/67)
## 0.7.2 (2022-08-16)
- Set task_type when starting a worker pool - [#66](https://github.com/ayrat555/fang/pull/66)
## 0.7.1 (2022-08-04)
- Fix a conflict in exports of the `blocking` and the `asynk` features - [#61](https://github.com/ayrat555/fang/pull/61)
## 0.7.0 (2022-08-03)
### [#21](https://github.com/ayrat555/fang/pull/21)
- Create a full functional async processing module for fang which uses `tokio` to summon processes that are more lighter than threads.
- Rename Job starting names into Task [#29](https://github.com/ayrat555/fang/pull/29).
- Export serde correctly - [commit](https://github.com/ayrat555/fang/pull/21/commits/cf2ce19c970e93d71e387526e882c67db53cea18).
- Implement Postgres backend for async.
## 0.6.0 (2022-07-01)
- Rust 1.62 is required for `typetag`
## 0.5.0 (2021-12-05)
- Add graceful shutdown - [#14](https://github.com/ayrat555/fang/pull/14)
## 0.4.2 (2021-11-30)
- Bump deps - [#13](https://github.com/ayrat555/fang/pull/13)
## 0.4.1 (2021-08-22)
- Do not use env-logger - [#11](https://github.com/ayrat555/fang/pull/11)
## 0.4.0 (2021-07-31)
### [#8](https://github.com/ayrat555/fang/pull/8):
- Maintain ConnectionPool for Worker Threads
- Rename Postgres into Queue
- Pass PgConnection into run function
- Add function to remove all tasks of the specified type
## 0.3.1 (2021-07-24)
- Add periodic tasks - [#5](https://github.com/ayrat555/fang/pull/5), [#7](https://github.com/ayrat555/fang/pull/7)
## 0.3.1 (2021-07-11)
- Simplify usage by re-exporting serde traits - [#3](https://github.com/ayrat555/fang/pull/3)
## 0.3.0 (2021-07-04)
- Execute different types of tasks in separate workers - [#1](https://github.com/ayrat555/fang/pull/1)
- Add retention mode for tasks - [#2](https://github.com/ayrat555/fang/pull/2)
## 0.2.0 (2021-06-24)
- The first release on crates.io

View file

@ -1,13 +1,11 @@
[package]
name = "fang"
version = "0.10.2"
name = "frango"
version = "0.1.0"
authors = [
"Ayrat Badykov <ayratin555@gmail.com>",
"Pepe Márquez <pepe.marquezromero@gmail.com>",
"Rafael Caricio <rafael@caricio.com>"
"Rafael Caricio <rafael@caricio.com>",
]
description = "Background job processing library for Rust"
repository = "https://github.com/rafaelcaricio/fang"
description = "Async background job processing library with Diesel and Tokio"
repository = "https://code.caric.io/rafaelcaricio/frango"
edition = "2021"
license = "MIT"
readme = "README.md"
@ -29,25 +27,9 @@ thiserror = "1.0"
typed-builder = "0.12"
typetag = "0.2"
uuid = { version = "1.1", features = ["v4"] }
[dependencies.diesel]
version = "2.0"
features = ["postgres", "serde_json", "chrono", "uuid", "r2d2"]
[dependencies.diesel-derive-enum]
version = "2.0.1"
features = ["postgres"]
[dependencies.diesel-async]
version = "0.2"
features = ["postgres", "bb8"]
[dependencies.tokio]
version = "1.25"
features = ["rt", "time", "macros"]
[dependencies.async-trait]
version = "0.1"
[dependencies.async-recursion]
version = "1"
async-trait = "0.1"
async-recursion = "1"
diesel = { version = "2.0", features = ["postgres", "serde_json", "chrono", "uuid"] }
diesel-derive-enum = { version = "2.0.1", features = ["postgres"] }
diesel-async = { version = "0.2", features = ["postgres", "bb8"] }
tokio = { version = "1.25", features = ["rt", "time", "macros"] }

View file

@ -10,3 +10,6 @@ log = "0.4.0"
tokio = { version = "1", features = ["full"] }
diesel-async = { version = "0.2", features = ["postgres", "bb8"] }
diesel = { version = "2.0", features = ["postgres"] }
async-trait = "0.1"
serde = { version = "1.0", features = ["derive"] }
typetag = "0.2"

View file

@ -1,13 +1,11 @@
use fang::async_trait;
use fang::queue::AsyncQueueable;
use fang::serde::{Deserialize, Serialize};
use fang::typetag;
use fang::runnable::AsyncRunnable;
use fang::FangError;
use fang::errors::FrangoError;
use std::time::Duration;
use async_trait::async_trait;
use serde::{Serialize, Deserialize};
#[derive(Serialize, Deserialize)]
#[serde(crate = "fang::serde")]
pub struct MyTask {
pub number: u16,
}
@ -19,7 +17,6 @@ impl MyTask {
}
#[derive(Serialize, Deserialize)]
#[serde(crate = "fang::serde")]
pub struct MyFailingTask {
pub number: u16,
}
@ -33,7 +30,7 @@ impl MyFailingTask {
#[async_trait]
#[typetag::serde]
impl AsyncRunnable for MyTask {
async fn run(&self, queue: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, queue: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
// let new_task = MyTask::new(self.number + 1);
// queue
// .insert_task(&new_task as &dyn AsyncRunnable)
@ -50,7 +47,7 @@ impl AsyncRunnable for MyTask {
#[async_trait]
#[typetag::serde]
impl AsyncRunnable for MyFailingTask {
async fn run(&self, queue: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, queue: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
// let new_task = MyFailingTask::new(self.number + 1);
// queue
// .insert_task(&new_task as &dyn AsyncRunnable)

View file

@ -7,7 +7,6 @@ use simple_async_worker::MyTask;
use std::time::Duration;
use diesel_async::pg::AsyncPgConnection;
use diesel_async::pooled_connection::{bb8::Pool, AsyncDieselConnectionManager};
use diesel::PgConnection;
#[tokio::main]
async fn main() {

View file

@ -3,21 +3,21 @@ use thiserror::Error;
/// An error that can happen during executing of tasks
#[derive(Debug)]
pub struct FangError {
pub struct FrangoError {
/// A description of an error
pub description: String,
}
impl From<AsyncQueueError> for FangError {
impl From<AsyncQueueError> for FrangoError {
fn from(error: AsyncQueueError) -> Self {
let message = format!("{error:?}");
FangError {
FrangoError {
description: message,
}
}
}
impl From<SerdeError> for FangError {
impl From<SerdeError> for FrangoError {
fn from(error: SerdeError) -> Self {
Self::from(AsyncQueueError::SerdeError(error))
}
@ -45,14 +45,6 @@ pub enum AsyncQueueError {
SerdeError(#[from] serde_json::Error),
#[error(transparent)]
CronError(#[from] CronError),
#[error("returned invalid result (expected {expected:?}, found {found:?})")]
ResultError { expected: u64, found: u64 },
#[error(
"AsyncQueue is not connected :( , call connect() method first and then perform operations"
)]
NotConnectedError,
#[error("Can not convert `std::time::Duration` to `chrono::Duration`")]
TimeError,
#[error("Can not perform this operation if task is not uniq, please check its definition in impl AsyncRunnable")]
TaskNotUniqError,
}

View file

@ -14,14 +14,22 @@ use sha2::{Digest, Sha256};
use uuid::Uuid;
impl Task {
pub async fn remove_all_scheduled_tasks(
pub async fn remove_all(
connection: &mut AsyncPgConnection,
) -> Result<u64, AsyncQueueError> {
Ok(diesel::delete(fang_tasks::table)
.execute(connection)
.await? as u64)
}
pub async fn remove_all_scheduled(
connection: &mut AsyncPgConnection,
) -> Result<u64, AsyncQueueError> {
let query = fang_tasks::table.filter(fang_tasks::scheduled_at.gt(Utc::now()));
Ok(diesel::delete(query).execute(connection).await? as u64)
}
pub async fn remove_task(
pub async fn remove(
connection: &mut AsyncPgConnection,
id: Uuid,
) -> Result<u64, AsyncQueueError> {
@ -29,7 +37,7 @@ impl Task {
Ok(diesel::delete(query).execute(connection).await? as u64)
}
pub async fn remove_task_by_metadata(
pub async fn remove_by_metadata(
connection: &mut AsyncPgConnection,
task: &dyn AsyncRunnable,
) -> Result<u64, AsyncQueueError> {
@ -42,7 +50,7 @@ impl Task {
Ok(diesel::delete(query).execute(connection).await? as u64)
}
pub async fn remove_tasks_type(
pub async fn remove_by_type(
connection: &mut AsyncPgConnection,
task_type: &str,
) -> Result<u64, AsyncQueueError> {
@ -50,7 +58,7 @@ impl Task {
Ok(diesel::delete(query).execute(connection).await? as u64)
}
pub async fn find_task_by_id(
pub async fn find_by_id(
connection: &mut AsyncPgConnection,
id: Uuid,
) -> Result<Task, AsyncQueueError> {
@ -61,7 +69,7 @@ impl Task {
Ok(task)
}
pub async fn fail_task(
pub async fn fail_with_message(
connection: &mut AsyncPgConnection,
task: Task,
error_message: &str,
@ -99,7 +107,7 @@ impl Task {
Ok(task)
}
pub async fn fetch_task_of_type(
pub async fn fetch_by_type(
connection: &mut AsyncPgConnection,
task_type: Option<String>,
) -> Option<Task> {
@ -120,7 +128,7 @@ impl Task {
.ok()
}
pub async fn update_task_state(
pub async fn update_state(
connection: &mut AsyncPgConnection,
task: Task,
state: FangTaskState,
@ -135,7 +143,7 @@ impl Task {
.await?)
}
pub async fn insert_task(
pub async fn insert(
connection: &mut AsyncPgConnection,
params: &dyn AsyncRunnable,
scheduled_at: DateTime<Utc>,
@ -157,7 +165,7 @@ impl Task {
let uniq_hash = Self::calculate_hash(metadata.to_string());
match Self::find_task_by_uniq_hash(connection, &uniq_hash).await {
match Self::find_by_uniq_hash(connection, &uniq_hash).await {
Some(task) => Ok(task),
None => {
let new_task = NewTask::builder()
@ -183,7 +191,7 @@ impl Task {
hex::encode(result)
}
pub async fn find_task_by_uniq_hash(
pub async fn find_by_uniq_hash(
connection: &mut AsyncPgConnection,
uniq_hash: &str,
) -> Option<Task> {

View file

@ -2,7 +2,6 @@ use crate::errors::AsyncQueueError;
use crate::errors::CronError;
use crate::fang_task_state::FangTaskState;
use crate::runnable::AsyncRunnable;
use crate::schema::fang_tasks;
use crate::task::Task;
use crate::Scheduled::*;
use async_trait::async_trait;
@ -11,7 +10,7 @@ use cron::Schedule;
use diesel::result::Error::QueryBuilderError;
use diesel_async::scoped_futures::ScopedFutureExt;
use diesel_async::AsyncConnection;
use diesel_async::{pg::AsyncPgConnection, pooled_connection::bb8::Pool, RunQueryDsl};
use diesel_async::{pg::AsyncPgConnection, pooled_connection::bb8::Pool};
use std::str::FromStr;
use typed_builder::TypedBuilder;
use uuid::Uuid;
@ -103,7 +102,7 @@ impl AsyncQueueable for PgAsyncQueue {
.get()
.await
.map_err(|e| QueryBuilderError(e.into()))?;
Task::find_task_by_id(&mut connection, id).await
Task::find_by_id(&mut connection, id).await
}
async fn fetch_and_touch_task(
@ -118,11 +117,11 @@ impl AsyncQueueable for PgAsyncQueue {
connection
.transaction::<Option<Task>, AsyncQueueError, _>(|conn| {
async move {
let Some(found_task) = Task::fetch_task_of_type(conn, task_type).await else {
let Some(found_task) = Task::fetch_by_type(conn, task_type).await else {
return Ok(None);
};
match Task::update_task_state(conn, found_task, FangTaskState::InProgress).await
match Task::update_state(conn, found_task, FangTaskState::InProgress).await
{
Ok(updated_task) => Ok(Some(updated_task)),
Err(err) => Err(err),
@ -139,7 +138,7 @@ impl AsyncQueueable for PgAsyncQueue {
.get()
.await
.map_err(|e| QueryBuilderError(e.into()))?;
Ok(Task::insert_task(&mut connection, task, Utc::now()).await?)
Ok(Task::insert(&mut connection, task, Utc::now()).await?)
}
async fn schedule_task(&mut self, task: &dyn AsyncRunnable) -> Result<Task, AsyncQueueError> {
@ -166,7 +165,7 @@ impl AsyncQueueable for PgAsyncQueue {
}
};
Ok(Task::insert_task(&mut connection, task, scheduled_at).await?)
Ok(Task::insert(&mut connection, task, scheduled_at).await?)
}
async fn remove_all_tasks(&mut self) -> Result<u64, AsyncQueueError> {
@ -175,10 +174,7 @@ impl AsyncQueueable for PgAsyncQueue {
.get()
.await
.map_err(|e| QueryBuilderError(e.into()))?;
Ok(diesel::delete(fang_tasks::table)
.execute(&mut connection)
.await? as u64)
Task::remove_all(&mut connection).await
}
async fn remove_all_scheduled_tasks(&mut self) -> Result<u64, AsyncQueueError> {
@ -187,7 +183,7 @@ impl AsyncQueueable for PgAsyncQueue {
.get()
.await
.map_err(|e| QueryBuilderError(e.into()))?;
let result = Task::remove_all_scheduled_tasks(&mut connection).await?;
let result = Task::remove_all_scheduled(&mut connection).await?;
Ok(result)
}
@ -197,7 +193,7 @@ impl AsyncQueueable for PgAsyncQueue {
.get()
.await
.map_err(|e| QueryBuilderError(e.into()))?;
let result = Task::remove_task(&mut connection, id).await?;
let result = Task::remove(&mut connection, id).await?;
Ok(result)
}
@ -211,7 +207,7 @@ impl AsyncQueueable for PgAsyncQueue {
.get()
.await
.map_err(|e| QueryBuilderError(e.into()))?;
let result = Task::remove_task_by_metadata(&mut connection, task).await?;
let result = Task::remove_by_metadata(&mut connection, task).await?;
Ok(result)
} else {
Err(AsyncQueueError::TaskNotUniqError)
@ -224,7 +220,7 @@ impl AsyncQueueable for PgAsyncQueue {
.get()
.await
.map_err(|e| QueryBuilderError(e.into()))?;
let result = Task::remove_tasks_type(&mut connection, task_type).await?;
let result = Task::remove_by_type(&mut connection, task_type).await?;
Ok(result)
}
@ -238,7 +234,7 @@ impl AsyncQueueable for PgAsyncQueue {
.get()
.await
.map_err(|e| QueryBuilderError(e.into()))?;
let task = Task::update_task_state(&mut connection, task, state).await?;
let task = Task::update_state(&mut connection, task, state).await?;
Ok(task)
}
@ -252,7 +248,7 @@ impl AsyncQueueable for PgAsyncQueue {
.get()
.await
.map_err(|e| QueryBuilderError(e.into()))?;
let task = Task::fail_task(&mut connection, task, error_message).await?;
let task = Task::fail_with_message(&mut connection, task, error_message).await?;
Ok(task)
}
@ -275,7 +271,7 @@ impl AsyncQueueable for PgAsyncQueue {
#[cfg(test)]
mod async_queue_tests {
use super::*;
use crate::errors::FangError;
use crate::errors::FrangoError;
use crate::Scheduled;
use async_trait::async_trait;
use chrono::prelude::*;
@ -294,7 +290,7 @@ mod async_queue_tests {
#[typetag::serde]
#[async_trait]
impl AsyncRunnable for AsyncTask {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
Ok(())
}
}
@ -307,7 +303,7 @@ mod async_queue_tests {
#[typetag::serde]
#[async_trait]
impl AsyncRunnable for AsyncUniqTask {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
Ok(())
}
@ -325,7 +321,7 @@ mod async_queue_tests {
#[typetag::serde]
#[async_trait]
impl AsyncRunnable for AsyncTaskSchedule {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
Ok(())
}

View file

@ -1,4 +1,4 @@
use crate::errors::FangError;
use crate::errors::FrangoError;
use crate::queue::AsyncQueueable;
use crate::Scheduled;
use async_trait::async_trait;
@ -10,7 +10,7 @@ pub const RETRIES_NUMBER: i32 = 20;
#[async_trait]
pub trait AsyncRunnable: Send + Sync {
/// Execute the task. This method should define its logic
async fn run(&self, client: &mut dyn AsyncQueueable) -> Result<(), FangError>;
async fn run(&self, client: &mut dyn AsyncQueueable) -> Result<(), FrangoError>;
/// Define the type of the task.
/// The `common` task type is used by default

View file

@ -1,4 +1,4 @@
use crate::errors::FangError;
use crate::errors::FrangoError;
use crate::fang_task_state::FangTaskState;
use crate::queue::AsyncQueueable;
use crate::runnable::AsyncRunnable;
@ -29,7 +29,7 @@ impl<AQueue> AsyncWorker<AQueue>
where
AQueue: AsyncQueueable + Clone + Sync + 'static,
{
async fn run(&mut self, task: Task, runnable: Box<dyn AsyncRunnable>) -> Result<(), FangError> {
async fn run(&mut self, task: Task, runnable: Box<dyn AsyncRunnable>) -> Result<(), FrangoError> {
let result = runnable.run(&mut self.queue).await;
match result {
@ -54,8 +54,8 @@ where
async fn finalize_task(
&mut self,
task: Task,
result: &Result<(), FangError>,
) -> Result<(), FangError> {
result: &Result<(), FrangoError>,
) -> Result<(), FrangoError> {
match self.retention_mode {
RetentionMode::KeepAll => match result {
Ok(_) => {
@ -89,7 +89,7 @@ where
tokio::time::sleep(self.sleep_params.sleep_period).await;
}
pub(crate) async fn run_tasks(&mut self) -> Result<(), FangError> {
pub(crate) async fn run_tasks(&mut self) -> Result<(), FrangoError> {
loop {
//fetch task
match self
@ -124,7 +124,7 @@ where
}
#[cfg(test)]
pub async fn run_tasks_until_none(&mut self) -> Result<(), FangError> {
pub async fn run_tasks_until_none(&mut self) -> Result<(), FrangoError> {
loop {
match self
.queue
@ -160,7 +160,7 @@ where
#[cfg(test)]
mod async_worker_tests {
use super::*;
use crate::errors::FangError;
use crate::errors::FrangoError;
use crate::queue::AsyncQueueable;
use crate::queue::PgAsyncQueue;
use crate::worker::Task;
@ -181,7 +181,7 @@ mod async_worker_tests {
#[typetag::serde]
#[async_trait]
impl AsyncRunnable for WorkerAsyncTask {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
Ok(())
}
}
@ -194,7 +194,7 @@ mod async_worker_tests {
#[typetag::serde]
#[async_trait]
impl AsyncRunnable for WorkerAsyncTaskSchedule {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
Ok(())
}
fn cron(&self) -> Option<Scheduled> {
@ -210,10 +210,10 @@ mod async_worker_tests {
#[typetag::serde]
#[async_trait]
impl AsyncRunnable for AsyncFailedTask {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
let message = format!("number {} is wrong :(", self.number);
Err(FangError {
Err(FrangoError {
description: message,
})
}
@ -229,10 +229,10 @@ mod async_worker_tests {
#[typetag::serde]
#[async_trait]
impl AsyncRunnable for AsyncRetryTask {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
let message = "Failed".to_string();
Err(FangError {
Err(FrangoError {
description: message,
})
}
@ -248,7 +248,7 @@ mod async_worker_tests {
#[typetag::serde]
#[async_trait]
impl AsyncRunnable for AsyncTaskType1 {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
Ok(())
}
@ -263,7 +263,7 @@ mod async_worker_tests {
#[typetag::serde]
#[async_trait]
impl AsyncRunnable for AsyncTaskType2 {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FangError> {
async fn run(&self, _queueable: &mut dyn AsyncQueueable) -> Result<(), FrangoError> {
Ok(())
}