mirror of
https://git.asonix.dog/asonix/background-jobs.git
synced 2024-11-28 06:50:59 +00:00
Prepare release
This commit is contained in:
parent
a22d10242a
commit
a7b494ca39
11 changed files with 99 additions and 29 deletions
|
@ -1,7 +1,7 @@
|
||||||
[package]
|
[package]
|
||||||
name = "background-jobs"
|
name = "background-jobs"
|
||||||
description = "Background Jobs implemented with tokio and futures"
|
description = "Background Jobs implemented with tokio and futures"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
license = "GPL-3.0"
|
license = "GPL-3.0"
|
||||||
authors = ["asonix <asonix@asonix.dog>"]
|
authors = ["asonix <asonix@asonix.dog>"]
|
||||||
repository = "https://git.asonix.dog/asonix/background-jobs"
|
repository = "https://git.asonix.dog/asonix/background-jobs"
|
||||||
|
|
47
README.md
47
README.md
|
@ -9,7 +9,7 @@ might not be the best experience.
|
||||||
#### Add Background Jobs to your project
|
#### Add Background Jobs to your project
|
||||||
```toml
|
```toml
|
||||||
[dependencies]
|
[dependencies]
|
||||||
background-jobs = "0.3"
|
background-jobs = "0.4"
|
||||||
failure = "0.1"
|
failure = "0.1"
|
||||||
futures = "0.1"
|
futures = "0.1"
|
||||||
tokio = "0.1"
|
tokio = "0.1"
|
||||||
|
@ -129,9 +129,9 @@ use server_jobs_example::queue_set;
|
||||||
fn main() -> Result<(), Error> {
|
fn main() -> Result<(), Error> {
|
||||||
// Run our job server
|
// Run our job server
|
||||||
tokio::run(ServerConfig::init(
|
tokio::run(ServerConfig::init(
|
||||||
|
1,
|
||||||
"127.0.0.1",
|
"127.0.0.1",
|
||||||
5555,
|
5555,
|
||||||
1,
|
|
||||||
queue_set(),
|
queue_set(),
|
||||||
"example-db",
|
"example-db",
|
||||||
));
|
));
|
||||||
|
@ -211,7 +211,7 @@ fn main() -> Result<(), Error> {
|
||||||
|
|
||||||
// Queue each job
|
// Queue each job
|
||||||
for job in jobs {
|
for job in jobs {
|
||||||
spawner.queue_sync::<MyProcessor>(job)?
|
spawner.queue_sync::<MyProcessor, _>(job)?
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
@ -226,9 +226,9 @@ but it's untested and hard to say. You can override this behavior by specifying
|
||||||
Cargo.toml
|
Cargo.toml
|
||||||
```toml
|
```toml
|
||||||
[Dependencies.background-jobs]
|
[Dependencies.background-jobs]
|
||||||
version = "0.1"
|
version = "0.4"
|
||||||
default-features = false
|
default-features = false
|
||||||
features = ["background-jobs-server", "background-jobs-server/futures-zmq"]
|
features = ["no_unix"]
|
||||||
```
|
```
|
||||||
|
|
||||||
[`futures-zmq`](https://crates.io/crates/futures-zmq) Is designed to be a drop-in replacement for
|
[`futures-zmq`](https://crates.io/crates/futures-zmq) Is designed to be a drop-in replacement for
|
||||||
|
@ -236,7 +236,42 @@ tokio-zmq that works on non-unix and non-tokio platforms. The reason why it isn'
|
||||||
that it's slower than tokio-zmq, and in all likelihood, the production environment for projects
|
that it's slower than tokio-zmq, and in all likelihood, the production environment for projects
|
||||||
depending on this one will be linux.
|
depending on this one will be linux.
|
||||||
|
|
||||||
#### Not using a ZeroMQ+LMDB based client/server model
|
#### Actix
|
||||||
|
Another implementation of a jobs processor is also provided by this library under a feature flag.
|
||||||
|
```toml
|
||||||
|
[dependencies.background-jobs]
|
||||||
|
version = "0.4"
|
||||||
|
default-features = false
|
||||||
|
features = ["actix"]
|
||||||
|
```
|
||||||
|
|
||||||
|
This provides an in-process implementation of a jobs server and worker setup. Here's some example usage.
|
||||||
|
```rust
|
||||||
|
use background_jobs::{Processor, ServerConfig, WorkerConfig};
|
||||||
|
|
||||||
|
let sys = actix::System::new("my-actix-thing");
|
||||||
|
|
||||||
|
let queue_handle = ServerConfig::new(1, db_path.into()).start::<MyState>();
|
||||||
|
|
||||||
|
let state = MyState {
|
||||||
|
queue_handle: queue_handle.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut worker_config = WorkerConfig::new(state);
|
||||||
|
WorkerConfig::register(&mut worker_config, FetchProcessor);
|
||||||
|
WorkerConfig::register(&mut worker_config, InstanceProcessor);
|
||||||
|
WorkerConfig::register(&mut worker_config, OpenProcessor);
|
||||||
|
WorkerConfig::set_processor_count(
|
||||||
|
&mut worker_config,
|
||||||
|
<InstanceProcessor as Processor<MyState>>::QUEUE,
|
||||||
|
16,
|
||||||
|
);
|
||||||
|
WorkerConfig::start(worker_config, queue_handle.clone());
|
||||||
|
|
||||||
|
let _ = sys.run();
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Bringing your own server/worker implementation
|
||||||
If you want to create your own jobs processor based on this idea, you can depend on the
|
If you want to create your own jobs processor based on this idea, you can depend on the
|
||||||
`background-jobs-core` crate, which provides the LMDB storage, Processor and Job traits, as well as some
|
`background-jobs-core` crate, which provides the LMDB storage, Processor and Job traits, as well as some
|
||||||
other useful types for implementing a jobs processor.
|
other useful types for implementing a jobs processor.
|
||||||
|
|
|
@ -15,5 +15,5 @@ serde_derive = "1.0"
|
||||||
tokio = "0.1"
|
tokio = "0.1"
|
||||||
|
|
||||||
[dependencies.background-jobs]
|
[dependencies.background-jobs]
|
||||||
version = "0.3"
|
version = "0.4"
|
||||||
path = "../.."
|
path = "../.."
|
||||||
|
|
|
@ -88,7 +88,7 @@ impl NewJobInfo {
|
||||||
///
|
///
|
||||||
/// Although exposed publically, this type should only really be handled by the library itself, and
|
/// Although exposed publically, this type should only really be handled by the library itself, and
|
||||||
/// is impossible to create outside of a
|
/// is impossible to create outside of a
|
||||||
/// [Processor](https://docs.rs/background-jobs/0.3.0/background_jobs/trait.Processor.html)'s
|
/// [Processor](https://docs.rs/background-jobs/0.4.0/background_jobs/trait.Processor.html)'s
|
||||||
/// new_job method.
|
/// new_job method.
|
||||||
pub struct JobInfo {
|
pub struct JobInfo {
|
||||||
/// ID of the job
|
/// ID of the job
|
||||||
|
|
|
@ -34,11 +34,11 @@ use crate::{Backoff, Job, JobError, MaxRetries, NewJobInfo};
|
||||||
/// - The job's default queue
|
/// - The job's default queue
|
||||||
/// - The job's default maximum number of retries
|
/// - The job's default maximum number of retries
|
||||||
/// - The job's [backoff
|
/// - The job's [backoff
|
||||||
/// strategy](https://docs.rs/background-jobs/0.3.0/background_jobs/enum.Backoff.html)
|
/// strategy](https://docs.rs/background-jobs/0.4.0/background_jobs/enum.Backoff.html)
|
||||||
///
|
///
|
||||||
/// Processors also provide the default mechanism for running a job, and the only mechanism for
|
/// Processors also provide the default mechanism for running a job, and the only mechanism for
|
||||||
/// creating a
|
/// creating a
|
||||||
/// [JobInfo](https://docs.rs/background-jobs-core/0.3.0/background_jobs_core/struct.JobInfo.html),
|
/// [JobInfo](https://docs.rs/background-jobs-core/0.4.0/background_jobs_core/struct.JobInfo.html),
|
||||||
/// which is the type required for queuing jobs to be executed.
|
/// which is the type required for queuing jobs to be executed.
|
||||||
///
|
///
|
||||||
/// ### Example
|
/// ### Example
|
||||||
|
@ -170,7 +170,7 @@ where
|
||||||
/// Patterns like this could be useful if you want to use the same job type for multiple
|
/// Patterns like this could be useful if you want to use the same job type for multiple
|
||||||
/// scenarios. Defining the `process` method for multiple `Processor`s with different
|
/// scenarios. Defining the `process` method for multiple `Processor`s with different
|
||||||
/// before/after logic for the same
|
/// before/after logic for the same
|
||||||
/// [`Job`](https://docs.rs/background-jobs/0.3.0/background_jobs/trait.Job.html) type is
|
/// [`Job`](https://docs.rs/background-jobs/0.4.0/background_jobs/trait.Job.html) type is
|
||||||
/// supported.
|
/// supported.
|
||||||
fn process(
|
fn process(
|
||||||
&self,
|
&self,
|
||||||
|
|
|
@ -28,16 +28,16 @@ use crate::{JobError, JobInfo, Processor};
|
||||||
/// A generic function that processes a job
|
/// A generic function that processes a job
|
||||||
///
|
///
|
||||||
/// Instead of storing
|
/// Instead of storing
|
||||||
/// [`Processor`](https://docs.rs/background-jobs/0.3.0/background_jobs/trait.Processor.html) type
|
/// [`Processor`](https://docs.rs/background-jobs/0.4.0/background_jobs/trait.Processor.html) type
|
||||||
/// directly, the
|
/// directly, the
|
||||||
/// [`ProcessorMap`](https://docs.rs/background-jobs-core/0.3.0/background_jobs_core/struct.ProcessorMap.html)
|
/// [`ProcessorMap`](https://docs.rs/background-jobs-core/0.4.0/background_jobs_core/struct.ProcessorMap.html)
|
||||||
/// struct stores these `ProcessFn` types that don't expose differences in Job types.
|
/// struct stores these `ProcessFn` types that don't expose differences in Job types.
|
||||||
pub type ProcessFn =
|
pub type ProcessFn =
|
||||||
Box<dyn Fn(Value) -> Box<dyn Future<Item = (), Error = JobError> + Send> + Send + Sync>;
|
Box<dyn Fn(Value) -> Box<dyn Future<Item = (), Error = JobError> + Send> + Send + Sync>;
|
||||||
|
|
||||||
/// A type for storing the relationships between processor names and the processor itself
|
/// A type for storing the relationships between processor names and the processor itself
|
||||||
///
|
///
|
||||||
/// [`Processor`s](https://docs.rs/background-jobs/0.3.0/background_jobs/trait.Processor.html) must
|
/// [`Processor`s](https://docs.rs/background-jobs/0.4.0/background_jobs/trait.Processor.html) must
|
||||||
/// be registered with the `ProcessorMap` in the initialization phase of an application before
|
/// be registered with the `ProcessorMap` in the initialization phase of an application before
|
||||||
/// workers are spawned in order to handle queued jobs.
|
/// workers are spawned in order to handle queued jobs.
|
||||||
pub struct ProcessorMap<S>
|
pub struct ProcessorMap<S>
|
||||||
|
@ -65,7 +65,7 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register a
|
/// Register a
|
||||||
/// [`Processor`](https://docs.rs/background-jobs/0.3.0/background_jobs/trait.Processor.html) with
|
/// [`Processor`](https://docs.rs/background-jobs/0.4.0/background_jobs/trait.Processor.html) with
|
||||||
/// this `ProcessorMap`.
|
/// this `ProcessorMap`.
|
||||||
///
|
///
|
||||||
/// `ProcessorMap`s are useless if no processors are registerd before workers are spawned, so
|
/// `ProcessorMap`s are useless if no processors are registerd before workers are spawned, so
|
||||||
|
|
|
@ -46,7 +46,7 @@ where
|
||||||
///
|
///
|
||||||
/// This trait should be implemented sparingly, but is provided so that synchronous tasks may be
|
/// This trait should be implemented sparingly, but is provided so that synchronous tasks may be
|
||||||
/// executed. If you have the ability to implement the
|
/// executed. If you have the ability to implement the
|
||||||
/// [`Job`](https://docs.rs/background-jobs/0.3.0/background_jobs/trait.Job.html) trait directly,
|
/// [`Job`](https://docs.rs/background-jobs/0.4.0/background_jobs/trait.Job.html) trait directly,
|
||||||
/// you should.
|
/// you should.
|
||||||
///
|
///
|
||||||
/// ### Example
|
/// ### Example
|
||||||
|
|
|
@ -123,7 +123,7 @@ struct MissingQueue(String);
|
||||||
///
|
///
|
||||||
/// `ServerConfig` is used to spin up the infrastructure to manage queueing and storing jobs, but
|
/// `ServerConfig` is used to spin up the infrastructure to manage queueing and storing jobs, but
|
||||||
/// it does not provide functionality to execute jobs. For that, you must create a
|
/// it does not provide functionality to execute jobs. For that, you must create a
|
||||||
/// [`Worker`](https://docs.rs/background-jobs-server/0.3.0/background_jobs_server/struct.WorkerConfig.html)
|
/// [`Worker`](https://docs.rs/background-jobs-server/0.4.0/background_jobs_server/struct.WorkerConfig.html)
|
||||||
/// that will connect to the running server.
|
/// that will connect to the running server.
|
||||||
///
|
///
|
||||||
/// This type doesn't have any associated data, but is used as a proxy for starting the
|
/// This type doesn't have any associated data, but is used as a proxy for starting the
|
||||||
|
@ -139,9 +139,9 @@ struct MissingQueue(String);
|
||||||
/// queue_set.insert("default".to_owned());
|
/// queue_set.insert("default".to_owned());
|
||||||
///
|
///
|
||||||
/// let start_server = ServerConfig::init(
|
/// let start_server = ServerConfig::init(
|
||||||
|
/// 1,
|
||||||
/// "127.0.0.1",
|
/// "127.0.0.1",
|
||||||
/// 5555,
|
/// 5555,
|
||||||
/// 1,
|
|
||||||
/// queue_set,
|
/// queue_set,
|
||||||
/// "example-db",
|
/// "example-db",
|
||||||
/// );
|
/// );
|
||||||
|
|
|
@ -38,7 +38,7 @@ use zmq::{Context, Message};
|
||||||
///
|
///
|
||||||
/// tokio::spawn(
|
/// tokio::spawn(
|
||||||
/// spawner
|
/// spawner
|
||||||
/// .queue::<MyProcessor>(job)
|
/// .queue::<MyProcessor, _>(job)
|
||||||
/// .map_err(|_| ()),
|
/// .map_err(|_| ()),
|
||||||
/// );
|
/// );
|
||||||
/// ```
|
/// ```
|
||||||
|
|
|
@ -34,7 +34,7 @@ use self::{config::Worker, portmap::PortMap};
|
||||||
///
|
///
|
||||||
/// A worker handles the processing of jobs, but not the queueing or storing of jobs. It connects
|
/// A worker handles the processing of jobs, but not the queueing or storing of jobs. It connects
|
||||||
/// to a server (crated with
|
/// to a server (crated with
|
||||||
/// [`ServerConfig`](https://docs.rs/background-jobs-server/0.3.0/background_jobs_server/struct.ServerConfig.html))
|
/// [`ServerConfig`](https://docs.rs/background-jobs-server/0.4.0/background_jobs_server/struct.ServerConfig.html))
|
||||||
/// and receives work from there.
|
/// and receives work from there.
|
||||||
///
|
///
|
||||||
/// ```rust
|
/// ```rust
|
||||||
|
@ -122,7 +122,7 @@ where
|
||||||
/// Register a processor with this worker
|
/// Register a processor with this worker
|
||||||
///
|
///
|
||||||
/// For more information, see
|
/// For more information, see
|
||||||
/// [`Processor`](https://docs.rs/background-jobs/0.3.0/background_jobs/enum.Processor.html).
|
/// [`Processor`](https://docs.rs/background-jobs/0.4.0/background_jobs/enum.Processor.html).
|
||||||
pub fn register_processor<P>(&mut self, processor: P)
|
pub fn register_processor<P>(&mut self, processor: P)
|
||||||
where
|
where
|
||||||
P: Processor<S> + Send + Sync + 'static,
|
P: Processor<S> + Send + Sync + 'static,
|
||||||
|
|
49
src/lib.rs
49
src/lib.rs
|
@ -28,7 +28,7 @@
|
||||||
//! #### Add Background Jobs to your project
|
//! #### Add Background Jobs to your project
|
||||||
//! ```toml
|
//! ```toml
|
||||||
//! [dependencies]
|
//! [dependencies]
|
||||||
//! background-jobs = "0.3"
|
//! background-jobs = "0.4"
|
||||||
//! failure = "0.1"
|
//! failure = "0.1"
|
||||||
//! futures = "0.1"
|
//! futures = "0.1"
|
||||||
//! tokio = "0.1"
|
//! tokio = "0.1"
|
||||||
|
@ -204,7 +204,7 @@
|
||||||
//! // Queue each job
|
//! // Queue each job
|
||||||
//! tokio::run(lazy(move || {
|
//! tokio::run(lazy(move || {
|
||||||
//! for job in jobs {
|
//! for job in jobs {
|
||||||
//! tokio::spawn(spawner.queue::<MyProcessor>(job).map_err(|_| ()));
|
//! tokio::spawn(spawner.queue::<MyProcessor, _>(job).map_err(|_| ()));
|
||||||
//! }
|
//! }
|
||||||
//!
|
//!
|
||||||
//! Ok(())
|
//! Ok(())
|
||||||
|
@ -223,9 +223,9 @@
|
||||||
//! hard to say. You can override this behavior by specifying the following in your Cargo.toml
|
//! hard to say. You can override this behavior by specifying the following in your Cargo.toml
|
||||||
//! ```toml
|
//! ```toml
|
||||||
//! [Dependencies.background-jobs]
|
//! [Dependencies.background-jobs]
|
||||||
//! version = "0.1"
|
//! version = "0.4"
|
||||||
//! default-features = false
|
//! default-features = false
|
||||||
//! features = ["background-jobs-server", "background-jobs-server/futures-zmq"]
|
//! features = ["no_unix"]
|
||||||
//! ```
|
//! ```
|
||||||
//!
|
//!
|
||||||
//! [`futures-zmq`](https://crates.io/crates/futures-zmq) Is designed to be a drop-in replacement
|
//! [`futures-zmq`](https://crates.io/crates/futures-zmq) Is designed to be a drop-in replacement
|
||||||
|
@ -233,10 +233,45 @@
|
||||||
//! by default is that it's slower than tokio-zmq, and in all likelihood, the production
|
//! by default is that it's slower than tokio-zmq, and in all likelihood, the production
|
||||||
//! environment for projects depending on this one will be linux.
|
//! environment for projects depending on this one will be linux.
|
||||||
//!
|
//!
|
||||||
//! #### Not using a ZeroMQ+LMDB based client/server model
|
//! #### Actix
|
||||||
|
//! Another implementation of a jobs processor is also provided by this library under a feature flag.
|
||||||
|
//! ```toml
|
||||||
|
//! [dependencies.background-jobs]
|
||||||
|
//! version = "0.4"
|
||||||
|
//! default-features = false
|
||||||
|
//! features = ["actix"]
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! This provides an in-process implementation of a jobs server and worker setup. Here's some example usage.
|
||||||
|
//! ```rust,ignore
|
||||||
|
//! use background_jobs::{Processor, ServerConfig, WorkerConfig};
|
||||||
|
//!
|
||||||
|
//! let sys = actix::System::new("my-actix-thing");
|
||||||
|
//!
|
||||||
|
//! let queue_handle = ServerConfig::new(1, db_path.into()).start::<MyState>();
|
||||||
|
//!
|
||||||
|
//! let state = MyState {
|
||||||
|
//! queue_handle: queue_handle.clone(),
|
||||||
|
//! };
|
||||||
|
//!
|
||||||
|
//! let mut worker_config = WorkerConfig::new(state);
|
||||||
|
//! WorkerConfig::register(&mut worker_config, FetchProcessor);
|
||||||
|
//! WorkerConfig::register(&mut worker_config, InstanceProcessor);
|
||||||
|
//! WorkerConfig::register(&mut worker_config, OpenProcessor);
|
||||||
|
//! WorkerConfig::set_processor_count(
|
||||||
|
//! &mut worker_config,
|
||||||
|
//! <InstanceProcessor as Processor<MyState>>::QUEUE,
|
||||||
|
//! 16,
|
||||||
|
//! );
|
||||||
|
//! WorkerConfig::start(worker_config, queue_handle.clone());
|
||||||
|
//!
|
||||||
|
//! let _ = sys.run();
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! #### Bringing your own server/worker implementation
|
||||||
//! If you want to create your own jobs processor based on this idea, you can depend on the
|
//! If you want to create your own jobs processor based on this idea, you can depend on the
|
||||||
//! `background-jobs-core` crate, which provides the LMDB storage, Processor and Job traits, as
|
//! `background-jobs-core` crate, which provides the LMDB storage, Processor and Job traits, as well as some
|
||||||
//! well as some other useful types for implementing a jobs processor.
|
//! other useful types for implementing a jobs processor.
|
||||||
|
|
||||||
pub use background_jobs_core::{Backoff, Job, MaxRetries, Processor};
|
pub use background_jobs_core::{Backoff, Job, MaxRetries, Processor};
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue