Reduce reliance on futures-util

This commit is contained in:
asonix 2023-08-23 11:59:42 -05:00
parent fb3ba0f3cd
commit 8f50a15b25
18 changed files with 79 additions and 41 deletions

1
Cargo.lock generated
View file

@ -1743,6 +1743,7 @@ dependencies = [
"console-subscriber",
"dashmap",
"flume",
"futures-core",
"futures-util",
"hex",
"md-5",

View file

@ -27,7 +27,8 @@ config = "0.13.0"
console-subscriber = "0.1"
dashmap = "5.1.0"
flume = "0.11.0"
futures-util = "0.3.17"
futures-core = "0.3"
futures-util = { version = "0.3.17", default-features = false }
hex = "0.4.3"
md-5 = "0.10.5"
metrics = "0.21.1"

View file

@ -4,7 +4,8 @@ use crate::{
store::Store,
};
use actix_web::web::Bytes;
use futures_util::{Stream, TryStreamExt};
use futures_core::Stream;
use futures_util::TryStreamExt;
use mime::APPLICATION_OCTET_STREAM;
use tracing::{Instrument, Span};

View file

@ -2,7 +2,7 @@ use actix_web::{
body::MessageBody,
web::{Bytes, BytesMut},
};
use futures_util::Stream;
use futures_core::Stream;
use std::{
collections::{vec_deque::IntoIter, VecDeque},
convert::Infallible,

View file

@ -12,7 +12,7 @@ use crate::{
process::Process,
};
use actix_web::web::Bytes;
use futures_util::Stream;
use futures_core::Stream;
use tokio::io::AsyncReadExt;
use super::{Discovery, DiscoveryLite};

View file

@ -2,7 +2,7 @@
mod tests;
use actix_web::web::Bytes;
use futures_util::Stream;
use futures_core::Stream;
use tokio::io::AsyncReadExt;
use crate::{

View file

@ -1,4 +1,4 @@
use futures_util::stream::Stream;
use futures_core::Stream;
use std::{
pin::Pin,
task::{Context, Poll},

View file

@ -6,9 +6,10 @@ pub(crate) use tokio_file::File;
#[cfg(not(feature = "io-uring"))]
mod tokio_file {
use crate::{store::file_store::FileError, Either};
use crate::{store::file_store::FileError, stream::IntoStreamer, Either};
use actix_web::web::{Bytes, BytesMut};
use futures_util::{Stream, StreamExt, TryStreamExt};
use futures_core::Stream;
use futures_util::TryStreamExt;
use std::{io::SeekFrom, path::Path};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt};
use tokio_util::codec::{BytesCodec, FramedRead};
@ -43,7 +44,8 @@ mod tokio_file {
where
S: Stream<Item = std::io::Result<Bytes>>,
{
futures_util::pin_mut!(stream);
let stream = std::pin::pin!(stream);
let mut stream = stream.into_streamer();
while let Some(res) = stream.next().await {
let mut bytes = res?;
@ -102,9 +104,9 @@ mod tokio_file {
#[cfg(feature = "io-uring")]
mod io_uring {
use crate::store::file_store::FileError;
use crate::{store::file_store::FileError, stream::IntoStreamer};
use actix_web::web::{Bytes, BytesMut};
use futures_util::stream::{Stream, StreamExt};
use futures_core::Stream;
use std::{
convert::TryInto,
fs::Metadata,
@ -181,7 +183,8 @@ mod io_uring {
where
S: Stream<Item = std::io::Result<Bytes>>,
{
futures_util::pin_mut!(stream);
let stream = std::pin::pin!(stream);
let mut stream = stream.into_streamer();
let mut cursor: u64 = 0;
while let Some(res) = stream.next().await {

View file

@ -5,9 +5,10 @@ use crate::{
formats::{InternalFormat, Validations},
repo::{Alias, ArcRepo, DeleteToken, Hash},
store::Store,
stream::IntoStreamer,
};
use actix_web::web::Bytes;
use futures_util::{Stream, StreamExt};
use futures_core::Stream;
use tracing::{Instrument, Span};
mod hasher;
@ -26,12 +27,14 @@ where
}
#[tracing::instrument(skip(stream))]
async fn aggregate<S>(mut stream: S) -> Result<Bytes, Error>
async fn aggregate<S>(stream: S) -> Result<Bytes, Error>
where
S: Stream<Item = Result<Bytes, Error>> + Unpin,
{
let mut buf = BytesStream::new();
let mut stream = stream.into_streamer();
while let Some(res) = stream.next().await {
buf.add_bytes(res?);
}

View file

@ -34,10 +34,8 @@ use actix_web::{
http::header::{CacheControl, CacheDirective, LastModified, Range, ACCEPT_RANGES},
web, App, HttpRequest, HttpResponse, HttpResponseBuilder, HttpServer,
};
use futures_util::{
stream::{empty, once},
Stream, StreamExt, TryStreamExt,
};
use futures_core::Stream;
use futures_util::{StreamExt, TryStreamExt};
use metrics_exporter_prometheus::PrometheusBuilder;
use middleware::Metrics;
use once_cell::sync::Lazy;
@ -46,7 +44,6 @@ use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
use reqwest_tracing::TracingMiddleware;
use rusty_s3::UrlStyle;
use std::{
future::ready,
path::Path,
path::PathBuf,
sync::Arc,
@ -72,7 +69,7 @@ use self::{
repo::{sled::SledRepo, Alias, DeleteToken, Hash, Repo, UploadId, UploadResult},
serde_str::Serde,
store::{file_store::FileStore, object_store::ObjectStore, Identifier, Store},
stream::{StreamLimit, StreamTimeout},
stream::{empty, once, StreamLimit, StreamTimeout},
};
pub use self::config::{ConfigSource, PictRsConfiguration};
@ -845,12 +842,9 @@ async fn process<S: Store + 'static>(
return Err(UploadError::Range.into());
}
} else if not_found {
(
HttpResponse::NotFound(),
Either::right(once(ready(Ok(bytes)))),
)
(HttpResponse::NotFound(), Either::right(once(Ok(bytes))))
} else {
(HttpResponse::Ok(), Either::right(once(ready(Ok(bytes)))))
(HttpResponse::Ok(), Either::right(once(Ok(bytes))))
};
Ok(srv_response(

View file

@ -1,4 +1,3 @@
use futures_util::StreamExt;
use std::{
rc::Rc,
sync::atomic::{AtomicU64, Ordering},
@ -10,6 +9,7 @@ use crate::{
error::{Error, UploadError},
repo::{ArcRepo, Hash},
store::{Identifier, Store},
stream::IntoStreamer,
};
pub(super) async fn migrate_store<S1, S2>(
@ -103,8 +103,7 @@ where
}
// Hashes are read in a consistent order
let stream = repo.hashes().await;
let mut stream = Box::pin(stream);
let mut stream = repo.hashes().await.into_streamer();
let state = Rc::new(MigrateState {
repo: repo.clone(),

View file

@ -5,8 +5,8 @@ use crate::{
repo::{Alias, ArcRepo, DeleteToken, Hash},
serde_str::Serde,
store::{Identifier, Store},
stream::IntoStreamer,
};
use futures_util::StreamExt;
pub(super) fn perform<'a, S>(
repo: &'a ArcRepo,
@ -136,7 +136,7 @@ async fn alias(repo: &ArcRepo, alias: Alias, token: DeleteToken) -> Result<(), E
#[tracing::instrument(skip_all)]
async fn all_variants(repo: &ArcRepo) -> Result<(), Error> {
let mut hash_stream = Box::pin(repo.hashes().await);
let mut hash_stream = repo.hashes().await.into_streamer();
while let Some(res) = hash_stream.next().await {
let hash = res?;
@ -151,7 +151,7 @@ async fn outdated_variants(repo: &ArcRepo, config: &Configuration) -> Result<(),
let now = time::OffsetDateTime::now_utc();
let since = now.saturating_sub(config.media.retention.variants.to_duration());
let mut variant_stream = Box::pin(repo.older_variants(since).await?);
let mut variant_stream = repo.older_variants(since).await?.into_streamer();
while let Some(res) = variant_stream.next().await {
let (hash, variant) = res?;
@ -166,7 +166,7 @@ async fn outdated_proxies(repo: &ArcRepo, config: &Configuration) -> Result<(),
let now = time::OffsetDateTime::now_utc();
let since = now.saturating_sub(config.media.retention.proxy.to_duration());
let mut alias_stream = Box::pin(repo.older_aliases(since).await?);
let mut alias_stream = repo.older_aliases(since).await?.into_streamer();
while let Some(res) = alias_stream.next().await {
let alias = res?;

View file

@ -1,13 +1,13 @@
use crate::{
error::{Error, UploadError},
store::Store,
stream::once,
};
use actix_web::{
http::header::{ByteRangeSpec, ContentRange, ContentRangeSpec, Range},
web::Bytes,
};
use futures_util::stream::{once, Stream};
use std::future::ready;
use futures_core::Stream;
pub(crate) fn chop_bytes(
byte_range: &ByteRangeSpec,
@ -17,7 +17,7 @@ pub(crate) fn chop_bytes(
if let Some((start, end)) = byte_range.to_satisfiable_range(length) {
// END IS INCLUSIVE
let end = end as usize + 1;
return Ok(once(ready(Ok(bytes.slice(start as usize..end)))));
return Ok(once(Ok(bytes.slice(start as usize..end))));
}
Err(UploadError::Range.into())

View file

@ -4,7 +4,7 @@ use crate::{
repo::{Alias, DeleteToken},
store::{Identifier, StoreError},
};
use futures_util::Stream;
use futures_core::Stream;
use std::fmt::Debug;
pub(crate) use self::sled::SledRepo;

View file

@ -1,6 +1,6 @@
use actix_web::web::Bytes;
use base64::{prelude::BASE64_STANDARD, Engine};
use futures_util::stream::Stream;
use futures_core::Stream;
use std::{fmt::Debug, sync::Arc};
use tokio::io::{AsyncRead, AsyncWrite};

View file

@ -4,7 +4,7 @@ use crate::{
store::Store,
};
use actix_web::web::Bytes;
use futures_util::stream::Stream;
use futures_core::Stream;
use std::{
path::{Path, PathBuf},
pin::Pin,

View file

@ -2,6 +2,7 @@ use crate::{
bytes_stream::BytesStream,
repo::{Repo, SettingsRepo},
store::Store,
stream::IntoStreamer,
};
use actix_rt::task::JoinError;
use actix_web::{
@ -13,7 +14,8 @@ use actix_web::{
web::Bytes,
};
use base64::{prelude::BASE64_STANDARD, Engine};
use futures_util::{Stream, StreamExt, TryStreamExt};
use futures_core::Stream;
use futures_util::TryStreamExt;
use reqwest::{header::RANGE, Body, Response};
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
use rusty_s3::{actions::S3Action, Bucket, BucketError, Credentials, UrlStyle};
@ -143,6 +145,8 @@ where
{
let mut buf = BytesStream::new();
let mut stream = stream.into_streamer();
while buf.len() < CHUNK_SIZE {
if let Some(res) = stream.next().await {
buf.add_bytes(res?)
@ -404,7 +408,7 @@ impl Store for ObjectStore {
));
}
let mut stream = response.bytes_stream();
let mut stream = response.bytes_stream().into_streamer();
while let Some(res) = stream.next().await {
let mut bytes = res.map_err(payload_to_io_error)?;

View file

@ -1,8 +1,9 @@
use actix_rt::{task::JoinHandle, time::Sleep};
use actix_web::web::Bytes;
use futures_util::Stream;
use futures_core::Stream;
use std::{
future::Future,
marker::PhantomData,
pin::Pin,
sync::{
atomic::{AtomicBool, Ordering},
@ -12,6 +13,37 @@ use std::{
time::Duration,
};
pub(crate) struct Empty<T>(PhantomData<T>);
impl<T> Stream for Empty<T> {
type Item = T;
fn poll_next(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Option<Self::Item>> {
Poll::Ready(None)
}
}
pub(crate) fn empty<T>() -> Empty<T> {
Empty(PhantomData)
}
pub(crate) struct Once<T>(Option<T>);
impl<T> Stream for Once<T>
where
T: Unpin,
{
type Item = T;
fn poll_next(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<Option<Self::Item>> {
Poll::Ready(self.0.take())
}
}
pub(crate) fn once<T>(value: T) -> Once<T> {
Once(Some(value))
}
pub(crate) type LocalBoxStream<'a, T> = Pin<Box<dyn Stream<Item = T> + 'a>>;
pub(crate) trait StreamLimit {