sluice-0.5.5/.cargo_vcs_info.json0000644000000001120000000000000123200ustar { "git": { "sha1": "4391d3dfac0b38bc9da4b17c2caf9f0c9c223cb5" } } sluice-0.5.5/.github/FUNDING.yml000064400000000000000000000000230000000000000142440ustar 00000000000000github: [sagebind] sluice-0.5.5/.github/dependabot.yml000064400000000000000000000002210000000000000152570ustar 00000000000000version: 2 updates: - package-ecosystem: cargo directory: "/" schedule: interval: daily time: "11:00" open-pull-requests-limit: 10 sluice-0.5.5/.github/workflows/build.yml000064400000000000000000000015270000000000000163200ustar 00000000000000name: build on: [push, pull_request] jobs: test: strategy: matrix: platform: - ubuntu-latest - macos-latest - windows-latest runs-on: ${{ matrix.platform }} env: RUST_BACKTRACE: 1 steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: stable default: true - run: cargo test check: strategy: matrix: platform: - ubuntu-latest - macos-latest - windows-latest runs-on: ${{ matrix.platform }} env: RUST_BACKTRACE: 1 steps: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: "1.41.1" default: true - run: cargo check sluice-0.5.5/.gitignore000064400000000000000000000000400000000000000130560ustar 00000000000000 /target/ **/*.rs.bk Cargo.lock sluice-0.5.5/Cargo.toml0000644000000024550000000000000103320ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies # # If you believe there's an error in this file please file an # issue against the rust-lang/cargo repository. If you're # editing this file be aware that the upstream Cargo.toml # will likely look very different (and much more reasonable) [package] edition = "2018" name = "sluice" version = "0.5.5" authors = ["Stephen M. Coakley "] description = "Efficient ring buffer for byte buffers, FIFO queues, and SPSC channels" documentation = "https://docs.rs/sluice" readme = "README.md" keywords = ["buffer", "pipe", "channel"] categories = ["asynchronous", "concurrency", "data-structures"] license = "MIT" repository = "https://github.com/sagebind/sluice" [[bench]] name = "pipe" harness = false [dependencies.async-channel] version = "1" [dependencies.futures-core] version = "0.3" [dependencies.futures-io] version = "0.3" [dev-dependencies.criterion] version = "0.3" [dev-dependencies.futures] version = "0.3" [dev-dependencies.quickcheck] version = "1.0" [dev-dependencies.quickcheck_macros] version = "1.0" sluice-0.5.5/Cargo.toml.orig000064400000000000000000000012160000000000000137630ustar 00000000000000[package] name = "sluice" version = "0.5.5" authors = ["Stephen M. Coakley "] edition = "2018" description = "Efficient ring buffer for byte buffers, FIFO queues, and SPSC channels" documentation = "https://docs.rs/sluice" repository = "https://github.com/sagebind/sluice" readme = "README.md" keywords = ["buffer", "pipe", "channel"] categories = ["asynchronous", "concurrency", "data-structures"] license = "MIT" [dependencies] async-channel = "1" futures-core = "0.3" futures-io = "0.3" [dev-dependencies] criterion = "0.3" futures = "0.3" quickcheck = "1.0" quickcheck_macros = "1.0" [[bench]] name = "pipe" harness = false sluice-0.5.5/LICENSE000064400000000000000000000020630000000000000121020ustar 00000000000000MIT License Copyright (c) 2017 Stephen M. Coakley Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. sluice-0.5.5/README.md000064400000000000000000000010340000000000000123510ustar 00000000000000# Sluice Asynchronous byte buffers and pipes for concurrent I/O programming. [![Crates.io](https://img.shields.io/crates/v/sluice.svg)](https://crates.io/crates/sluice) [![Documentation](https://docs.rs/sluice/badge.svg)](https://docs.rs/sluice) ![License](https://img.shields.io/badge/license-MIT-blue.svg) ## [Documentation] Check the [documentation] for up-to-date usage and examples. ## License This library is licensed under the MIT license. See the [LICENSE](LICENSE) file for details. [Documentation]: https://docs.rs/sluice sluice-0.5.5/benches/pipe.rs000064400000000000000000000016030000000000000140060ustar 00000000000000use criterion::*; fn benchmark(c: &mut Criterion) { c.bench_function("write 100 1K chunks", |b| { use futures::prelude::*; let data = [1; 1024]; b.iter_batched( sluice::pipe::pipe, |(reader, mut writer)| { let producer = async { for _ in 0u8..100 { writer.write_all(&data).await.unwrap(); } writer.close().await.unwrap(); }; let consumer = async { let mut sink = futures::io::sink(); futures::io::copy(reader, &mut sink).await.unwrap(); }; futures::executor::block_on(future::join(producer, consumer)); }, BatchSize::SmallInput, ) }); } criterion_group!(benches, benchmark); criterion_main!(benches); sluice-0.5.5/src/lib.rs000064400000000000000000000010060000000000000127740ustar 00000000000000//! Asynchronous byte buffers and pipes for concurrent I/O programming. //! //! ## Pipes //! //! The primary feature offered by Sluice are _pipes_, which are asynchronous //! in-memory byte buffers that allow separate tasks to read and write from the //! buffer in parallel. //! //! See the `pipe` module for details. #![deny(unsafe_code)] #![warn( future_incompatible, missing_debug_implementations, missing_docs, rust_2018_idioms, unreachable_pub, unused, clippy::all, )] pub mod pipe; sluice-0.5.5/src/pipe/chunked.rs000064400000000000000000000214330000000000000146120ustar 00000000000000//! Generally a ring buffer is an efficient and appropriate data structure for //! asynchronously transmitting a stream of bytes between two threads that also //! gives you control over memory allocation to avoid consuming an unknown //! amount of memory. Setting a fixed memory limit also gives you a degree of //! flow control if the producer ends up being faster than the consumer. //! //! But for some use cases a ring buffer will not work if an application uses //! its own internal buffer management and requires you to consume either all of //! a "chunk" of bytes, or none of it. //! //! Because of these constraints, instead we use a quite unique type of buffer //! that uses a fixed number of growable buffers that are exchanged back and //! forth between a producer and a consumer. Since each buffer is a vector, it //! can grow to whatever size is required of it in order to fit a single chunk. //! //! To avoid the constant allocation overhead of creating a new buffer for every //! chunk, after a consumer finishes reading from a buffer, it returns the //! buffer to the producer over a channel to be reused. The number of buffers //! available in this system is fixed at creation time, so the only allocations //! that happen during reads and writes are occasional reallocation for each //! individual vector to fit larger chunks of bytes that don't already fit. use async_channel::{bounded, Sender, Receiver}; use futures_core::{FusedStream, Stream}; use futures_io::{AsyncBufRead, AsyncRead, AsyncWrite}; use std::{ io, io::{BufRead, Cursor, Write}, pin::Pin, task::{Context, Poll}, }; /// Create a new chunked pipe with room for a fixed number of chunks. /// /// The `count` parameter sets how many buffers are available in the pipe at /// once. Smaller values will reduce the number of allocations and reallocations /// may be required when writing and reduce overall memory usage. Larger values /// reduce the amount of waiting done between chunks if you have a producer and /// consumer that run at different speeds. /// /// If `count` is set to 1, then the pipe is essentially serial, since only the /// reader or writer can operate on the single buffer at one time and cannot be /// run in parallel. pub(crate) fn new(count: usize) -> (Reader, Writer) { let (buf_pool_tx, buf_pool_rx) = bounded(count); let (buf_stream_tx, buf_stream_rx) = bounded(count); // Fill up the buffer pool. for _ in 0..count { buf_pool_tx .try_send(Cursor::new(Vec::new())) .expect("buffer pool overflow"); } let reader = Reader { buf_pool_tx, buf_stream_rx, chunk: None, }; let writer = Writer { buf_pool_rx, buf_stream_tx, }; (reader, writer) } /// The reading half of a chunked pipe. pub(crate) struct Reader { /// A channel of incoming chunks from the writer. buf_pool_tx: Sender>>, /// A channel of chunk buffers that have been consumed and can be reused. buf_stream_rx: Receiver>>, /// A chunk currently being read from. chunk: Option>>, } impl AsyncRead for Reader { fn poll_read( mut self: Pin<&mut Self>, cx: &mut Context<'_>, mut buf: &mut [u8], ) -> Poll> { // Read into the internal buffer. match self.as_mut().poll_fill_buf(cx)? { // Not quite ready yet. Poll::Pending => Poll::Pending, // A chunk is available. Poll::Ready(chunk) => { // Copy as much of the chunk as we can to the destination // buffer. let amt = buf.write(chunk)?; // Mark however much was successfully copied as being consumed. self.consume(amt); Poll::Ready(Ok(amt)) } } } } impl AsyncBufRead for Reader { fn poll_fill_buf(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { // If the current chunk is consumed, first return it to the writer for // reuse. if let Some(chunk) = self.chunk.as_ref() { if chunk.position() >= chunk.get_ref().len() as u64 { let mut chunk = self.chunk.take().unwrap(); chunk.set_position(0); chunk.get_mut().clear(); if let Err(e) = self.buf_pool_tx.try_send(chunk) { // We pre-fill the buffer pool channel with an exact number // of buffers, so this can never happen. if e.is_full() { panic!("buffer pool overflow") } // If the writer disconnects, then we'll just discard this // buffer and any subsequent buffers until we've read // everything still in the pipe. else if e.is_closed() { // Nothing! } // Some other error occurred. else { return Poll::Ready(Err(io::ErrorKind::BrokenPipe.into())); } } } } // If we have no current chunk, then attempt to read one. if self.chunk.is_none() { // If the stream has terminated, then do not poll it again. if self.buf_stream_rx.is_terminated() { return Poll::Ready(Ok(&[])); } match Pin::new(&mut self.buf_stream_rx).poll_next(cx) { // Wait for a new chunk to be delivered. Poll::Pending => return Poll::Pending, // Pipe has closed, so return EOF. Poll::Ready(None) => return Poll::Ready(Ok(&[])), // Accept the new chunk. Poll::Ready(buf) => self.chunk = buf, } } // Return the current chunk, if any, as the buffer. #[allow(unsafe_code)] Poll::Ready(match unsafe { self.get_unchecked_mut().chunk.as_mut() } { Some(chunk) => chunk.fill_buf(), None => Ok(&[]), }) } fn consume(mut self: Pin<&mut Self>, amt: usize) { if let Some(chunk) = self.chunk.as_mut() { // Consume the requested amount from the current chunk. chunk.consume(amt); } } } impl Drop for Reader { fn drop(&mut self) { // Ensure we close the primary stream first before the pool stream so // that the writer knows the pipe is closed before trying to poll the // pool channel. self.buf_stream_rx.close(); self.buf_pool_tx.close(); } } /// Writing half of a chunked pipe. pub(crate) struct Writer { /// A channel of chunks to send to the reader. buf_pool_rx: Receiver>>, /// A channel of incoming buffers to write chunks to. buf_stream_tx: Sender>>, } impl AsyncWrite for Writer { fn poll_write( mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8], ) -> Poll> { // If the reading end of the pipe is closed then return an error now, // otherwise we'd be spending time writing the entire buffer only to // discover that it is closed afterward. if self.buf_stream_tx.is_closed() { return Poll::Ready(Err(io::ErrorKind::BrokenPipe.into())); } // Do not send empty buffers through the rotation. if buf.is_empty() { return Poll::Ready(Ok(0)); } // Attempt to grab an available buffer to write the chunk to. match Pin::new(&mut self.buf_pool_rx).poll_next(cx) { // Wait for the reader to finish reading a chunk. Poll::Pending => Poll::Pending, // Pipe has closed. Poll::Ready(None) => Poll::Ready(Err(io::ErrorKind::BrokenPipe.into())), // An available buffer has been found. Poll::Ready(Some(mut chunk)) => { // Write the buffer to the chunk. chunk.get_mut().extend_from_slice(buf); // Send the chunk to the reader. match self.buf_stream_tx.try_send(chunk) { Ok(()) => Poll::Ready(Ok(buf.len())), Err(e) => { if e.is_full() { panic!("buffer pool overflow") } else { Poll::Ready(Err(io::ErrorKind::BrokenPipe.into())) } } } } } } fn poll_flush(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } fn poll_close(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { self.buf_stream_tx.close(); Poll::Ready(Ok(())) } } sluice-0.5.5/src/pipe/mod.rs000064400000000000000000000047220000000000000137520ustar 00000000000000//! Asynchronous in-memory byte buffers aimed at producer-consumer problems. //! //! Pipes are like byte-oriented channels that implement I/O traits for reading //! and writing. use futures_io::{AsyncBufRead, AsyncRead, AsyncWrite}; use std::{ fmt, io, pin::Pin, task::{Context, Poll}, }; mod chunked; /// How many chunks should be available in a chunked pipe. Default is 4, which /// strikes a good balance of low memory usage and throughput. const DEFAULT_CHUNK_COUNT: usize = 4; /// Creates a new asynchronous pipe with the default configuration. /// /// The default implementation guarantees that when writing a slice of bytes, /// either the entire slice is written at once or not at all. Slices will never /// be partially written. pub fn pipe() -> (PipeReader, PipeWriter) { let (reader, writer) = chunked::new(DEFAULT_CHUNK_COUNT); (PipeReader { inner: reader }, PipeWriter { inner: writer }) } /// The reading end of an asynchronous pipe. pub struct PipeReader { inner: chunked::Reader, } impl AsyncRead for PipeReader { fn poll_read( mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut [u8], ) -> Poll> { Pin::new(&mut self.inner).poll_read(cx, buf) } } impl AsyncBufRead for PipeReader { #[allow(unsafe_code)] fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { unsafe { self.map_unchecked_mut(|s| &mut s.inner) }.poll_fill_buf(cx) } fn consume(mut self: Pin<&mut Self>, amt: usize) { Pin::new(&mut self.inner).consume(amt) } } impl fmt::Debug for PipeReader { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.pad("PipeReader") } } /// The writing end of an asynchronous pipe. pub struct PipeWriter { inner: chunked::Writer, } impl AsyncWrite for PipeWriter { fn poll_write( mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8], ) -> Poll> { Pin::new(&mut self.inner).poll_write(cx, buf) } fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.inner).poll_flush(cx) } fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { Pin::new(&mut self.inner).poll_close(cx) } } impl fmt::Debug for PipeWriter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.pad("PipeWriter") } } sluice-0.5.5/tests/pipe.rs000064400000000000000000000051570000000000000135510ustar 00000000000000use futures::{ executor::block_on, join, prelude::*, }; use quickcheck_macros::quickcheck; use sluice::pipe::pipe; use std::io; #[test] fn read_empty() { block_on(async { let (mut reader, writer) = pipe(); drop(writer); let mut out = String::new(); reader.read_to_string(&mut out).await.unwrap(); assert_eq!(out, ""); }) } #[test] fn read_then_write() { block_on(async { let (mut reader, mut writer) = pipe(); writer.write_all(b"hello world").await.unwrap(); let mut dest = [0; 6]; assert_eq!(reader.read(&mut dest).await.unwrap(), 6); assert_eq!(&dest, b"hello "); assert_eq!(reader.read(&mut dest).await.unwrap(), 5); assert_eq!(&dest[..5], b"world"); }) } #[test] fn reader_still_drainable_after_writer_disconnects() { block_on(async { let (mut reader, mut writer) = pipe(); writer.write_all(b"hello").await.unwrap(); drop(writer); let mut dest = [0; 5]; assert_eq!(reader.read(&mut dest).await.unwrap(), 5); assert_eq!(&dest, b"hello"); // Continue returning Ok(0) forever. for _ in 0..3 { assert_eq!(reader.read(&mut dest).await.unwrap(), 0); } }) } #[test] fn writer_errors_if_reader_is_dropped() { block_on(async { let (reader, mut writer) = pipe(); drop(reader); for _ in 0..3 { assert_eq!(writer.write(b"hello").await.unwrap_err().kind(), io::ErrorKind::BrokenPipe); } }) } #[test] fn pipe_lots_of_data() { block_on(async { let data = vec![0xff; 1_000_000]; let (mut reader, mut writer) = pipe(); join!( async { writer.write_all(&data).await.unwrap(); writer.close().await.unwrap(); }, async { let mut out = Vec::new(); reader.read_to_end(&mut out).await.unwrap(); assert_eq!(&out[..], &data[..]); }, ); }) } #[quickcheck] fn read_write_chunks_random(chunks: u8) { block_on(async { let data = [0; 8192]; let (mut reader, mut writer) = pipe(); join!( async { for _chunk in 0..chunks { writer.write_all(&data).await.unwrap(); } }, async { for _chunk in 0..chunks { let mut buf = data.clone(); reader.read(&mut buf).await.unwrap(); assert_eq!(&buf[..], &data[..]); } }, ); }) }