ordered-stream-0.2.0/.cargo_vcs_info.json0000644000000001360000000000100137500ustar { "git": { "sha1": "5c58ca1a3d680c2406f0c6a1520c3476c27048aa" }, "path_in_vcs": "" }ordered-stream-0.2.0/.gitignore000064400000000000000000000000241046102023000145240ustar 00000000000000/target /Cargo.lock ordered-stream-0.2.0/Cargo.toml0000644000000020660000000000100117520ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "ordered-stream" version = "0.2.0" authors = [ "Daniel De Graaf ", "Zeeshan Ali Khan ", ] description = "Streams that are ordered relative to external events" readme = "README.md" keywords = [ "async", "stream", "timestamp", ] license = "MIT OR Apache-2.0" repository = "https://github.com/danieldg/ordered-stream" [dependencies.futures-core] version = "0.3" [dependencies.pin-project-lite] version = "0.2" [dev-dependencies.futures-executor] version = "0.3.25" [dev-dependencies.futures-util] version = "0.3.25" ordered-stream-0.2.0/Cargo.toml.orig000064400000000000000000000007611046102023000154330ustar 00000000000000[package] name = "ordered-stream" version = "0.2.0" authors = ["Daniel De Graaf ", "Zeeshan Ali Khan "] license = "MIT OR Apache-2.0" edition = "2018" description = "Streams that are ordered relative to external events" repository = "https://github.com/danieldg/ordered-stream" keywords = ["async", "stream", "timestamp"] [dependencies] futures-core = "0.3" pin-project-lite = "0.2" [dev-dependencies] futures-executor = "0.3.25" futures-util = "0.3.25" ordered-stream-0.2.0/LICENSE-APACHE000064400000000000000000000251371046102023000144740ustar 00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ordered-stream-0.2.0/LICENSE-MIT000064400000000000000000000017771046102023000142100ustar 00000000000000Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ordered-stream-0.2.0/README.md000064400000000000000000000043521046102023000140230ustar 00000000000000[![](https://docs.rs/ordered-stream/badge.svg)](https://docs.rs/ordered-stream/) [![](https://img.shields.io/crates/v/ordered-stream)](https://crates.io/crates/ordered-stream) # Streams that produce elements with an associated ordering Say you have a bunch of events that all have a timestamp, sequence number, or other ordering attribute. If you get these events from multiple [`Stream`]s, then you should be able to produce a "composite" stream by joining each of the individual streams, so long as each originating stream is ordered. However, if you actually implement this, you discover that you need to buffer at least one element from each stream in order to avoid ordering inversions if the sources are independent (including just running in different tasks). This presents a problem if one of the sources rarely produces events: that slow source can stall all other streams in order to handle the case where the slowness is due to an earlier element instead of just having no elements. The [`OrderedStream`] trait provides a way to solve this problem: if you can ask a stream if it will ever have any events that should be delivered before a given event, then you can often avoid blocking the composite stream when data is ready. ```rust use futures_core::Stream; use ordered_stream::FromStream; use ordered_stream::JoinMultiple; use ordered_stream::OrderedStream; use ordered_stream::OrderedStreamExt; use std::pin::Pin; use std::time::SystemTime; pub struct Message { time: SystemTime, level: u8, data: String, source: String, } pub struct RemoteLogSource { stream: Pin>>, min_level: u8, } pub async fn display_logs(logs: &mut [RemoteLogSource]) { let mut streams: Vec<_> = logs .iter_mut() .map(|s| { let min = s.min_level; FromStream::with_ordering(&mut s.stream, |m| m.time) .filter(move |m| m.level >= min) .peekable() }) .collect(); let mut joined = JoinMultiple(streams); while let Some(msg) = joined.next().await { println!("{:?}: {}", msg.time, msg.data); } } ``` [`Stream`]: https://docs.rs/futures/latest/futures/stream/trait.Stream.html ordered-stream-0.2.0/src/adapters.rs000064400000000000000000000716121046102023000155070ustar 00000000000000use crate::*; use core::future::Future; use core::pin::Pin; use core::task::{Context, Poll}; use futures_core::{FusedStream, Stream}; /// Helpers for chaining [`OrderedStream`]s. pub trait OrderedStreamExt: OrderedStream { /// Apply a closure to the data. /// /// This does not change the ordering. fn map(self, f: F) -> Map where Self: Sized, F: FnMut(Self::Data) -> R, { Map { stream: self, f } } /// Apply a closure to the items that has access to the ordering data. fn map_item(self, f: F) -> MapItem where Self: Sized, F: FnMut(&Self::Ordering, Self::Data) -> R, { MapItem { stream: self, f } } /// Apply a closure to the items that can change the type of the ordering value. /// /// A bidirectional mapping for ordering values is required in order to remap `before` values. /// It is the caller's responsibility to ensure that the items in the mapped stream still meet /// the ordering requirements that [`OrderedStream`] expects. fn map_ordering( self, map_into: MapInto, map_from: MapFrom, ) -> MapOrdering where Self: Sized, MapInto: FnMut(Self::Ordering, Self::Data) -> (NewOrdering, NewData), MapFrom: FnMut(&NewOrdering) -> Option, NewOrdering: Ord, { MapOrdering { stream: self, map_into, map_from, } } fn filter(self, filter: F) -> Filter where Self: Sized, F: FnMut(&Self::Data) -> bool, { Filter { stream: self, filter, } } fn filter_map(self, filter: F) -> FilterMap where Self: Sized, F: FnMut(Self::Data) -> Option, { FilterMap { stream: self, filter, } } /// Apply a closure that produces a [`Future`] to items, running the future on each item in /// sequence before processing the next. /// /// This has the side effect of buffering items that are not before the requested ordering /// point; you can use [`ready`](core::future::ready) as the closure to take advantage of this /// behavior if you don't want to buffer items yourself. fn then(self, then: F) -> Then where Self: Sized, F: FnMut(Self::Data) -> Fut, Fut: Future, { Then { stream: self, then, future: ThenItem::Idle, } } /// Convert this into a [`Stream`], discarding the ordering information. fn into_stream(self) -> IntoStream where Self: Sized, { IntoStream { stream: self } } /// Convert this into a [`Stream`], keeping the ordering objects. fn into_tuple_stream(self) -> IntoTupleStream where Self: Sized, { IntoTupleStream { stream: self } } /// Convert this into a [`Stream`], keeping only the ordering objects. fn into_ordering(self) -> IntoOrdering where Self: Sized, { IntoOrdering { stream: self } } /// Return the next item in this stream. fn next(&mut self) -> Next<'_, Self> where Self: Unpin, { Next { stream: Pin::new(self), } } /// Return a [`PollResult`] corresponding to the next item in the stream. fn next_before<'a>(&'a mut self, before: Option<&'a Self::Ordering>) -> NextBefore<'a, Self> where Self: Unpin, { NextBefore { stream: Pin::new(self), before, } } fn peekable(self) -> Peekable where Self: Sized, { Peekable { stream: self, item: None, is_terminated: false, } } } impl OrderedStreamExt for T {} pin_project_lite::pin_project! { /// An [`OrderedStream`] wrapper around a [`Stream`]. /// /// This does not use any future or past knowledge of elements, and so is suitable if the /// stream rarely or never blocks. Prefer using [`FromStream`] if you plan to filter or join /// this stream and want other streams to be able to make progress while this one blocks. #[derive(Debug)] pub struct FromStreamDirect { #[pin] stream: S, split_item: F, } } impl FromStreamDirect { /// Create a new [`OrderedStream`] by applying a `split_item` closure to each element /// produced by the original stream. pub fn new(stream: S, split_item: F) -> Self where S: Stream, F: FnMut(S::Item) -> (Ordering, Data), Ordering: Ord, { Self { stream, split_item } } /// Helper function to simplify the creation of a stream when you have a get_ordering function. pub fn with_ordering( stream: S, mut get_ordering: F, ) -> FromStreamDirect (Ordering, S::Item)> where S: Stream, F: FnMut(&S::Item) -> Ordering, Ordering: Ord, { FromStreamDirect::new(stream, move |data| { let ordering = get_ordering(&data); (ordering, data) }) } } impl OrderedStream for FromStreamDirect where S: Stream, F: FnMut(S::Item) -> (Ordering, Data), Ordering: Ord, { type Data = Data; type Ordering = Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, _: Option<&Self::Ordering>, ) -> Poll> { let this = self.project(); let split_item = this.split_item; this.stream.poll_next(cx).map(|opt| match opt { None => PollResult::Terminated, Some(data) => { let (ordering, data) = split_item(data); PollResult::Item { data, ordering } } }) } fn size_hint(&self) -> (usize, Option) { self.stream.size_hint() } } impl FusedOrderedStream for FromStreamDirect where S: FusedStream, F: FnMut(S::Item) -> (Ordering, Data), Ordering: Ord, { fn is_terminated(&self) -> bool { self.stream.is_terminated() } } pin_project_lite::pin_project! { /// An [`OrderedStream`] wrapper around a [`Stream`]. /// /// Unlike [`FromStream`], the items in the [`Stream`] are themselves ordered with no /// additional data. #[derive(Debug)] pub struct FromSortedStream { #[pin] pub stream: S, } } impl FromSortedStream { /// Create a new [`OrderedStream`] by applying a `split_item` closure to each element /// produced by the original stream. pub fn new(stream: S) -> Self where S: Stream, S::Item: Ord, { Self { stream } } } impl OrderedStream for FromSortedStream where S: Stream, S::Item: Ord, { type Data = (); type Ordering = S::Item; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, _: Option<&Self::Ordering>, ) -> Poll> { let this = self.project(); this.stream.poll_next(cx).map(|opt| match opt { None => PollResult::Terminated, Some(ordering) => PollResult::Item { data: (), ordering }, }) } fn size_hint(&self) -> (usize, Option) { self.stream.size_hint() } } impl FusedOrderedStream for FromSortedStream where S: FusedStream, S::Item: Ord, { fn is_terminated(&self) -> bool { self.stream.is_terminated() } } pin_project_lite::pin_project! { /// An [`OrderedStream`] wrapper around a [`Stream`]. /// /// This caches the last-used ordering point returned by the stream and uses it to produce /// NoneBefore results. This makes it suitable for using to adapt streams that are filtered /// or mapped before joining. It still relies on the original stream producing a later-ordered /// element to allow other streams to progress, however. #[derive(Debug)] pub struct FromStream { #[pin] stream: S, split_item: F, last: Option, } } impl FromStream where S: Stream, Ordering: Ord + Clone, { /// Create a new [`OrderedStream`] by applying a `split_item` closure to each element /// produced by the original stream. pub fn new(stream: S, split_item: F) -> Self where F: FnMut(S::Item) -> (Ordering, Data), { FromStream { stream, split_item, last: None, } } /// Helper function to simplify the creation of a stream when you have a get_ordering function. pub fn with_ordering( stream: S, mut get_ordering: F, ) -> FromStream (Ordering, S::Item), Ordering> where F: FnMut(&S::Item) -> Ordering, { FromStream::new(stream, move |data| { let ordering = get_ordering(&data); (ordering, data) }) } } impl OrderedStream for FromStream where S: Stream, F: FnMut(S::Item) -> (Ordering, Data), Ordering: Ord + Clone, { type Data = Data; type Ordering = Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { let this = self.project(); let split_item = this.split_item; let last = this.last; if let (Some(last), Some(before)) = (last.as_ref(), before) { if last >= before { return Poll::Ready(PollResult::NoneBefore); } } this.stream.poll_next(cx).map(|opt| match opt { None => PollResult::Terminated, Some(item) => { let (ordering, data) = split_item(item); *last = Some(ordering.clone()); PollResult::Item { data, ordering } } }) } fn position_hint(&self) -> Option> { self.last.as_ref().map(MaybeBorrowed::Borrowed) } fn size_hint(&self) -> (usize, Option) { self.stream.size_hint() } } impl FusedOrderedStream for FromStream where S: FusedStream, F: FnMut(S::Item) -> (Ordering, Data), Ordering: Ord + Clone, { fn is_terminated(&self) -> bool { self.stream.is_terminated() } } pin_project_lite::pin_project! { /// A [`Stream`] for the [`into_stream`](OrderedStreamExt::into_stream) function. #[derive(Debug)] pub struct IntoStream { #[pin] stream: S, } } impl Stream for IntoStream { type Item = S::Data; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { self.project() .stream .poll_next_before(cx, None) .map(|r| r.into_data()) } fn size_hint(&self) -> (usize, Option) { self.stream.size_hint() } } impl FusedStream for IntoStream where S: FusedOrderedStream, { fn is_terminated(&self) -> bool { self.stream.is_terminated() } } pin_project_lite::pin_project! { /// A [`Stream`] for the [`into_tuple_stream`](OrderedStreamExt::into_tuple_stream) function. #[derive(Debug)] pub struct IntoTupleStream { #[pin] stream: S, } } impl Stream for IntoTupleStream { type Item = (S::Ordering, S::Data); fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { self.project() .stream .poll_next_before(cx, None) .map(|r| r.into_tuple()) } fn size_hint(&self) -> (usize, Option) { self.stream.size_hint() } } impl FusedStream for IntoTupleStream where S: FusedOrderedStream, { fn is_terminated(&self) -> bool { self.stream.is_terminated() } } pin_project_lite::pin_project! { /// A [`Stream`] for the [`into_ordering`](OrderedStreamExt::into_ordering) function. #[derive(Debug)] pub struct IntoOrdering { #[pin] stream: S, } } impl Stream for IntoOrdering { type Item = S::Ordering; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { self.project() .stream .poll_next_before(cx, None) .map(|r| r.into_tuple().map(|t| t.0)) } fn size_hint(&self) -> (usize, Option) { self.stream.size_hint() } } impl FusedStream for IntoOrdering where S: FusedOrderedStream, { fn is_terminated(&self) -> bool { self.stream.is_terminated() } } pin_project_lite::pin_project! { /// An [`OrderedStream`] wrapper around an [`OrderedFuture`]. #[derive(Debug)] pub struct FromFuture { #[pin] future: Option, } } impl From for FromFuture { fn from(future: F) -> Self { Self { future: Some(future), } } } impl OrderedStream for FromFuture { type Data = F::Output; type Ordering = F::Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { let mut this = self.project(); match this.future.as_mut().as_pin_mut() { Some(future) => match future.poll_before(cx, before) { Poll::Ready(Some((ordering, data))) => { this.future.set(None); Poll::Ready(PollResult::Item { data, ordering }) } Poll::Ready(None) => Poll::Ready(PollResult::NoneBefore), Poll::Pending => Poll::Pending, }, None => Poll::Ready(PollResult::Terminated), } } fn position_hint(&self) -> Option> { self.future.as_ref().and_then(|f| f.position_hint()) } fn size_hint(&self) -> (usize, Option) { if self.future.is_some() { (1, Some(1)) } else { (0, Some(0)) } } } impl FusedOrderedStream for FromFuture { fn is_terminated(&self) -> bool { self.future.is_none() } } pin_project_lite::pin_project! { /// A stream for the [`map`](OrderedStreamExt::map) function. #[derive(Debug)] pub struct Map { #[pin] stream: S, f: F, } } impl Map { /// Convert to source stream. pub fn into_inner(self) -> S { self.stream } } impl OrderedStream for Map where S: OrderedStream, F: FnMut(S::Data) -> R, { type Data = R; type Ordering = S::Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { let this = self.project(); let f = this.f; this.stream .poll_next_before(cx, before) .map(|res| res.map_data(f)) } fn position_hint(&self) -> Option> { self.stream.position_hint() } fn size_hint(&self) -> (usize, Option) { self.stream.size_hint() } } pin_project_lite::pin_project! { /// A stream for the [`map_item`](OrderedStreamExt::map_item) function. #[derive(Debug)] pub struct MapItem { #[pin] stream: S, f: F, } } impl MapItem { /// Convert to source stream. pub fn into_inner(self) -> S { self.stream } } impl OrderedStream for MapItem where S: OrderedStream, F: FnMut(&S::Ordering, S::Data) -> R, { type Data = R; type Ordering = S::Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { let this = self.project(); let f = this.f; this.stream .poll_next_before(cx, before) .map(|res| match res { PollResult::Item { data, ordering } => { let data = f(&ordering, data); PollResult::Item { data, ordering } } PollResult::NoneBefore => PollResult::NoneBefore, PollResult::Terminated => PollResult::Terminated, }) } fn position_hint(&self) -> Option> { self.stream.position_hint() } fn size_hint(&self) -> (usize, Option) { self.stream.size_hint() } } pin_project_lite::pin_project! { /// A stream for the [`map_ordering`](OrderedStreamExt::map_ordering) function. #[derive(Debug)] pub struct MapOrdering { #[pin] stream: S, map_into: MapInto, map_from: MapFrom, } } impl MapOrdering { /// Convert to source stream. pub fn into_inner(self) -> S { self.stream } } impl OrderedStream for MapOrdering where S: OrderedStream, MapInto: FnMut(S::Ordering, S::Data) -> (NewOrdering, NewData), MapFrom: FnMut(&NewOrdering) -> Option, NewOrdering: Ord, { type Data = NewData; type Ordering = NewOrdering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { let this = self.project(); let map_into = this.map_into; let before = before.and_then(this.map_from); this.stream .poll_next_before(cx, before.as_ref()) .map(|res| match res { PollResult::Item { data, ordering } => { let (ordering, data) = map_into(ordering, data); PollResult::Item { data, ordering } } PollResult::NoneBefore => PollResult::NoneBefore, PollResult::Terminated => PollResult::Terminated, }) } fn size_hint(&self) -> (usize, Option) { self.stream.size_hint() } } pin_project_lite::pin_project! { /// A stream for the [`filter`](OrderedStreamExt::filter) function. #[derive(Debug)] pub struct Filter { #[pin] stream: S, filter: F, } } impl Filter { /// Convert to source stream. pub fn into_inner(self) -> S { self.stream } } impl OrderedStream for Filter where S: OrderedStream, F: FnMut(&S::Data) -> bool, { type Data = S::Data; type Ordering = S::Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { let mut this = self.project(); loop { match this.stream.as_mut().poll_next_before(cx, before).into() { PollState::Pending => return Poll::Pending, PollState::Terminated => return Poll::Ready(PollResult::Terminated), PollState::NoneBefore => return Poll::Ready(PollResult::NoneBefore), PollState::Item(data, ordering) => { if (this.filter)(&data) { return Poll::Ready(PollResult::Item { data, ordering }); } } } } } fn position_hint(&self) -> Option> { self.stream.position_hint() } fn size_hint(&self) -> (usize, Option) { (0, self.stream.size_hint().1) } } pin_project_lite::pin_project! { /// A stream for the [`filter_map`](OrderedStreamExt::filter_map) function. #[derive(Debug)] pub struct FilterMap { #[pin] stream: S, filter: F, } } impl FilterMap { /// Convert to source stream. pub fn into_inner(self) -> S { self.stream } } impl OrderedStream for FilterMap where S: OrderedStream, F: FnMut(S::Data) -> Option, { type Data = R; type Ordering = S::Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { let mut this = self.project(); loop { match this.stream.as_mut().poll_next_before(cx, before).into() { PollState::Pending => return Poll::Pending, PollState::Terminated => return Poll::Ready(PollResult::Terminated), PollState::NoneBefore => return Poll::Ready(PollResult::NoneBefore), PollState::Item(data, ordering) => match (this.filter)(data) { Some(data) => return Poll::Ready(PollResult::Item { data, ordering }), None => continue, }, } } } fn position_hint(&self) -> Option> { self.stream.position_hint() } fn size_hint(&self) -> (usize, Option) { (0, self.stream.size_hint().1) } } pin_project_lite::pin_project! { #[project = ThenProj] #[project_replace = ThenDone] #[derive(Debug)] enum ThenItem { Running { #[pin] future: Fut, ordering: T }, Idle, } } pin_project_lite::pin_project! { /// A stream for the [`then`](OrderedStreamExt::then) function. #[derive(Debug)] pub struct Then where S: OrderedStream { #[pin] stream: S, then: F, #[pin] future: ThenItem, } } impl OrderedStream for Then where S: OrderedStream, F: FnMut(S::Data) -> Fut, Fut: Future, { type Data = Fut::Output; type Ordering = S::Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { let mut this = self.project(); loop { if let ThenProj::Running { future, ordering } = this.future.as_mut().project() { // Because we know the next ordering, we can answer questions about it now. if let Some(before) = before { if *ordering >= *before { return Poll::Ready(PollResult::NoneBefore); } } if let Poll::Ready(data) = future.poll(cx) { if let ThenDone::Running { ordering, .. } = this.future.as_mut().project_replace(ThenItem::Idle) { return Poll::Ready(PollResult::Item { data, ordering }); } } else { return Poll::Pending; } } match this.stream.as_mut().poll_next_before(cx, before).into() { PollState::Pending => return Poll::Pending, PollState::Terminated => return Poll::Ready(PollResult::Terminated), PollState::NoneBefore => return Poll::Ready(PollResult::NoneBefore), PollState::Item(data, ordering) => { this.future.set(ThenItem::Running { future: (this.then)(data), ordering, }); } } } } fn position_hint(&self) -> Option> { match &self.future { ThenItem::Running { ordering, .. } => Some(MaybeBorrowed::Borrowed(ordering)), ThenItem::Idle => self.stream.position_hint(), } } fn size_hint(&self) -> (usize, Option) { let (min, max) = self.stream.size_hint(); match self.future { ThenItem::Running { .. } => (min.saturating_add(1), max.and_then(|v| v.checked_add(1))), ThenItem::Idle => (min, max), } } } /// A future for the [`next`](OrderedStreamExt::next) function. #[derive(Debug)] pub struct Next<'a, S: ?Sized> { stream: Pin<&'a mut S>, } impl<'a, S: ?Sized> Unpin for Next<'a, S> {} impl<'a, S> Future for Next<'a, S> where S: OrderedStream + ?Sized, { type Output = Option; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { self.stream .as_mut() .poll_next_before(cx, None) .map(PollResult::into_data) } } /// A future for the [`next_before`](OrderedStreamExt::next_before) function. #[derive(Debug)] pub struct NextBefore<'a, S> where S: OrderedStream + ?Sized, { stream: Pin<&'a mut S>, before: Option<&'a S::Ordering>, } impl<'a, S: OrderedStream + ?Sized> Unpin for NextBefore<'a, S> {} impl<'a, S> Future for NextBefore<'a, S> where S: OrderedStream + ?Sized, { type Output = PollResult; fn poll( mut self: Pin<&mut Self>, cx: &mut Context<'_>, ) -> Poll> { let before = self.before; self.stream.as_mut().poll_next_before(cx, before) } } pin_project_lite::pin_project! { /// A stream for the [`peekable`](OrderedStreamExt::peekable) function. #[derive(Debug)] pub struct Peekable { #[pin] stream: S, is_terminated: bool, item: Option<(S::Ordering, S::Data)>, } } impl Peekable { /// Convert into the source stream. /// /// This method returns the source stream along with any buffered item and its /// ordering. pub fn into_inner(self) -> (S, Option<(S::Data, S::Ordering)>) { (self.stream, self.item.map(|(o, d)| (d, o))) } /// The current item, without polling pub(crate) fn item(&self) -> Option<&(S::Ordering, S::Data)> { self.item.as_ref() } /// Peek on the next item in the stream pub fn poll_peek_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&S::Ordering>, ) -> Poll> { let mut this = self.project(); if *this.is_terminated { return Poll::Ready(PollResult::Terminated); } let stream = this.stream.as_mut(); if this.item.is_none() { match stream.poll_next_before(cx, before) { Poll::Ready(PollResult::Item { ordering, data }) => { *this.item = Some((ordering, data)); } Poll::Ready(PollResult::NoneBefore) => return Poll::Ready(PollResult::NoneBefore), Poll::Ready(PollResult::Terminated) => { *this.is_terminated = true; return Poll::Ready(PollResult::Terminated); } Poll::Pending => return Poll::Pending, } } let item = this.item.as_mut().unwrap(); Poll::Ready(PollResult::Item { ordering: &item.0, data: &mut item.1, }) } } impl OrderedStream for Peekable { type Ordering = S::Ordering; type Data = S::Data; fn poll_next_before( mut self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&S::Ordering>, ) -> Poll> { match self.as_mut().poll_peek_before(cx, before) { Poll::Ready(PollResult::Item { .. }) => { let (ordering, data) = self.project().item.take().unwrap(); Poll::Ready(PollResult::Item { ordering, data }) } Poll::Ready(PollResult::NoneBefore) => Poll::Ready(PollResult::NoneBefore), Poll::Ready(PollResult::Terminated) => Poll::Ready(PollResult::Terminated), Poll::Pending => Poll::Pending, } } fn position_hint(&self) -> Option> { match &self.item { Some((ordering, _)) => Some(MaybeBorrowed::Borrowed(ordering)), None => self.stream.position_hint(), } } fn size_hint(&self) -> (usize, Option) { let (min, max) = if self.is_terminated { (0, Some(0)) } else { self.stream.size_hint() }; if self.item.is_some() { (min.saturating_add(1), max.and_then(|v| v.checked_add(1))) } else { (min, max) } } } impl FusedOrderedStream for Peekable { fn is_terminated(&self) -> bool { self.is_terminated } } ordered-stream-0.2.0/src/join.rs000064400000000000000000000452351046102023000146450ustar 00000000000000use crate::*; use core::mem; use core::pin::Pin; use core::task::{Context, Poll}; pin_project_lite::pin_project! { /// A stream for the [`join`](fn.join.html) function. #[derive(Debug)] pub struct Join where A: OrderedStream, B: OrderedStream, { #[pin] stream_a: A, #[pin] stream_b: B, state: JoinState, } } /// Join two streams while preserving the overall ordering of elements. /// /// You can think of this as implementing the "merge" step of a merge sort on the two streams, /// producing a single stream that is sorted given two sorted streams. If the streams return /// [`PollResult::NoneBefore`] as intended, then the joined stream will be able to produce items /// when only one of the sources has unblocked. pub fn join(stream_a: A, stream_b: B) -> Join where A: OrderedStream, B: OrderedStream, { Join { stream_a, stream_b, state: JoinState::None, } } #[derive(Debug)] enum JoinState { None, A(A, T), B(B, T), OnlyPollA, OnlyPollB, Terminated, } impl JoinState { fn take_split(&mut self) -> (PollState, PollState) { match mem::replace(self, JoinState::None) { JoinState::None => (PollState::Pending, PollState::Pending), JoinState::A(a, t) => (PollState::Item(a, t), PollState::Pending), JoinState::B(b, t) => (PollState::Pending, PollState::Item(b, t)), JoinState::OnlyPollA => (PollState::Pending, PollState::Terminated), JoinState::OnlyPollB => (PollState::Terminated, PollState::Pending), JoinState::Terminated => (PollState::Terminated, PollState::Terminated), } } } /// A helper equivalent to Poll> but easier to match pub(crate) enum PollState { Item(I, T), Pending, NoneBefore, Terminated, } impl PollState { fn ordering(&self) -> Option<&T> { match self { Self::Item(_, t) => Some(t), _ => None, } } fn update( &mut self, before: Option<&T>, other_token: Option<&T>, retry: bool, run: impl FnOnce(Option<&T>) -> Poll>, ) -> bool { match self { // Do not re-poll if we have an item already or if we are terminated Self::Item { .. } | Self::Terminated => return false, // No need to re-poll if we already declared no items <= before Self::NoneBefore if retry => return false, _ => {} } // Run the poll with the earlier of the two tokens to avoid transitioning to Pending (which // will stall the Join) when we could have transitioned to NoneBefore. let ordering = match (before, other_token) { (Some(u), Some(o)) => { if *u > *o { // The other ordering is earlier - so a retry might let us upgrade a Pending to a // NoneBefore Some(o) } else if retry { // A retry will not improve matters, so don't bother return false; } else { Some(u) } } (Some(t), None) | (None, Some(t)) => Some(t), (None, None) => None, }; *self = run(ordering).into(); matches!(self, Self::Item { .. }) } } impl From> for Poll> { fn from(poll: PollState) -> Self { match poll { PollState::Item(data, ordering) => Poll::Ready(PollResult::Item { data, ordering }), PollState::Pending => Poll::Pending, PollState::NoneBefore => Poll::Ready(PollResult::NoneBefore), PollState::Terminated => Poll::Ready(PollResult::Terminated), } } } impl From>> for PollState { fn from(poll: Poll>) -> Self { match poll { Poll::Ready(PollResult::Item { data, ordering }) => Self::Item(data, ordering), Poll::Ready(PollResult::NoneBefore) => Self::NoneBefore, Poll::Ready(PollResult::Terminated) => Self::Terminated, Poll::Pending => Self::Pending, } } } impl Join where A: OrderedStream, B: OrderedStream, { /// Split into the source streams. /// /// This method returns the source streams along with any buffered item and its /// ordering. pub fn into_inner(self) -> (A, B, Option<(A::Data, A::Ordering)>) { let item = match self.state { JoinState::A(a, o) => Some((a, o)), JoinState::B(b, o) => Some((b, o)), _ => None, }; (self.stream_a, self.stream_b, item) } /// Provide direct access to the underlying stream. /// /// This may be useful if the stream provides APIs beyond [OrderedStream]. Note that the join /// itself may be buffering an item from this stream, so you should consult /// [Self::peek_buffered] and, if needed, [Self::take_buffered] before polling it directly. pub fn stream_a(self: Pin<&mut Self>) -> Pin<&mut A> { self.project().stream_a } /// Provide direct access to the underlying stream. /// /// This may be useful if the stream provides APIs beyond [OrderedStream]. Note that the join /// itself may be buffering an item from this stream, so you should consult /// [Self::peek_buffered] and, if needed, [Self::take_buffered] before polling it directly. pub fn stream_b(self: Pin<&mut Self>) -> Pin<&mut B> { self.project().stream_b } /// Allow access to the buffered item, if any. /// /// At most one of the two sides will be `Some`. The returned item is a candidate for being /// the next item returned by the joined stream, but it could not be returned by the most /// recent [`OrderedStream::poll_next_before`] call. pub fn peek_buffered( self: Pin<&mut Self>, ) -> ( Option<(&mut A::Data, &A::Ordering)>, Option<(&mut B::Data, &B::Ordering)>, ) { match self.project().state { JoinState::A(a, o) => (Some((a, o)), None), JoinState::B(b, o) => (None, Some((b, o))), _ => (None, None), } } /// Remove the buffered item, if one is present. /// /// This does not poll either underlying stream. See [Self::peek_buffered] for details on why /// buffering exists. pub fn take_buffered(self: Pin<&mut Self>) -> Option<(A::Data, A::Ordering)> { let state = self.project().state; match mem::replace(state, JoinState::None) { JoinState::A(a, o) => Some((a, o)), JoinState::B(b, o) => Some((b, o)), other => { *state = other; None } } } } impl OrderedStream for Join where A: OrderedStream, B: OrderedStream, { type Data = A::Data; type Ordering = A::Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { let mut this = self.project(); let (mut poll_a, mut poll_b) = this.state.take_split(); poll_a.update(before, poll_b.ordering(), false, |ordering| { this.stream_a.as_mut().poll_next_before(cx, ordering) }); if poll_b.update(before, poll_a.ordering(), false, |ordering| { this.stream_b.as_mut().poll_next_before(cx, ordering) }) { // If B just got an item, it's possible that A already knows that it won't have any // items before that item; we couldn't ask that question before. Ask it now. poll_a.update(before, poll_b.ordering(), true, |ordering| { this.stream_a.as_mut().poll_next_before(cx, ordering) }); } match (poll_a, poll_b) { // Both are ready - we can judge ordering directly (simplest case). The first one is // returned while the other one is buffered for the next poll. (PollState::Item(a, ta), PollState::Item(b, tb)) => { if ta <= tb { *this.state = JoinState::B(b, tb); Poll::Ready(PollResult::Item { data: a, ordering: ta, }) } else { *this.state = JoinState::A(a, ta); Poll::Ready(PollResult::Item { data: b, ordering: tb, }) } } // If both sides are terminated, so are we. (PollState::Terminated, PollState::Terminated) => { *this.state = JoinState::Terminated; Poll::Ready(PollResult::Terminated) } // If one side is terminated, we can produce items directly from the other side. (a, PollState::Terminated) => { *this.state = JoinState::OnlyPollA; a.into() } (PollState::Terminated, b) => { *this.state = JoinState::OnlyPollB; b.into() } // If one side is pending, we can't return Ready until that gets resolved. Because we // have already requested that our child streams wake us when it is possible to make // any kind of progress, we meet the requirements to return Poll::Pending. (PollState::Item(a, t), PollState::Pending) => { *this.state = JoinState::A(a, t); Poll::Pending } (PollState::Pending, PollState::Item(b, t)) => { *this.state = JoinState::B(b, t); Poll::Pending } (PollState::Pending, PollState::Pending) => Poll::Pending, (PollState::Pending, PollState::NoneBefore) => Poll::Pending, (PollState::NoneBefore, PollState::Pending) => Poll::Pending, // If both sides report NoneBefore, so can we. (PollState::NoneBefore, PollState::NoneBefore) => Poll::Ready(PollResult::NoneBefore), (PollState::Item(data, ordering), PollState::NoneBefore) => { // B was polled using either the Some value of (before) or using A's ordering. // // If before is set and is earlier than A's ordering, then B might later produce a // value with (bt >= before && bt < at), so we can't return A's item yet and must // buffer it. However, we can return None because neither stream will produce // items before the ordering passed in before. // // If before is either None or after A's ordering, B's NoneBefore return represents a // promise to not produce an item before A's, so we can return A's item now. match before { Some(before) if ordering > *before => { *this.state = JoinState::A(data, ordering); Poll::Ready(PollResult::NoneBefore) } _ => Poll::Ready(PollResult::Item { data, ordering }), } } (PollState::NoneBefore, PollState::Item(data, ordering)) => { // A was polled using either the Some value of (before) or using B's ordering. // // By a mirror of the above argument, this NoneBefore result gives us permission to // produce either B's item or NoneBefore. match before { Some(before) if ordering > *before => { *this.state = JoinState::B(data, ordering); Poll::Ready(PollResult::NoneBefore) } _ => Poll::Ready(PollResult::Item { data, ordering }), } } } } fn position_hint(&self) -> Option> { let (a, b) = match &self.state { JoinState::None => (self.stream_a.position_hint(), self.stream_b.position_hint()), JoinState::A(_, t) => ( Some(MaybeBorrowed::Borrowed(t)), self.stream_b.position_hint(), ), JoinState::B(_, t) => ( self.stream_b.position_hint(), Some(MaybeBorrowed::Borrowed(t)), ), JoinState::OnlyPollA => return self.stream_a.position_hint(), JoinState::OnlyPollB => return self.stream_b.position_hint(), JoinState::Terminated => return None, }; // We can only provide a hint if we have a valid hint for both sides match (a, b) { (Some(a), Some(b)) if *a <= *b => Some(a), (Some(_), Some(b)) => Some(b), _ => None, } } fn size_hint(&self) -> (usize, Option) { let extra = match &self.state { JoinState::None => 0, JoinState::A { .. } => 1, JoinState::B { .. } => 1, JoinState::OnlyPollA => return self.stream_a.size_hint(), JoinState::OnlyPollB => return self.stream_b.size_hint(), JoinState::Terminated => return (0, Some(0)), }; let (al, ah) = self.stream_a.size_hint(); let (bl, bh) = self.stream_b.size_hint(); let min = al.saturating_add(bl).saturating_add(extra); let max = ah .and_then(|a| bh.and_then(|b| a.checked_add(b))) .and_then(|h| h.checked_add(extra)); (min, max) } } impl FusedOrderedStream for Join where A: OrderedStream, B: OrderedStream, { fn is_terminated(&self) -> bool { matches!(self.state, JoinState::Terminated) } } #[cfg(test)] mod test { extern crate alloc; use crate::join; use crate::FromStream; use crate::OrderedStream; use crate::OrderedStreamExt; use crate::PollResult; use alloc::rc::Rc; use core::cell::Cell; use core::pin::Pin; use core::task::{Context, Poll}; use futures_executor::block_on; use futures_util::pin_mut; use futures_util::stream::iter; #[derive(Debug, PartialEq)] pub struct Message { serial: u32, } #[test] fn join_two() { block_on(async { let stream1 = iter([ Message { serial: 1 }, Message { serial: 4 }, Message { serial: 5 }, ]); let stream2 = iter([ Message { serial: 2 }, Message { serial: 3 }, Message { serial: 6 }, ]); let mut joined = join( FromStream::with_ordering(stream1, |m| m.serial), FromStream::with_ordering(stream2, |m| m.serial), ); for i in 0..6 { let msg = joined.next().await.unwrap(); assert_eq!(msg.serial, i as u32 + 1); } }); } #[test] fn join_one_slow() { futures_executor::block_on(async { pub struct DelayStream(Rc>); impl OrderedStream for DelayStream { type Ordering = u32; type Data = Message; fn poll_next_before( self: Pin<&mut Self>, _: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { match self.0.get() { 0 => Poll::Pending, 1 if matches!(before, Some(&1)) => Poll::Ready(PollResult::NoneBefore), 1 => Poll::Pending, 2 => { self.0.set(3); Poll::Ready(PollResult::Item { data: Message { serial: 4 }, ordering: 4, }) } _ => Poll::Ready(PollResult::Terminated), } } } let stream1 = iter([ Message { serial: 1 }, Message { serial: 3 }, Message { serial: 5 }, ]); let stream1 = FromStream::with_ordering(stream1, |m| m.serial); let go = Rc::new(Cell::new(0)); let stream2 = DelayStream(go.clone()); let join = join(stream1, stream2); let waker = futures_util::task::noop_waker(); let mut ctx = core::task::Context::from_waker(&waker); pin_mut!(join); // When the DelayStream has no information about what it contains, join returns Pending // (since there could be a serial-0 message output of DelayStream) assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Pending ); go.set(1); // Now the DelayStream will return NoneBefore on serial 1 assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Item { data: Message { serial: 1 }, ordering: 1, }) ); // however, it does not (yet) do so for serial 3 assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Pending ); go.set(2); assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Item { data: Message { serial: 3 }, ordering: 3, }) ); assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Item { data: Message { serial: 4 }, ordering: 4, }) ); assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Item { data: Message { serial: 5 }, ordering: 5, }) ); assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Terminated) ); }); } } ordered-stream-0.2.0/src/lib.rs000064400000000000000000000256661046102023000144620ustar 00000000000000#![no_std] #![doc = include_str!("../README.md")] use core::pin::Pin; use core::task::{Context, Poll}; /// A stream that produces items that are ordered according to some token. /// /// The main advantage of this trait over the standard `Stream` trait is the ability to implement a /// [`join`](join()) function that does not either block until both source streams produce an item /// or contain a race condition when rejoining streams that originated from a common well-ordered /// source. pub trait OrderedStream { /// The type ordered by this stream. /// /// Each stream must produce values that are in ascending order according to this function, /// although there is no requirement that the values be strictly ascending. type Ordering: Ord; /// The unordered data carried by this stream /// /// This is split from the `Ordering` type to allow specifying a smaller or cheaper-to-generate /// type as the ordering key. This is especially useful if you generate values to pass in to /// `before`. type Data; /// Attempt to pull out the next value of this stream, registering the current task for wakeup /// if needed, and returning `NoneBefore` if it is known that the stream will not produce any /// more values ordered before the given point. /// /// # Return value /// /// There are several possible return values, each indicating a distinct stream state depending /// on the value passed in `before`: /// /// - If `before` was `None`, `Poll::Pending` means that this stream's next value is not ready /// yet. Implementations will ensure that the current task is notified when the next value may /// be ready. /// /// - If `before` was `Some`, `Poll::Pending` means that this stream's next value is not ready /// and that it is not yet known if the stream will produce a value ordered prior to the given /// ordering value. Implementations will ensure that the current task is notified when either /// the next value is ready or once it is known that no such value will be produced. /// /// - `Poll::Ready(PollResult::Item)` means that the stream has successfully produced /// an item. The stream may produce further values on subsequent `poll_next_before` calls. /// The returned ordering value must not be less than any prior ordering value returned by this /// stream. The returned ordering value **may** be greater than the value passed to `before`. /// /// - `Poll::Ready(PollResult::Terminated)` means that the stream has terminated, and /// `poll_next_before` should not be invoked again. /// /// - `Poll::Ready(PollResult::NoneBefore)` means that the stream will not produce /// any further ordering tokens less than the given token. Subsequent `poll_next_before` calls /// may still produce additional items, but their tokens will be greater than or equal to the /// given token. It does not make sense to return this value if `before` was `None`. fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll>; /// The minimum value of the ordering for any future items. /// /// If this does not return `None`, the returned ordering must be less than or equal to the /// ordering of any future item returned from [`Self::poll_next_before`]. This value should /// (but is not required to) be greater than or equal to the ordering of the most recent item /// returned. fn position_hint(&self) -> Option> { None } /// Returns the bounds on the remaining length of the stream. fn size_hint(&self) -> (usize, Option) { (0, None) } } /// A value that is either borrowed or owned. /// /// This is similar to `std::borrow::Cow`, but does not require the ability to convert from /// borrowed to owned. #[derive(Debug)] pub enum MaybeBorrowed<'a, T> { Borrowed(&'a T), Owned(T), } impl<'a, T> AsRef for MaybeBorrowed<'a, T> { fn as_ref(&self) -> &T { match self { Self::Borrowed(t) => t, Self::Owned(t) => t, } } } impl<'a, T> core::ops::Deref for MaybeBorrowed<'a, T> { type Target = T; fn deref(&self) -> &T { match self { Self::Borrowed(t) => t, Self::Owned(t) => t, } } } impl

OrderedStream for Pin

where P: core::ops::DerefMut + Unpin, P::Target: OrderedStream, { type Data = ::Data; type Ordering = ::Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { self.get_mut().as_mut().poll_next_before(cx, before) } fn position_hint(&self) -> Option> { (**self).position_hint() } fn size_hint(&self) -> (usize, Option) { (**self).size_hint() } } impl OrderedStream for Option where S: OrderedStream, { type Data = S::Data; type Ordering = S::Ordering; fn poll_next_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { match self.as_pin_mut() { Some(s) => s.poll_next_before(cx, before), None => Poll::Ready(PollResult::Terminated), } } fn position_hint(&self) -> Option> { self.as_ref().and_then(|s| s.position_hint()) } fn size_hint(&self) -> (usize, Option) { self.as_ref().map_or((0, Some(0)), |s| s.size_hint()) } } /// An [`OrderedStream`] that tracks if the underlying stream should be polled. pub trait FusedOrderedStream: OrderedStream { /// Returns `true` if the stream should no longer be polled. fn is_terminated(&self) -> bool; } impl

FusedOrderedStream for Pin

where P: core::ops::DerefMut + Unpin, P::Target: FusedOrderedStream, { fn is_terminated(&self) -> bool { (**self).is_terminated() } } impl FusedOrderedStream for Option where S: FusedOrderedStream, { fn is_terminated(&self) -> bool { self.as_ref().map_or(true, |s| s.is_terminated()) } } /// The result of a [`OrderedStream::poll_next_before`] operation. #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] pub enum PollResult { /// An item with a corresponding ordering token. Item { data: Data, ordering: Ordering }, /// This stream will not return any items prior to the given point. NoneBefore, /// This stream is terminated and should not be polled again. Terminated, } impl PollResult { /// Extract the data from the result. pub fn into_data(self) -> Option { match self { Self::Item { data, .. } => Some(data), _ => None, } } /// Extract the item from the result. pub fn into_tuple(self) -> Option<(T, D)> { match self { Self::Item { data, ordering } => Some((ordering, data)), _ => None, } } /// Apply a closure to the data. pub fn map_data(self, f: impl FnOnce(D) -> R) -> PollResult { match self { Self::Item { data, ordering } => PollResult::Item { data: f(data), ordering, }, Self::NoneBefore => PollResult::NoneBefore, Self::Terminated => PollResult::Terminated, } } } impl PollResult> { /// Extract the error of a [`Result`] item. pub fn transpose_result(self) -> Result, E> { self.transpose_result_item().map_err(|(_, e)| e) } /// Extract the error and ordering from a [`Result`] item. pub fn transpose_result_item(self) -> Result, (T, E)> { match self { Self::Item { data: Ok(data), ordering, } => Ok(PollResult::Item { data, ordering }), Self::Item { data: Err(data), ordering, } => Err((ordering, data)), Self::NoneBefore => Ok(PollResult::NoneBefore), Self::Terminated => Ok(PollResult::Terminated), } } } /// A [`Future`](core::future::Future) that produces an item with an associated ordering. /// /// This is equivalent to an [`OrderedStream`] that always produces exactly one item. This trait /// is not very useful on its own; see [`FromFuture`] to convert it to a stream. /// /// It is valid to implement both [`Future`](core::future::Future) and [`OrderedFuture`] on the /// same type. In this case, unless otherwise documented by the implementing type, neither poll /// function should be invoked after either returns an output value. pub trait OrderedFuture { /// See [`OrderedStream::Ordering`]. type Ordering: Ord; /// See [`OrderedStream::Data`]. type Output; /// Attempt to pull out the value of this future, registering the current task for wakeup if /// needed, and returning `None` if it is known that the future will not produce a value /// ordered before the given point. /// /// # Return value /// /// There are several possible return values, each indicating a distinct state depending on the /// value passed in `before`: /// /// - If `before` was `None`, `Poll::Pending` means that this future's value is not ready yet. /// Implementations will ensure that the current task is notified when the next value may be /// ready. /// /// - If `before` was `Some`, `Poll::Pending` means that this future's value is not ready and /// that it is not yet known if the value will be ordered prior to the given ordering value. /// Implementations will ensure that the current task is notified when either the next value is /// ready or once it is known that no such value will be produced. /// /// - `Poll::Ready(Some(Data))` means that the future has successfully terminated. The /// returned ordering value **may** be greater than the value passed to `before`. The /// `poll_before` function should not be invoked again. /// /// - `Poll::Ready(None)` means that this future will not produce an ordering token less than /// the given token. It is an error to return `None` if `before` was `None`. fn poll_before( self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll>; /// The minimum value of the ordering of the item. /// /// See [`OrderedStream::position_hint`] for details. fn position_hint(&self) -> Option> { None } } mod adapters; pub use adapters::*; mod join; pub use join::*; mod multi; pub use multi::*; ordered-stream-0.2.0/src/multi.rs000064400000000000000000000303051046102023000150300ustar 00000000000000use crate::*; use core::ops::DerefMut; use core::pin::Pin; use core::task::{Context, Poll}; fn poll_multiple_step( streams: I, cx: &mut Context<'_>, before: Option<&S::Ordering>, mut retry: Option<&mut Option>, ) -> Poll> where I: IntoIterator>, P: DerefMut>, S: OrderedStream, S::Ordering: Clone, { // The stream with the earliest item that is actually before the given point let mut best: Option> = None; // true if we have a stream that has not terminated let mut has_data = false; let mut has_pending = false; let mut skip_retry = false; for mut stream in streams { let best_before = best.as_ref().and_then(|p| p.item().map(|i| &i.0)); let current_bound = match (before, best_before) { (Some(given), Some(best)) if given <= best => Some(given), (_, Some(best)) => Some(best), (given, None) => given, }; // improved is true if have improved the `before` bound from the initial value match stream.as_mut().poll_peek_before(cx, current_bound) { Poll::Pending => { has_pending = true; skip_retry = true; } Poll::Ready(PollResult::Terminated) => continue, Poll::Ready(PollResult::NoneBefore) => { has_data = true; } Poll::Ready(PollResult::Item { ordering, .. }) => { has_data = true; match current_bound { Some(max) if max < ordering => continue, _ => {} } match (&mut retry, before, has_pending) { (Some(retry), Some(initial_bound), true) if ordering < initial_bound => { // We have just improved the initial bound, so the streams that // previously returned Pending might be able to return NoneBefore in a // retry. This is only useful if there are no later Pending returns, so // those will set skip_retry. **retry = Some(ordering.clone()); skip_retry = false; } (Some(retry), None, true) => { **retry = Some(ordering.clone()); skip_retry = false; } _ => {} } best = Some(stream); } } } if skip_retry { retry.map(|r| *r = None); } match best { _ if has_pending => Poll::Pending, None if has_data => Poll::Ready(PollResult::NoneBefore), None => Poll::Ready(PollResult::Terminated), // This is guaranteed to return PollResult::Item Some(mut stream) => stream.as_mut().poll_next_before(cx, before), } } /// Join a collection of [`OrderedStream`]s. /// /// This is similar to repeatedly using [`join()`] on all the streams in the contained collection. /// It is not optimized to avoid polling streams that are not ready, so it works best if the number /// of streams is relatively small. // // Unlike `FutureUnordered` or `SelectAll`, the ordering properties that this struct provides can // easily require that all items in the collection be consulted before returning any item. An // example of such a situation is a series of streams that all generate timestamps (locally) for // their items and only return `NoneBefore` for past timestamps. If only one stream produces an // item for each call to `JoinMultiple::poll_next_before`, that timestamp must be checked against // every other stream, and no amount of preparatory work or hints will help this. // // On the other hand, if all streams provide a position hint that matches their next item, it is // possible to build a priority queue to sort the streams and reduce the cost of a single poll from // `n` to `log(n)`. This does require maintaining a snapshot of the hints (so S::Ordering: Clone), // and will significantly increase the worst-case workload, so it should be a distinct type. #[derive(Debug, Default, Clone)] pub struct JoinMultiple(pub C); impl Unpin for JoinMultiple {} impl OrderedStream for JoinMultiple where for<'a> &'a mut C: IntoIterator>, S: OrderedStream + Unpin, S::Ordering: Clone, { type Ordering = S::Ordering; type Data = S::Data; fn poll_next_before( mut self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&S::Ordering>, ) -> Poll> { let mut retry = None; let rv = poll_multiple_step( self.as_mut().get_mut().0.into_iter().map(Pin::new), cx, before, Some(&mut retry), ); if rv.is_pending() && retry.is_some() { poll_multiple_step( self.get_mut().0.into_iter().map(Pin::new), cx, retry.as_ref(), None, ) } else { rv } } } impl FusedOrderedStream for JoinMultiple where for<'a> &'a mut C: IntoIterator>, for<'a> &'a C: IntoIterator>, S: OrderedStream + Unpin, S::Ordering: Clone, { fn is_terminated(&self) -> bool { self.0.into_iter().all(|peekable| peekable.is_terminated()) } } pin_project_lite::pin_project! { /// Join a collection of pinned [`OrderedStream`]s. /// /// This is identical to [`JoinMultiple`], but accepts [`OrderedStream`]s that are not [`Unpin`] by /// requiring that the collection provide a pinned [`IntoIterator`] implementation. /// /// This is not a feature available in most `std` collections. If you wish to use them, you /// should use `Box::pin` to make the stream [`Unpin`] before inserting it in the collection, /// and then use [`JoinMultiple`] on the resulting collection. #[derive(Debug,Default,Clone)] pub struct JoinMultiplePin { #[pin] pub streams: C, } } impl JoinMultiplePin { pub fn as_pin_mut(self: Pin<&mut Self>) -> Pin<&mut C> { self.project().streams } } impl OrderedStream for JoinMultiplePin where for<'a> Pin<&'a mut C>: IntoIterator>>, S: OrderedStream, S::Ordering: Clone, { type Ordering = S::Ordering; type Data = S::Data; fn poll_next_before( mut self: Pin<&mut Self>, cx: &mut Context<'_>, before: Option<&S::Ordering>, ) -> Poll> { let mut retry = None; let rv = poll_multiple_step(self.as_mut().as_pin_mut(), cx, before, Some(&mut retry)); if rv.is_pending() && retry.is_some() { poll_multiple_step(self.as_pin_mut(), cx, retry.as_ref(), None) } else { rv } } } #[cfg(test)] mod test { extern crate alloc; use crate::{FromStream, JoinMultiple, OrderedStream, OrderedStreamExt, PollResult}; use alloc::{boxed::Box, rc::Rc, vec, vec::Vec}; use core::{cell::Cell, pin::Pin, task::Context, task::Poll}; use futures_core::Stream; use futures_util::{pin_mut, stream::iter}; #[derive(Debug, PartialEq)] pub struct Message { serial: u32, } #[test] fn join_mutiple() { futures_executor::block_on(async { pub struct RemoteLogSource { stream: Pin>>, } let mut logs = [ RemoteLogSource { stream: Box::pin(iter([ Message { serial: 1 }, Message { serial: 4 }, Message { serial: 5 }, ])), }, RemoteLogSource { stream: Box::pin(iter([ Message { serial: 2 }, Message { serial: 3 }, Message { serial: 6 }, ])), }, ]; let streams: Vec<_> = logs .iter_mut() .map(|s| FromStream::with_ordering(&mut s.stream, |m| m.serial).peekable()) .collect(); let mut joined = JoinMultiple(streams); for i in 0..6 { let msg = joined.next().await.unwrap(); assert_eq!(msg.serial, i as u32 + 1); } }); } #[test] fn join_one_slow() { futures_executor::block_on(async { pub struct DelayStream(Rc>); impl OrderedStream for DelayStream { type Ordering = u32; type Data = Message; fn poll_next_before( self: Pin<&mut Self>, _: &mut Context<'_>, before: Option<&Self::Ordering>, ) -> Poll> { match self.0.get() { 0 => Poll::Pending, 1 if matches!(before, Some(&1)) => Poll::Ready(PollResult::NoneBefore), 1 => Poll::Pending, 2 => { self.0.set(3); Poll::Ready(PollResult::Item { data: Message { serial: 4 }, ordering: 4, }) } _ => Poll::Ready(PollResult::Terminated), } } } let stream1 = iter([ Message { serial: 1 }, Message { serial: 3 }, Message { serial: 5 }, ]); let stream1 = FromStream::with_ordering(stream1, |m| m.serial); let go = Rc::new(Cell::new(0)); let stream2 = DelayStream(go.clone()); let stream1: Pin>> = Box::pin(stream1); let stream2: Pin>> = Box::pin(stream2); let streams = vec![stream1.peekable(), stream2.peekable()]; let join = JoinMultiple(streams); let waker = futures_util::task::noop_waker(); let mut ctx = core::task::Context::from_waker(&waker); pin_mut!(join); // When the DelayStream has no information about what it contains, join returns Pending // (since there could be a serial-0 message output of DelayStream) assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Pending ); go.set(1); // Now the DelayStream will return NoneBefore on serial 1 assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Item { data: Message { serial: 1 }, ordering: 1, }) ); // however, it does not (yet) do so for serial 3 assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Pending ); go.set(2); assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Item { data: Message { serial: 3 }, ordering: 3, }) ); assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Item { data: Message { serial: 4 }, ordering: 4, }) ); assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Item { data: Message { serial: 5 }, ordering: 5, }) ); assert_eq!( join.as_mut().poll_next_before(&mut ctx, None), Poll::Ready(PollResult::Terminated) ); }); } }