netbuf-0.4.1/.gitignore010064400037200003720000000000341326446552100132130ustar0000000000000000/target /Cargo.lock /.vagga netbuf-0.4.1/.travis.yml010064400037200003720000000022231326446552100133360ustar0000000000000000sudo: false dist: trusty language: rust cache: - cargo before_cache: - rm -r $TRAVIS_BUILD_DIR/target/debug jobs: include: - os: linux rust: stable - os: linux rust: beta - os: linux rust: nightly # deploy - stage: publish os: linux rust: stable env: # CARGO_TOKEN - secure: "ZFVUuLLY6PZwc8kjUfGeEK8rrWa01sXEkB3APIxTnQQSP/VPkNcIRZtIjY3KPYwWBZqrrICclEzMJ7llgk58Sk8rfLI33mk4br1QFq+R0eCeW4YnDu7jTpZqjyXhWwStzE83RbXMqdElMnOntrQQLp2t2qNfqcZ58T9knt2qJfNM5MUNqN3ytP30as+nYRBSvv3mrYqKTe24tj4YwfUMFNeVr48FUPo99kOXeRfhaVpJVlm/id7mnYVq3VoXexTLMT/TTq5r5838SxMkl+8VD34wPerNrWsHyFunbn+nl/0W51iJ12+KxBjQb2FmJIkerZREkOOIkcdk8d3qeEIFTOlW7aSCapMDehVj+4/UtvnIXb1MClxphKFC6+Rpy52zzfufh7IJmjDzwzCUfGvTqbGWsghwQzxRp4yAoN8x0JQSRi6Zes5N0njZ7gFEdihBkwfmc/XYyOyGgpwkpOeHd/aqLT8kpZGDhgevzQtBleMgJuC5bob4+jvVAF3Q2NP6h4AriUvOgH14aCaW1/EAV4TUkUAvDIe5nGb6rT6WgBYjFInMD/Ny5dEENg6wDfqX3OAmHDPaYlIcLnb7XaRI/s9t2bSw/6kRRVlkgM4dwEtptD7u8MzF190m22fiGWCy7LvXjI2xME3kQAxZnElV+RNK7j5qaByL4tqsyiBh/7E=" install: true script: true deploy: - provider: script script: 'cargo publish --verbose --token=$CARGO_TOKEN' on: tags: true netbuf-0.4.1/Cargo.toml.orig010064400037200003720000000010631326446552100141150ustar0000000000000000[package] name = "netbuf" description = """ The simple to use, growable, contiguous buffer object with right assumptions and interface. Tuned for using it for network buffers. """ readme = "README.md" keywords = ["buf", "netbuf", "network", "buffer", "vec"] categories = ["network-programming", "memory-management"] version = "0.4.1" authors = ["Paul Colomiets "] license = "MIT/Apache-2.0" documentation = "http://tailhook.github.io/netbuf/" repository = "http://github.com/tailhook/netbuf" [dev-dependencies] mockstream = "0.0.2" netbuf-0.4.1/Cargo.toml0000644000000021020000000000000103510ustar00# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g. crates.io) dependencies # # If you believe there's an error in this file please file an # issue against the rust-lang/cargo repository. If you're # editing this file be aware that the upstream Cargo.toml # will likely look very different (and much more reasonable) [package] name = "netbuf" version = "0.4.1" authors = ["Paul Colomiets "] description = " The simple to use, growable, contiguous buffer object with right\n assumptions and interface. Tuned for using it for network buffers.\n" documentation = "http://tailhook.github.io/netbuf/" readme = "README.md" keywords = ["buf", "netbuf", "network", "buffer", "vec"] categories = ["network-programming", "memory-management"] license = "MIT/Apache-2.0" repository = "http://github.com/tailhook/netbuf" [dev-dependencies.mockstream] version = "0.0.2" netbuf-0.4.1/LICENSE-APACHE010064400037200003720000000261361326446552100131620ustar0000000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. netbuf-0.4.1/LICENSE-MIT010064400037200003720000000020421326446552100126600ustar0000000000000000Copyright (c) 2015 Paul Colomiets Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. netbuf-0.4.1/README.md010064400037200003720000000016461326446552100125140ustar0000000000000000Netbuf ====== [Documentation](https://docs.rs/netbuf) | [Github](https://github.com/tailhook/netbuf) | [Crate](https://crates.io/crates/netbuf) The network buffer for usage in asynchronous network applications. Has right interface for reading from network into the ``Buf`` and for parsing packets directly from it. It also has capacity management tuned for network application (i.e. it starts growing fast, but deallocated for idle connections). License ======= Licensed under either of * Apache License, Version 2.0, (./LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) * MIT license (./LICENSE-MIT or http://opensource.org/licenses/MIT) at your option. Contribution ------------ Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. netbuf-0.4.1/bulk.yaml010064400037200003720000000002051326446552100130440ustar0000000000000000minimum-bulk: v0.4.5 versions: - file: Cargo.toml block-start: ^\[package\] block-end: ^\[.*\] regex: ^version\s*=\s*"(\S+)" netbuf-0.4.1/src/buf.rs010064400037200003720000001026351326446552100131460ustar0000000000000000use std::ptr::{copy_nonoverlapping, copy}; use std::ops::{Index, IndexMut, RangeFrom, RangeTo, RangeFull, Range}; use std::cmp::{min, max}; use std::io::{Read, Write, Result}; use std::fmt::{self, Debug}; use std::mem::swap; use range::RangeArgument; const READ_MIN: usize = 4096; const ALLOC_MIN: usize = 16384; /// /// A buffer object to be used for reading from network /// /// Assumptions: /// /// 1. Buffer needs to be growable as sometimes requests are large /// /// 2. Buffer should deallocate when empty as /// most of the time connections are idle /// /// a. Deallocations are cheap as we have a cool memory allocator /// (jemalloc) /// /// b. First allocation should be big (i.e. kilobytes not few bytes) /// /// 3. Should be easy to peek and get a slice as it makes packet parsing easy /// /// 4. Cheap removing bytes at the start of the buf /// pub struct Buf { data: Option>, consumed: usize, remaining: usize, } // TODO(tailhook) use std::slice::bytes::copy_memory; fn copy_memory(src: &[u8], dst: &mut [u8]) { assert!(src.len() == dst.len()); unsafe { copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), dst.len()); } } impl Buf { /// Create empty buffer. It has no preallocated size. The underlying memory /// chunk is always deallocated when there are no useful bytes in the /// buffer. pub fn new() -> Buf { Buf { data: None, consumed: 0, remaining: 0, } } fn reserve(&mut self, bytes: usize) { self.data = self.data.take().map(|slice| { let old_cap = slice.len(); let old_bytes = old_cap - self.consumed() - self.remaining(); if self.consumed() > 0 { // let's allocate new slice and move let min_size = old_bytes + bytes; let mut vec = Vec::with_capacity(max(min_size, old_cap.saturating_mul(2))); let cap = vec.capacity(); unsafe { vec.set_len(cap) }; copy_memory(&slice[self.consumed()..old_cap - self.remaining()], &mut vec[..old_bytes]); self.remaining = cap - old_bytes; self.consumed = 0; Some(vec.into_boxed_slice()) } else { // just reallocate let mut vec = slice.into_vec(); vec.reserve(bytes); let cap = vec.capacity(); unsafe { vec.set_len(cap) }; self.remaining = cap - old_bytes; Some(vec.into_boxed_slice()) } }).unwrap_or_else(|| { let mut vec = Vec::with_capacity(max(bytes, ALLOC_MIN)); let cap = vec.capacity(); unsafe { vec.set_len(cap) }; self.remaining = cap; Some(vec.into_boxed_slice()) }) } fn reserve_exact(&mut self, bytes: usize) { self.data = self.data.take().map(|slice| { let old_cap = slice.len(); let old_bytes = old_cap - self.consumed() - self.remaining(); if self.consumed() > 0 { // let's allocate new slice and move let size = old_bytes + bytes; let mut vec = Vec::with_capacity(size); let cap = vec.capacity(); unsafe { vec.set_len(cap) }; copy_memory(&slice[self.consumed()..old_cap - self.remaining()], &mut vec[..old_bytes]); self.remaining = cap - old_bytes; self.consumed = 0; Some(vec.into_boxed_slice()) } else { // just reallocate let mut vec = slice.into_vec(); vec.reserve_exact(bytes); let cap = vec.capacity(); unsafe { vec.set_len(cap) }; self.remaining = cap - old_bytes; Some(vec.into_boxed_slice()) } }).unwrap_or_else(|| { let mut vec = Vec::with_capacity(bytes); let cap = vec.capacity(); unsafe { vec.set_len(cap) }; self.remaining = cap; Some(vec.into_boxed_slice()) }) } fn remaining(&self) -> usize { self.remaining as usize } fn consumed(&self) -> usize { self.consumed as usize } /// Mark the first `bytes` of the buffer as read. Basically it's shaving /// off bytes from the buffer, but does it efficiently. When there are /// no more bytes in the buffer it's deallocated. /// /// Note: Buffer currently doesn't shrink when calling this method. It's /// assumed that all bytes will be consumed shortly. In case you're /// appending to the buffer after consume, old data is discarded. /// /// # Panics /// /// Panics if `bytes` is larger than current length of buffer pub fn consume(&mut self, bytes: usize) { let ln = self.len(); assert!(bytes <= ln); if bytes == ln { *self = Buf::new(); } else { self.consumed += bytes; } } /// Allows removing an arbitrary range of bytes /// /// A more comprehensive version of `consume()`. It's occasionally useful /// if your data is in frames/chunks but you want to buffer the whole body /// anyway. E.g. in http chunked encoding you have each chunk prefixed by /// its length, but that doesn't mean you can't buffer the whole request /// into memory. This method allows you to continue reading the next chunk /// into the same buffer while removing the chunk length. /// /// Note: it's not super efficient, as it requires to move (copy) bytes /// after the range, in case range is neither at the start nor at the /// end of buffer. Still it should be faster than copying everything /// to yet another buffer. /// /// We never shrink the buffer here (except when it becomes empty, to /// keep this invariant), assuming that you will receive more data into /// the buffer shortly. /// /// The `RangeArgument` type is a temporary type until rust provides /// the one in standard library, you should use the range syntax directly: /// /// ```ignore /// buf.remove_range(5..7) /// ``` /// /// # Panics /// /// Panics if range is invalid for the buffer pub fn remove_range>(&mut self, range: R) { use range::RangeArgument::*; match range.into() { RangeTo(x) | Range(0, x) => self.consume(x), RangeFrom(0) => *self = Buf::new(), RangeFrom(x) => { let ln = self.len(); assert!(x <= ln); self.remaining += ln - x; } Range(x, y) => { let ln = self.len(); if x == y { return; } assert!(x < y); let removed_bytes = y - x; assert!(y <= ln); let start = self.consumed() + x; let end = self.consumed() + y; if let Some(ref mut data) = self.data { let dlen = data.len(); unsafe { copy(data[end..].as_ptr(), data[start..dlen - removed_bytes].as_mut_ptr(), dlen - end); } self.remaining += removed_bytes; } else { panic!("Not-existent buffere where data exists"); } } } } /// Capacity of the buffer. I.e. the bytes it is allocated for. Use for /// debugging or for calculating memory usage. Note it's not guaranteed /// that you can write `buf.capacity() - buf.len()` bytes without resize pub fn capacity(&self) -> usize { self.data.as_ref().map(|x| x.len()).unwrap_or(0) } /// Number of useful bytes in the buffer pub fn len(&self) -> usize { self.data.as_ref() .map(|x| x.len() - self.consumed() - self.remaining()) .unwrap_or(0) } /// If buffer is empty. Potentially a little bit faster than /// getting `len()` pub fn is_empty(&self) -> bool { self.data.is_none() } fn future_slice<'x>(&'x mut self) -> &'x mut [u8] { let rem = self.remaining(); self.data.as_mut() .map(|x| { let upto = x.len(); &mut x[upto - rem .. upto] }) .unwrap() } /// Extend buffer. Note unlike `Write::write()` and `read_from()` this /// method reserves as small as possible a chunk of memory. So it's /// inefficien to grow with this method. You may use the Write trait to /// grow incrementally. pub fn extend(&mut self, buf: &[u8]) { if buf.len() == 0 { return; } if self.remaining() < buf.len() { self.reserve_exact(buf.len()); } copy_memory(buf, &mut self.future_slice()[..buf.len()]); self.remaining -= buf.len(); } /// Read some bytes from stream (object implementing `Read`) into buffer /// /// Note this does *not* continue to read until getting `WouldBlock`. It /// passes all errors as is. It preallocates some chunk to read into /// buffer. It may be possible that the stream still has bytes buffered /// after this method returns. This method is expected either to be called /// until `WouldBlock` is returned or is used with level-triggered polling. pub fn read_from(&mut self, stream: &mut R) -> Result { if self.remaining() < READ_MIN { self.reserve(READ_MIN); } let bytes = match stream.read(self.future_slice()) { res @ Ok(0) | res @ Err(_) => { self.consume(0); return res; } Ok(x) => x, }; debug_assert!(bytes <= self.remaining()); self.remaining -= bytes; Ok(bytes) } /// Reads no more than max bytes into buffer and returns boolean flag /// of whether max bytes are reached /// /// Except for the limit on number of bytes and slightly different /// allocation strategy, this method has the same considerations as /// read_from /// /// Note this method might be used for two purposes: /// /// 1. Limit number of bytes buffered until parser can process data /// (for example HTTP header size, which is number of bytes read before /// `\r\n\r\n` delimiter reached) /// 2. Wait until exact number of bytes are fully received /// /// Since we support (1) we don't preallocate buffer for exact size of /// the `max` value. It also helps a little in case (2) for minimizing /// DDoS attack vector. If that doesn't suit you, you may wish to use /// `Vec::with_capacity()` for the purpose of (2). /// /// On the contrary we don't overallocate more than `max` bytes, so if /// you expect data to go after exact number of bytes read, it might be /// better to use `read_from()` and check buffer length. /// pub fn read_max_from(&mut self, max: usize, stream: &mut R) -> Result { let todo = max.saturating_sub(self.len()); if todo == 0 { return Ok(true); } if self.remaining() < READ_MIN { if self.capacity().saturating_mul(2) < max { // TODO is too large we may never need so big buffer self.reserve(READ_MIN); } else { self.reserve_exact(todo); } } let res = { let slc = self.future_slice(); let do_now = min(slc.len(), todo); stream.read(&mut slc[..do_now]) }; let bytes = match res { Ok(0) => { self.consume(0); return Ok(false); } Err(e) => { self.consume(0); return Err(e); } Ok(x) => x, }; debug_assert!(bytes <= self.remaining()); self.remaining -= bytes; Ok(self.len() >= max) } /// Write contents of buffer to the stream (object implementing /// the Write trait). We assume that stream is non-blocking, use /// `Write::write` (instead of `Write::write_all`) and return all errors /// to the caller (including `WouldBlock` or `Interrupted`). /// /// In addition to returning the number of bytes written, it also /// `consume()`s those bytes from the buffer, so it's safe to retry calling /// the method at any moment. Also it's a common pattern to append more /// data to the buffer between calls. pub fn write_to(&mut self, sock: &mut W) -> Result { let bytes = match sock.write(&self[..]) { Ok(bytes) => bytes, Err(e) => return Err(e), }; self.consume(bytes); Ok(bytes) } /// Splits buffer into two at the given index. /// /// `self` contains bytes `[0, at)` and the returned `Buf` contains /// bytes `[at, len)`. /// /// Reallocates smaller part of buffer. /// /// # Panics /// /// Panics if at > len. /// /// # Examples /// ```rust /// let mut buf = netbuf::Buf::new(); /// buf.extend(b"Hello World!"); /// let tail = buf.split_off(6); /// assert_eq!(&buf[..], b"Hello "); /// assert_eq!(&tail[..], b"World!"); /// ``` pub fn split_off(&mut self, at: usize) -> Buf { assert!(at <= self.len()); let mut tail = Buf::new(); match self.data.as_ref().map(|x| x.len() - at > at) { Some(true) => { // re-allocate self; swap(&mut tail, self); if at > 0 { self.extend(&tail[..at]); tail.consume(at); } }, Some(false) => { // allocate for tail; tail.extend(&self[at..]); self.remove_range(at..); }, None => {} } tail } /// Returns the byte at the index, or `None` if the index is /// out of bounds. pub fn get(&self, index: usize) -> Option { if index >= self.len() { return None; } let consumed = self.consumed(); if let Some(ref data) = self.data { Some(data[consumed + index]) } else { None } } /// Returns a mutable reference to the byte at the index, or /// `None` if the index is out of bounds. pub fn get_mut(&mut self, index: usize) -> Option<&mut u8> { if index >= self.len() { return None; } let consumed = self.consumed(); if let Some(ref mut data) = self.data { Some(&mut data[consumed + index]) } else { None } } } impl Debug for Buf { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "Buf {{ len={}; consumed={}; remaining={} }}", self.len(), self.consumed(), self.remaining()) } } impl AsRef<[u8]> for Buf { fn as_ref(&self) -> &[u8] { &self[..] } } impl Into> for Buf { fn into(mut self) -> Vec { if self.consumed == 0 { self.data.take().map(|slice| { let mut vec = slice.into_vec(); if self.remaining > 0 { let nlen = vec.len() - self.remaining(); unsafe { vec.set_len(nlen) }; } vec }).unwrap_or_else(Vec::new) } else { self[..].to_vec() } } } impl Index for Buf { type Output = u8; fn index<'x>(&'x self, index: usize) -> &'x u8 { if let Some(ref data) = self.data { return &data[self.consumed() + index] } panic!("cannot index empty buffer"); } } impl Index for Buf { type Output = [u8]; fn index<'x>(&'x self, _idx: RangeFull) -> &'x[u8] { self.data.as_ref() .map(|x| &x[self.consumed()..x.len() - self.remaining()]) .unwrap_or(b"") } } impl Index> for Buf { type Output = [u8]; fn index<'x>(&'x self, slice: RangeTo) -> &'x[u8] { let idx = slice.end; if idx == 0 { return b""; } assert!(idx <= self.len()); &self.data.as_ref().unwrap()[self.consumed()..self.consumed() + idx] } } impl Index> for Buf { type Output = [u8]; fn index<'x>(&'x self, slice: RangeFrom) -> &'x[u8] { let idx = slice.start; if idx == self.len() { return b""; } assert!(idx <= self.len()); let buf = &self.data.as_ref().unwrap(); &buf[self.consumed() + idx .. buf.len() - self.remaining()] } } impl Index> for Buf { type Output = [u8]; fn index<'x>(&'x self, slice: Range) -> &'x[u8] { let start = slice.start; let end = slice.end; if end == 0 { return b""; } assert!(end <= self.len()); assert!(start <= end); let buf = &self.data.as_ref().unwrap(); &buf[self.consumed() + start .. self.consumed() + end] } } impl IndexMut for Buf { fn index_mut<'x>(&'x mut self, index: usize) -> &'x mut u8 { let consumed = self.consumed(); if let Some(ref mut data) = self.data { return &mut data[consumed + index] } panic!("cannot index empty buffer"); } } impl IndexMut for Buf { fn index_mut<'x>(&'x mut self, _idx: RangeFull) -> &'x mut[u8] { let consumed = self.consumed(); let remaining = self.remaining(); self.data.as_mut() .map(|x| { let len = x.len(); &mut x[consumed..len - remaining] }) .unwrap_or(&mut []) } } impl IndexMut> for Buf { fn index_mut<'x>(&'x mut self, slice: RangeTo) -> &'x mut[u8] { let idx = slice.end; if idx == 0 { return &mut []; } assert!(idx <= self.len()); let consumed = self.consumed(); &mut self.data.as_mut().unwrap()[consumed..consumed + idx] } } impl IndexMut> for Buf { fn index_mut<'x>(&'x mut self, slice: RangeFrom) -> &'x mut[u8] { let idx = slice.start; if idx == self.len() { return &mut []; } assert!(idx <= self.len()); let consumed = self.consumed(); let remaining = self.remaining(); let buf = self.data.as_mut().unwrap(); let len = buf.len(); &mut buf[consumed + idx .. len - remaining] } } impl IndexMut> for Buf { fn index_mut<'x>(&'x mut self, slice: Range) -> &'x mut[u8] { let start = slice.start; let end = slice.end; if end == 0 { return &mut []; } assert!(end <= self.len()); assert!(start <= end); let consumed = self.consumed(); let buf = self.data.as_mut().unwrap(); &mut buf[consumed + start .. consumed + end] } } impl Write for Buf { fn write(&mut self, buf: &[u8]) -> Result { if buf.len() == 0 { return Ok(0); } if self.remaining() < buf.len() { self.reserve(buf.len()); } copy_memory(buf, &mut self.future_slice()[..buf.len()]); self.remaining -= buf.len(); Ok(buf.len()) } fn flush(&mut self) -> Result<()> { Ok(()) } } impl Default for Buf { fn default() -> Self { Buf::new() } } #[cfg(test)] mod test { use std::mem::size_of; use std::io::{self, Read, Write}; use super::Buf; use super::ALLOC_MIN; use mockstream::SharedMockStream; struct ReadErr; impl Read for ReadErr { fn read(&mut self, _: &mut [u8]) -> Result { Err(io::Error::new(io::ErrorKind::WouldBlock, "fake would block")) } } #[test] #[cfg(target_arch="x86_64")] fn size32() { assert_eq!(size_of::(), 32); } #[test] #[cfg(target_arch="x86")] fn size32() { assert_eq!(size_of::(), 16); } #[test] fn empty() { let buf = Buf::new(); assert_eq!(&buf[..], b""); assert_eq!(format!("{:?}", buf), "Buf { len=0; consumed=0; remaining=0 }") } #[test] fn read_from_empty() { let mut s = SharedMockStream::new(); let mut buf = Buf::new(); assert_eq!(buf.read_from(&mut s).unwrap(), 0); assert!(buf.is_empty()); assert_eq!(&buf[..], b""); assert_eq!(format!("{:?}", buf), "Buf { len=0; consumed=0; remaining=0 }") } #[test] fn read_from_err() { let mut buf = Buf::new(); buf.read_from(&mut ReadErr).unwrap_err(); assert!(buf.is_empty()); assert_eq!(&buf[..], b""); assert_eq!(format!("{:?}", buf), "Buf { len=0; consumed=0; remaining=0 }") } #[test] fn read_max_from_empty() { let mut s = SharedMockStream::new(); let mut buf = Buf::new(); assert_eq!(buf.read_max_from(1024, &mut s).unwrap(), false); assert!(buf.is_empty()); assert_eq!(&buf[..], b""); assert_eq!(format!("{:?}", buf), "Buf { len=0; consumed=0; remaining=0 }") } #[test] fn read_max_from_err() { let mut buf = Buf::new(); buf.read_max_from(1024, &mut ReadErr).unwrap_err(); assert!(buf.is_empty()); assert_eq!(&buf[..], b""); assert_eq!(format!("{:?}", buf), "Buf { len=0; consumed=0; remaining=0 }") } #[test] fn read_from() { let mut s = SharedMockStream::new(); s.push_bytes_to_read(b"hello"); let mut buf = Buf::new(); assert_eq!(buf.read_from(&mut s).unwrap(), 5); assert_eq!(&buf[..], b"hello"); } #[test] fn read_max_from() { let mut s = SharedMockStream::new(); s.push_bytes_to_read(b"hello"); let mut buf = Buf::new(); s.push_bytes_to_read(b"hello world"); assert!(!buf.read_max_from(1024*1024, &mut s).unwrap()); assert_eq!(&buf[..], b"hellohello world"); assert!(buf.capacity() < ALLOC_MIN*2); s.push_bytes_to_read(b" from me!Oh, crap!"); assert!(buf.read_max_from(25, &mut s).unwrap()); assert_eq!(&buf[..], b"hellohello world from me!"); assert!(buf.capacity() < ALLOC_MIN*2); assert!(buf.read_max_from(25, &mut s).unwrap()); assert_eq!(&buf[..], b"hellohello world from me!"); assert!(buf.capacity() < ALLOC_MIN*2); buf.consume(5); assert!(buf.read_max_from(22, &mut s).unwrap()); assert_eq!(&buf[..], b"hello world from me!Oh"); assert!(buf.capacity() < ALLOC_MIN*2); } #[test] fn two_reads() { let mut s = SharedMockStream::new(); s.push_bytes_to_read(b"hello"); let mut buf = Buf::new(); buf.read_from(&mut s).unwrap(); s.push_bytes_to_read(b" world"); buf.read_from(&mut s).unwrap(); assert_eq!(&buf[..], b"hello world"); assert_eq!(buf.capacity(), ALLOC_MIN); } #[test] fn realloc() { let mut s = SharedMockStream::new(); s.push_bytes_to_read(b"hello"); let mut buf = Buf::new(); buf.read_from(&mut s).unwrap(); s.push_bytes_to_read(&b"abcdefg".iter().cloned().cycle().take(1024*1024) .collect::>()[..]); buf.read_from(&mut s).unwrap(); assert_eq!(&buf[..9], b"helloabcd"); assert_eq!(buf.len(), ALLOC_MIN); assert_eq!(buf.capacity(), ALLOC_MIN); buf.read_from(&mut s).unwrap(); assert_eq!(buf.len(), buf.capacity()); assert!(buf.capacity() >= 32768); println!("Capacity {}", buf.capacity()); buf.read_from(&mut s).unwrap(); println!("Capacity {}", buf.capacity()); buf.read_from(&mut s).unwrap(); println!("Capacity {}", buf.capacity()); buf.read_from(&mut s).unwrap(); println!("Capacity {}", buf.capacity()); buf.read_from(&mut s).unwrap(); println!("Capacity {}", buf.capacity()); buf.read_from(&mut s).unwrap(); println!("Capacity {}", buf.capacity()); buf.read_from(&mut s).unwrap(); println!("Capacity {}", buf.capacity()); assert_eq!(buf.len(), 1048576 + 5); assert!(buf.capacity() >= buf.len()); assert!(buf.capacity() <= 2100000); assert_eq!(&buf[buf.len()-10..], b"bcdefgabcd"); } #[test] fn two_writes() { let mut buf = Buf::new(); assert_eq!(buf.write(b"hello").unwrap(), 5); assert_eq!(buf.write(b" world").unwrap(), 6); assert_eq!(&buf[..], b"hello world"); assert_eq!(buf.capacity(), ALLOC_MIN); } #[test] fn two_extends() { let mut buf = Buf::new(); buf.extend(b"hello"); buf.extend(b" world"); assert_eq!(&buf[..], b"hello world"); assert_eq!(buf.capacity(), 11); } #[test] fn write_extend() { let mut buf = Buf::new(); buf.write(b"hello").unwrap(); buf.extend(b" world"); assert_eq!(&buf[..], b"hello world"); assert_eq!(buf.capacity(), ALLOC_MIN); } #[test] fn extend_write() { let mut buf = Buf::new(); buf.extend(b"hello"); buf.write(b" world").unwrap(); assert_eq!(&buf[..], b"hello world"); let cap = buf.capacity(); // We don't know exact allocation policy for vec // So this check is fuzzy assert!(cap >= 11); assert!(cap < 256); } #[test] fn extend_empty() { let mut buf = Buf::new(); buf.extend(b""); assert_eq!(&buf[..], b""); assert_eq!(buf.len(), 0); assert_eq!(buf.capacity(), 0); } #[test] fn into() { let mut buf = Buf::new(); buf.extend(b"hello"); buf.write(b" world").unwrap(); let vec: Vec = buf.into(); assert_eq!(&vec[..], b"hello world"); let cap = vec.capacity(); // We don't know exact allocation policy for vec // So this check is fuzzy assert!(cap >= 11); assert!(cap < 256); } #[test] fn consumed_into() { let mut buf = Buf::new(); buf.extend(b"hello"); buf.write(b" world").unwrap(); buf.consume(6); let vec: Vec = buf.into(); assert_eq!(&vec[..], b"world"); assert_eq!(vec.capacity(), 5); } #[test] fn write_to() { let mut s = SharedMockStream::new(); let mut buf = Buf::new(); buf.extend(b"hello world"); assert_eq!(buf.write_to(&mut s).unwrap(), 11); assert_eq!(&s.pop_bytes_written()[..], b"hello world"); } #[test] fn empty_write_to_empty() { let mut buf = Buf::new(); let mut empty = Buf::new(); assert_eq!(empty.write_to(&mut buf).unwrap(), 0); } #[test] fn extend_consume_extend() { let mut buf = Buf::new(); buf.extend(b"hello"); buf.consume(3); buf.extend(b" world"); assert_eq!(&buf[..], b"lo world"); assert_eq!(buf.capacity(), 8); } #[test] fn extend_consume_write() { let mut buf = Buf::new(); buf.extend(b"hello"); buf.consume(3); buf.write(b"Some larger string for you").unwrap(); assert_eq!(&buf[..], b"loSome larger string for you"); assert_eq!(buf.capacity(), 28); } #[test] fn consume_all() { let mut buf = Buf::new(); buf.extend(b"hello"); buf.write(b" world").unwrap(); buf.consume(11); assert_eq!(buf.capacity(), 0); assert_eq!(&buf[..], b""); let vec: Vec = buf.into(); assert_eq!(&vec[..], b""); assert_eq!(vec.capacity(), 0); } #[test] fn index() { let mut buf = Buf::new(); buf.extend(b"Hello World!"); assert_eq!(buf[0], b'H'); assert_eq!(buf[6], b'W'); buf.consume(2); assert_eq!(buf[2], b'o'); assert_eq!(buf[8], b'd'); } #[test] fn index_mut() { let mut buf = Buf::new(); buf.extend(b"Hello World!"); assert_eq!(buf[0], b'H'); buf[0] = b'h'; assert_eq!(buf[6], b'W'); buf[6] = b'M'; assert_eq!(&buf[..], b"hello Morld!"); buf.consume(2); buf[1] = b'e'; buf[5] = b'e'; buf[8] = b'e'; assert_eq!(&buf[..], b"leo Merle!"); } #[test] fn get() { let mut buf = Buf::new(); assert_eq!(buf.get(0), None); buf.extend(b"Hello World!"); assert_eq!(buf.get(0), Some(b'H')); assert_eq!(buf.get(6), Some(b'W')); assert_eq!(buf.get(11), Some(b'!')); assert_eq!(buf.get(12), None); buf.consume(2); assert_eq!(buf.get(2), Some(b'o')); assert_eq!(buf.get(8), Some(b'd')); assert_eq!(buf.get(9), Some(b'!')); assert_eq!(buf.get(10), None); } #[test] fn get_mut() { let mut buf = Buf::new(); assert_eq!(buf.get_mut(0), None); buf.extend(b"Hello World!"); assert_eq!(buf.get_mut(0), Some(&mut b'H')); if let Some(x) = buf.get_mut(0) { *x = b'h'; } assert_eq!(buf.get_mut(6), Some(&mut b'W')); if let Some(x) = buf.get_mut(6) { *x = b'M'; } assert_eq!(&buf[..], b"hello Morld!"); assert_eq!(buf.get_mut(11), Some(&mut b'!')); assert_eq!(buf.get_mut(12), None); buf.consume(2); if let Some(x) = buf.get_mut(1) { *x = b'e'; } if let Some(x) = buf.get_mut(5) { *x = b'e'; } if let Some(x) = buf.get_mut(8) { *x = b'e'; } assert_eq!(&buf[..], b"leo Merle!"); assert_eq!(buf.get_mut(9), Some(&mut b'!')); assert_eq!(buf.get_mut(10), None); } #[test] fn ranges() { let mut buf = Buf::new(); buf.extend(b"Hello world!"); assert_eq!(&buf[..], b"Hello world!"); assert_eq!(&buf[..0], b""); assert_eq!(&buf[..5], b"Hello"); assert_eq!(&buf[..12], b"Hello world!"); assert_eq!(&buf[..7], b"Hello w"); assert_eq!(&buf[0..], b"Hello world!"); assert_eq!(&buf[3..], b"lo world!"); assert_eq!(&buf[6..], b"world!"); assert_eq!(&buf[12..], b""); assert_eq!(&buf[0..0], b""); assert_eq!(&buf[2..8], b"llo wo"); assert_eq!(&buf[0..12], b"Hello world!"); assert_eq!(&buf[0..5], b"Hello"); assert_eq!(&buf[7..12], b"orld!"); assert_eq!(&buf[3..3], b""); assert_eq!(&buf[3..9], b"lo wor"); } #[test] fn consume_ranges() { let mut buf = Buf::new(); buf.extend(b"Crappy stuff"); buf.extend(b"Hello world!"); buf.consume(12); assert_eq!(&buf[..], b"Hello world!"); assert_eq!(&buf[..0], b""); assert_eq!(&buf[..5], b"Hello"); assert_eq!(&buf[..12], b"Hello world!"); assert_eq!(&buf[..7], b"Hello w"); assert_eq!(&buf[0..], b"Hello world!"); assert_eq!(&buf[3..], b"lo world!"); assert_eq!(&buf[6..], b"world!"); assert_eq!(&buf[12..], b""); assert_eq!(&buf[2..8], b"llo wo"); assert_eq!(&buf[0..12], b"Hello world!"); assert_eq!(&buf[0..5], b"Hello"); assert_eq!(&buf[7..12], b"orld!"); assert_eq!(&buf[3..3], b""); assert_eq!(&buf[3..9], b"lo wor"); } #[test] fn remove_ranges() { let mut buf = Buf::new(); buf.extend(b"Crappy stuff"); buf.extend(b"Hello world!"); assert_eq!(&buf[..], b"Crappy stuffHello world!"); buf.remove_range(..4); assert_eq!(&buf[..], b"py stuffHello world!"); buf.remove_range(3..8); assert_eq!(&buf[..], b"py Hello world!"); buf.remove_range(7..14); assert_eq!(&buf[..], b"py Hell!"); buf.remove_range(7..); assert_eq!(&buf[..], b"py Hell"); let end = buf.len(); buf.remove_range(2..end); assert_eq!(&buf[..], b"py"); } #[test] fn mut_ranges() { let mut buf = Buf::new(); buf.extend(b"Crappy stuff"); buf.extend(b"Hello world!"); buf.consume(12); buf[..].clone_from_slice(b"HELLO WORLD."); assert_eq!(&buf[..], b"HELLO WORLD."); buf[..5].clone_from_slice(b"hell!"); assert_eq!(&buf[..], b"hell! WORLD."); buf[4..10].clone_from_slice(b"o worl"); assert_eq!(&buf[..], b"hello worlD."); buf[10..].clone_from_slice(b"d!"); assert_eq!(&buf[..], b"hello world!"); } #[test] fn split_off_realloc_self() { let mut buf = Buf::new(); buf.reserve_exact(20); buf.extend(b"Hello World"); let tail = buf.split_off(5); assert_eq!(&buf[..], b"Hello"); assert_eq!(buf.consumed(), 0); assert_eq!(buf.len(), 5); assert_eq!(&tail[..], b" World"); assert_eq!(tail.consumed(), 5); assert_eq!(tail.len(), 6); } #[test] fn split_off_alloc_tail() { let mut buf = Buf::new(); buf.reserve_exact(20); buf.extend(b"Hello World!..."); assert_eq!(buf.capacity(), 20); let tail = buf.split_off(11); assert_eq!(&buf[..], b"Hello World"); assert_eq!(buf.capacity(), 20); assert_eq!(buf.consumed(), 0); assert_eq!(buf.len(), 11); assert_eq!(&tail[..], b"!..."); assert_eq!(tail.capacity(), 4); assert_eq!(tail.consumed(), 0); assert_eq!(tail.len(), 4); let mut buf = Buf::new(); buf.extend(b"abcdefg"); let tail = buf.split_off(0); assert_eq!(&tail[..], b"abcdefg"); assert_eq!(&buf[..], b""); } } netbuf-0.4.1/src/lib.rs010064400037200003720000000031411326446552100131300ustar0000000000000000//! # Netbuf //! //! [Documentation](https://docs.rs/netbuf) | //! [Github](https://github.com/tailhook/netbuf) | //! [Crate](https://crates.io/crates/netbuf) //! //! This module currently includes single `Buf` struct for holding buffers. //! Comparing to `Vec` class buffer has different allocation policy and has //! a marker of consumed data (i.e. already processed by protocol parser or //! already written to socket) //! //! The `Buf` is deemed good both for input and output network buffer. //! //! It also contains helper methods `read_from` and `write_to` which are used //! to read and append bytes from stream that implements Read and write bytes //! from buffer to a stream which implements Write respectively. //! //! Note there are basically three ways to fill the buffer: //! //! * `Buf::read_from` -- preallocates some chunk and gives it to object //! implemeting Read //! * `Write::write` -- writes chunk to buffer assuming more data will follow //! shortly, i.e. it does large preallocations //! * `Buf::extend` -- writes chunk to buffer assuming it will not grow in the //! near perspective, so it allocates minimum chunk to hold the data //! //! In other words you should use: //! //! * `Buf::read_from` -- to read from the network //! * `Write::write` -- when you are constructing object directly to the buffer //! incrementally //! * `Buf::extend` -- when you put whole object in place and give it to the //! network code for sending //! //! More documentation is found in `Buf` object itself #[cfg(test)] extern crate mockstream; mod buf; mod range; pub use buf::Buf; pub use range::RangeArgument; netbuf-0.4.1/src/range.rs010064400037200003720000000014101326446552100134530ustar0000000000000000use std::ops::{Range, RangeFrom, RangeTo, RangeFull}; /// Temporary type until the one in stdlib is made stable pub enum RangeArgument { RangeFrom(usize), Range(usize, usize), RangeTo(usize), } impl From> for RangeArgument { fn from(r: Range) -> RangeArgument { RangeArgument::Range(r.start, r.end) } } impl From> for RangeArgument { fn from(r: RangeFrom) -> RangeArgument { RangeArgument::RangeFrom(r.start) } } impl From> for RangeArgument { fn from(r: RangeTo) -> RangeArgument { RangeArgument::RangeTo(r.end) } } impl From for RangeArgument { fn from(_: RangeFull) -> RangeArgument { RangeArgument::RangeFrom(0) } } netbuf-0.4.1/vagga.yaml010064400037200003720000000041041326446552100131760ustar0000000000000000commands: make: !Command description: Build netbuf library container: ubuntu run: [cargo, build] test: !Command description: Test netbuf library container: ubuntu environ: RUST_BACKTRACE: 1 run: [cargo, test] coverage: !Command description: Run coverage tests for netbuf library container: ubuntu environ: RUST_BACKTRACE: 1 run: | cargo test --no-run rm -rf target/cov kcov target/cov target/debug/netbuf-638268726e961e5a _bulk: !Command description: Run `bulk` command (for version bookkeeping) container: ubuntu run: [bulk] cargo: !Command description: Run any cargo command container: ubuntu run: [cargo] docs: !Command description: Generate documentation container: ubuntu run: [cargo, doc] containers: ubuntu: setup: - !Ubuntu xenial - !UbuntuUniverse - !Install [make, wget, ca-certificates, vim, libssl-dev, build-essential] - !TarInstall url: "https://static.rust-lang.org/dist/rust-1.25.0-x86_64-unknown-linux-gnu.tar.gz" script: "./install.sh --prefix=/usr \ --components=rustc,rust-std-x86_64-unknown-linux-gnu,cargo" - !BuildDeps [libcurl4-openssl-dev, libelf-dev, libdw-dev, cmake, python, pkg-config] - !Install [libdw1, libelf1, libcurl3] - &bulk !Tar url: "https://github.com/tailhook/bulk/releases/download/v0.4.11/bulk-v0.4.11.tar.gz" sha256: b718bb8448e726690c94d98d004bf7575f7a429106ec26ad3faf11e0fd9a7978 path: / - !GitInstall url: git://github.com/SimonKagstrom/kcov script: | cmake . make make install environ: HOME: /work/target nightly: setup: - !Ubuntu xenial - !UbuntuUniverse - !Install [make, wget, ca-certificates, libssl-dev, build-essential] - !TarInstall url: "http://static.rust-lang.org/dist/rust-nightly-x86_64-unknown-linux-gnu.tar.gz" script: "./install.sh --prefix=/usr --components=rustc,cargo" environ: HOME: /work/target