bmap-parser-0.1.0/.cargo_vcs_info.json0000644000000001510000000000100132400ustar { "git": { "sha1": "fd8700e257b06fb58f5bddc5d7db0c2bb56fdc96" }, "path_in_vcs": "bmap-parser" }bmap-parser-0.1.0/Cargo.toml0000644000000024630000000000100112460ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "bmap-parser" version = "0.1.0" authors = ["Sjoerd Simons "] description = "bmap-parser is a library for Rust that allows you to copy files or flash block devices safely" readme = "README.md" license = "MIT AND Apache-2.0" repository = "https://github.com/collabora/bmap-rs" [dependencies.anyhow] version = "1.0.40" optional = true [dependencies.async-trait] version = "0.1.58" [dependencies.digest] version = "0.10.5" [dependencies.flate2] version = "1.0.20" [dependencies.futures] version = "0.3.25" [dependencies.quick-xml] version = "0.26.0" features = ["serialize"] [dependencies.serde] version = "1.0.147" features = ["derive"] [dependencies.sha2] version = "0.10.6" features = ["asm"] [dependencies.strum] version = "0.24.1" features = ["derive"] [dependencies.thiserror] version = "1.0.24" bmap-parser-0.1.0/Cargo.toml.orig000064400000000000000000000014671046102023000147320ustar 00000000000000[package] name = "bmap-parser" version = "0.1.0" authors = ["Sjoerd Simons "] edition = "2018" license = "MIT AND Apache-2.0" description = "bmap-parser is a library for Rust that allows you to copy files or flash block devices safely" repository = "https://github.com/collabora/bmap-rs" readme = "../README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] thiserror = "1.0.24" quick-xml = { version = "0.26.0", features = [ "serialize" ] } serde = { version = "1.0.147", features = [ "derive" ] } anyhow = { version = "1.0.40", optional = true } sha2 = { version = "0.10.6", features = [ "asm" ] } strum = { version = "0.24.1", features = [ "derive"] } digest = "0.10.5" flate2 = "1.0.20" async-trait = "0.1.58" futures = "0.3.25" bmap-parser-0.1.0/README.md000064400000000000000000000014231046102023000133120ustar 00000000000000# bmap-rs The bmap-rs project aims to implement tools related to bmap. The project is written in rust. The inspiration for it is an existing project that is written in python called [bmap-tools](https://salsa.debian.org/debian/bmap-tools). Right now the implemented function is copying system images files using bmap, which is safer and faster than regular cp ro dd. That can be used to flash images into block devices. ## Usage bmap-rs supports 1 subcommand: - "copy" - copy a file to another file using a bmap file. ```bash bmap-rs copy ``` The bmap file is automatically searched in the source directory. The recommendation is to name it as the source but with bmap extension. ## License bmap-rs is licensed under dual Apache-2.0 and MIT licenses. bmap-parser-0.1.0/src/bmap/xml.rs000064400000000000000000000065121046102023000147130ustar 00000000000000use crate::bmap::{BmapBuilder, BmapBuilderError, HashType, HashValue}; use quick_xml::de::{from_str, DeError}; use serde::Deserialize; use std::str::FromStr; use thiserror::Error; #[derive(Debug, Deserialize)] struct Range { chksum: String, #[serde(rename = "$value")] range: String, } #[derive(Debug, Deserialize)] struct BlockMap { #[serde(rename = "Range")] ranges: Vec, } #[allow(dead_code)] #[derive(Debug, Deserialize)] struct Bmap { version: String, #[serde(rename = "ImageSize")] image_size: u64, #[serde(rename = "BlockSize")] block_size: u64, #[serde(rename = "BlocksCount")] blocks_count: u64, #[serde(rename = "MappedBlocksCount")] mapped_blocks_count: u64, #[serde(rename = "ChecksumType")] checksum_type: String, #[serde(rename = "BmapFileChecksum")] bmap_file_checksum: String, #[serde(rename = "BlockMap")] block_map: BlockMap, } #[derive(Debug, Error)] pub enum XmlError { #[error("Failed to parse bmap XML: {0}")] XmlParsError(#[from] DeError), #[error("Invalid bmap file: {0}")] InvalidFIleError(#[from] BmapBuilderError), #[error("Unknown checksum type: {0}")] UnknownChecksumType(String), #[error("Invalid checksum: {0}")] InvalidChecksum(String), } const fn hexdigit_to_u8(c: u8) -> Option { match c { b'a'..=b'f' => Some(c - b'a' + 0xa), b'A'..=b'F' => Some(c - b'A' + 0xa), b'0'..=b'9' => Some(c - b'0'), _ => None, } } fn str_to_digest(s: String, digest: &mut [u8]) -> Result<(), XmlError> { let l = digest.len(); if s.len() != l * 2 { return Err(XmlError::InvalidChecksum(format!( "No enough chars: {} {}", s, s.len() ))); } for (i, chunk) in s.as_bytes().chunks(2).enumerate() { let hi = match hexdigit_to_u8(chunk[0]) { Some(v) => v, None => return Err(XmlError::InvalidChecksum(s)), }; let lo = match hexdigit_to_u8(chunk[1]) { Some(v) => v, None => return Err(XmlError::InvalidChecksum(s)), }; digest[i] = hi << 4 | lo; } Ok(()) } pub(crate) fn from_xml(xml: &str) -> Result { let b: Bmap = from_str(xml)?; let mut builder = BmapBuilder::default(); let hash_type = b.checksum_type; let hash_type = HashType::from_str(&hash_type).map_err(|_| XmlError::UnknownChecksumType(hash_type))?; builder .image_size(b.image_size) .block_size(b.block_size) .blocks(b.blocks_count) .checksum_type(hash_type) .mapped_blocks(b.mapped_blocks_count); for range in b.block_map.ranges { let mut split = range.range.trim().splitn(2, '-'); let start = match split.next() { Some(s) => s.parse().unwrap(), None => unimplemented!("woops"), }; let end = match split.next() { Some(s) => s.parse().unwrap(), None => start, }; let checksum = match hash_type { HashType::Sha256 => { let mut v = [0; 32]; str_to_digest(range.chksum, &mut v)?; HashValue::Sha256(v) } }; builder.add_block_range(start, end, checksum); } builder.build().map_err(std::convert::Into::into) } bmap-parser-0.1.0/src/bmap.rs000064400000000000000000000116151046102023000141130ustar 00000000000000use strum::{Display, EnumDiscriminants, EnumString}; use thiserror::Error; mod xml; #[derive(Copy, Clone, Debug, PartialEq, Eq, EnumString, Display)] #[strum(serialize_all = "lowercase")] #[non_exhaustive] pub enum HashType { Sha256, } #[derive(Copy, Clone, Debug, PartialEq, Eq, EnumDiscriminants)] #[non_exhaustive] pub enum HashValue { Sha256([u8; 32]), } impl HashValue { pub fn to_type(&self) -> HashType { match self { HashValue::Sha256(_) => HashType::Sha256, } } pub fn as_slice(&self) -> &[u8] { match self { HashValue::Sha256(v) => v, } } } #[derive(Clone, Debug, PartialEq, Eq)] pub struct BlockRange { offset: u64, length: u64, checksum: HashValue, } impl BlockRange { pub fn checksum(&self) -> HashValue { self.checksum } pub fn offset(&self) -> u64 { self.offset } pub fn length(&self) -> u64 { self.length } } #[derive(Clone, Debug)] pub struct Bmap { image_size: u64, block_size: u64, blocks: u64, mapped_blocks: u64, checksum_type: HashType, blockmap: Vec, } impl Bmap { pub fn builder() -> BmapBuilder { BmapBuilder::default() } pub fn from_xml(xml: &str) -> Result { xml::from_xml(xml) } pub fn image_size(&self) -> u64 { self.image_size } pub const fn block_size(&self) -> u64 { self.block_size } pub fn blocks(&self) -> u64 { self.blocks } pub fn mapped_blocks(&self) -> u64 { self.mapped_blocks } pub fn checksum_type(&self) -> HashType { self.checksum_type } pub fn block_map(&self) -> impl ExactSizeIterator + Iterator { self.blockmap.iter() } pub fn total_mapped_size(&self) -> u64 { self.block_size * self.mapped_blocks } } #[derive(Clone, Debug, Error)] pub enum BmapBuilderError { #[error("Image size missing")] MissingImageSize, #[error("Block size missing")] MissingBlockSize, #[error("Blocks missing")] MissingBlocks, #[error("Mapped blocks missing")] MissingMappedBlocks, #[error("Checksum type missing")] MissingChecksumType, #[error("No block ranges")] NoBlockRanges, } #[derive(Clone, Debug, Default)] pub struct BmapBuilder { image_size: Option, block_size: Option, blocks: Option, checksum_type: Option, mapped_blocks: Option, blockmap: Vec, } impl BmapBuilder { pub fn image_size(&mut self, size: u64) -> &mut Self { self.image_size = Some(size); self } pub fn block_size(&mut self, block_size: u64) -> &mut Self { self.block_size = Some(block_size); self } pub fn blocks(&mut self, blocks: u64) -> &mut Self { self.blocks = Some(blocks); self } pub fn mapped_blocks(&mut self, blocks: u64) -> &mut Self { self.mapped_blocks = Some(blocks); self } pub fn checksum_type(&mut self, checksum_type: HashType) -> &mut Self { self.checksum_type = Some(checksum_type); self } pub fn add_block_range(&mut self, start: u64, end: u64, checksum: HashValue) -> &mut Self { let bs = self.block_size.expect("Blocksize needs to be set first"); let total = self.image_size.expect("Image size needs to be set first"); let offset = start * bs; let length = (total - offset).min((end - start + 1) * bs); self.add_byte_range(offset, length, checksum) } pub fn add_byte_range(&mut self, offset: u64, length: u64, checksum: HashValue) -> &mut Self { let range = BlockRange { offset, length, checksum, }; self.blockmap.push(range); self } pub fn build(self) -> Result { let image_size = self.image_size.ok_or(BmapBuilderError::MissingImageSize)?; let block_size = self.block_size.ok_or(BmapBuilderError::MissingBlockSize)?; let blocks = self.blocks.ok_or(BmapBuilderError::MissingBlocks)?; let mapped_blocks = self .mapped_blocks .ok_or(BmapBuilderError::MissingMappedBlocks)?; let checksum_type = self .checksum_type .ok_or(BmapBuilderError::MissingChecksumType)?; let blockmap = self.blockmap; Ok(Bmap { image_size, block_size, blocks, mapped_blocks, checksum_type, blockmap, }) } } #[cfg(test)] mod test { use super::*; use std::str::FromStr; #[test] fn hashes() { assert_eq!("sha256", &HashType::Sha256.to_string()); assert_eq!(HashType::Sha256, HashType::from_str("sha256").unwrap()); let h = HashValue::Sha256([0; 32]); assert_eq!(HashType::Sha256, h.to_type()); } } bmap-parser-0.1.0/src/discarder.rs000064400000000000000000000051021046102023000151260ustar 00000000000000use crate::{AsyncSeekForward, SeekForward}; use async_trait::async_trait; use futures::io::{AsyncRead, AsyncReadExt}; use std::io::Read; use std::io::Result as IOResult; use std::pin::Pin; use std::task::{Context, Poll}; /// Adaptor that implements SeekForward on types only implementing Read by discarding data pub struct Discarder { reader: R, } impl Discarder { pub fn new(reader: R) -> Self { Self { reader } } pub fn into_inner(self) -> R { self.reader } } impl Read for Discarder { fn read(&mut self, buf: &mut [u8]) -> IOResult { self.reader.read(buf) } } impl SeekForward for Discarder { fn seek_forward(&mut self, forward: u64) -> IOResult<()> { let mut buf = [0; 4096]; let mut left = forward as usize; while left > 0 { let toread = left.min(buf.len()); let r = self.reader.read(&mut buf[0..toread])?; left -= r; } Ok(()) } } pub struct AsyncDiscarder { reader: R, } impl AsyncDiscarder { pub fn new(reader: R) -> Self { Self { reader } } pub fn into_inner(self) -> R { self.reader } } impl AsyncRead for AsyncDiscarder { fn poll_read( mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut [u8], ) -> Poll> { Pin::new(&mut self.reader).poll_read(cx, buf) } } #[async_trait(?Send)] impl AsyncSeekForward for AsyncDiscarder { async fn async_seek_forward(&mut self, forward: u64) -> IOResult<()> { let mut buf = [0; 4096]; let mut left = forward as usize; while left > 0 { let toread = left.min(buf.len()); let r = self.read(&mut buf[0..toread]).await?; left -= r; } Ok(()) } } #[cfg(test)] mod test { use super::*; use std::slice; #[test] fn discard() { let mut data = Vec::with_capacity(256); for byte in 0u8..=255 { data.push(byte); } let mut discarder = Discarder::new(data.as_slice()); let _ = &[0u64, 5, 16, 31, 63, 200, 255] .iter() .fold(0, |pos, offset| { let mut byte: u8 = 1; discarder.seek_forward((offset - pos) as u64).unwrap(); assert_eq!(1, discarder.read(slice::from_mut(&mut byte)).unwrap()); assert_eq!(*offset, byte as u64); *offset + 1 }); } } bmap-parser-0.1.0/src/lib.rs000064400000000000000000000104571046102023000137450ustar 00000000000000mod bmap; pub use crate::bmap::*; mod discarder; pub use crate::discarder::*; use async_trait::async_trait; use futures::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt}; use futures::TryFutureExt; use sha2::{Digest, Sha256}; use thiserror::Error; use std::io::Result as IOResult; use std::io::{Read, Seek, SeekFrom, Write}; /// Trait that can only seek further forwards pub trait SeekForward { fn seek_forward(&mut self, offset: u64) -> IOResult<()>; } impl SeekForward for T { fn seek_forward(&mut self, forward: u64) -> IOResult<()> { self.seek(SeekFrom::Current(forward as i64))?; Ok(()) } } #[async_trait(?Send)] pub trait AsyncSeekForward { async fn async_seek_forward(&mut self, offset: u64) -> IOResult<()>; } #[async_trait(?Send)] impl AsyncSeekForward for T { async fn async_seek_forward(&mut self, forward: u64) -> IOResult<()> { self.seek(SeekFrom::Current(forward as i64)).await?; Ok(()) } } #[derive(Debug, Error)] pub enum CopyError { #[error("Failed to Read: {0}")] ReadError(std::io::Error), #[error("Failed to Write: {0}")] WriteError(std::io::Error), #[error("Checksum error")] ChecksumError, #[error("Unexpected EOF on input")] UnexpectedEof, } pub fn copy(input: &mut I, output: &mut O, map: &Bmap) -> Result<(), CopyError> where I: Read + SeekForward, O: Write + SeekForward, { let mut hasher = match map.checksum_type() { HashType::Sha256 => Sha256::new(), }; let mut v = Vec::new(); // TODO benchmark a reasonable size for this v.resize(8 * 1024 * 1024, 0); let buf = v.as_mut_slice(); let mut position = 0; for range in map.block_map() { let forward = range.offset() - position; input.seek_forward(forward).map_err(CopyError::ReadError)?; output .seek_forward(forward) .map_err(CopyError::WriteError)?; let mut left = range.length() as usize; while left > 0 { let toread = left.min(buf.len()); let r = input .read(&mut buf[0..toread]) .map_err(CopyError::ReadError)?; if r == 0 { return Err(CopyError::UnexpectedEof); } hasher.update(&buf[0..r]); output .write_all(&buf[0..r]) .map_err(CopyError::WriteError)?; left -= r; } let digest = hasher.finalize_reset(); if range.checksum().as_slice() != digest.as_slice() { return Err(CopyError::ChecksumError); } position = range.offset() + range.length(); } Ok(()) } pub async fn copy_async(input: &mut I, output: &mut O, map: &Bmap) -> Result<(), CopyError> where I: AsyncRead + AsyncSeekForward + Unpin, O: AsyncWrite + AsyncSeekForward + Unpin, { let mut hasher = match map.checksum_type() { HashType::Sha256 => Sha256::new(), }; let mut v = Vec::new(); // TODO benchmark a reasonable size for this v.resize(8 * 1024 * 1024, 0); let buf = v.as_mut_slice(); let mut position = 0; for range in map.block_map() { let forward = range.offset() - position; input .async_seek_forward(forward) .map_err(CopyError::ReadError) .await?; output.flush().map_err(CopyError::WriteError).await?; output .async_seek_forward(forward) .map_err(CopyError::WriteError) .await?; let mut left = range.length() as usize; while left > 0 { let toread = left.min(buf.len()); let r = input .read(&mut buf[0..toread]) .map_err(CopyError::ReadError) .await?; if r == 0 { return Err(CopyError::UnexpectedEof); } hasher.update(&buf[0..r]); output .write_all(&buf[0..r]) .await .map_err(CopyError::WriteError)?; left -= r; } let digest = hasher.finalize_reset(); if range.checksum().as_slice() != digest.as_slice() { return Err(CopyError::ChecksumError); } position = range.offset() + range.length(); } Ok(()) } bmap-parser-0.1.0/tests/copy.rs000064400000000000000000000111011046102023000145070ustar 00000000000000use bmap_parser::{Bmap, Discarder, SeekForward}; use flate2::read::GzDecoder; use sha2::{Digest, Sha256}; use std::env; use std::fs::File; use std::io::Result as IOResult; use std::io::{Error, ErrorKind, Read, Write}; use std::path::PathBuf; #[derive(Clone, Debug)] struct OutputMockRange { offset: u64, data: Vec, } impl OutputMockRange { fn new(offset: u64) -> Self { Self { offset, data: Vec::new(), } } fn write(&mut self, data: &[u8]) { self.data.extend_from_slice(data); } fn sha256(&self) -> [u8; 32] { Sha256::digest(&self.data).into() } } #[derive(Clone, Debug)] struct OutputMock { size: u64, offset: u64, ranges: Vec, } impl OutputMock { fn new(size: u64) -> Self { Self { size, offset: 0, ranges: Vec::new(), } } fn add_range(&mut self, offset: u64) -> &mut OutputMockRange { self.ranges.push(OutputMockRange::new(offset)); self.ranges.last_mut().unwrap() } fn sha256(&mut self) -> [u8; 32] { fn pad(hasher: &mut Sha256, mut topad: u64) { const ZEROES: [u8; 4096] = [0; 4096]; while topad > 0 { let len = ZEROES.len() as u64; let len = len.min(topad); hasher.update(&ZEROES[0..len as usize]); topad -= len; } } let mut hasher = Sha256::new(); let mut offset = 0; for range in self.ranges.iter() { if offset < range.offset { pad(&mut hasher, range.offset - offset); offset = range.offset; } hasher.update(&range.data); offset += range.data.len() as u64; } pad(&mut hasher, self.size - offset); hasher.finalize().into() } } impl Write for OutputMock { fn write(&mut self, data: &[u8]) -> IOResult { let maxsize = self.size as usize; let range = match self.ranges.last_mut() { Some(last) if last.offset == self.offset => last, _ => self.add_range(self.offset), }; if range.offset as usize + range.data.len() + data.len() > maxsize { return Err(Error::new(ErrorKind::Other, "Writing outside of space")); } range.write(data); Ok(data.len()) } fn flush(&mut self) -> IOResult<()> { Ok(()) } } impl SeekForward for OutputMock { fn seek_forward(&mut self, forward: u64) -> IOResult<()> { self.offset += if let Some(last) = self.ranges.last() { last.data.len() as u64 + forward } else { forward }; Ok(()) } } fn setup_data(basename: &str) -> (Bmap, impl Read + SeekForward) { let mut datadir = PathBuf::new(); datadir.push(env::var("CARGO_MANIFEST_DIR").unwrap()); datadir.push("tests/data"); let mut bmapfile = datadir.clone(); bmapfile.push(format!("{}.bmap", basename)); let mut b = File::open(&bmapfile).expect(&format!("Failed to open bmap file:{:?}", bmapfile)); let mut xml = String::new(); b.read_to_string(&mut xml).unwrap(); let bmap = Bmap::from_xml(&xml).unwrap(); let mut datafile = datadir.clone(); datafile.push(format!("{}.gz", basename)); let g = File::open(&datafile).expect(&format!("Failed to open data file:{:?}", datafile)); let gz = GzDecoder::new(g); let gz = Discarder::new(gz); (bmap, gz) } fn sha256_reader(mut reader: R) -> [u8; 32] { let mut buffer = [0; 4096]; let mut hasher = Sha256::new(); loop { let r = reader.read(&mut buffer).unwrap(); if r == 0 { break; } hasher.update(&buffer[0..r]); } hasher.finalize().into() } #[test] fn copy() { let (bmap, mut input) = setup_data("test.img"); let mut output = OutputMock::new(bmap.image_size()); bmap_parser::copy(&mut input, &mut output, &bmap).unwrap(); assert_eq!(bmap_parser::HashType::Sha256, bmap.checksum_type()); assert_eq!(bmap.block_map().len(), output.ranges.len()); // Assert that written ranges match the ranges in the map file for (map, range) in bmap.block_map().zip(output.ranges.iter()) { assert_eq!(map.offset(), range.offset); assert_eq!(map.length(), range.data.len() as u64); assert_eq!(map.checksum().as_slice(), range.sha256()); } let (_, mut input) = setup_data("test.img"); // Assert that the full gzipped content match the written output assert_eq!(sha256_reader(&mut input), output.sha256()) } bmap-parser-0.1.0/tests/data/simple.bmap000064400000000000000000000017011046102023000162370ustar 00000000000000 4198400 4096 1025 680 sha256 0000000000000000000000000000000000000000000000000000000000000000 0 8-16 32-64 128-256 512-1024 bmap-parser-0.1.0/tests/data/test.img.bmap000064400000000000000000000052241046102023000165040ustar 00000000000000 16777216 4096 4096 1026 sha256 d374877d61522c62fe76f6eaad4aa9e84dc1a74575ea529a9076cfafab23ca77 256 1024-1535 2356-2484 2560-2687 3584-3839 bmap-parser-0.1.0/tests/data/test.img.gz000064400000000000000000000002041046102023000161760ustar 00000000000000version https://git-lfs.github.com/spec/v1 oid sha256:2f4ab24d109c4a9b83232c1dd3772b76865ba03dae22f5bb54ad54880bae1ab5 size 4217156 bmap-parser-0.1.0/tests/parse.rs000064400000000000000000000013511046102023000146550ustar 00000000000000use bmap_parser::Bmap; use digest::Digest; use sha2::Sha256; #[test] fn parse() { let xml = include_str!("data/simple.bmap"); let bmap = Bmap::from_xml(xml).unwrap(); assert_eq!(4096, bmap.block_size()); assert_eq!(1025, bmap.blocks()); assert_eq!(1025 * 4096, bmap.image_size()); assert_eq!(680, bmap.mapped_blocks()); let mut block = 0; for range in bmap.block_map() { assert_eq!(block * 4096, range.offset()); assert_eq!((block + 1) * 4096, range.length()); let digest = Sha256::digest(format!("{}", block).as_bytes()); assert_eq!(digest.as_slice(), range.checksum().as_slice()); block = if block == 0 { 8 } else { block * 4 }; } assert_eq!(2048, block); }