wezterm-blob-leases-0.1.0/.cargo_vcs_info.json0000644000000001610000000000100147130ustar { "git": { "sha1": "b956a6ac304c707fbab6534effbaa7d5dd154114" }, "path_in_vcs": "wezterm-blob-leases" }wezterm-blob-leases-0.1.0/Cargo.toml0000644000000022410000000000100127120ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2021" name = "wezterm-blob-leases" version = "0.1.0" description = "Manage image blob caching/leasing for wezterm" license = "MIT" repository = "https://github.com/wez/wezterm" [dependencies.getrandom] version = "0.2" [dependencies.mac_address] version = "1.1" [dependencies.once_cell] version = "1.8" [dependencies.serde] version = "1.0" features = ["derive"] optional = true [dependencies.sha2] version = "0.10" [dependencies.tempfile] version = "3.4" optional = true [dependencies.thiserror] version = "1.0" [dependencies.uuid] version = "1.3" features = [ "v1", "rng", ] [features] default = [] serde = [ "dep:serde", "uuid/serde", ] simple_tempdir = ["dep:tempfile"] wezterm-blob-leases-0.1.0/Cargo.toml.orig000064400000000000000000000012041046102023000163710ustar 00000000000000[package] name = "wezterm-blob-leases" version = "0.1.0" edition = "2021" repository = "https://github.com/wez/wezterm" description = "Manage image blob caching/leasing for wezterm" license = "MIT" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] getrandom = "0.2" once_cell = "1.8" mac_address = "1.1" serde = {version="1.0", features=["derive"], optional=true} sha2 = "0.10" tempfile = {version="3.4", optional=true} thiserror = "1.0" uuid = {version="1.3", features=["v1", "rng"]} [features] default = [] serde = ["dep:serde", "uuid/serde"] simple_tempdir = ["dep:tempfile"] wezterm-blob-leases-0.1.0/src/content_id.rs000064400000000000000000000017051046102023000167730ustar 00000000000000#[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use sha2::Digest; /// Identifies data within the store. /// This is an (unspecified) hash of the content #[derive(Clone, Copy, Eq, PartialEq, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct ContentId([u8; 32]); impl ContentId { pub fn for_bytes(bytes: &[u8]) -> Self { let mut hasher = sha2::Sha256::new(); hasher.update(bytes); Self(hasher.finalize().into()) } pub fn as_hash_bytes(&self) -> [u8; 32] { self.0 } } impl std::fmt::Display for ContentId { fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { write!(fmt, "sha256-")?; for byte in &self.0 { write!(fmt, "{byte:x}")?; } Ok(()) } } impl std::fmt::Debug for ContentId { fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { write!(fmt, "ContentId({self})") } } wezterm-blob-leases-0.1.0/src/error.rs000064400000000000000000000007341046102023000157770ustar 00000000000000use crate::ContentId; use thiserror::Error; #[derive(Error, Debug)] pub enum Error { #[error("Lease Expired, data is no longer accessible")] LeaseExpired, #[error("Content with id {0} not found")] ContentNotFound(ContentId), #[error("Io error in BlobLease: {0}")] Io(#[from] std::io::Error), #[error("Storage has already been initialized")] AlreadyInitializedStorage, #[error("Storage has not been initialized")] StorageNotInit, } wezterm-blob-leases-0.1.0/src/lease.rs000064400000000000000000000071761046102023000157460ustar 00000000000000use crate::{get_storage, BoxedReader, ContentId, Error, LeaseId}; use std::sync::Arc; /// A lease represents a handle to data in the store. /// The lease will help to keep the data alive in the store. /// Depending on the policy configured for the store, it /// may guarantee to keep the data intact for its lifetime, /// or in some cases, it the store is being thrashed and at /// capacity, it may have been evicted. #[derive(Clone, Debug, PartialEq, Eq)] pub struct BlobLease { inner: Arc, } #[derive(Debug, PartialEq, Eq)] struct LeaseInner { pub content_id: ContentId, pub lease_id: LeaseId, } impl BlobLease { pub(crate) fn make_lease(content_id: ContentId, lease_id: LeaseId) -> Self { Self { inner: Arc::new(LeaseInner { content_id, lease_id, }), } } /// Returns a copy of the data, owned by the caller pub fn get_data(&self) -> Result, Error> { let storage = get_storage()?; storage.get_data(self.inner.content_id, self.inner.lease_id) } /// Returns a reader that can be used to stream/seek into /// the data pub fn get_reader(&self) -> Result { let storage = get_storage()?; storage.get_reader(self.inner.content_id, self.inner.lease_id) } pub fn content_id(&self) -> ContentId { self.inner.content_id } } impl Drop for LeaseInner { fn drop(&mut self) { if let Ok(storage) = get_storage() { storage .advise_lease_dropped(self.lease_id, self.content_id) .ok(); } } } /// Serialize a lease as the corresponding data bytes. /// This can fail during serialization if the lease is /// stale, but not during deserialization, as deserialiation /// will store the data implicitly. #[cfg(feature = "serde")] pub mod lease_bytes { use super::*; use crate::BlobManager; use serde::{de, ser, Deserialize, Serialize}; /// Serialize a lease as its bytes pub fn serialize(lease: &BlobLease, serializer: S) -> Result where S: ser::Serializer, { let data = lease .get_data() .map_err(|err| ser::Error::custom(format!("{err:#}")))?; data.serialize(serializer) } /// Deserialize a lease from bytes. pub fn deserialize<'de, D>(d: D) -> Result where D: de::Deserializer<'de>, { let data = as Deserialize>::deserialize(d)?; BlobManager::store(&data).map_err(|err| de::Error::custom(format!("{err:#}"))) } } /// Serialize a lease to/from its content id. /// This can fail in either direction if the lease is stale /// during serialization, or if the data for that content id /// is not available during deserialization. #[cfg(feature = "serde")] pub mod lease_content_id { use super::*; use crate::BlobManager; use serde::{de, ser, Deserialize, Serialize}; /// Serialize a lease as its content id pub fn serialize(lease: &BlobLease, serializer: S) -> Result where S: ser::Serializer, { lease.inner.content_id.serialize(serializer) } /// Deserialize a lease from a content id. /// Will fail unless the content id is already available /// to the local storage manager pub fn deserialize<'de, D>(d: D) -> Result where D: de::Deserializer<'de>, { let content_id = ::deserialize(d)?; BlobManager::get_by_content_id(content_id) .map_err(|err| de::Error::custom(format!("{err:#}"))) } } wezterm-blob-leases-0.1.0/src/lease_id.rs000064400000000000000000000015761046102023000164200ustar 00000000000000use once_cell::sync::Lazy; use uuid::Uuid; /// Represents an individual lease #[derive(Clone, Copy, Eq, PartialEq, Debug)] pub struct LeaseId { uuid: Uuid, pid: u32, } impl std::fmt::Display for LeaseId { fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { write!(fmt, "lease:pid={},{}", self.pid, self.uuid.hyphenated()) } } fn get_mac_address() -> [u8; 6] { match mac_address::get_mac_address() { Ok(Some(addr)) => addr.bytes(), _ => { let mut mac = [0u8; 6]; getrandom::getrandom(&mut mac).ok(); mac } } } impl LeaseId { pub fn new() -> Self { static MAC: Lazy<[u8; 6]> = Lazy::new(get_mac_address); let uuid = Uuid::now_v1(&*MAC); let pid = std::process::id(); Self { uuid, pid } } pub fn pid(&self) -> u32 { self.pid } } wezterm-blob-leases-0.1.0/src/lib.rs000064400000000000000000000003401046102023000154050ustar 00000000000000mod content_id; mod error; mod lease; mod lease_id; mod manager; mod storage; pub mod simple_tempdir; pub use content_id::*; pub use error::*; pub use lease::*; pub use lease_id::*; pub use manager::*; pub use storage::*; wezterm-blob-leases-0.1.0/src/manager.rs000064400000000000000000000015521046102023000162570ustar 00000000000000use crate::{get_storage, BlobLease, ContentId, Error, LeaseId}; pub struct BlobManager {} impl BlobManager { /// Store data into the store, de-duplicating it and returning /// a BlobLease that can be used to reference and access it. pub fn store(data: &[u8]) -> Result { let storage = get_storage()?; let lease_id = LeaseId::new(); let content_id = ContentId::for_bytes(data); storage.store(content_id, data, lease_id)?; Ok(BlobLease::make_lease(content_id, lease_id)) } /// Attempt to resolve by content id pub fn get_by_content_id(content_id: ContentId) -> Result { let storage = get_storage()?; let lease_id = LeaseId::new(); storage.lease_by_content(content_id, lease_id)?; Ok(BlobLease::make_lease(content_id, lease_id)) } } wezterm-blob-leases-0.1.0/src/simple_tempdir.rs000064400000000000000000000110221046102023000176530ustar 00000000000000#![cfg(feature = "simple_tempdir")] use crate::{BlobStorage, BoxedReader, BufSeekRead, ContentId, Error, LeaseId}; use std::collections::HashMap; use std::fs::File; use std::io::{BufReader, Write}; use std::path::PathBuf; use std::sync::Mutex; use tempfile::TempDir; pub struct SimpleTempDir { root: TempDir, refs: Mutex>, } impl SimpleTempDir { pub fn new() -> Result { let root = tempfile::Builder::new() .prefix("wezterm-blob-lease-") .rand_bytes(8) .tempdir()?; Ok(Self { root, refs: Mutex::new(HashMap::new()), }) } fn path_for_content(&self, content_id: ContentId) -> Result { let path = self.root.path().join(format!("{content_id}")); std::fs::create_dir_all(path.parent().unwrap())?; Ok(path) } fn add_ref(&self, content_id: ContentId) { *self.refs.lock().unwrap().entry(content_id).or_insert(0) += 1; } fn del_ref(&self, content_id: ContentId) { let mut refs = self.refs.lock().unwrap(); match refs.get_mut(&content_id) { Some(count) if *count == 1 => { if let Ok(path) = self.path_for_content(content_id) { if let Err(err) = std::fs::remove_file(&path) { eprintln!("Failed to remove {}: {err:#}", path.display()); } } *count = 0; } Some(count) => { *count -= 1; } None => { // Shouldn't really happen... } } } } impl BlobStorage for SimpleTempDir { fn store(&self, content_id: ContentId, data: &[u8], _lease_id: LeaseId) -> Result<(), Error> { let mut refs = self.refs.lock().unwrap(); let path = self.path_for_content(content_id)?; let mut file = tempfile::Builder::new() .prefix("new-") .rand_bytes(5) .tempfile_in(&self.root.path())?; file.write_all(data)?; file.persist(&path) .map_err(|persist_err| persist_err.error)?; *refs.entry(content_id).or_insert(0) += 1; Ok(()) } fn lease_by_content(&self, content_id: ContentId, _lease_id: LeaseId) -> Result<(), Error> { let _refs = self.refs.lock().unwrap(); let path = self.path_for_content(content_id)?; if path.exists() { self.add_ref(content_id); Ok(()) } else { Err(Error::ContentNotFound(content_id)) } } fn get_data(&self, content_id: ContentId, _lease_id: LeaseId) -> Result, Error> { let _refs = self.refs.lock().unwrap(); let path = self.path_for_content(content_id)?; Ok(std::fs::read(&path)?) } fn get_reader(&self, content_id: ContentId, lease_id: LeaseId) -> Result { struct Reader { file: BufReader, content_id: ContentId, lease_id: LeaseId, } impl BufSeekRead for Reader {} impl std::io::BufRead for Reader { fn fill_buf(&mut self) -> std::io::Result<&[u8]> { self.file.fill_buf() } fn consume(&mut self, amount: usize) { self.file.consume(amount) } } impl std::io::Read for Reader { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { self.file.read(buf) } } impl std::io::Seek for Reader { fn seek(&mut self, whence: std::io::SeekFrom) -> std::io::Result { self.file.seek(whence) } } impl Drop for Reader { fn drop(&mut self) { if let Ok(s) = crate::get_storage() { s.advise_lease_dropped(self.lease_id, self.content_id).ok(); } } } let path = self.path_for_content(content_id)?; let file = BufReader::new(std::fs::File::open(&path)?); self.add_ref(content_id); Ok(Box::new(Reader { file, content_id, lease_id, })) } fn advise_lease_dropped(&self, _lease_id: LeaseId, content_id: ContentId) -> Result<(), Error> { self.del_ref(content_id); Ok(()) } fn advise_of_pid(&self, _pid: u32) -> Result<(), Error> { Ok(()) } fn advise_pid_terminated(&self, _pid: u32) -> Result<(), Error> { Ok(()) } } wezterm-blob-leases-0.1.0/src/storage.rs000064400000000000000000000055651046102023000163210ustar 00000000000000use crate::{ContentId, Error, LeaseId}; use std::io::{BufRead, Seek}; use std::sync::{Arc, Mutex}; static STORAGE: Mutex>> = Mutex::new(None); pub trait BufSeekRead: BufRead + Seek {} pub type BoxedReader = Box; /// Implements the actual storage mechanism for blobs pub trait BlobStorage { /// Store data with the provided content_id. /// lease_id is provided by the caller to identify this store. /// The underlying store is expected to dedup storing data with the same /// content_id. fn store(&self, content_id: ContentId, data: &[u8], lease_id: LeaseId) -> Result<(), Error>; /// Resolve the data associated with content_id. /// If found, establish a lease with the given lease_id. /// If not found, returns Err(Error::ContentNotFound) fn lease_by_content(&self, content_id: ContentId, lease_id: LeaseId) -> Result<(), Error>; /// Retrieves the data identified by content_id. /// lease_id is provided in order to advise the storage system /// which lease fetched it, so that it can choose to record that /// information to track the liveness of a lease fn get_data(&self, content_id: ContentId, lease_id: LeaseId) -> Result, Error>; /// Retrieves the data identified by content_id as a readable+seekable /// buffered handle. /// /// lease_id is provided in order to advise the storage system /// which lease fetched it, so that it can choose to record that /// information to track the liveness of a lease. /// /// The returned handle serves to extend the lifetime of the lease. fn get_reader(&self, content_id: ContentId, lease_id: LeaseId) -> Result; /// Advises the storage manager that a particular lease has been dropped. fn advise_lease_dropped(&self, lease_id: LeaseId, content_id: ContentId) -> Result<(), Error>; /// Advises the storage manager that a given process id is now, or /// continues to be, alive and a valid consumer of the store. fn advise_of_pid(&self, pid: u32) -> Result<(), Error>; /// Advises the storage manager that a given process id is, or will /// very shortly, terminate and will cease to be a valid consumer /// of the store. /// It may choose to do something to invalidate all leases with /// a corresponding pid. fn advise_pid_terminated(&self, pid: u32) -> Result<(), Error>; } pub fn register_storage( storage: Arc, ) -> Result<(), Error> { STORAGE.lock().unwrap().replace(storage); Ok(()) } pub fn get_storage() -> Result, Error> { STORAGE .lock() .unwrap() .as_ref() .map(|s| s.clone()) .ok_or_else(|| Error::StorageNotInit) } pub fn clear_storage() { STORAGE.lock().unwrap().take(); }