clone-file-0.1.0/.cargo_vcs_info.json0000644000000001360000000000100130470ustar { "git": { "sha1": "52a45651e5580de9b0c982d53eff25cc4901a0e4" }, "path_in_vcs": "" }clone-file-0.1.0/.github/workflows/build.yml000064400000000000000000000012321046102023000170540ustar 00000000000000on: push: branches: ["master"] name: Build and Test jobs: build-and-test: name: Build and Test runs-on: ubuntu-latest steps: - name: Checkout sources uses: actions/checkout@v2 - name: Install stable toolchain uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: stable override: true - name: Run cargo check uses: actions-rs/cargo@v1 with: command: check - name: Install external test dependencies run: | sudo apt update sudo apt install btrfs-progs - name: Run tests run: ./test.sh clone-file-0.1.0/.gitignore000064400000000000000000000005001046102023000136220ustar 00000000000000# Generated by Cargo # will have compiled files and executables /target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk clone-file-0.1.0/Cargo.toml0000644000000020370000000000100110470ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2021" name = "clone-file" version = "0.1.0" authors = ["Ilia Pozdnyakov "] description = "functions to clone files and file ranges with as little overhead as possible" readme = "README.md" keywords = [ "reflink", "ficlone", "ficlonerange", ] categories = ["filesystem"] license = "MIT" repository = "https://github.com/iliazeus/clone-file" [dev-dependencies.anyhow] version = "1.0.70" features = ["backtrace"] [target."cfg(target_os = \"linux\")".dependencies.nix] version = "0.26.2" features = ["ioctl"] default-features = false clone-file-0.1.0/Cargo.toml.orig000064400000000000000000000010701046102023000145240ustar 00000000000000[package] name = "clone-file" version = "0.1.0" description = "functions to clone files and file ranges with as little overhead as possible" repository = "https://github.com/iliazeus/clone-file" authors = ["Ilia Pozdnyakov "] license = "MIT" keywords = ["reflink", "ficlone", "ficlonerange"] categories = ["filesystem"] edition = "2021" [target.'cfg(target_os = "linux")'.dependencies] nix = { version = "0.26.2", default-features = false, features = ["ioctl"] } [dev-dependencies] anyhow = { version = "1.0.70", features = ["backtrace"] } clone-file-0.1.0/LICENSE000064400000000000000000000020601046102023000126420ustar 00000000000000MIT License Copyright (c) 2023 Ilia Pozdnyakov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. clone-file-0.1.0/README.md000064400000000000000000000033441046102023000131220ustar 00000000000000# clone-file A simple way to use your filesystem's [reflink] features. [reflink]: https://en.wikipedia.org/wiki/Data_deduplication#reflink ```rust // Clone a file using a reflink, or error if it can not be done. clone_file("src.bin", "dest.bin"); // Try to clone a file, falling back to a regular copy. clone_or_copy_file("src.bin", "dest.bin"); // Clone a sub-range of a file using a reflink, or error if it can not be done. clone_file_range( "src.bin", /* offset: */ 4 << 10, /* length: */ 2 << 20, "dest.bin", /* dest offset: */ 0 ); // Try to clone a sub-range of a file, falling back to a naive copy. clone_or_copy_file_range( "src.bin", /* offset: */ 4 << 10, /* length: */ 2 << 20, "dest.bin", /* dest offset: */ 0 ); ``` ## Implementation details ### Linux On Linux, `FICLONE` and `FICLONERANGE` ioctls are used. Please refer to [`man 2 ioctl_ficlonerange`] for details and limitations. Tested with the `btrfs` filesystem. [`man 2 ioctl_ficlonerange`]: https://man7.org/linux/man-pages/man2/ioctl_ficlonerange.2.html ### Others The `clone_file` and `clone_file_range` functions are currently not implemented for other platforms. However, the fallback functions `clone_or_copy_file` and `clone_or_copy_file_range` will work, falling back to naive copies. ## Running tests To test the cloning, we need a filesystem that supports reflinks. This requires a bit of a setup, which is implemented in the `test.sh` script. It expects a Linux system, with a `btrfs-progs` packages installed. It creates a 200MiB loopback device, formats it into `btrfs`, then creates the neccessary test data and runs `cargo tests`. It then cleans up the loopback and the mount. The tests are intentionally set up to run only through `test.sh` clone-file-0.1.0/src/common.rs000064400000000000000000000033521046102023000142670ustar 00000000000000use std::fs::{self, File}; use std::io::{self, Read, Seek, SeekFrom}; use std::path::Path; /// Tries to clone the file with [crate::clone_file], falling back to [fs::copy] on error /// /// Returns `Ok(None)` on successful clone, `Ok(Some(copied_byte_count))` on successful [fs::copy]. pub fn clone_or_copy_file, Q: AsRef>( src: P, dest: Q, ) -> io::Result> { if let Ok(_) = crate::clone_file(&src, &dest) { Ok(None) } else { fs::copy(&src, &dest).and_then(|x| Ok(Some(x))) } } /// Tries to clone a range of bytes to another file, falling back to [copy_file_range] on error /// /// Returns `Ok(None)` on successful clone, `Ok(Some(copied_byte_count))` on successful naive copy. pub fn clone_or_copy_file_range, Q: AsRef>( src: P, src_offset: u64, src_length: u64, dest: Q, dest_offset: u64, ) -> io::Result> { if let Ok(_) = crate::clone_file_range(&src, src_offset, src_length, &dest, dest_offset) { Ok(None) } else { todo!() } } /// Naively copy a range of bytes from one file to another /// /// This function is mainly implemented as a fallback for [clone_or_copy_file_range]. /// /// Returns `Ok(copied_byte_count)` on success. pub fn copy_file_range, Q: AsRef>( src: P, src_offset: u64, src_length: u64, dest: Q, dest_offset: u64, ) -> io::Result { let mut src_file = File::open(src)?; let mut dest_file = File::options().write(true).create(true).open(dest)?; src_file.seek(SeekFrom::Start(src_offset))?; let mut src_file = src_file.take(src_length); dest_file.seek(SeekFrom::Start(dest_offset))?; io::copy(&mut src_file, &mut dest_file) } clone-file-0.1.0/src/lib.rs000064400000000000000000000004001046102023000135340ustar 00000000000000mod common; pub use common::*; #[cfg(any(doc, target_os = "linux"))] pub mod linux; #[cfg(target_os = "linux")] pub use linux::*; #[cfg(any(doc, not(target_os = "linux")))] pub mod unsupported; #[cfg(not(target_os = "linux"))] pub use unsupported::*; clone-file-0.1.0/src/linux.rs000064400000000000000000000042101046102023000141300ustar 00000000000000use std::fs::File; use std::io; use std::os::fd::AsRawFd; use std::path::Path; /// Clone a file using the FICLONE syscall /// /// This is mainly tested on the `btrfs` filesystem. /// /// For the details on the limitations of this method, see [`man 2 ioctl_ficlonerange`]. /// /// [`man 2 ioctl_ficlonerange`]: https://man7.org/linux/man-pages/man2/ioctl_ficlonerange.2.html pub fn clone_file, Q: AsRef>(src: P, dest: Q) -> io::Result<()> { let src_file = File::open(src)?; let dest_file = File::options() .write(true) .create(true) .truncate(true) .open(dest)?; match unsafe { ioctl::ficlone(dest_file.as_raw_fd(), src_file.as_raw_fd() as u64) } { Ok(_) => Ok(()), Err(_) => Err(io::Error::last_os_error()), } } /// Clone a range from a file using the FICLONERANGE syscall /// /// This is mainly tested on the `btrfs` filesystem. /// /// For the details on the limitations of this method, see [`man 2 ioctl_ficlonerange`]. /// /// One of the more common limitations is that `src_offset`, `src_length` and `dest_offset` /// must be multiples of the filesystem block size. Expect this function to fail if that is not the case. /// /// [`man 2 ioctl_ficlonerange`]: https://man7.org/linux/man-pages/man2/ioctl_ficlonerange.2.html pub fn clone_file_range, Q: AsRef>( src: P, src_offset: u64, src_length: u64, dest: Q, dest_offset: u64, ) -> io::Result<()> { let src_file = File::open(src)?; let dest_file = File::options().write(true).create(true).open(dest)?; let args = ioctl::FileCloneRange { src_fd: src_file.as_raw_fd() as i64, src_offset, src_length, dest_offset, }; match unsafe { ioctl::ficlonerange(dest_file.as_raw_fd(), &args) } { Ok(_) => Ok(()), Err(_) => Err(io::Error::last_os_error()), } } mod ioctl { pub struct FileCloneRange { pub src_fd: i64, pub src_offset: u64, pub src_length: u64, pub dest_offset: u64, } nix::ioctl_write_int!(ficlone, 0x94, 9); nix::ioctl_write_ptr!(ficlonerange, 0x94, 13, FileCloneRange); } clone-file-0.1.0/src/unsupported.rs000064400000000000000000000013731046102023000153700ustar 00000000000000use std::env; use std::io; use std::path::Path; /// This function is not implemented for this platform pub fn clone_file, Q: AsRef>(src: P, dest: Q) -> io::Result<()> { operation_not_supported() } /// This function is not implemented for this platform pub fn clone_file_range, Q: AsRef>( src: P, src_offset: u64, src_length: u64, dest: Q, dest_offset: u64, ) -> io::Result<()> { operation_not_supported() } fn operation_not_supported() -> io::Result<()> { Err(io::Error::new( io::ErrorKind::Other, format!( "Operation not supported on {}-{}-{}", env::consts::ARCH, env::consts::OS, env::consts::FAMILY ), )) } clone-file-0.1.0/test.sh000075500000000000000000000015201046102023000131530ustar 00000000000000#!/usr/bin/env bash set -E setup() { # volume and file sizes are specifically chosen so that naive copy will not have enough space fallocate -l 200M .btrfs.img mkfs.btrfs -f .btrfs.img mkdir -p .btrfs-mnt sudo mount -o loop .btrfs.img .btrfs-mnt sudo chmod a+rwx .btrfs-mnt cat /dev/random | head -c 50M > .btrfs-mnt/src.bin # chopping 16K from start and 16K from end cat .btrfs-mnt/src.bin | tail -c +16K | tail -c +2 | head -c -16K > .btrfs-mnt/src-part.bin } test() { cargo test diff -q .btrfs-mnt/src.bin .btrfs-mnt/dest.bin diff -q .btrfs-mnt/src-part.bin .btrfs-mnt/dest-part.bin diff -q .btrfs-mnt/clone-range-small.bin .btrfs-mnt/copy-range-small.bin } teardown() { code=$? trap - ERR sudo umount .btrfs-mnt rm -r .btrfs-mnt rm .btrfs.img exit $code } setup trap teardown ERR test teardown clone-file-0.1.0/tests/btrfs-loopback.rs000064400000000000000000000025761046102023000162710ustar 00000000000000use clone_file::*; use std::fs; const BASE_DIR: &'static str = env!("CARGO_MANIFEST_DIR"); fn ensure_test_sh() -> anyhow::Result<()> { use anyhow::Context; fs::metadata(format!("{BASE_DIR}/.btrfs-mnt")).context("please run tests with test.sh")?; Ok(()) } #[cfg(target_os = "linux")] #[test] fn clone_file_works() -> anyhow::Result<()> { ensure_test_sh()?; clone_file( format!("{BASE_DIR}/.btrfs-mnt/src.bin"), format!("{BASE_DIR}/.btrfs-mnt/dest.bin"), )?; Ok(()) } #[cfg(target_os = "linux")] #[test] fn clone_file_range_works() -> anyhow::Result<()> { ensure_test_sh()?; clone_file_range( format!("{BASE_DIR}/.btrfs-mnt/src.bin"), // chopping 16K from start and 16K from end 16 << 10, (50 << 20) - (16 << 10) * 2, format!("{BASE_DIR}/.btrfs-mnt/dest-part.bin"), 0, )?; Ok(()) } #[cfg(target_os = "linux")] #[test] fn clone_range_consistent_with_copy_range() -> anyhow::Result<()> { ensure_test_sh()?; clone_file_range( format!("{BASE_DIR}/.btrfs-mnt/src.bin"), 0, 16 << 10, format!("{BASE_DIR}/.btrfs-mnt/clone-range-small.bin"), 0, )?; copy_file_range( format!("{BASE_DIR}/.btrfs-mnt/src.bin"), 0, 16 << 10, format!("{BASE_DIR}/.btrfs-mnt/copy-range-small.bin"), 0, )?; Ok(()) }