dircpy-0.3.16/.cargo_vcs_info.json0000644000000001360000000000100124150ustar { "git": { "sha1": "5d11e6b14b20ebcf10570dc3556e1da181fede93" }, "path_in_vcs": "" }dircpy-0.3.16/.github/workflows/test_linux.yml000064400000000000000000000003701046102023000175230ustar 00000000000000on: [push] name: Test Linux jobs: check: name: Test ubuntu-latest runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Run cargo check uses: actions-rs/cargo@v1 with: command: checkdircpy-0.3.16/.github/workflows/test_windows.yml000064400000000000000000000003741046102023000200620ustar 00000000000000on: [push] name: Test Windows jobs: check: name: Test windows-latest runs-on: windows-latest steps: - uses: actions/checkout@v2 - name: Run cargo check uses: actions-rs/cargo@v1 with: command: checkdircpy-0.3.16/.gitignore000064400000000000000000000001021046102023000131660ustar 00000000000000/target Cargo.lock .history sample output* sample* bench_data destdircpy-0.3.16/Cargo.toml0000644000000023100000000000100104070ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "dircpy" version = "0.3.16" authors = ["Johann Woelper "] description = "Copy directories recursively with flexible options." readme = "README.md" keywords = [ "copy", "recursive", "filesystem", "file", ] license = "MIT" repository = "https://github.com/woelper/dircpy/" [[bench]] name = "copy_rustlang" harness = false [dependencies.jwalk] version = "0.8" optional = true [dependencies.log] version = "0.4" [dependencies.walkdir] version = "2.4" [dev-dependencies.criterion] version = "0.5" [dev-dependencies.env_logger] version = "0.11" [dev-dependencies.reqwest] version = "0.11" features = ["blocking"] [dev-dependencies.unzip] version = "0.1" [features] default = ["jwalk"] dircpy-0.3.16/Cargo.toml.orig000064400000000000000000000013131046102023000140720ustar 00000000000000[package] name = "dircpy" version = "0.3.16" authors = ["Johann Woelper "] edition = "2018" license = "MIT" description = "Copy directories recursively with flexible options." repository = "https://github.com/woelper/dircpy/" keywords = ["copy", "recursive", "filesystem", "file"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] default = ["jwalk"] [dependencies] walkdir = "2.4" log = "0.4" # rayon = "1.4.0" jwalk = { version = "0.8", optional = true } [dev-dependencies] unzip = "0.1" reqwest = { version = "0.11", features = ["blocking"] } env_logger = "0.11" criterion = "0.5" [[bench]] name = "copy_rustlang" harness = false dircpy-0.3.16/LICENSE000064400000000000000000000020571046102023000122160ustar 00000000000000MIT License Copyright (c) 2020 Johann Woelper Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. dircpy-0.3.16/README.md000064400000000000000000000022531046102023000124660ustar 00000000000000# dircpy [![Crates.io](https://img.shields.io/crates/v/dircpy.svg)](https://crates.io/crates/dircpy) [![license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/woelper/dircpy/blob/master/LICENSE) [![Docs Status](https://docs.rs/dircpy/badge.svg)](https://docs.rs/dircpy) ![Crates.io](https://img.shields.io/crates/d/dircpy?label=crates.io%20downloads) [![Test Linux](https://github.com/woelper/dircpy/actions/workflows/test_linux.yml/badge.svg)](https://github.com/woelper/dircpy/actions/workflows/test_linux.yml) [![Test Windows](https://github.com/woelper/dircpy/actions/workflows/test_windows.yml/badge.svg)](https://github.com/woelper/dircpy/actions/workflows/test_windows.yml) A cross-platform library to recursively copy directories, with some convenience added. ```rust use dircpy::*; // Most basic example: copy_dir("src", "dest"); // Simple builder example: CopyBuilder::new("src", "dest") .run() .unwrap(); // Copy recursively, only including certain files: CopyBuilder::new("src", "dest") .overwrite_if_newer(true) .overwrite_if_size_differs(true) .with_include_filter(".txt") .with_include_filter(".csv") .run() .unwrap(); ``` dircpy-0.3.16/benches/copy_rustlang.rs000064400000000000000000000101571046102023000160570ustar 00000000000000use criterion::*; use dircpy::CopyBuilder; use env_logger; use log::*; use std::fs::File; use unzip::Unzipper; //const SAMPLE_DATA: &str = "https://github.com/rust-lang/rust/archive/master.zip"; const SAMPLE_DATA: &str = "https://github.com/rust-lang/cargo/archive/master.zip"; const SOURCE: &str = "bench_data/source"; const DEST: &str = "bench_data/dest"; fn random_string() -> String { format!("{:?}", std::time::Instant::now()) } fn download_and_unpack(url: &str, name: &str) { let archive = format!("{}.zip", name); if !std::path::Path::new(&archive).is_file() { info!("Downloading {:?}", url); let mut resp = reqwest::blocking::get(url).unwrap(); let mut out = File::create(&archive).expect("failed to create file"); std::io::copy(&mut resp, &mut out).expect("failed to copy content"); } else { info!("Did not download, archive already present"); } info!("Unzipping..."); Unzipper::new(File::open(&archive).unwrap(), name) .unzip() .unwrap(); info!("Done. Ready."); } fn setup(_: &mut Criterion) { std::env::set_var("RUST_LOG", "INFO"); let _ = env_logger::builder().try_init(); std::fs::create_dir_all(SOURCE).unwrap(); download_and_unpack(SAMPLE_DATA, SOURCE); } fn teardown(_: &mut Criterion) { std::env::set_var("RUST_LOG", "INFO"); let _ = env_logger::builder().try_init(); // One-time setup code goes here info!("CLEANUP"); // let _ = std::fs::remove_dir_all(source); let _ = std::fs::remove_dir_all(DEST); // let _ = std::fs::remove_file(archive); info!("DONE"); } fn test_cp(c: &mut Criterion) { std::env::set_var("RUST_LOG", "INFO"); let _ = env_logger::builder().try_init(); // One-time setup code goes here c.bench_function("cp -r", |b| { // Per-sample (note that a sample can be many iterations) setup goes here b.iter(|| { // Measured code goes here std::process::Command::new("cp") .arg("-r") .arg(SOURCE) .arg(&format!("{}{}", DEST, random_string())) .output().unwrap(); }); }); } fn test_dircpy_single(c: &mut Criterion) { // One-time setup code goes here // download_and_unpack(SAMPLE_DATA, source); c.bench_function("cpy single threaded", |b| { // Per-sample (note that a sample can be many iterations) setup goes here b.iter(|| { // Measured code goes here CopyBuilder::new(&SOURCE, &format!("{}{}", DEST, random_string())) .overwrite(true) .run() .unwrap(); }); }); } fn test_dircpy_parallel(c: &mut Criterion) { // One-time setup code goes here #[cfg(feature = "jwalk")] c.bench_function("cpy multi-threaded", |b| { // Per-sample (note that a sample can be many iterations) setup goes here b.iter(|| { // Measured code goes here CopyBuilder::new(&SOURCE, &format!("{}{}", DEST, random_string())) .overwrite(true) .run_par() .unwrap(); }); }); } fn test_lms(c: &mut Criterion) { std::env::set_var("RUST_LOG", "INFO"); let _ = env_logger::builder().try_init(); // One-time setup code goes here // download_and_unpack(SAMPLE_DATA, source); c.bench_function("lms", |b| { // Per-sample (note that a sample can be many iterations) setup goes here b.iter(|| { // Measured code goes here std::process::Command::new("lms") .arg("cp") .arg(SOURCE) .arg(&format!("{}{}", DEST, random_string())) .output().unwrap(); }); }); } criterion_group! { name = benches; // This can be any expression that returns a `Criterion` object. config = Criterion::default() .sample_size(10) // .sampling_mode() .warm_up_time(std::time::Duration::from_secs(4)) .measurement_time(std::time::Duration::from_secs(6)) ; targets = setup, test_dircpy_single, test_dircpy_parallel, test_cp, test_lms, teardown } criterion_main!(benches); dircpy-0.3.16/run_bench.sh000064400000000000000000000000761046102023000135070ustar 00000000000000cargo install lms cargo bench echo "Cleanup" rm -rf bench_datadircpy-0.3.16/src/lib.rs000064400000000000000000000315211046102023000131120ustar 00000000000000//! Recursively copy a directory from a to b. //! ``` //! use dircpy::*; //! //! // Most basic example: //! copy_dir("src", "dest").unwrap(); //! //! // Simple builder example: //!CopyBuilder::new("src", "dest") //!.run() //!.unwrap(); //! //! // Copy recursively, only including certain files: //!CopyBuilder::new("src", "dest") //!.overwrite_if_newer(true) //!.overwrite_if_size_differs(true) //!.with_include_filter(".txt") //!.with_include_filter(".csv") //!.run() //!.unwrap(); //! ``` use log::*; // use rayon::prelude::*; #[cfg(feature = "jwalk")] use jwalk::WalkDir as JWalkDir; use std::fs::{copy, read_link}; use std::io::{Error, ErrorKind}; use std::path::{Path, PathBuf}; use std::time::SystemTime; use walkdir::WalkDir; #[cfg(test)] mod tests; #[derive(Debug, Clone)] /// Recursively copy a directory from a to b. /// ``` /// use dircpy::*; /// /// // Most basic example: /// copy_dir("src", "dest"); /// /// // Simple builder example: ///CopyBuilder::new("src", "dest") ///.run() ///.unwrap(); /// /// // Copy recursively, only including certain files: ///CopyBuilder::new("src", "dest") ///.overwrite_if_newer(true) ///.overwrite_if_size_differs(true) ///.with_include_filter(".txt") ///.with_include_filter(".csv") ///.run() ///.unwrap(); /// ``` pub struct CopyBuilder { /// The source directory pub source: PathBuf, /// The destination directory pub destination: PathBuf, /// Overwrite all files in target, if already existing overwrite_all: bool, /// Overwrite target files if they are newer overwrite_if_newer: bool, /// Overwrite target files if they differ in size overwrite_if_size_differs: bool, /// A list of include filters exclude_filters: Vec, /// A list of exclude filters include_filters: Vec, } /// Determine if the modification date of file_a is newer than that of file_b fn is_file_newer(file_a: &Path, file_b: &Path) -> bool { match (file_a.symlink_metadata(), file_b.symlink_metadata()) { (Ok(meta_a), Ok(meta_b)) => { meta_a.modified().unwrap_or_else(|_| SystemTime::now()) > meta_b.modified().unwrap_or(SystemTime::UNIX_EPOCH) } _ => false, } } /// Determine if file_a and file_b's size differs. fn is_filesize_different(file_a: &Path, file_b: &Path) -> bool { match (file_a.symlink_metadata(), file_b.symlink_metadata()) { (Ok(meta_a), Ok(meta_b)) => meta_a.len() != meta_b.len(), _ => false, } } #[cfg(feature = "jwalk")] fn copy_file(source: &Path, options: CopyBuilder) -> Result<(), std::io::Error> { let abs_source = options.source.canonicalize()?; let abs_dest = options.destination.canonicalize()?; let rel_dest = source .strip_prefix(&abs_source) .map_err(|e| Error::new(ErrorKind::Other, format!("Could not strip prefix: {:?}", e)))?; let dest_entry = abs_dest.join(rel_dest); if source.is_file() { // the source exists // Early out if target is present and overwrite is off if !options.overwrite_all && dest_entry.is_file() && !options.overwrite_if_newer && !options.overwrite_if_size_differs { return Ok(()); } for f in &options.exclude_filters { if source.to_string_lossy().contains(f) { return Ok(()); } } for f in &options.include_filters { if !source.to_string_lossy().contains(f) { return Ok(()); } } // File is not present: copy it if !dest_entry.is_file() { debug!( "Dest not present: CP {} DST {}", source.display(), dest_entry.display() ); copy(source, dest_entry)?; return Ok(()); } // File newer? if options.overwrite_if_newer { if is_file_newer(source, &dest_entry) { debug!( "Source newer: CP {} DST {}", source.display(), dest_entry.display() ); copy(source, &dest_entry)?; } return Ok(()); } // Different size? if options.overwrite_if_size_differs { if is_filesize_different(source, &dest_entry) { debug!( "Source differs: CP {} DST {}", source.display(), dest_entry.display() ); copy(source, &dest_entry)?; } return Ok(()); } // The regular copy operation debug!("CP {} DST {}", source.display(), dest_entry.display()); copy(source, dest_entry)?; } else if source.is_dir() && !dest_entry.is_dir() { debug!("MKDIR {}", source.display()); std::fs::create_dir_all(dest_entry)?; } Ok(()) } impl CopyBuilder { /// Construct a new CopyBuilder with `source` and `dest`. pub fn new, Q: AsRef>(source: P, dest: Q) -> CopyBuilder { CopyBuilder { source: source.as_ref().to_path_buf(), destination: dest.as_ref().to_path_buf(), overwrite_all: false, overwrite_if_newer: false, overwrite_if_size_differs: false, exclude_filters: vec![], include_filters: vec![], } } /// Overwrite target files (off by default) pub fn overwrite(self, overwrite: bool) -> CopyBuilder { CopyBuilder { overwrite_all: overwrite, ..self } } /// Overwrite if the source is newer (off by default) pub fn overwrite_if_newer(self, overwrite_only_newer: bool) -> CopyBuilder { CopyBuilder { overwrite_if_newer: overwrite_only_newer, ..self } } /// Overwrite if size between source and dest differs (off by default) pub fn overwrite_if_size_differs(self, overwrite_if_size_differs: bool) -> CopyBuilder { CopyBuilder { overwrite_if_size_differs, ..self } } /// Do not copy files that contain this string pub fn with_exclude_filter(self, f: &str) -> CopyBuilder { let mut filters = self.exclude_filters.clone(); filters.push(f.to_owned()); CopyBuilder { exclude_filters: filters, ..self } } /// Only copy files that contain this string. pub fn with_include_filter(self, f: &str) -> CopyBuilder { let mut filters = self.include_filters.clone(); filters.push(f.to_owned()); CopyBuilder { include_filters: filters, ..self } } /// Execute the copy operation pub fn run(&self) -> Result<(), std::io::Error> { if !self.destination.is_dir() { debug!("MKDIR {:?}", &self.destination); std::fs::create_dir_all(&self.destination)?; } let abs_source = self.source.canonicalize()?; let abs_dest = self.destination.canonicalize()?; debug!( "Building copy operation: SRC {} DST {}", abs_source.display(), abs_dest.display() ); 'files: for entry in WalkDir::new(&abs_source) .into_iter() .filter_entry(|e| e.path() != abs_dest) .filter_map(|e| e.ok()) { let rel_dest = entry.path().strip_prefix(&abs_source).map_err(|e| { Error::new(ErrorKind::Other, format!("Could not strip prefix: {:?}", e)) })?; let dest_entry = abs_dest.join(rel_dest); if entry.path().symlink_metadata().is_ok() && !entry.file_type().is_dir() { // the source exists, but isn't a directory // Early out if target is present and overwrite is off if !self.overwrite_all && dest_entry.symlink_metadata().is_ok() && !self.overwrite_if_newer && !self.overwrite_if_size_differs { continue; } for f in &self.exclude_filters { debug!("EXCL {} for {:?}", f, entry); if entry.path().to_string_lossy().contains(f) { continue 'files; } } for f in &self.include_filters { if !entry.path().to_string_lossy().contains(f) { continue 'files; } } // File is not present: copy it in any case let dest_exists = dest_entry.symlink_metadata().is_ok(); if !dest_exists { debug!( "Dest not present: CP {} DST {}", entry.path().display(), dest_entry.display() ); } // File newer? if dest_exists && self.overwrite_if_newer { if is_file_newer(entry.path(), &dest_entry) { debug!( "Source newer: CP {} DST {}", entry.path().display(), dest_entry.display() ); } else { continue; } } // Different size? if dest_exists && self.overwrite_if_size_differs { if is_filesize_different(entry.path(), &dest_entry) { debug!( "Source differs: CP {} DST {}", entry.path().display(), dest_entry.display() ); } else { continue; } } if entry.file_type().is_file() { // The regular copy operation debug!("CP {} DST {}", entry.path().display(), dest_entry.display()); copy(entry.path(), dest_entry)?; } else if entry.file_type().is_symlink() { debug!( "CP LNK {} DST {}", entry.path().display(), dest_entry.display() ); let target = read_link(entry.path())?; #[cfg(unix)] std::os::unix::fs::symlink(target, dest_entry)? } else { unimplemented!( "File {} has unhandled type {:?}", entry.path().display(), entry.file_type() ); } } else if entry.path().is_dir() && !dest_entry.is_dir() { debug!("MKDIR {}", entry.path().display()); std::fs::create_dir_all(dest_entry)?; } } Ok(()) } /// Execute the copy operation in parallel. The usage of this function is discouraged /// until proven to work faster. #[cfg(feature = "jwalk")] pub fn run_par(&self) -> Result<(), std::io::Error> { if !self.destination.is_dir() { debug!("MKDIR {:?}", &self.destination); std::fs::create_dir_all(&self.destination)?; } let abs_source = self.source.canonicalize()?; let abs_dest = self.destination.canonicalize()?; debug!( "Building copy operation: SRC {} DST {}", abs_source.display(), abs_dest.display() ); for entry in JWalkDir::new(&abs_source) .into_iter() .filter_map(|e| e.ok()) { let _ = copy_file(&entry.path(), self.clone()); } Ok(()) } } /// Copy a directory from `source` to `dest`, creating `dest`, with all options. pub fn copy_dir_advanced, Q: AsRef>( source: P, dest: Q, overwrite_all: bool, overwrite_if_newer: bool, overwrite_if_size_differs: bool, exclude_filters: Vec, include_filters: Vec, ) -> Result<(), std::io::Error> { CopyBuilder { source: source.as_ref().to_path_buf(), destination: dest.as_ref().to_path_buf(), overwrite_all, overwrite_if_newer, overwrite_if_size_differs, exclude_filters, include_filters, } .run() } /// Copy a directory from `source` to `dest`, creating `dest`, with minimal options. pub fn copy_dir, Q: AsRef>(source: P, dest: Q) -> Result<(), std::io::Error> { CopyBuilder { source: source.as_ref().to_path_buf(), destination: dest.as_ref().to_path_buf(), overwrite_all: false, overwrite_if_newer: false, overwrite_if_size_differs: false, exclude_filters: vec![], include_filters: vec![], } .run() } dircpy-0.3.16/src/tests.rs000064400000000000000000000106621046102023000135110ustar 00000000000000use super::*; use std::fs::create_dir_all; use std::fs::File; #[cfg(unix)] use std::os::unix::fs::{symlink, PermissionsExt}; #[test] fn copy_basic() { create_dir_all("source/level1/level2/level3").unwrap(); File::create("source/test").unwrap(); File::create("source/level1/other_file").unwrap(); #[cfg(unix)] { File::create("source/exec_file").unwrap(); std::fs::set_permissions("source/exec_file", std::fs::Permissions::from_mode(0o755)) .unwrap(); symlink("exec_file", "source/symlink").unwrap(); symlink("does_not_exist", "source/dangling_symlink").unwrap(); } CopyBuilder::new("source", "dest") .overwrite(true) .overwrite_if_newer(true) .run() .unwrap(); #[cfg(unix)] { let f = File::open("dest/exec_file").unwrap(); let metadata = f.metadata().unwrap(); let permissions = metadata.permissions(); println!("permissions: {:o}", permissions.mode()); assert_eq!(permissions.mode(), 33261); assert_eq!( Path::new("exec_file"), read_link("dest/symlink").unwrap().as_path() ); assert_eq!( Path::new("does_not_exist"), read_link("dest/dangling_symlink").unwrap().as_path() ); } // clean up std::fs::remove_dir_all("source").unwrap(); std::fs::remove_dir_all("dest").unwrap(); } #[test] fn copy_subdir() { std::env::set_var("RUST_LOG", "debug"); let _ = env_logger::try_init(); create_dir_all("source/subdir").unwrap(); create_dir_all("source/this_should_copy").unwrap(); File::create("source/this_should_copy/file.doc").unwrap(); File::create("source/a.jpg").unwrap(); File::create("source/b.jpg").unwrap(); File::create("source/d.txt").unwrap(); CopyBuilder::new("source", "source/subdir") .run() .unwrap(); // std::fs::remove_dir_all("source").unwrap(); } #[test] fn copy_exclude() { std::env::set_var("RUST_LOG", "DEBUG"); let _ = env_logger::builder().try_init(); let src = "ex_src"; let dst = "ex_dest"; create_dir_all(src).unwrap(); File::create(format!("{}/foo", src)).unwrap(); File::create(format!("{}/bar", src)).unwrap(); CopyBuilder::new(src, dst) .overwrite(true) .overwrite_if_newer(true) .with_exclude_filter("foo") .run() .unwrap(); assert!(!Path::new(&format!("{}/foo", dst)).is_file()); // clean up std::fs::remove_dir_all(src).unwrap(); std::fs::remove_dir_all(dst).unwrap(); } #[test] fn copy_include() { std::env::set_var("RUST_LOG", "DEBUG"); let _ = env_logger::builder().try_init(); let src = "in_src"; let dst = "in_dest"; create_dir_all(src).unwrap(); File::create(format!("{}/foo", src)).unwrap(); File::create(format!("{}/bar", src)).unwrap(); CopyBuilder::new(src, dst) .overwrite(true) .overwrite_if_newer(true) .with_include_filter("foo") .run() .unwrap(); assert!(Path::new(&format!("{}/foo", dst)).is_file()); assert!(!Path::new(&format!("{}/bar", dst)).is_file()); // clean up std::fs::remove_dir_all(src).unwrap(); std::fs::remove_dir_all(dst).unwrap(); } #[test] fn copy_cargo() { let url = "https://github.com/rust-lang/cargo/archive/master.zip"; let sample_dir = "cargo"; let output_dir = format!("{}_output", sample_dir); let archive = format!("{}.zip", sample_dir); println!("Expanding {}", archive); let mut resp = reqwest::blocking::get(url).unwrap(); let mut out = File::create(&archive).expect("failed to create file"); std::io::copy(&mut resp, &mut out).expect("failed to copy content"); let reader = std::fs::File::open(&archive).unwrap(); unzip::Unzipper::new(reader, sample_dir) .unzip() .expect("Could not expand cargo sources"); let num_input_files = WalkDir::new(&sample_dir) .into_iter() .filter_map(|e| e.ok()) .count(); CopyBuilder::new( &Path::new(sample_dir).canonicalize().unwrap(), &PathBuf::from(&output_dir), ) .run() .unwrap(); let num_output_files = WalkDir::new(&output_dir) .into_iter() .filter_map(|e| e.ok()) .count(); assert_eq!(num_output_files, num_input_files); std::fs::remove_dir_all(sample_dir).unwrap(); std::fs::remove_dir_all(output_dir).unwrap(); std::fs::remove_file(archive).unwrap(); }