tree_magic_mini-3.0.3/.cargo_vcs_info.json0000644000000001120000000000100141440ustar { "git": { "sha1": "8e190ab9f1ed5c14b0ec1a7e84d3d02ddf3aa185" } } tree_magic_mini-3.0.3/.gitignore000064400000000000000000000000470072674642500147630ustar 00000000000000target Cargo.lock graph.dot graph.svg tree_magic_mini-3.0.3/CHANGELOG.md000064400000000000000000000041140072674642500146030ustar 00000000000000# tree_magic_mini 3.0.0 * Split GPL-licensed files into a separate optional dependency. The main crate is now MIT-licensed, and searches for data files installed on the system at run-time by default. If you enable the `with-gpl-data` feature, then the data files will be hard-coded into the library at compile time. Programs that use this feature must be distributed according to the terms of the GNU GPL 2.0 or later. # tree_magic_mini 2.0.0 * Change license to GPL-2.0-or-later for compatibility with upstream xdg-shared-mime-info license. # tree_magic_mini 1.0.1 * Update to nom 6. # tree_magic_mini 1.0.0 * Forked and changed name to `tree_magic_mini` * Updated dependencies. * Reduced copying and memory allocation, for a slight increase in speed and decrease in memory use. * Reduced API surface. Some previously public APIs are now internal. * Removed the optional `cli` feature and `tmagic` binary. # 0.2.3 Upgraded package versions to latest (except nom, which is currently stuck at 3.x) and fixed the paths in the doc tests # 0.2.2 Yanked due to accidental breaking API change # 0.2.1 Incorporated fix by Bram Sanders to prevent panic on non-existent file. # 0.2.0 Major changes, front-end and back. - Added `is_alias` function - `from_*` functions excluding `from_*_node` now return MIME, not Option - New feature flag: `staticmime`. Changes type of MIME from String to &'static str - Bundled magic file, so it works on Windows as well. - Split `fdo_magic` checker into `fdo_magic::sys` and `fdo_magic::builtin` - `len` argument removed from `*_u8` functions - Tests and benchmarks added. - Fixed horribly broken logic in `fdo_magic` checker - Checks the most common types before obscure types - Changed hasher to `fnv`. - Added support for handling aliases in input - `tmagic` command has more features - Major speed improvements # 0.1.1 - *Changed public interface*: Added `from_u8` export function - *Changed public interface*: Changed len argument for `u8` functions from `u32` to `usize` - Minor speed improvements in `fdo_magic` checker # 0.1.0 Initial release tree_magic_mini-3.0.3/Cargo.toml0000644000000026160000000000100121550ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "tree_magic_mini" version = "3.0.3" authors = ["Matt Brubeck ", "Allison Hancock "] exclude = ["tests/*", "benches/*/"] description = "Determines the MIME type of a file by traversing a filetype tree." documentation = "https://docs.rs/tree_magic_mini/" readme = "README.md" keywords = ["mime", "filesystem", "media-types"] license = "MIT" repository = "https://github.com/mbrubeck/tree_magic/" [[bench]] name = "from_u8" harness = false [[bench]] name = "match_u8" harness = false [dependencies.bytecount] version = "0.6.0" [dependencies.fnv] version = "1.0" [dependencies.lazy_static] version = "1.4" [dependencies.nom] version = "7.0" [dependencies.once_cell] version = "1.0" [dependencies.petgraph] version = "0.6.0" [dependencies.tree_magic_db] version = "3.0" optional = true [dev-dependencies.bencher] version = "0.1.0" [features] with-gpl-data = ["tree_magic_db"] tree_magic_mini-3.0.3/Cargo.toml.orig000064400000000000000000000015610072674642500156640ustar 00000000000000[package] name = "tree_magic_mini" version = "3.0.3" authors = [ "Matt Brubeck ", "Allison Hancock ", ] description = "Determines the MIME type of a file by traversing a filetype tree." repository = "https://github.com/mbrubeck/tree_magic/" documentation = "https://docs.rs/tree_magic_mini/" readme = "README.md" keywords = ["mime", "filesystem", "media-types"] license = "MIT" exclude = ["tests/*", "benches/*/"] edition = "2018" [dependencies] petgraph = "0.6.0" nom = "7.0" lazy_static = "1.4" fnv = "1.0" bytecount = "0.6.0" once_cell = "1.0" tree_magic_db = { version = "3.0", path = "./magic_db" , optional = true } [features] with-gpl-data = ["tree_magic_db"] [dev-dependencies] bencher = "0.1.0" [workspace] members = ["magic_db"] [[bench]] name = "from_u8" harness = false [[bench]] name = "match_u8" harness = false tree_magic_mini-3.0.3/LICENSE000064400000000000000000000020560072674642500140020ustar 00000000000000MIT License Copyright (c) 2017 Aaron Hancock Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. tree_magic_mini-3.0.3/README.md000064400000000000000000000216400072674642500142540ustar 00000000000000# tree_magic_mini `tree_magic_mini` is a Rust crate that determines the MIME type a given file or byte stream. Read the documentation at https://docs.rs/tree_magic_mini/ This is a fork of the [tree_magic](https://crates.io/crates/tree_magic) crate by Allison Hancock. It includes the following changes: * Updated dependencies. * Reduced copying and memory allocation, for a slight increase in speed and decrease in memory use. * Reduced API surface. Some previously public APIs are now internal. * Removed the optional `cli` feature and `tmagic` binary. * Split GPL-licensed data files into a separate optional crate. These changes were made both to make the library more efficient, and to simplify the effort to maintain and optimize of this fork. I would like to eventually merge these changes back to the original `tree_magic` crate, and/or restore some of the removed features if there is demand for that. ## Licensing and the MIME database By default, `tree_magic_mini` will attempt to load the shared MIME info database from the standard locations at runtime. If you won't have the database files available, or would like to include them in your binary for simplicity, you can optionally embed the database information if you enable the `tree_magic_db` feature. **As the magic database files themselves are licensed under the GPL, you must make sure your project uses a compatible license if you enable this behaviour.** --- Continue reading for the original `tree_magic` documentation. ## About tree_magic Unlike the typical approach that libmagic and file(1) uses, this loads all the file types in a tree based on subclasses. (EX: `application/vnd.openxmlformats-officedocument.wordprocessingml.document` (MS Office 2007) subclasses `application/zip` which subclasses `application/octet-stream`) Then, instead of checking the file against *every* file type, it can traverse down the tree and only check the file types that make sense to check. (After all, the fastest check is the check that never gets run.) This library also provides the ability to check if a file is a certain type without going through the process of checking it against every file type. ## Performance This is fast. FAST. This is a test of my Downloads folder (sorry, can't find a good publicly available set of random files) on OpenSUSE Tumbleweed. `tmagic` was compiled with `cargo build --release`, and `file` came from the OpenSUSE repos. This is a warm run, which means I've ran both programs through a few times. System is a dual-core Intel Core i7 640M, and results were measured with `time`. Program | real | user | sys --------|------|------|----- tmagic 0.2.0 | 0m0.063s | 0m0.052s | 0m0.004s file-5.30 --mime-type | 0m0.924s | 0.800s | 0.116s There's a couple things that lead to this. Mainly: - Less types to parse due to graph approach. - First 4K of file is loaded then passed to all parsers, instead of constantly reloading from disk. (When doing that, the time was more around ~0.130s.) - The most common types (image/png, image/jpeg, application/zip, etc.) are checked before the exotic ones. - Everything that can be processed in a lazy_static! is. Nightly users can also run `cargo bench` for some benchmarks. For tree_magic 0.2.0 on the same hardware: test from_u8::application_zip ... bench: 17,086 ns/iter (+/- 845) test from_u8::image_gif ... bench: 5,027 ns/iter (+/- 520) test from_u8::image_png ... bench: 4,421 ns/iter (+/- 1,795) test from_u8::text_plain ... bench: 112,578 ns/iter (+/- 11,778) test match_u8::application_zip ... bench: 222 ns/iter (+/- 144) test match_u8::image_gif ... bench: 140 ns/iter (+/- 14) test match_u8::image_png ... bench: 139 ns/iter (+/- 18) test match_u8::text_plain ... bench: 44 ns/iter (+/- 3) However, it should be noted that the FreeDesktop.org magic files less filetypes than the magic files used by libmagic. (On my system tree_magic supports 400 types, while `/usr/share/misc/magic` contains 855 `!:mime` tags.) It is, however, significantly easier to parse, as it only covers magic numbers and not attributes or anything like that. See the TODO section for plans to fix this. ## Compatibility This has been tested using Rust Stable and Nightly on Windows 7 and OpenSUSE Tumbleweed Linux. All mime information and relation information is loaded from the Shared MIME-info Database as described at https://specifications.freedesktop.org/shared-mime-info-spec/shared-mime-info-spec-latest.html. If you beleive that this is not present on your system, turn off the `sys_fdo_magic` feature flag. This provides the most common file types, but it's still missing some important ones, like LibreOffice or MS Office 2007+ support or ISO files. Expect this to improve, especially as the `zip` checker is added. ### Architecture `tree_magic` is split up into different "checker" modules. Each checker handles a certain set of filetypes, and only those. For instance, the `basetype` checker handles the `inode/*` and `text/plain` types, while the `fdo_magic` checker handles anything with a magic number. Th idea here is that instead of following the `libmagic` route of having one magic descriptor format that fits every file, we can specialize and choose the checker that suits the file format best. During library initialization, each checker is queried for the types is supports and the parent->child relations between them. During this time, the checkers can load any rules, schemas, etc. into memory. A big philosophy here is that **time during the checking phase is many times more valuable than during the init phase**. The library only gets initialized once, and the library can check thousands of files during a program's lifetime. From the list of file types and relations, a directed graph is built, and each node is added to a hash map. The library user can use these directly to find parents, children, etc. of a given MIME if needed. When a file needs to be checked against a certain MIME (match_*), each checker is queried to see if it supports that type, and if so, it runs the checker. If the checker returns true, it must be that type. When a file needs it's MIME type found (from_*), the library starts at the `all/all` node of the type graph (or whichever node the user specifies) and walks down the tree. If a match is found, it continues searching down that branch. If no match is found, it retrieves the deepest MIME type found. ## TODO ### Improve fdo-magic checker Right now the `fdo-magic` checker does not handle endianess. It also does not handle magic files stored in the user's home directory. ### Additional checkers It is planned to have custom file checking functions for many types. Here's some ideas: - `zip`: Everything that subclasses `application/zip` can be determined further by peeking at the zip's directory listing. - `grep`: Text files such as program scripts and configuration files could be parsed with a regex (or whatever works best). - `json`, `toml`, `xml`, etc: Check the given file against a schema and return true if it matches. (By this point there should be few enough potential matches that it should be okay to load the entire file) - (specialized parsers): Binary (or text) files without any sort of magic can be checked for compliance against a quick and dirty `nom` parser instead of the weird heuristics used by libmagic. To add additional checker types, add a new module exporting: - `init::get_supported() -> Vec<(String)>` - `init::get_subclasses() -> Vec` - `test::from_u8(&[u8], &str) -> bool` - `test::from_filpath(&str, &str) -> Result` and then add references to those functions into the CHECKERS lazy_static! in `lib.rs`. The bottommost entries get searched first. ### Caching Going forward, it is essential for a checker (like `basetype`'s metadata, or that json/toml/xml example) to be able to cache an in-memory representation of the file, so it doesn't have to get re-loaded and re-parsed for every new type. With the current architecture, this is rather difficult to implement. ### Multiple file types There are some weird files out there ( [Polyglot quines](https://en.wikipedia.org/wiki/Polyglot_(computing)) come to mind. ) that are multiple file types. This might be worth handling for security reasons. (It's not a huge priority, though.) ### Parallel processing Right now this is single-threaded. This is an embarasingly parallel task (multiple files, multiple types, multiple rules for each type...), so there should be a great speed benefit. ## TO NOT DO ### File attributes `libmagic` and `file`, by default, print descriptive strings detailing the file type and, for things like JPEG images or ELF files, a whole bunch of metadata. This is not something `tree_magic` will ever support, as it is entirely unnecessary. Support for attributes would best be handled in a seperate crate that, given a MIME, can extract metadata in a predictable, machine readable format. tree_magic_mini-3.0.3/benches/from_u8.rs000064400000000000000000000012230072674642500163240ustar 00000000000000use tree_magic_mini as tree_magic; #[macro_use] extern crate bencher; use bencher::Bencher; ///Image tests fn image_gif(b: &mut Bencher) { b.iter(|| tree_magic::from_u8(include_bytes!("image/gif"))); } fn image_png(b: &mut Bencher) { b.iter(|| tree_magic::from_u8(include_bytes!("image/png"))); } /// Archive tests fn application_zip(b: &mut Bencher) { b.iter(|| tree_magic::from_u8(include_bytes!("application/zip"))); } /// Text tests fn text_plain(b: &mut Bencher) { b.iter(|| tree_magic::from_u8(include_bytes!("text/plain"))); } benchmark_group!(benches, image_gif, image_png, application_zip, text_plain); benchmark_main!(benches); tree_magic_mini-3.0.3/benches/match_u8.rs000064400000000000000000000013270072674642500164620ustar 00000000000000use tree_magic_mini as tree_magic; #[macro_use] extern crate bencher; use bencher::Bencher; ///Image benchmarks fn image_gif(b: &mut Bencher) { b.iter(|| tree_magic::match_u8("image/gif", include_bytes!("image/gif"))); } fn image_png(b: &mut Bencher) { b.iter(|| tree_magic::match_u8("image/png", include_bytes!("image/png"))); } /// Archive tests fn application_zip(b: &mut Bencher) { b.iter(|| tree_magic::match_u8("application/zip", include_bytes!("application/zip"))); } /// Text tests fn text_plain(b: &mut Bencher) { b.iter(|| tree_magic::match_u8("text/plain", include_bytes!("text/plain"))); } benchmark_group!(benches, image_gif, image_png, application_zip, text_plain); benchmark_main!(benches); tree_magic_mini-3.0.3/src/basetype/check.rs000064400000000000000000000037760072674642500170350ustar 00000000000000use crate::{read_bytes, MIME}; use fnv::FnvHashMap; use std::path::Path; pub(crate) struct BaseType; impl crate::Checker for BaseType { fn from_u8(&self, file: &[u8], mimetype: &str) -> bool { from_u8(file, mimetype) } fn from_filepath(&self, filepath: &Path, mimetype: &str) -> bool { from_filepath(filepath, mimetype) } fn get_supported(&self) -> Vec { super::init::get_supported() } fn get_subclasses(&self) -> Vec<(MIME, MIME)> { super::init::get_subclasses() } fn get_aliaslist(&self) -> FnvHashMap { super::init::get_aliaslist() } } /// If there are any null bytes, return False. Otherwise return True. fn is_text_plain_from_u8(b: &[u8]) -> bool { bytecount::count(b, 0) == 0 } // TODO: Hoist the main logic here somewhere else. This'll get redundant fast! fn is_text_plain_from_filepath(filepath: &Path) -> bool { let b = match read_bytes(filepath, 512) { Ok(x) => x, Err(_) => return false, }; is_text_plain_from_u8(b.as_slice()) } #[allow(unused_variables)] pub fn from_u8(b: &[u8], mimetype: &str) -> bool { if mimetype == "application/octet-stream" || mimetype == "all/allfiles" { // Both of these are the case if we have a bytestream at all return true; } if mimetype == "text/plain" { is_text_plain_from_u8(b) } else { // ...how did we get bytes for this? false } } pub fn from_filepath(filepath: &Path, mimetype: &str) -> bool { use std::fs; // Being bad with error handling here, // but if you can't open it it's probably not a file. let meta = match fs::metadata(filepath) { Ok(x) => x, Err(_) => { return false; } }; match mimetype { "all/all" => true, "all/allfiles" | "application/octet-stream" => meta.is_file(), "inode/directory" => meta.is_dir(), "text/plain" => is_text_plain_from_filepath(filepath), _ => false, } } tree_magic_mini-3.0.3/src/basetype/init.rs000064400000000000000000000007350072674642500167130ustar 00000000000000use crate::MIME; use fnv::FnvHashMap; pub fn get_supported() -> Vec { super::TYPES.to_vec() } /// Returns Vec of parent->child relations pub fn get_subclasses() -> Vec<(MIME, MIME)> { vec![ ("all/all", "all/allfiles"), ("all/all", "inode/directory"), ("all/allfiles", "application/octet-stream"), ("application/octet-stream", "text/plain"), ] } pub fn get_aliaslist() -> FnvHashMap { FnvHashMap::default() } tree_magic_mini-3.0.3/src/basetype/mod.rs000064400000000000000000000003400072674642500165170ustar 00000000000000//! Handles "base types" such as inode/* and text/plain const TYPES: [&str; 5] = [ "all/all", "all/allfiles", "inode/directory", "text/plain", "application/octet-stream", ]; pub mod check; pub mod init; tree_magic_mini-3.0.3/src/fdo_magic/builtin/check.rs000064400000000000000000000036670072674642500205760ustar 00000000000000use crate::{fdo_magic, read_bytes, MIME}; use fnv::FnvHashMap; use petgraph::prelude::*; use std::path::Path; pub(crate) struct FdoMagic; impl crate::Checker for FdoMagic { fn from_u8(&self, file: &[u8], mimetype: &str) -> bool { from_u8(file, mimetype) } fn from_filepath(&self, filepath: &Path, mimetype: &str) -> bool { from_filepath(filepath, mimetype) } fn get_supported(&self) -> Vec { super::init::get_supported() } fn get_subclasses(&self) -> Vec<(MIME, MIME)> { super::init::get_subclasses() } fn get_aliaslist(&self) -> FnvHashMap { super::init::get_aliaslist() } } /// Test against all rules #[allow(unused_variables)] pub fn from_u8(file: &[u8], mimetype: &str) -> bool { // Get magic ruleset let graph = match super::ALLRULES.get(mimetype) { Some(item) => item, None => return false, // No rule for this mime }; // Check all rulesets for x in graph.externals(Incoming) { if fdo_magic::check::from_u8_walker(file, mimetype, graph, x, true) { return true; } } false } /// This only exists for the case of a direct match_filepath call /// and even then we could probably get rid of this... #[allow(unused_variables)] pub fn from_filepath(filepath: &Path, mimetype: &str) -> bool { // Get magic ruleset let magic_rules = match super::ALLRULES.get(mimetype) { Some(item) => item, None => return false, // No rule for this mime }; // Get # of bytes to read let mut scanlen = 0; for x in magic_rules.raw_nodes() { let y = &x.weight; let tmplen = y.start_off as usize + y.val.len() + y.region_len as usize; if tmplen > scanlen { scanlen = tmplen; } } let b = match read_bytes(filepath, scanlen) { Ok(x) => x, Err(_) => return false, }; from_u8(b.as_slice(), mimetype) } tree_magic_mini-3.0.3/src/fdo_magic/builtin/init.rs000064400000000000000000000026030072674642500204510ustar 00000000000000use crate::MIME; use fnv::FnvHashMap; #[cfg(not(feature = "with-gpl-data"))] use super::runtime; fn aliases() -> &'static str { #[cfg(feature = "with-gpl-data")] return tree_magic_db::aliases(); #[cfg(not(feature = "with-gpl-data"))] return runtime::aliases(); } fn subclasses() -> &'static str { #[cfg(feature = "with-gpl-data")] return tree_magic_db::subclasses(); #[cfg(not(feature = "with-gpl-data"))] return runtime::subclasses(); } pub fn get_aliaslist() -> FnvHashMap { aliases() .lines() .map(|line| { let mut parts = line.split_whitespace(); let a = parts.next().unwrap(); let b = parts.next().unwrap(); (a, b) }) .collect() } /// Get list of supported MIME types pub fn get_supported() -> Vec { super::ALLRULES.keys().cloned().collect() } /// Get list of parent -> child subclass links pub fn get_subclasses() -> Vec<(MIME, MIME)> { subclasses() .lines() .map(|line| { let mut parts = line.split_whitespace(); let child = parts.next().unwrap(); let child = super::ALIASES.get(child).copied().unwrap_or(child); let parent = parts.next().unwrap(); let parent = super::ALIASES.get(parent).copied().unwrap_or(parent); (parent, child) }) .collect() } tree_magic_mini-3.0.3/src/fdo_magic/builtin/mod.rs000064400000000000000000000020240072674642500202620ustar 00000000000000//! Read magic file bundled in crate use super::MagicRule; use crate::MIME; use fnv::FnvHashMap; use lazy_static::lazy_static; use petgraph::prelude::*; /// Preload alias list lazy_static! { static ref ALIASES: FnvHashMap = init::get_aliaslist(); } /// Load magic file before anything else. lazy_static! { static ref ALLRULES: FnvHashMap, u32>> = rules(); } pub mod check; pub mod init; #[cfg(not(feature = "with-gpl-data"))] mod runtime; fn rules() -> FnvHashMap, u32>> { #[cfg(feature = "with-gpl-data")] return static_rules(); #[cfg(not(feature = "with-gpl-data"))] return runtime_rules(); } #[cfg(feature = "with-gpl-data")] fn static_rules() -> FnvHashMap, u32>> { super::ruleset::from_u8(tree_magic_db::magic()).unwrap_or_default() } #[cfg(not(feature = "with-gpl-data"))] fn runtime_rules() -> FnvHashMap, u32>> { runtime::rules().unwrap_or_default() } tree_magic_mini-3.0.3/src/fdo_magic/builtin/runtime.rs000064400000000000000000000054230072674642500211740ustar 00000000000000///! Enable loading the magic database files at runtime rather than embedding the GPLed database use std::fs::File; use std::io::Read; use fnv::FnvHashMap; use once_cell::sync::OnceCell; use petgraph::prelude::DiGraph; use super::MagicRule; use crate::fdo_magic::ruleset; use crate::MIME; static RUNTIME_RULES: OnceCell>> = OnceCell::new(); static ALIAS_STRING: OnceCell = OnceCell::new(); static SUBCLASS_STRING: OnceCell = OnceCell::new(); /// Load the magic database from the predefined locations in the XDG standard fn load_xdg_shared_magic() -> Result>, String> { const SEARCH_PATHS: &[&str; 3] = &[ "/usr/share/mime/magic", "/usr/local/share/mime/magic", "$HOME/.local/share/mime/magic", ]; let files: Vec> = SEARCH_PATHS .iter() .map(|p| File::open(p).ok()) .filter_map(|f| f) .map(|mut f| { let mut buf = vec![]; f.read_to_end(&mut buf) .map_err(|e| format!("Failed to read magic file bytes: {:#?}", e))?; Ok(buf) }) .collect::>()?; if files.is_empty() { Err("No MIME magic files found in the XDG default paths".to_string()) } else { Ok(files) } } /// Load a number of files at `paths` and concatenate them together with a newline fn load_concat_strings(paths: &[&str]) -> String { let strings: Vec = paths .iter() .map(|p| File::open(p).ok()) .filter_map(|f| f) .map(|mut f| { let mut s = String::new(); f.read_to_string(&mut s) .expect("Failed to read aliases from file"); s }) .collect(); strings.join("\n") } /// Load the magic aliases from the XDG standard locations and concatenate them together fn load_aliases() -> String { const SEARCH_PATHS: &[&str; 3] = &[ "/usr/share/mime/aliases", "/usr/local/share/mime/aliases", "$HOME/.local/share/mime/aliases", ]; load_concat_strings(SEARCH_PATHS) } /// Load the subclass definitions from the XDG standard locations and concatenate them together fn load_subclasses() -> String { const SEARCH_PATHS: &[&str; 3] = &[ "/usr/share/mime/subclasses", "/usr/local/share/mime/subclasses", "$HOME/.local/share/mime/subclasses", ]; load_concat_strings(SEARCH_PATHS) } pub(crate) fn aliases() -> &'static str { ALIAS_STRING.get_or_init(load_aliases) } pub(crate) fn subclasses() -> &'static str { SUBCLASS_STRING.get_or_init(load_subclasses) } pub(crate) fn rules() -> Result, u32>>, String> { let files = RUNTIME_RULES.get_or_try_init(load_xdg_shared_magic)?; ruleset::from_multiple(files) } tree_magic_mini-3.0.3/src/fdo_magic/check.rs000064400000000000000000000100570072674642500171170ustar 00000000000000use petgraph::prelude::*; fn from_u8_singlerule(file: &[u8], rule: &super::MagicRule) -> bool { // Check if we're even in bounds let bound_min = rule.start_off as usize; let bound_max = rule.start_off as usize + rule.val.len() + rule.region_len as usize; if (file.len()) < bound_max { return false; } if rule.region_len == 0 { //println!("Region == 0"); match rule.mask { None => { //println!("\tMask == None"); let x: Vec = file .iter() .skip(bound_min) .take(bound_max - bound_min) .copied() .collect(); //println!("\t{:?} / {:?}", x, rule.val); //println!("\tIndent: {}, Start: {}", rule.indent_level, rule.start_off); return rule.val.iter().eq(x.iter()); } Some(ref mask) => { //println!("\tMask == Some, len == {}", mask.len()); //println!("\tIndent: {}, Start: {}", rule.indent_level, rule.start_off); let mut x: Vec = file .iter() .skip(bound_min) // Skip to start of area .take(bound_max - bound_min) // Take until end of area - region length .copied() .collect(); // Convert to vector let mut val: Vec = rule.val.iter().copied().collect(); //println!("\t{:?} / {:?}", x, rule.val); assert_eq!(x.len(), mask.len()); for i in 0..std::cmp::min(x.len(), mask.len()) { x[i] &= mask[i]; val[i] &= mask[i]; } //println!("\t & {:?} => {:?}", mask, x); return rule.val.iter().eq(x.iter()); } } } else { //println!("\tRegion == {}", rule.region_len); //println!("\tIndent: {}, Start: {}", rule.indent_level, rule.start_off); // Define our testing slice let x: &Vec = &file.iter().take(file.len()).copied().collect(); let testarea: Vec = x .iter() .skip(bound_min) .take(bound_max - bound_min) .copied() .collect(); //println!("{:?}, {:?}, {:?}\n", file, testarea, rule.val); // Search down until we find a hit let mut y = Vec::::with_capacity(testarea.len()); for x in testarea.windows(rule.val.len()) { y.clear(); // Apply mask to value let rule_mask = &rule.mask; match *rule_mask { Some(ref mask) => { for i in 0..rule.val.len() { y.push(x[i] & mask[i]); } } None => y = x.to_vec(), } if y.iter().eq(rule.val.iter()) { return true; } } } false } /// Test every given rule by walking graph /// TODO: Not loving the code duplication here. pub fn from_u8_walker( file: &[u8], mimetype: &str, graph: &DiGraph, node: NodeIndex, isroot: bool, ) -> bool { let n = graph.neighbors_directed(node, Outgoing); if isroot { let rule = &graph[node]; // Check root if !from_u8_singlerule(&file, rule) { return false; } // Return if that was the only test if n.clone().count() == 0 { return true; } // Otherwise next indent level is lower, so continue } // Check subrules recursively for y in n { let rule = &graph[y]; if from_u8_singlerule(&file, rule) { // Check next indent level if needed if graph.neighbors_directed(y, Outgoing).count() != 0 { return from_u8_walker(file, mimetype, graph, y, false); // Next indent level is lower, so this must be it } else { return true; } } } false } tree_magic_mini-3.0.3/src/fdo_magic/mod.rs000064400000000000000000000004600072674642500166160ustar 00000000000000// Common routines for all fdo_magic parsers pub mod builtin; #[derive(Debug, Clone)] pub struct MagicRule<'a> { pub indent_level: u32, pub start_off: u32, pub val: &'a [u8], pub mask: Option<&'a [u8]>, pub word_len: u32, pub region_len: u32, } pub mod check; pub mod ruleset; tree_magic_mini-3.0.3/src/fdo_magic/ruleset.rs000064400000000000000000000070360072674642500175300ustar 00000000000000use super::MagicRule; use fnv::FnvHashMap; use nom::{ bytes::complete::{is_not, tag, take, take_while}, character::is_digit, combinator::{map, map_res, opt}, multi::many0, number::complete::be_u16, sequence::{delimited, preceded, terminated, tuple}, IResult, }; use petgraph::prelude::*; use std::str; // Singular magic ruleset fn magic_rules(input: &[u8]) -> IResult<&[u8], MagicRule<'_>> { let int_or = |default| { map(take_while(is_digit), move |digits| { str::from_utf8(digits).unwrap().parse().unwrap_or(default) }) }; let (input, (indent_level, start_off, val_len)) = tuple(( terminated(int_or(0), tag(">")), terminated(int_or(0), tag("=")), be_u16, ))(input)?; let (input, (val, mask, word_len, region_len)) = terminated( tuple(( take(val_len), opt(preceded(tag("&"), take(val_len))), opt(preceded(tag("~"), int_or(1))), opt(preceded(tag("+"), int_or(0))), )), tag("\n"), )(input)?; Ok(( input, MagicRule { indent_level, start_off, val, mask, word_len: word_len.unwrap_or(1), region_len: region_len.unwrap_or(0), }, )) } /// Converts a magic file given as a &[u8] array /// to a vector of MagicEntry structs fn ruleset(input: &[u8]) -> IResult<&[u8], Vec<(&str, Vec>)>> { // Parse the MIME type from "[priority: mime]" let mime = map_res( terminated( delimited( delimited(tag("["), is_not(":"), tag(":")), // priority is_not("]"), // mime tag("]"), ), tag("\n"), ), str::from_utf8, ); let magic_entry = tuple((mime, many0(magic_rules))); preceded(tag("MIME-Magic\0\n"), many0(magic_entry))(input) } fn gen_graph(magic_rules: Vec>) -> DiGraph, u32> { use petgraph::prelude::*; // Whip up a graph real quick let mut graph = DiGraph::::new(); let mut rulestack = Vec::<(MagicRule, NodeIndex)>::new(); for x in magic_rules { let xnode = graph.add_node(x.clone()); loop { let y = rulestack.pop(); match y { None => { break; } Some(rule) => { if rule.0.indent_level < x.indent_level { graph.add_edge(rule.1, xnode, 1); rulestack.push(rule); break; } } }; } rulestack.push((x, xnode)); } graph } #[cfg(feature = "with-gpl-data")] pub fn from_u8(b: &[u8]) -> Result, u32>>, String> { let tuplevec = ruleset(b).map_err(|e| e.to_string())?.1; let res = tuplevec .into_iter() .map(|x| (x.0, gen_graph(x.1))) .collect(); Ok(res) } #[cfg(not(feature = "with-gpl-data"))] /// Parse multiple ruleset magic files and aggregate the tuples into a single graph pub fn from_multiple<'a>( files: &'a [Vec], ) -> Result, u32>>, String> { let mut tuplevec = vec![]; for slice in files { tuplevec.append(&mut ruleset(slice.as_ref()).map_err(|e| e.to_string())?.1); } let res = tuplevec .into_iter() .map(|x| (x.0, gen_graph(x.1))) .collect(); Ok(res) } tree_magic_mini-3.0.3/src/lib.rs000064400000000000000000000341360072674642500147040ustar 00000000000000//! `tree_magic_mini` is a Rust crate that determines the MIME type a given file or byte stream. //! //! This is a fork of the [tree_magic](https://crates.io/crates/tree_magic) //! crate by Allison Hancock. It includes the following changes: //! //! * Updated dependencies. //! * Reduced copying and memory allocation, for a slight increase in speed and //! decrease in memory use. //! * Reduced API surface. Some previously public APIs are now internal. //! * Removed the optional `cli` feature and `tmagic` binary. //! //! # About tree_magic //! //! `tree_magic` is designed to be more efficient and to have less false positives compared //! to the old approach used by `libmagic`, or old-fashioned file extension comparisons. //! //! Instead, this loads all known MIME types into a tree based on subclasses. Then, instead //! of checking against *every* file type, `tree_magic` will traverse down the tree and //! only check the files that make sense to check. //! //! # Features //! //! - Very fast perfomance (~150ns to check one file against one type, //! between 5,000ns and 100,000ns to find a MIME type.) //! - Check if a file *is* a certain type. //! - Handles aliases (ex: `application/zip` vs `application/x-zip-compressed`) //! - Can delegate different file types to different "checkers", reducing false positives //! by choosing a different method of attack. //! //! ## Licensing and the MIME database //! //! By default, `tree_magic_mini` will attempt to load the shared MIME info //! database from the standard locations at runtime. //! //! If you won't have the database files available, or would like to include them //! in your binary for simplicity, you can optionally embed the database //! information if you enable the `tree_magic_db` feature. //! //! **As the magic database files themselves are licensed under the GPL, you must //! make sure your project uses a compatible license if you enable this behaviour.** //! //! # Example //! ```rust //! // Load a GIF file //! let input: &[u8] = include_bytes!("../tests/image/gif"); //! //! // Find the MIME type of the GIF //! let result = tree_magic_mini::from_u8(input); //! assert_eq!(result, "image/gif"); //! //! // Check if the MIME and the file are a match //! let result = tree_magic_mini::match_u8("image/gif", input); //! assert_eq!(result, true); //! ``` #![allow(unused_doc_comments)] use fnv::FnvHashMap; use fnv::FnvHashSet; use lazy_static::lazy_static; use petgraph::prelude::*; use std::path::Path; mod basetype; mod fdo_magic; type MIME = &'static str; /// Check these types first /// TODO: Poll these from the checkers? Feels a bit arbitrary const TYPEORDER: [&str; 6] = [ "image/png", "image/jpeg", "image/gif", "application/zip", "application/x-msdos-executable", "application/pdf", ]; pub(crate) trait Checker: Send + Sync { fn from_u8(&self, file: &[u8], mimetype: &str) -> bool; fn from_filepath(&self, filepath: &Path, mimetype: &str) -> bool; fn get_supported(&self) -> Vec; fn get_subclasses(&self) -> Vec<(MIME, MIME)>; fn get_aliaslist(&self) -> FnvHashMap; } static CHECKERS: &[&'static dyn Checker] = &[ &fdo_magic::builtin::check::FdoMagic, &basetype::check::BaseType, ]; /// Mappings between modules and supported mimes lazy_static! { static ref CHECKER_SUPPORT: FnvHashMap = { let mut out = FnvHashMap::::default(); for &c in CHECKERS { for m in c.get_supported() { out.insert(m, c); } } out }; } lazy_static! { static ref ALIASES: FnvHashMap = { let mut out = FnvHashMap::::default(); for &c in CHECKERS { out.extend(c.get_aliaslist()); } out }; } /// Information about currently loaded MIME types /// /// The `graph` contains subclass relations between all given mimes. /// (EX: `application/json` -> `text/plain` -> `application/octet-stream`) /// This is a `petgraph` DiGraph, so you can walk the tree if needed. /// /// The `hash` is a mapping between MIME types and nodes on the graph. /// The root of the graph is "all/all", so start traversing there unless /// you need to jump to a particular node. struct TypeStruct { graph: DiGraph, } lazy_static! { /// The TypeStruct autogenerated at library init, and used by the library. static ref TYPE: TypeStruct = graph_init(); } // Initialize filetype graph fn graph_init() -> TypeStruct { let mut graph = DiGraph::::new(); let mut added_mimes = FnvHashMap::::default(); // Get list of MIME types and MIME relations let mut mimelist = Vec::::new(); let mut edgelist_raw = Vec::<(MIME, MIME)>::new(); for &c in CHECKERS { mimelist.extend(c.get_supported()); edgelist_raw.extend(c.get_subclasses()); } mimelist.sort_unstable(); mimelist.dedup(); let mimelist = mimelist; // Create all nodes for mimetype in mimelist.iter() { let node = graph.add_node(mimetype); added_mimes.insert(mimetype, node); } let mut edge_list = FnvHashSet::<(NodeIndex, NodeIndex)>::with_capacity_and_hasher( edgelist_raw.len(), Default::default(), ); for x in edgelist_raw { let child_raw = x.0; let parent_raw = x.1; let parent = match added_mimes.get(&parent_raw) { Some(node) => *node, None => { continue; } }; let child = match added_mimes.get(&child_raw) { Some(node) => *node, None => { continue; } }; edge_list.insert((child, parent)); } graph.extend_with_edges(&edge_list); //Add to applicaton/octet-stream, all/all, or text/plain, depending on top-level //(We'll just do it here because having the graph makes it really nice) let added_mimes_tmp = added_mimes.clone(); let node_text = match added_mimes_tmp.get("text/plain") { Some(x) => *x, None => { let node = graph.add_node("text/plain"); added_mimes.insert("text/plain", node); node } }; let node_octet = match added_mimes_tmp.get("application/octet-stream") { Some(x) => *x, None => { let node = graph.add_node("application/octet-stream"); added_mimes.insert("application/octet-stream", node); node } }; let node_allall = match added_mimes_tmp.get("all/all") { Some(x) => *x, None => { let node = graph.add_node("all/all"); added_mimes.insert("all/all", node); node } }; let node_allfiles = match added_mimes_tmp.get("all/allfiles") { Some(x) => *x, None => { let node = graph.add_node("all/allfiles"); added_mimes.insert("all/allfiles", node); node } }; let mut edge_list_2 = FnvHashSet::<(NodeIndex, NodeIndex)>::default(); for mimenode in graph.externals(Incoming) { let mimetype = &graph[mimenode]; let toplevel = mimetype.split('/').next().unwrap_or(""); if mimenode == node_text || mimenode == node_octet || mimenode == node_allfiles || mimenode == node_allall { continue; } if toplevel == "text" { edge_list_2.insert((node_text, mimenode)); } else if toplevel == "inode" { edge_list_2.insert((node_allall, mimenode)); } else { edge_list_2.insert((node_octet, mimenode)); } } // Don't add duplicate entries graph.extend_with_edges(edge_list_2.difference(&edge_list)); TypeStruct { graph } } /// Just the part of from_*_node that walks the graph fn typegraph_walker(parentnode: NodeIndex, input: &T, matchfn: F) -> Option where T: ?Sized, F: Fn(&str, &T) -> bool, { // Pull most common types towards top let mut children: Vec = TYPE .graph .neighbors_directed(parentnode, Outgoing) .collect(); for i in 0..children.len() { let x = children[i]; if TYPEORDER.contains(&&*TYPE.graph[x]) { children.remove(i); children.insert(0, x); } } // Walk graph for childnode in children { let mimetype = &TYPE.graph[childnode]; let result = matchfn(mimetype, input); match result { true => match typegraph_walker(childnode, input, matchfn) { Some(foundtype) => return Some(foundtype), None => return Some(mimetype), }, false => continue, } } None } /// Transforms an alias into it's real type fn get_alias(mimetype: &str) -> &str { match ALIASES.get(mimetype) { Some(x) => x, None => mimetype, } } /// Internal function. Checks if an alias exists, and if it does, /// then runs `from_u8`. fn match_u8_noalias(mimetype: &str, bytes: &[u8]) -> bool { match CHECKER_SUPPORT.get(mimetype) { None => false, Some(y) => y.from_u8(bytes, mimetype), } } /// Checks if the given bytestream matches the given MIME type. /// /// Returns true or false if it matches or not. If the given MIME type is not known, /// the function will always return false. /// If mimetype is an alias of a known MIME, the file will be checked agains that MIME. /// /// # Examples /// ```rust /// // Load a GIF file /// let input: &[u8] = include_bytes!("../tests/image/gif"); /// /// // Check if the MIME and the file are a match /// let result = tree_magic_mini::match_u8("image/gif", input); /// assert_eq!(result, true); /// ``` pub fn match_u8(mimetype: &str, bytes: &[u8]) -> bool { match_u8_noalias(get_alias(mimetype), bytes) } /// Gets the type of a file from a raw bytestream, starting at a certain node /// in the type graph. /// /// Returns MIME as string wrapped in Some if a type matches, or /// None if no match is found under the given node. /// Retreive the node from the `TYPE.hash` HashMap, using the MIME as the key. /// /// # Panics /// Will panic if the given node is not found in the graph. /// As the graph is immutable, this should not happen if the node index comes from /// TYPE.hash. fn from_u8_node(parentnode: NodeIndex, bytes: &[u8]) -> Option { typegraph_walker(parentnode, bytes, match_u8_noalias) } /// Gets the type of a file from a byte stream. /// /// Returns MIME as string. /// /// # Examples /// ```rust /// // Load a GIF file /// let input: &[u8] = include_bytes!("../tests/image/gif"); /// /// // Find the MIME type of the GIF /// let result = tree_magic_mini::from_u8(input); /// assert_eq!(result, "image/gif"); /// ``` pub fn from_u8(bytes: &[u8]) -> MIME { let node = match TYPE.graph.externals(Incoming).next() { Some(foundnode) => foundnode, None => panic!("No filetype definitions are loaded."), }; from_u8_node(node, bytes).unwrap() } /// Internal function. Checks if an alias exists, and if it does, /// then runs `from_filepath`. fn match_filepath_noalias(mimetype: &str, filepath: &Path) -> bool { match CHECKER_SUPPORT.get(mimetype) { None => false, Some(c) => c.from_filepath(&filepath, mimetype), } } /// Check if the given filepath matches the given MIME type. /// /// Returns true or false if it matches or not, or an Error if the file could /// not be read. If the given MIME type is not known, it will always return false. /// /// # Examples /// ```rust /// use std::path::Path; /// /// // Get path to a GIF file /// let path: &Path = Path::new("tests/image/gif"); /// /// // Check if the MIME and the file are a match /// let result = tree_magic_mini::match_filepath("image/gif", path); /// assert_eq!(result, true); /// ``` pub fn match_filepath(mimetype: &str, filepath: &Path) -> bool { match_filepath_noalias(get_alias(mimetype), filepath) } /// Gets the type of a file from a filepath, starting at a certain node /// in the type graph. /// /// Returns MIME as string wrapped in Some if a type matches, or /// None if the file is not found or cannot be opened. /// Retreive the node from the `TYPE.hash` FnvHashMap, using the MIME as the key. /// /// # Panics /// Will panic if the given node is not found in the graph. /// As the graph is immutable, this should not happen if the node index comes from /// `TYPE.hash`. fn from_filepath_node(parentnode: NodeIndex, filepath: &Path) -> Option { // We're actually just going to thunk this down to a u8 // unless we're checking via basetype for speed reasons. // Ensure it's at least a application/octet-stream if !match_filepath("application/octet-stream", filepath) { // Check the other base types return typegraph_walker(parentnode, filepath, match_filepath_noalias); } // Load the first 2K of file and parse as u8 // for batch processing like this let b = match read_bytes(filepath, 2048) { Ok(x) => x, Err(_) => return None, }; from_u8_node(parentnode, b.as_slice()) } /// Gets the type of a file from a filepath. /// /// Does not look at file name or extension, just the contents. /// Returns MIME as string wrapped in Some if a type matches, or /// None if the file is not found or cannot be opened. /// /// # Examples /// ```rust /// use std::path::Path; /// /// // Get path to a GIF file /// let path: &Path = Path::new("tests/image/gif"); /// /// // Find the MIME type of the GIF /// let result = tree_magic_mini::from_filepath(path); /// assert_eq!(result, Some("image/gif")); /// ``` pub fn from_filepath(filepath: &Path) -> Option { let node = match TYPE.graph.externals(Incoming).next() { Some(foundnode) => foundnode, None => panic!("No filetype definitions are loaded."), }; from_filepath_node(node, filepath) } /// Reads the given number of bytes from a file fn read_bytes(filepath: &Path, bytecount: usize) -> Result, std::io::Error> { use std::fs::File; use std::io::prelude::*; let mut b = Vec::::with_capacity(bytecount); let f = File::open(filepath)?; f.take(bytecount as u64).read_to_end(&mut b)?; Ok(b) }