diff --git a/Cargo.lock b/Cargo.lock index 0a73d39..c1b793b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,7 +168,7 @@ checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" [[package]] name = "fif" -version = "0.2.5" +version = "0.2.6" dependencies = [ "cached", "cfg-if", @@ -182,6 +182,7 @@ dependencies = [ "rayon", "smartstring", "snailquote", + "tempfile", "walkdir", "xdg-mime", ] @@ -341,6 +342,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85" +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -383,6 +390,46 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core", +] + [[package]] name = "rayon" version = "1.5.0" @@ -427,6 +474,15 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "ryu" version = "1.0.5" @@ -484,6 +540,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "termcolor" version = "1.1.2" diff --git a/Cargo.toml b/Cargo.toml index 7945b58..d417170 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "fif" description = "A command-line tool for detecting and optionally correcting files with incorrect extensions." -version = "0.2.5" +version = "0.2.6" authors = ["Lynnesbian "] edition = "2018" license = "GPL-3.0-or-later" @@ -32,8 +32,11 @@ rayon = { version = "1.5.0", optional = true } exitcode = "1.1.2" cfg-if = "1.0.0" +[target.'cfg(any(unix, target_os="redox"))'.dependencies] +xdg-mime = "0.3" + +[patch.crates-io] # use git version while waiting on a release incorporating https://github.com/ebassi/xdg-mime-rs/commit/de5a6dd -[target.'cfg(not(target_os = "windows"))'.dependencies] xdg-mime = {git = "https://github.com/ebassi/xdg-mime-rs", version = "0.3", rev = "de5a6dd" } [dependencies.clap] @@ -50,6 +53,9 @@ features = ["termcolor", "atty"] version = "0.23.0" default-features = false +[dev-dependencies] +tempfile = "3.2.0" + [profile.release] lto = "thin" diff --git a/src/formats.rs b/src/formats.rs index 0a2d71b..a6668f6 100644 --- a/src/formats.rs +++ b/src/formats.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use snailquote::escape; use crate::scanerror::ScanError; -use crate::Findings; +use crate::{Findings, BACKEND}; const VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION"); @@ -140,11 +140,13 @@ impl Format for Script { } fn header(&self, _: &Entries, f: &mut W) -> io::Result<()> { - write!( + writeln!( f, - "#!/usr/bin/env sh\n# Generated by fif {}.\n\nset -e\n\n", - VERSION.unwrap_or("???") - ) + "#!/usr/bin/env sh\n# Generated by fif {} ({} backend)", + VERSION.unwrap_or("???"), + BACKEND + )?; + writeln!(f, "\nset -e\n") } fn footer(&self, _: &Entries, f: &mut W) -> io::Result<()> { diff --git a/src/main.rs b/src/main.rs index 0ebb771..cfd23c7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,11 +41,16 @@ mod mimedb; mod parameters; mod scanerror; +#[cfg(test)] +mod tests; + cfg_if! { - if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] { + if #[cfg(any(all(not(unix), not(feature = "xdg-mime-backend")), all(unix, feature = "infer-backend")))] { static MIMEDB: OnceCell = OnceCell::new(); + const BACKEND: &str = "Infer"; } else { static MIMEDB: OnceCell = OnceCell::new(); + const BACKEND: &str = "XDG-Mime"; } } @@ -69,8 +74,8 @@ cfg_if! { } } -fn wanted_file(args: ¶meters::Parameters, exts: &[&str], entry: &DirEntry) -> bool { - if !args.scan_hidden && is_hidden(entry) { +fn wanted_file(scan_hidden: bool, exts: &[&str], entry: &DirEntry) -> bool { + if !scan_hidden && is_hidden(entry) { // skip hidden files and directories. this check is performed first because it's very lightweight. return false; } @@ -153,47 +158,11 @@ fn scan_from_walkdir(entries: &[DirEntry]) -> Vec = if let Some(exts) = &args.exts { - exts.iter().map(|s| s.as_str()).collect() - } else if let Some(exts) = &args.ext_set { - exts.extensions().to_vec() - } else { - unreachable!() - }; - - debug!("Checking files with extensions: {:?}", extensions); - - let stepper = WalkDir::new(&args.dirs).into_iter(); +fn scan_directory(dirs: &PathBuf, exts: &Vec<&str>, scan_hidden: bool) -> Option> { + let stepper = WalkDir::new(dirs).into_iter(); let mut probably_fatal_error = false; let entries: Vec = stepper - .filter_entry(|e| wanted_file(&args, &extensions, e)) // filter out unwanted files + .filter_entry(|e| wanted_file(scan_hidden, exts, e)) // filter out unwanted files .filter_map(|e| { if let Err(err) = &e { debug!("uh oh spaghettio!! {:#?}", e); @@ -220,14 +189,66 @@ fn main() { .filter(|e| !e.file_type().is_dir()) // remove directories from the final list .collect(); - if entries.is_empty() { - if probably_fatal_error { - // no need to log anything for fatal errors - fif will already have printed something obvious like - // "[ERROR] /fake/path: No such file or directory (os error 2)". we can assume that if this has happened, the dir - // given as input doesn't exist or is otherwise unreadable. - exit(exitcode::NOINPUT); - } + if probably_fatal_error { + None + } else { + Some(entries) + } +} +fn init_db() { + cfg_if! { + if #[cfg(any(all(not(unix), not(feature = "xdg-mime-backend")), all(unix, feature = "infer-backend")))] { + MIMEDB + .set(mimedb::InferDb::init()) + .or(Err("Failed to initialise Infer backend!")) + .unwrap(); + } else { + MIMEDB + .set(mimedb::XdgDb::init()) + .or(Err("Failed to initialise XDG Mime backend!")) + .unwrap(); + } + } +} + +fn main() { + let args = parameters::Parameters::parse(); + + let mut builder = env_logger::Builder::from_default_env(); + builder + // .format(|buf, r| writeln!(buf, "{} - {}", r.level(), r.args())) + .format_module_path(false) // don't include module in logs, as it's not necessary + .format_timestamp(None) // don't include timestamps (unnecessary, and the feature flag is disabled anyway) + // .target(env_logger::Target::Stdout) // log to stdout rather than stderr + .init(); + + init_db(); + + debug!("Iterating directory: {:?}", args.dirs); + + let extensions: Vec<&str> = if let Some(exts) = &args.exts { + exts.iter().map(|s| s.as_str()).collect() + } else if let Some(exts) = &args.ext_set { + exts.extensions().to_vec() + } else { + unreachable!() + }; + + debug!("Checking files with extensions: {:?}", extensions); + + let entries = scan_directory(&args.dirs, &extensions, args.scan_hidden); + + if entries.is_none() { + // no need to log anything for fatal errors - fif will already have printed something obvious like + // "[ERROR] /fake/path: No such file or directory (os error 2)". we can assume that if this has happened, the dir + // given as input doesn't exist or is otherwise unreadable. + exit(exitcode::NOINPUT); + } + + let entries = entries.unwrap(); + + if entries.is_empty() { warn!("No files matching requested options found."); exit(exitcode::DATAERR); } diff --git a/src/tests/mod.rs b/src/tests/mod.rs new file mode 100644 index 0000000..3d841dd --- /dev/null +++ b/src/tests/mod.rs @@ -0,0 +1,129 @@ +use crate::inspectors::mime_extension_lookup; +use crate::mimedb::*; +use crate::{extension_from_path, init_db, scan_directory, scan_from_walkdir}; + +use cfg_if::cfg_if; +use mime_guess::mime::{APPLICATION_OCTET_STREAM, APPLICATION_PDF, IMAGE_JPEG, IMAGE_PNG}; +use mime_guess::Mime; +use smartstring::alias::String; +use std::collections::HashMap; +use std::path::Path; + +const JPEG_BYTES: &[u8] = b"\xFF\xD8\xFF"; +const PNG_BYTES: &[u8] = b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"; +const PDF_BYTES: &[u8] = b"%PDF-"; +const ZIP_BYTES: &[u8] = b"PK\x03\x04"; + +cfg_if! { + if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] { + fn get_mime_db() -> InferDb { + InferDb::init() + } + } else { + fn get_mime_db() -> XdgDb { + XdgDb::init() + } + } +} + +fn application_zip() -> Mime { + use std::str::FromStr; + Mime::from_str("application/zip").unwrap() +} + +#[test] +fn get_ext() { + let mut ext_checks = HashMap::new(); + ext_checks.insert(Path::new("test.txt"), Some(String::from("txt"))); + ext_checks.insert(Path::new("test.zip"), Some(String::from("zip"))); + ext_checks.insert(Path::new("test.tar.gz"), Some(String::from("gz"))); + ext_checks.insert(Path::new("test."), Some(String::from(""))); + ext_checks.insert(Path::new("test"), None); + ext_checks.insert(Path::new(".hidden"), None); + + for (path, ext) in ext_checks { + assert_eq!(extension_from_path(path), ext) + } +} + +#[test] +fn detect_type() { + let db = get_mime_db(); + assert_eq!(db.get_type(JPEG_BYTES), Some(IMAGE_JPEG)); + assert_eq!(db.get_type(PNG_BYTES), Some(IMAGE_PNG)); + assert_eq!(db.get_type(PDF_BYTES), Some(APPLICATION_PDF)); + assert_eq!(db.get_type(ZIP_BYTES), Some(application_zip())); +} + +#[test] +fn recommend_ext() { + assert!(mime_extension_lookup(IMAGE_JPEG) + .unwrap() + .contains(&String::from("jpg"))); + assert!(mime_extension_lookup(IMAGE_PNG).unwrap().contains(&String::from("png"))); + assert!(mime_extension_lookup(APPLICATION_PDF) + .unwrap() + .contains(&String::from("pdf"))); + assert!(mime_extension_lookup(application_zip()) + .unwrap() + .contains(&String::from("zip"))); +} + +#[test] +fn simple_directory() { + use std::fs::File; + use std::io::Write; + use tempfile::tempdir; + + let mut files = HashMap::new(); + files.insert("test.jpg", JPEG_BYTES); + files.insert("test.jpeg", JPEG_BYTES); + files.insert("test.png", PNG_BYTES); + files.insert("test.pdf", PDF_BYTES); + files.insert("test.zip", ZIP_BYTES); + files.insert("wrong.jpg", PNG_BYTES); + + let dir = tempdir().expect("Failed to create temporary directory."); + + for (name, bytes) in &files { + let mut file = File::create(dir.path().join(name)).expect(&*format!("Failed to create file: {}", name)); + + file + .write_all(bytes) + .expect(&*format!("Failed to write to file: {}", name)); + drop(file); + } + + let entries = scan_directory( + &dir.path().to_path_buf(), + &vec!["jpg", "jpeg", "png", "pdf", "zip"], + true, + ) + .expect("Directory scan failed."); + + assert_eq!(entries.len(), files.len()); + + // initialise global mime DB + init_db(); + + let results = scan_from_walkdir(&entries); + for result in results { + let result = result.expect("Error while scanning file"); + if !result.valid { + // this should be "wrong.jpg", which is a misnamed png file + assert_eq!(result.mime, IMAGE_PNG); + continue; + } + + assert_eq!( + result.mime, + match extension_from_path(&*result.file).as_deref() { + Some("jpg") | Some("jpeg") => IMAGE_JPEG, + Some("png") => IMAGE_PNG, + Some("pdf") => APPLICATION_PDF, + Some("zip") => application_zip(), + Some(_) | None => APPLICATION_OCTET_STREAM, // general "fallback" type + } + ); + } +}