lynnesbian/fif
2
1
Fork 0
Code Releases Activity

tests and more!

Browse source 
- added unit tests
- replaced `target_os="linux"` with `unix`
- updated Cargo.toml to patch xdg-mime properly
This commit is contained in:
Lynne Megido 2021-02-28 19:47:18 +10:00
parent 967592b22a
commit d76ed2585d
Signed by: lynnesbian
GPG key ID: F0A184B5213D9F90
5 changed files with 285 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
Cargo.lock generated
View file

@ -168,7 +168,7 @@ checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193"
[[package]]
name = "fif"
version = "0.2.5"
version = "0.2.6"
dependencies = [
"cached",
"cfg-if",
@ -182,6 +182,7 @@ dependencies = [
"rayon",
"smartstring",
"snailquote",
"tempfile",
"walkdir",
"xdg-mime",
]
@ -341,6 +342,12 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85"
[[package]]
name = "ppv-lite86"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "proc-macro-error"
version = "1.0.4"
@ -383,6 +390,46 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
"rand_hc",
]
[[package]]
name = "rand_chacha"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
dependencies = [
"getrandom",
]
[[package]]
name = "rand_hc"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
dependencies = [
"rand_core",
]
[[package]]
name = "rayon"
version = "1.5.0"
@ -427,6 +474,15 @@ dependencies = [
"redox_syscall",
]
[[package]]
name = "remove_dir_all"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
dependencies = [
"winapi",
]
[[package]]
name = "ryu"
version = "1.0.5"
@ -484,6 +540,20 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "tempfile"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
dependencies = [
"cfg-if",
"libc",
"rand",
"redox_syscall",
"remove_dir_all",
"winapi",
]
[[package]]
name = "termcolor"
version = "1.1.2"

10
Cargo.toml
View file

@ -1,7 +1,7 @@
[package]
name = "fif"
description = "A command-line tool for detecting and optionally correcting files with incorrect extensions."
version = "0.2.5"
version = "0.2.6"
authors = ["Lynnesbian <lynne@bune.city>"]
edition = "2018"
license = "GPL-3.0-or-later"
@ -32,8 +32,11 @@ rayon = { version = "1.5.0", optional = true }
exitcode = "1.1.2"
cfg-if = "1.0.0"
[target.'cfg(any(unix, target_os="redox"))'.dependencies]
xdg-mime = "0.3"
[patch.crates-io]
# use git version while waiting on a release incorporating https://github.com/ebassi/xdg-mime-rs/commit/de5a6dd
[target.'cfg(not(target_os = "windows"))'.dependencies]
xdg-mime = {git = "https://github.com/ebassi/xdg-mime-rs", version = "0.3", rev = "de5a6dd" }
[dependencies.clap]
@ -50,6 +53,9 @@ features = ["termcolor", "atty"]
version = "0.23.0"
default-features = false
[dev-dependencies]
tempfile = "3.2.0"
[profile.release]
lto = "thin"

12
src/formats.rs
View file

@ -6,7 +6,7 @@ use std::path::PathBuf;
use snailquote::escape;
use crate::scanerror::ScanError;
use crate::Findings;
use crate::{Findings, BACKEND};
const VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION");
@ -140,11 +140,13 @@ impl Format for Script {
}
fn header<W: Write>(&self, _: &Entries, f: &mut W) -> io::Result<()> {
write!(
writeln!(
f,
"#!/usr/bin/env sh\n# Generated by fif {}.\n\nset -e\n\n",
VERSION.unwrap_or("???")
)
"#!/usr/bin/env sh\n# Generated by fif {} ({} backend)",
VERSION.unwrap_or("???"),
BACKEND
)?;
writeln!(f, "\nset -e\n")
}
fn footer<W: Write>(&self, _: &Entries, f: &mut W) -> io::Result<()> {

119
src/main.rs
View file

@ -41,11 +41,16 @@ mod mimedb;
mod parameters;
mod scanerror;
#[cfg(test)]
mod tests;
cfg_if! {
if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
if #[cfg(any(all(not(unix), not(feature = "xdg-mime-backend")), all(unix, feature = "infer-backend")))] {
static MIMEDB: OnceCell<mimedb::InferDb> = OnceCell::new();
const BACKEND: &str = "Infer";
} else {
static MIMEDB: OnceCell<mimedb::XdgDb> = OnceCell::new();
const BACKEND: &str = "XDG-Mime";
}
}
@ -69,8 +74,8 @@ cfg_if! {
}
}
fn wanted_file(args: &parameters::Parameters, exts: &[&str], entry: &DirEntry) -> bool {
if !args.scan_hidden && is_hidden(entry) {
fn wanted_file(scan_hidden: bool, exts: &[&str], entry: &DirEntry) -> bool {
if !scan_hidden && is_hidden(entry) {
// skip hidden files and directories. this check is performed first because it's very lightweight.
return false;
}
@ -153,47 +158,11 @@ fn scan_from_walkdir(entries: &[DirEntry]) -> Vec<Result<Findings, (ScanError, P
}
}
fn main() {
let args = parameters::Parameters::parse();
let mut builder = env_logger::Builder::from_default_env();
builder
// .format(|buf, r| writeln!(buf, "{} - {}", r.level(), r.args()))
.format_module_path(false) // don't include module in logs, as it's not necessary
.format_timestamp(None) // don't include timestamps (unnecessary, and the feature flag is disabled anyway)
// .target(env_logger::Target::Stdout) // log to stdout rather than stderr
.init();
cfg_if! {
if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
MIMEDB
.set(mimedb::InferDb::init())
.or(Err("Failed to initialise Infer backend!"))
.unwrap();
} else {
MIMEDB
.set(mimedb::XdgDb::init())
.or(Err("Failed to initialise XDG Mime backend!"))
.unwrap();
}
}
debug!("Iterating directory: {:?}", args.dirs);
let extensions: Vec<&str> = if let Some(exts) = &args.exts {
exts.iter().map(|s| s.as_str()).collect()
} else if let Some(exts) = &args.ext_set {
exts.extensions().to_vec()
} else {
unreachable!()
};
debug!("Checking files with extensions: {:?}", extensions);
let stepper = WalkDir::new(&args.dirs).into_iter();
fn scan_directory(dirs: &PathBuf, exts: &Vec<&str>, scan_hidden: bool) -> Option<Vec<DirEntry>> {
let stepper = WalkDir::new(dirs).into_iter();
let mut probably_fatal_error = false;
let entries: Vec<DirEntry> = stepper
.filter_entry(|e| wanted_file(&args, &extensions, e)) // filter out unwanted files
.filter_entry(|e| wanted_file(scan_hidden, exts, e)) // filter out unwanted files
.filter_map(|e| {
if let Err(err) = &e {
debug!("uh oh spaghettio!! {:#?}", e);
@ -220,14 +189,66 @@ fn main() {
.filter(|e| !e.file_type().is_dir()) // remove directories from the final list
.collect();
if entries.is_empty() {
if probably_fatal_error {
// no need to log anything for fatal errors - fif will already have printed something obvious like
// "[ERROR] /fake/path: No such file or directory (os error 2)". we can assume that if this has happened, the dir
// given as input doesn't exist or is otherwise unreadable.
exit(exitcode::NOINPUT);
}
if probably_fatal_error {
None
} else {
Some(entries)
}
}
fn init_db() {
cfg_if! {
if #[cfg(any(all(not(unix), not(feature = "xdg-mime-backend")), all(unix, feature = "infer-backend")))] {
MIMEDB
.set(mimedb::InferDb::init())
.or(Err("Failed to initialise Infer backend!"))
.unwrap();
} else {
MIMEDB
.set(mimedb::XdgDb::init())
.or(Err("Failed to initialise XDG Mime backend!"))
.unwrap();
}
}
}
fn main() {
let args = parameters::Parameters::parse();
let mut builder = env_logger::Builder::from_default_env();
builder
// .format(|buf, r| writeln!(buf, "{} - {}", r.level(), r.args()))
.format_module_path(false) // don't include module in logs, as it's not necessary
.format_timestamp(None) // don't include timestamps (unnecessary, and the feature flag is disabled anyway)
// .target(env_logger::Target::Stdout) // log to stdout rather than stderr
.init();
init_db();
debug!("Iterating directory: {:?}", args.dirs);
let extensions: Vec<&str> = if let Some(exts) = &args.exts {
exts.iter().map(|s| s.as_str()).collect()
} else if let Some(exts) = &args.ext_set {
exts.extensions().to_vec()
} else {
unreachable!()
};
debug!("Checking files with extensions: {:?}", extensions);
let entries = scan_directory(&args.dirs, &extensions, args.scan_hidden);
if entries.is_none() {
// no need to log anything for fatal errors - fif will already have printed something obvious like
// "[ERROR] /fake/path: No such file or directory (os error 2)". we can assume that if this has happened, the dir
// given as input doesn't exist or is otherwise unreadable.
exit(exitcode::NOINPUT);
}
let entries = entries.unwrap();
if entries.is_empty() {
warn!("No files matching requested options found.");
exit(exitcode::DATAERR);
}

129
src/tests/mod.rs Normal file
View file

@ -0,0 +1,129 @@
use crate::inspectors::mime_extension_lookup;
use crate::mimedb::*;
use crate::{extension_from_path, init_db, scan_directory, scan_from_walkdir};
use cfg_if::cfg_if;
use mime_guess::mime::{APPLICATION_OCTET_STREAM, APPLICATION_PDF, IMAGE_JPEG, IMAGE_PNG};
use mime_guess::Mime;
use smartstring::alias::String;
use std::collections::HashMap;
use std::path::Path;
const JPEG_BYTES: &[u8] = b"\xFF\xD8\xFF";
const PNG_BYTES: &[u8] = b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A";
const PDF_BYTES: &[u8] = b"%PDF-";
const ZIP_BYTES: &[u8] = b"PK\x03\x04";
cfg_if! {
if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
fn get_mime_db() -> InferDb {
InferDb::init()
}
} else {
fn get_mime_db() -> XdgDb {
XdgDb::init()
}
}
}
fn application_zip() -> Mime {
use std::str::FromStr;
Mime::from_str("application/zip").unwrap()
}
#[test]
fn get_ext() {
let mut ext_checks = HashMap::new();
ext_checks.insert(Path::new("test.txt"), Some(String::from("txt")));
ext_checks.insert(Path::new("test.zip"), Some(String::from("zip")));
ext_checks.insert(Path::new("test.tar.gz"), Some(String::from("gz")));
ext_checks.insert(Path::new("test."), Some(String::from("")));
ext_checks.insert(Path::new("test"), None);
ext_checks.insert(Path::new(".hidden"), None);
for (path, ext) in ext_checks {
assert_eq!(extension_from_path(path), ext)
}
}
#[test]
fn detect_type() {
let db = get_mime_db();
assert_eq!(db.get_type(JPEG_BYTES), Some(IMAGE_JPEG));
assert_eq!(db.get_type(PNG_BYTES), Some(IMAGE_PNG));
assert_eq!(db.get_type(PDF_BYTES), Some(APPLICATION_PDF));
assert_eq!(db.get_type(ZIP_BYTES), Some(application_zip()));
}
#[test]
fn recommend_ext() {
assert!(mime_extension_lookup(IMAGE_JPEG)
.unwrap()
.contains(&String::from("jpg")));
assert!(mime_extension_lookup(IMAGE_PNG).unwrap().contains(&String::from("png")));
assert!(mime_extension_lookup(APPLICATION_PDF)
.unwrap()
.contains(&String::from("pdf")));
assert!(mime_extension_lookup(application_zip())
.unwrap()
.contains(&String::from("zip")));
}
#[test]
fn simple_directory() {
use std::fs::File;
use std::io::Write;
use tempfile::tempdir;
let mut files = HashMap::new();
files.insert("test.jpg", JPEG_BYTES);
files.insert("test.jpeg", JPEG_BYTES);
files.insert("test.png", PNG_BYTES);
files.insert("test.pdf", PDF_BYTES);
files.insert("test.zip", ZIP_BYTES);
files.insert("wrong.jpg", PNG_BYTES);
let dir = tempdir().expect("Failed to create temporary directory.");
for (name, bytes) in &files {
let mut file = File::create(dir.path().join(name)).expect(&*format!("Failed to create file: {}", name));
file
.write_all(bytes)
.expect(&*format!("Failed to write to file: {}", name));
drop(file);
}
let entries = scan_directory(
&dir.path().to_path_buf(),
&vec!["jpg", "jpeg", "png", "pdf", "zip"],
true,
)
.expect("Directory scan failed.");
assert_eq!(entries.len(), files.len());
// initialise global mime DB
init_db();
let results = scan_from_walkdir(&entries);
for result in results {
let result = result.expect("Error while scanning file");
if !result.valid {
// this should be "wrong.jpg", which is a misnamed png file
assert_eq!(result.mime, IMAGE_PNG);
continue;
}
assert_eq!(
result.mime,
match extension_from_path(&*result.file).as_deref() {
Some("jpg") | Some("jpeg") => IMAGE_JPEG,
Some("png") => IMAGE_PNG,
Some("pdf") => APPLICATION_PDF,
Some("zip") => application_zip(),
Some(_) | None => APPLICATION_OCTET_STREAM, // general "fallback" type
}
);
}
}