.gitignore

@@ -5,4 +5,3 @@ fif_*
 *.sh
 !clippy.sh
 cargo-timing*.html
-todo.txt

Cargo.lock

@@ -39,6 +39,12 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "cc"
+version = "1.0.67"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -100,26 +106,28 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.3"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
+checksum = "d60ab4a8dba064f2fbb5aa270c28da5cf4bbd0e72dae1140a6b0353a779dbe00"
 dependencies = [
  "cfg-if",
  "crossbeam-utils",
  "lazy_static",
+ "loom",
  "memoffset",
  "scopeguard",
 ]
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.3"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
+checksum = "bae8f328835f8f5a6ceb6a7842a7f2d0c03692adb5c889347235d59194731fe3"
 dependencies = [
  "autocfg",
  "cfg-if",
  "lazy_static",
+ "loom",
 ]
 
 [[package]]
@@ -168,10 +176,9 @@ checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193"
 
 [[package]]
 name = "fif"
-version = "0.2.5"
+version = "0.2.3"
 dependencies = [
  "cached",
- "cfg-if",
  "clap",
  "env_logger",
  "exitcode",
@@ -186,6 +193,19 @@ dependencies = [
  "xdg-mime",
 ]
 
+[[package]]
+name = "generator"
+version = "0.6.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc"
+dependencies = [
+ "cc",
+ "libc",
+ "log",
+ "rustc_version",
+ "winapi",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.2"
@@ -277,6 +297,17 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "loom"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d44c73b4636e497b4917eb21c33539efa3816741a2d3ff26c6316f1b529481a4"
+dependencies = [
+ "cfg-if",
+ "generator",
+ "scoped-tls",
+]
+
 [[package]]
 name = "memchr"
 version = "2.3.4"
@@ -331,9 +362,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.7.0"
+version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10acf907b94fc1b1a152d08ef97e7759650268cf986bf127f387e602b02c7e5a"
+checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
 
 [[package]]
 name = "os_str_bytes"
@@ -427,6 +458,15 @@ dependencies = [
  "redox_syscall",
 ]
 
+[[package]]
+name = "rustc_version"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.5"
@@ -442,12 +482,33 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "scoped-tls"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "semver"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
+dependencies = [
+ "semver-parser",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
+
 [[package]]
 name = "smartstring"
 version = "0.2.6"

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "fif"
 description = "A command-line tool for detecting and optionally correcting files with incorrect extensions."
-version = "0.2.5"
+version = "0.2.3"
 authors = ["Lynnesbian <lynne@bune.city>"]
 edition = "2018"
 license = "GPL-3.0-or-later"
@@ -15,10 +15,10 @@ exclude = [".idea/", "Cross.toml", "*.sh"]
 #license-file = "LICENSE"
 
 [features]
-default = ["multi-threaded"]
+default = ["multi-threaded", "infer-backend"]
 multi-threaded = ["rayon"]
-infer-backend = []
+infer-backend = ["infer"]
-xdg-mime-backend = []
+xdg-mime-backend = ["xdg-mime"]
 
 [dependencies]
 walkdir = "2.3.1"
@@ -27,14 +27,13 @@ smartstring = "0.2.6"
 mime_guess = "2.0.3"
 snailquote = "0.3.0"
 once_cell = "1.5.2"
-infer = { version = "0.3.4" }
 rayon = { version = "1.5.0", optional = true }
+infer = { version = "0.3.4", optional = true }
 exitcode = "1.1.2"
-cfg-if = "1.0.0"
 
 # use git version while waiting on a release incorporating https://github.com/ebassi/xdg-mime-rs/commit/de5a6dd
 [target.'cfg(not(target_os = "windows"))'.dependencies]
-xdg-mime = {git = "https://github.com/ebassi/xdg-mime-rs", version = "0.3", rev = "de5a6dd" }
+xdg-mime = {git = "https://github.com/ebassi/xdg-mime-rs", version = "0.3", rev = "de5a6dd", optional = true }
 
 [dependencies.clap]
 version = "3.0.0-beta.2"

src/extensionset.rs

@@ -16,17 +16,13 @@ impl ExtensionSet {
 			Self::Images => mime_guess::get_mime_extensions_str("image/*"),
 			Self::Videos => mime_guess::get_mime_extensions_str("video/*"),
 			Self::Audio => mime_guess::get_mime_extensions_str("audio/*"),
-			Self::Documents => Some(
+			Self::Documents => Some(&[
-				&[
+				"doc", "docx", "ppt", "pptx", "xls", "xlsx", "csv", "tsv", "pdf", "odt", "ods", "odp",
-					"pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "csv", "tsv", "odt", "ods", "odp", "oda", "rtf", "ps",
+			][..]),
-				][..],
-			),
 			// many compressed file types follow the name scheme "application/x.+compressed.*" - maybe this can be used
 			// somehow to extract extensions for compressed files from mime_guess?
 			Self::Archives => Some(&["zip", "tar", "gz", "zst", "xz", "rar", "7z", "bz", "bz2"][..]),
 			_ => todo!(),
-		}
+		}.unwrap().to_vec()
-		.unwrap()
-		.to_vec()
 	}
 }

src/formats.rs

@@ -105,38 +105,37 @@ impl Format for Script {
 	}
 
 	fn rename<W: Write>(&self, f: &mut W, from: &PathBuf, to: &PathBuf) -> io::Result<()> {
-		smart_write(
+		smart_write(f, &[
-			f,
-			&[
 			"mv -v -i -- ".into(),
 			from.into(),
 			Writable::Space,
 			to.into(),
-				Writable::Newline,
+			Writable::Newline
-			],
+		])
-		)
 	}
 
 	fn no_known_extension<W: Write>(&self, f: &mut W, path: &PathBuf) -> io::Result<()> {
-		smart_write(
+		smart_write(f, &[
-			f,
+			"echo No known extension for ".into(),
-			&["echo No known extension for ".into(), path.into(), Writable::Newline],
+			path.into(),
-		)
+			Writable::Newline
+		])
 	}
 
 	fn unreadable<W: Write>(&self, f: &mut W, path: &PathBuf) -> io::Result<()> {
-		smart_write(f, &["# Failed to read ".into(), path.into(), Writable::Newline])
+		smart_write(f, &[
+			"# Failed to read ".into(),
+			path.into(),
+			Writable::Newline
+		])
 	}
 
 	fn unknown_type<W: Write>(&self, f: &mut W, path: &PathBuf) -> io::Result<()> {
-		smart_write(
+		smart_write(f, &[
-			f,
-			&[
 			"# Failed to detect mime type for ".into(),
 			path.into(),
-				Writable::Newline,
+			Writable::Newline
-			],
+		])
-		)
 	}
 
 	fn header<W: Write>(&self, _: &Entries, f: &mut W) -> io::Result<()> {

src/inspectors.rs

@@ -10,12 +10,24 @@ use smartstring::alias::String;
 
 use crate::mimedb::MimeDb;
 
-// rather than reading once into a large buffer, it tends to be faster to first try identifying the file from a small
+// use log::{debug, warn};
-// chunk read from the top, and *then* proceeding with the large buffer. many file formats can be easily identified by
-// the first 128 bytes. of course, not all formats can, and some (OOXML...) require reading a long ways in.
 
-const INITIAL_BUF_SIZE: usize = 128;
+// from looking at the files in https://github.com/bojand/infer/tree/master/src/matchers, the format with the largest
-const BUF_SIZE: usize = 4096;
+// buffer size requirement for identification requires 262 bytes, and the next largest buffer necessary is only 131
+// bytes. as only two formats need more than 128 bytes, it would be fairly reasonable to only read 128 bytes.
+// unfortunately, the format that requires 262 bytes for identification is tar, an extremely popular format (in the *nix
+// world, at least). however, tar files almost always appear wrapped in other formats (.tar.gz, .tar.zst, etc) anyway,
+// so maybe it's fine...? maybe this should be configurable by the user? i don't know.
+// empirical testing (or rather, starting from 256 and incrementing until it worked) reveals that xdg-mime requires
+// at least 265 bytes to identify a tar file.
+
+// additionally, since many formats can by identified with ≤64 bytes, it's worth reading 64 bytes, checking for the mime
+// type, and then reading the full 512 bytes if necessary. in most cases, this will end up being faster on the whole,
+// even though two reads are needed for certain formats, unless the directory being scanned is predominantly made up of
+// such formats.
+
+const INITIAL_BUF_SIZE: usize = 64;
+const BUF_SIZE: usize = 512;
 
 pub fn mime_type<T: MimeDb>(db: &T, path: &Path) -> io::Result<Option<Mime>> {
 	let mut buffer = [0; INITIAL_BUF_SIZE];
@@ -26,27 +38,19 @@ pub fn mime_type<T: MimeDb>(db: &T, path: &Path) -> io::Result<Option<Mime>> {
 	#[allow(clippy::unused_io_amount)]
 	file.read(&mut buffer)?;
 
-	let r = db.get_type(&buffer).filter(|mime|
+	let r = db.get_type(&buffer);
-		// some mime types should be investigated further, reading up to BUF_SIZE even if they've been determined already
-		// one such type is XML - there's many more specific types that can be determined by reading further (such as SVG)
-		mime != &mime_guess::mime::TEXT_XML
-		// another is ZIP - many file formats (DOCX, ODT, JAR...) are just ZIP files with particular data structures.
-		// determining that a file is in one of the MS office formats in particular requires looking quite far into the
-		// file.
-		&& mime != &Mime::from_str("application/zip").unwrap());
 
 	if r.is_some() {
 		return Ok(r);
 	}
 
 	// attempt to read up to the BUF_SIZE bytes of the file.
-	// we've already read the first 128 bytes into a buffer, but i can't see an obvious way to reuse them in a way that's
+	// we've already read the first 64 bytes into a buffer, but i can't see an obvious way to reuse those 64 bytes that's
-	// faster than simply moving the seek position back to the start of the file and re-reading the whole BUF_SIZE bytes.
+	// faster than simply moving the seek position back to the start of the file and re-reading the whole 512 bytes.
-	// for example, reading (BUF_SIZE - INITIAL_BUF_SIZE) bytes, then concatenating the original INITIAL_BUF_SIZE buffer
+	// for example, starting with a buffer of 64 bytes, then creating a new 512 byte buffer from the contents of the first
-	// with this new one would presumably be faster - but it's not. i think it's more expensive to create the iterator,
+	// buffer with (512 - 64) blank bytes, then finally reading the rest, is much slower than simply reading the file
-	// collect the contents, etc. i'll have to look into this more. i don't at all doubt that there IS a way to do this
+	// twice. i don't at all doubt that there IS a way to do this efficiently, and i can think of a way in principle, but
-	// efficiently, and i can think of a way in principle, but i'm not sure how to express it in a way that is both
+	// i'm not sure how to express it in a way that is both idiomatic/safe and fast.
-	// idiomatic/safe and fast.
 	let mut buffer = [0; BUF_SIZE];
 	file.seek(SeekFrom::Start(0))?;
 	file.read(&mut buffer)?;
@@ -82,14 +86,19 @@ cached! {
 					// to have valid extensions.
 					[vec![String::from("xml"), String::from("svg")], possible_exts].concat()
 
-				} else if mime == Mime::from_str("application/msword").unwrap() {
-					// classic office files considered harmful
-					vec![String::from("doc"), String::from("xls"), String::from("ppt")]
-
 				} else if mime == Mime::from_str("application/zip").unwrap() {
-					// neither xdg-mime nor infer seem to be able to detect office XML files properly...
+					// until proper document support is added, treat all common document extensions as zips
-					[vec![String::from("zip"), String::from("docx"), String::from("xlsx"), String::from("pptx")], possible_exts].concat()
+					[vec![
-
+					String::from("zip"),
+					String::from("docx"),
+					String::from("pptx"),
+					String::from("xlsx"),
+					String::from("odt"),
+					String::from("ods"),
+					String::from("odp"),
+					String::from("pages"),
+					String::from("key"),
+					String::from("numbers")], possible_exts].concat()
 				} else {
 					possible_exts
 				})

src/main.rs

@@ -17,7 +17,6 @@
 use std::io::{stdout, BufWriter};
 use std::path::{Path, PathBuf};
 
-use cfg_if::cfg_if;
 use clap::Clap;
 use log::{debug, error, info, trace, warn};
 use once_cell::sync::OnceCell;
@@ -41,16 +40,14 @@ mod mimedb;
 mod parameters;
 mod scanerror;
 
-cfg_if! {
+#[cfg(feature = "infer-backend")]
-	if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
 static MIMEDB: OnceCell<mimedb::InferDb> = OnceCell::new();
-	} else {
-		static MIMEDB: OnceCell<mimedb::XdgDb> = OnceCell::new();
-	}
-}
 
-cfg_if! {
+#[cfg(feature = "xdg-mime-backend")]
-	if #[cfg(windows)] {
+static MIMEDB: OnceCell<mimedb::XdgDb> = OnceCell::new();
+
+// TODO: test if this actually works on a windows machine
+#[cfg(windows)]
 fn is_hidden(entry: &DirEntry) -> bool {
 	use std::os::windows::prelude::*;
 	std::fs::metadata(entry.path()) // try to get metadata for file
@@ -59,15 +56,14 @@ cfg_if! {
 			|f| f.file_attributes() & 0x2 > 0, // flag for hidden - https://docs.microsoft.com/windows/win32/fileio/file-attribute-constants
 		)
 }
-	} else {
+
+#[cfg(not(windows))]
 fn is_hidden(entry: &DirEntry) -> bool {
 	entry
 		.file_name()
 		.to_str()
 		.map_or(false, |f| f.starts_with('.') && f != ".")
 }
-	}
-}
 
 fn wanted_file(args: &parameters::Parameters, exts: &[&str], entry: &DirEntry) -> bool {
 	if !args.scan_hidden && is_hidden(entry) {
@@ -133,8 +129,8 @@ fn scan_file(entry: &DirEntry) -> Result<Findings, (ScanError, PathBuf)> {
 }
 
 fn scan_from_walkdir(entries: &[DirEntry]) -> Vec<Result<Findings, (ScanError, PathBuf)>> {
-	cfg_if! {
+	#[cfg(feature = "multi-threaded")]
-		if #[cfg(feature = "multi-threaded")] {
+	{
 		// rather than using a standard par_iter, split the entries into chunks of 32 first.
 		// this allows each spawned thread to handle 32 files before before closing, rather than creating a new thread for
 		// each file. this leads to a pretty substantial speedup that i'm pretty substantially happy about 0u0
@@ -147,9 +143,11 @@ fn scan_from_walkdir(entries: &[DirEntry]) -> Vec<Result<Findings, (ScanError, P
 					.collect::<Vec<_>>()
 			})
 			.collect()
-		} else {
-			entries.iter().map(|entry: &DirEntry| scan_file(entry)).collect()
 	}
+
+	#[cfg(not(feature = "multi-threaded"))]
+	{
+		entries.iter().map(|entry: &DirEntry| scan_file(entry)).collect()
 	}
 }
 
@@ -164,19 +162,17 @@ fn main() {
 		// .target(env_logger::Target::Stdout) // log to stdout rather than stderr
 		.init();
 
-	cfg_if! {
+	#[cfg(feature = "infer-backend")]
-		if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
 	MIMEDB
 		.set(mimedb::InferDb::init())
 		.or(Err("Failed to initialise Infer backend!"))
 		.unwrap();
-		} else {
+
+	#[cfg(feature = "xdg-mime-backend")]
 	MIMEDB
 		.set(mimedb::XdgDb::init())
 		.or(Err("Failed to initialise XDG Mime backend!"))
 		.unwrap();
-		}
-	}
 
 	debug!("Iterating directory: {:?}", args.dirs);
 
@@ -257,8 +253,7 @@ fn main() {
 	}
 
 	if results.is_empty() {
-		info!("All files have valid extensions!");
+		info!("All files have valid extensions!")
-		exit(0);
 	}
 
 	match args.output_format {

src/mimedb.rs

@@ -1,4 +1,6 @@
-use cfg_if::cfg_if;
+#[cfg(feature = "infer-backend")]
+use std::str::FromStr;
+
 use mime_guess::Mime;
 
 pub trait MimeDb {
@@ -6,21 +8,12 @@ pub trait MimeDb {
 	fn get_type(&self, data: &[u8]) -> Option<Mime>;
 }
 
-cfg_if! {
+#[cfg(feature = "infer-backend")]
-	if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
-		use std::str::FromStr;
-
 pub struct InferDb {
 	db: infer::Infer,
 }
 
-		fn open_document_check(buf: &[u8], kind: &str) -> bool {
+#[cfg(feature = "infer-backend")]
-			let mime = format!("application/vnd.oasis.opendocument.{}", kind);
-			let mime = mime.as_bytes();
-
-			buf.len() > 38 + mime.len() && buf.starts_with(b"PK\x03\x04") && buf[38..mime.len() + 38] == mime[..]
-		}
-
 impl MimeDb for InferDb {
 	fn init() -> Self {
 		let mut info = infer::Infer::new();
@@ -30,18 +23,6 @@ cfg_if! {
 			buf.len() > 23 && buf[..23] == b"\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A\x6A\x70\x32\x20"[..]
 		});
 
-				info.add("application/vnd.oasis.opendocument.text", "odt", |buf| {
-					open_document_check(buf, "text")
-				});
-
-				info.add("application/vnd.oasis.opendocument.spreadsheet", "ods", |buf| {
-					open_document_check(buf, "spreadsheet")
-				});
-
-				info.add("application/vnd.oasis.opendocument.presentation", "odp", |buf| {
-					open_document_check(buf, "presentation")
-				});
-
 		info.add("image/svg+xml", "svg", |buf| {
 			// before doing the moderately expensive SVG check, we should make sure that the input is actually SGML-ish
 			// by "SGML-ish", i mean starts with anywhere from zero to ∞-1 whitespace characters, and then a less than sign,
@@ -81,19 +62,21 @@ cfg_if! {
 		self.db.get(data).map(|f| Mime::from_str(f.mime_type()).unwrap())
 	}
 }
-	} else {
+
+#[cfg(feature = "xdg-mime-backend")]
 pub struct XdgDb {
 	db: xdg_mime::SharedMimeInfo,
 }
 
+#[cfg(feature = "xdg-mime-backend")]
 impl MimeDb for XdgDb {
 	fn init() -> Self {
-				Self { db: xdg_mime::SharedMimeInfo::new() }
+		Self {
+			db: xdg_mime::SharedMimeInfo::new(),
+		}
 	}
 
 	fn get_type(&self, data: &[u8]) -> Option<Mime> {
-				self.db.get_mime_type_for_data(data).map(|m| m.0)
+		self.db.get_mime_type_for_data(&data).map(|m| m.0)
-			}
-		}
 	}
 }

src/parameters.rs

@@ -11,7 +11,6 @@ pub enum OutputFormat {
 }
 
 #[derive(Clap, Debug)]
-#[clap(version = option_env!("CARGO_PKG_VERSION").unwrap_or("???"))]
 pub struct Parameters {
 	/// Only examine files with these extensions (Comma-separated list)
 	#[clap(