Compare commits
No commits in common. "fb204d9c9a69456eb82d9d18eecd6b42851fdd81" and "9a0113d1dc6c67f1d903c12970e76d46ac81a39d" have entirely different histories.
fb204d9c9a
...
9a0113d1dc
9 changed files with 256 additions and 216 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -5,4 +5,3 @@ fif_*
|
|||
*.sh
|
||||
!clippy.sh
|
||||
cargo-timing*.html
|
||||
todo.txt
|
||||
|
|
|
|||
77
Cargo.lock
generated
77
Cargo.lock
generated
|
|
@ -39,6 +39,12 @@ dependencies = [
|
|||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.67"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
|
|
@ -100,26 +106,28 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.3"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
|
||||
checksum = "d60ab4a8dba064f2fbb5aa270c28da5cf4bbd0e72dae1140a6b0353a779dbe00"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"lazy_static",
|
||||
"loom",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.3"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
|
||||
checksum = "bae8f328835f8f5a6ceb6a7842a7f2d0c03692adb5c889347235d59194731fe3"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"lazy_static",
|
||||
"loom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -168,10 +176,9 @@ checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193"
|
|||
|
||||
[[package]]
|
||||
name = "fif"
|
||||
version = "0.2.5"
|
||||
version = "0.2.3"
|
||||
dependencies = [
|
||||
"cached",
|
||||
"cfg-if",
|
||||
"clap",
|
||||
"env_logger",
|
||||
"exitcode",
|
||||
|
|
@ -186,6 +193,19 @@ dependencies = [
|
|||
"xdg-mime",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generator"
|
||||
version = "0.6.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"log",
|
||||
"rustc_version",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.2"
|
||||
|
|
@ -277,6 +297,17 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "loom"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d44c73b4636e497b4917eb21c33539efa3816741a2d3ff26c6316f1b529481a4"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"generator",
|
||||
"scoped-tls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.3.4"
|
||||
|
|
@ -331,9 +362,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.7.0"
|
||||
version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10acf907b94fc1b1a152d08ef97e7759650268cf986bf127f387e602b02c7e5a"
|
||||
checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
|
|
@ -427,6 +458,15 @@ dependencies = [
|
|||
"redox_syscall",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
|
||||
dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.5"
|
||||
|
|
@ -442,12 +482,33 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scoped-tls"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
|
||||
dependencies = [
|
||||
"semver-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver-parser"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||
|
||||
[[package]]
|
||||
name = "smartstring"
|
||||
version = "0.2.6"
|
||||
|
|
|
|||
13
Cargo.toml
13
Cargo.toml
|
|
@ -1,7 +1,7 @@
|
|||
[package]
|
||||
name = "fif"
|
||||
description = "A command-line tool for detecting and optionally correcting files with incorrect extensions."
|
||||
version = "0.2.5"
|
||||
version = "0.2.3"
|
||||
authors = ["Lynnesbian <lynne@bune.city>"]
|
||||
edition = "2018"
|
||||
license = "GPL-3.0-or-later"
|
||||
|
|
@ -15,10 +15,10 @@ exclude = [".idea/", "Cross.toml", "*.sh"]
|
|||
#license-file = "LICENSE"
|
||||
|
||||
[features]
|
||||
default = ["multi-threaded"]
|
||||
default = ["multi-threaded", "infer-backend"]
|
||||
multi-threaded = ["rayon"]
|
||||
infer-backend = []
|
||||
xdg-mime-backend = []
|
||||
infer-backend = ["infer"]
|
||||
xdg-mime-backend = ["xdg-mime"]
|
||||
|
||||
[dependencies]
|
||||
walkdir = "2.3.1"
|
||||
|
|
@ -27,14 +27,13 @@ smartstring = "0.2.6"
|
|||
mime_guess = "2.0.3"
|
||||
snailquote = "0.3.0"
|
||||
once_cell = "1.5.2"
|
||||
infer = { version = "0.3.4" }
|
||||
rayon = { version = "1.5.0", optional = true }
|
||||
infer = { version = "0.3.4", optional = true }
|
||||
exitcode = "1.1.2"
|
||||
cfg-if = "1.0.0"
|
||||
|
||||
# use git version while waiting on a release incorporating https://github.com/ebassi/xdg-mime-rs/commit/de5a6dd
|
||||
[target.'cfg(not(target_os = "windows"))'.dependencies]
|
||||
xdg-mime = {git = "https://github.com/ebassi/xdg-mime-rs", version = "0.3", rev = "de5a6dd" }
|
||||
xdg-mime = {git = "https://github.com/ebassi/xdg-mime-rs", version = "0.3", rev = "de5a6dd", optional = true }
|
||||
|
||||
[dependencies.clap]
|
||||
version = "3.0.0-beta.2"
|
||||
|
|
|
|||
|
|
@ -16,17 +16,13 @@ impl ExtensionSet {
|
|||
Self::Images => mime_guess::get_mime_extensions_str("image/*"),
|
||||
Self::Videos => mime_guess::get_mime_extensions_str("video/*"),
|
||||
Self::Audio => mime_guess::get_mime_extensions_str("audio/*"),
|
||||
Self::Documents => Some(
|
||||
&[
|
||||
"pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "csv", "tsv", "odt", "ods", "odp", "oda", "rtf", "ps",
|
||||
][..],
|
||||
),
|
||||
Self::Documents => Some(&[
|
||||
"doc", "docx", "ppt", "pptx", "xls", "xlsx", "csv", "tsv", "pdf", "odt", "ods", "odp",
|
||||
][..]),
|
||||
// many compressed file types follow the name scheme "application/x.+compressed.*" - maybe this can be used
|
||||
// somehow to extract extensions for compressed files from mime_guess?
|
||||
Self::Archives => Some(&["zip", "tar", "gz", "zst", "xz", "rar", "7z", "bz", "bz2"][..]),
|
||||
_ => todo!(),
|
||||
}
|
||||
.unwrap()
|
||||
.to_vec()
|
||||
}.unwrap().to_vec()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ fn smart_write<W: Write>(f: &mut W, writeables: &[Writable]) -> io::Result<()> {
|
|||
for writeable in writeables {
|
||||
match writeable {
|
||||
Writable::Space => write!(f, " ")?,
|
||||
Writable::Newline => writeln!(f,)?,
|
||||
Writable::Newline => writeln!(f, )?,
|
||||
Writable::String(s) => write!(f, "{}", s)?,
|
||||
Writable::Path(path) => {
|
||||
if let Some(string) = path.to_str() {
|
||||
|
|
@ -45,7 +45,7 @@ fn smart_write<W: Write>(f: &mut W, writeables: &[Writable]) -> io::Result<()> {
|
|||
} else {
|
||||
write!(f, "'''")?;
|
||||
#[cfg(unix)]
|
||||
f.write_all(&*path.as_os_str().as_bytes())?;
|
||||
f.write_all(&*path.as_os_str().as_bytes())?;
|
||||
#[cfg(windows)]
|
||||
write!(f, "{}", path.as_os_str().to_string_lossy())?; // TODO: implement bonked strings for windows
|
||||
// f.write_all(&*path.as_os_str().encode_wide().collect::<Vec<u16>>())?;
|
||||
|
|
@ -105,38 +105,37 @@ impl Format for Script {
|
|||
}
|
||||
|
||||
fn rename<W: Write>(&self, f: &mut W, from: &PathBuf, to: &PathBuf) -> io::Result<()> {
|
||||
smart_write(
|
||||
f,
|
||||
&[
|
||||
"mv -v -i -- ".into(),
|
||||
from.into(),
|
||||
Writable::Space,
|
||||
to.into(),
|
||||
Writable::Newline,
|
||||
],
|
||||
)
|
||||
smart_write(f, &[
|
||||
"mv -v -i -- ".into(),
|
||||
from.into(),
|
||||
Writable::Space,
|
||||
to.into(),
|
||||
Writable::Newline
|
||||
])
|
||||
}
|
||||
|
||||
fn no_known_extension<W: Write>(&self, f: &mut W, path: &PathBuf) -> io::Result<()> {
|
||||
smart_write(
|
||||
f,
|
||||
&["echo No known extension for ".into(), path.into(), Writable::Newline],
|
||||
)
|
||||
smart_write(f, &[
|
||||
"echo No known extension for ".into(),
|
||||
path.into(),
|
||||
Writable::Newline
|
||||
])
|
||||
}
|
||||
|
||||
fn unreadable<W: Write>(&self, f: &mut W, path: &PathBuf) -> io::Result<()> {
|
||||
smart_write(f, &["# Failed to read ".into(), path.into(), Writable::Newline])
|
||||
smart_write(f, &[
|
||||
"# Failed to read ".into(),
|
||||
path.into(),
|
||||
Writable::Newline
|
||||
])
|
||||
}
|
||||
|
||||
fn unknown_type<W: Write>(&self, f: &mut W, path: &PathBuf) -> io::Result<()> {
|
||||
smart_write(
|
||||
f,
|
||||
&[
|
||||
"# Failed to detect mime type for ".into(),
|
||||
path.into(),
|
||||
Writable::Newline,
|
||||
],
|
||||
)
|
||||
smart_write(f, &[
|
||||
"# Failed to detect mime type for ".into(),
|
||||
path.into(),
|
||||
Writable::Newline
|
||||
])
|
||||
}
|
||||
|
||||
fn header<W: Write>(&self, _: &Entries, f: &mut W) -> io::Result<()> {
|
||||
|
|
|
|||
|
|
@ -10,12 +10,24 @@ use smartstring::alias::String;
|
|||
|
||||
use crate::mimedb::MimeDb;
|
||||
|
||||
// rather than reading once into a large buffer, it tends to be faster to first try identifying the file from a small
|
||||
// chunk read from the top, and *then* proceeding with the large buffer. many file formats can be easily identified by
|
||||
// the first 128 bytes. of course, not all formats can, and some (OOXML...) require reading a long ways in.
|
||||
// use log::{debug, warn};
|
||||
|
||||
const INITIAL_BUF_SIZE: usize = 128;
|
||||
const BUF_SIZE: usize = 4096;
|
||||
// from looking at the files in https://github.com/bojand/infer/tree/master/src/matchers, the format with the largest
|
||||
// buffer size requirement for identification requires 262 bytes, and the next largest buffer necessary is only 131
|
||||
// bytes. as only two formats need more than 128 bytes, it would be fairly reasonable to only read 128 bytes.
|
||||
// unfortunately, the format that requires 262 bytes for identification is tar, an extremely popular format (in the *nix
|
||||
// world, at least). however, tar files almost always appear wrapped in other formats (.tar.gz, .tar.zst, etc) anyway,
|
||||
// so maybe it's fine...? maybe this should be configurable by the user? i don't know.
|
||||
// empirical testing (or rather, starting from 256 and incrementing until it worked) reveals that xdg-mime requires
|
||||
// at least 265 bytes to identify a tar file.
|
||||
|
||||
// additionally, since many formats can by identified with ≤64 bytes, it's worth reading 64 bytes, checking for the mime
|
||||
// type, and then reading the full 512 bytes if necessary. in most cases, this will end up being faster on the whole,
|
||||
// even though two reads are needed for certain formats, unless the directory being scanned is predominantly made up of
|
||||
// such formats.
|
||||
|
||||
const INITIAL_BUF_SIZE: usize = 64;
|
||||
const BUF_SIZE: usize = 512;
|
||||
|
||||
pub fn mime_type<T: MimeDb>(db: &T, path: &Path) -> io::Result<Option<Mime>> {
|
||||
let mut buffer = [0; INITIAL_BUF_SIZE];
|
||||
|
|
@ -26,27 +38,19 @@ pub fn mime_type<T: MimeDb>(db: &T, path: &Path) -> io::Result<Option<Mime>> {
|
|||
#[allow(clippy::unused_io_amount)]
|
||||
file.read(&mut buffer)?;
|
||||
|
||||
let r = db.get_type(&buffer).filter(|mime|
|
||||
// some mime types should be investigated further, reading up to BUF_SIZE even if they've been determined already
|
||||
// one such type is XML - there's many more specific types that can be determined by reading further (such as SVG)
|
||||
mime != &mime_guess::mime::TEXT_XML
|
||||
// another is ZIP - many file formats (DOCX, ODT, JAR...) are just ZIP files with particular data structures.
|
||||
// determining that a file is in one of the MS office formats in particular requires looking quite far into the
|
||||
// file.
|
||||
&& mime != &Mime::from_str("application/zip").unwrap());
|
||||
let r = db.get_type(&buffer);
|
||||
|
||||
if r.is_some() {
|
||||
return Ok(r);
|
||||
}
|
||||
|
||||
// attempt to read up to the BUF_SIZE bytes of the file.
|
||||
// we've already read the first 128 bytes into a buffer, but i can't see an obvious way to reuse them in a way that's
|
||||
// faster than simply moving the seek position back to the start of the file and re-reading the whole BUF_SIZE bytes.
|
||||
// for example, reading (BUF_SIZE - INITIAL_BUF_SIZE) bytes, then concatenating the original INITIAL_BUF_SIZE buffer
|
||||
// with this new one would presumably be faster - but it's not. i think it's more expensive to create the iterator,
|
||||
// collect the contents, etc. i'll have to look into this more. i don't at all doubt that there IS a way to do this
|
||||
// efficiently, and i can think of a way in principle, but i'm not sure how to express it in a way that is both
|
||||
// idiomatic/safe and fast.
|
||||
// we've already read the first 64 bytes into a buffer, but i can't see an obvious way to reuse those 64 bytes that's
|
||||
// faster than simply moving the seek position back to the start of the file and re-reading the whole 512 bytes.
|
||||
// for example, starting with a buffer of 64 bytes, then creating a new 512 byte buffer from the contents of the first
|
||||
// buffer with (512 - 64) blank bytes, then finally reading the rest, is much slower than simply reading the file
|
||||
// twice. i don't at all doubt that there IS a way to do this efficiently, and i can think of a way in principle, but
|
||||
// i'm not sure how to express it in a way that is both idiomatic/safe and fast.
|
||||
let mut buffer = [0; BUF_SIZE];
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
file.read(&mut buffer)?;
|
||||
|
|
@ -82,14 +86,19 @@ cached! {
|
|||
// to have valid extensions.
|
||||
[vec![String::from("xml"), String::from("svg")], possible_exts].concat()
|
||||
|
||||
} else if mime == Mime::from_str("application/msword").unwrap() {
|
||||
// classic office files considered harmful
|
||||
vec![String::from("doc"), String::from("xls"), String::from("ppt")]
|
||||
|
||||
} else if mime == Mime::from_str("application/zip").unwrap() {
|
||||
// neither xdg-mime nor infer seem to be able to detect office XML files properly...
|
||||
[vec![String::from("zip"), String::from("docx"), String::from("xlsx"), String::from("pptx")], possible_exts].concat()
|
||||
|
||||
// until proper document support is added, treat all common document extensions as zips
|
||||
[vec![
|
||||
String::from("zip"),
|
||||
String::from("docx"),
|
||||
String::from("pptx"),
|
||||
String::from("xlsx"),
|
||||
String::from("odt"),
|
||||
String::from("ods"),
|
||||
String::from("odp"),
|
||||
String::from("pages"),
|
||||
String::from("key"),
|
||||
String::from("numbers")], possible_exts].concat()
|
||||
} else {
|
||||
possible_exts
|
||||
})
|
||||
|
|
|
|||
109
src/main.rs
109
src/main.rs
|
|
@ -17,7 +17,6 @@
|
|||
use std::io::{stdout, BufWriter};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use cfg_if::cfg_if;
|
||||
use clap::Clap;
|
||||
use log::{debug, error, info, trace, warn};
|
||||
use once_cell::sync::OnceCell;
|
||||
|
|
@ -41,32 +40,29 @@ mod mimedb;
|
|||
mod parameters;
|
||||
mod scanerror;
|
||||
|
||||
cfg_if! {
|
||||
if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
|
||||
static MIMEDB: OnceCell<mimedb::InferDb> = OnceCell::new();
|
||||
} else {
|
||||
static MIMEDB: OnceCell<mimedb::XdgDb> = OnceCell::new();
|
||||
}
|
||||
#[cfg(feature = "infer-backend")]
|
||||
static MIMEDB: OnceCell<mimedb::InferDb> = OnceCell::new();
|
||||
|
||||
#[cfg(feature = "xdg-mime-backend")]
|
||||
static MIMEDB: OnceCell<mimedb::XdgDb> = OnceCell::new();
|
||||
|
||||
// TODO: test if this actually works on a windows machine
|
||||
#[cfg(windows)]
|
||||
fn is_hidden(entry: &DirEntry) -> bool {
|
||||
use std::os::windows::prelude::*;
|
||||
std::fs::metadata(entry.path()) // try to get metadata for file
|
||||
.map_or(
|
||||
false, // if getting metadata/attributes fails, assume it's not hidden
|
||||
|f| f.file_attributes() & 0x2 > 0, // flag for hidden - https://docs.microsoft.com/windows/win32/fileio/file-attribute-constants
|
||||
)
|
||||
}
|
||||
|
||||
cfg_if! {
|
||||
if #[cfg(windows)] {
|
||||
fn is_hidden(entry: &DirEntry) -> bool {
|
||||
use std::os::windows::prelude::*;
|
||||
std::fs::metadata(entry.path()) // try to get metadata for file
|
||||
.map_or(
|
||||
false, // if getting metadata/attributes fails, assume it's not hidden
|
||||
|f| f.file_attributes() & 0x2 > 0, // flag for hidden - https://docs.microsoft.com/windows/win32/fileio/file-attribute-constants
|
||||
)
|
||||
}
|
||||
} else {
|
||||
fn is_hidden(entry: &DirEntry) -> bool {
|
||||
entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.map_or(false, |f| f.starts_with('.') && f != ".")
|
||||
}
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
fn is_hidden(entry: &DirEntry) -> bool {
|
||||
entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.map_or(false, |f| f.starts_with('.') && f != ".")
|
||||
}
|
||||
|
||||
fn wanted_file(args: ¶meters::Parameters, exts: &[&str], entry: &DirEntry) -> bool {
|
||||
|
|
@ -133,23 +129,25 @@ fn scan_file(entry: &DirEntry) -> Result<Findings, (ScanError, PathBuf)> {
|
|||
}
|
||||
|
||||
fn scan_from_walkdir(entries: &[DirEntry]) -> Vec<Result<Findings, (ScanError, PathBuf)>> {
|
||||
cfg_if! {
|
||||
if #[cfg(feature = "multi-threaded")] {
|
||||
// rather than using a standard par_iter, split the entries into chunks of 32 first.
|
||||
// this allows each spawned thread to handle 32 files before before closing, rather than creating a new thread for
|
||||
// each file. this leads to a pretty substantial speedup that i'm pretty substantially happy about 0u0
|
||||
entries
|
||||
.par_chunks(32) // split into chunks of 32
|
||||
.flat_map(|chunk| {
|
||||
chunk // return Vec<...> instead of Chunk<Vec<...>>
|
||||
.iter() // iter over the chunk, which is a slice of DirEntry structs
|
||||
.map(|entry| scan_file(entry))
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect()
|
||||
} else {
|
||||
entries.iter().map(|entry: &DirEntry| scan_file(entry)).collect()
|
||||
}
|
||||
#[cfg(feature = "multi-threaded")]
|
||||
{
|
||||
// rather than using a standard par_iter, split the entries into chunks of 32 first.
|
||||
// this allows each spawned thread to handle 32 files before before closing, rather than creating a new thread for
|
||||
// each file. this leads to a pretty substantial speedup that i'm pretty substantially happy about 0u0
|
||||
entries
|
||||
.par_chunks(32) // split into chunks of 32
|
||||
.flat_map(|chunk| {
|
||||
chunk // return Vec<...> instead of Chunk<Vec<...>>
|
||||
.iter() // iter over the chunk, which is a slice of DirEntry structs
|
||||
.map(|entry| scan_file(entry))
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "multi-threaded"))]
|
||||
{
|
||||
entries.iter().map(|entry: &DirEntry| scan_file(entry)).collect()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -164,19 +162,17 @@ fn main() {
|
|||
// .target(env_logger::Target::Stdout) // log to stdout rather than stderr
|
||||
.init();
|
||||
|
||||
cfg_if! {
|
||||
if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
|
||||
MIMEDB
|
||||
.set(mimedb::InferDb::init())
|
||||
.or(Err("Failed to initialise Infer backend!"))
|
||||
.unwrap();
|
||||
} else {
|
||||
MIMEDB
|
||||
.set(mimedb::XdgDb::init())
|
||||
.or(Err("Failed to initialise XDG Mime backend!"))
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "infer-backend")]
|
||||
MIMEDB
|
||||
.set(mimedb::InferDb::init())
|
||||
.or(Err("Failed to initialise Infer backend!"))
|
||||
.unwrap();
|
||||
|
||||
#[cfg(feature = "xdg-mime-backend")]
|
||||
MIMEDB
|
||||
.set(mimedb::XdgDb::init())
|
||||
.or(Err("Failed to initialise XDG Mime backend!"))
|
||||
.unwrap();
|
||||
|
||||
debug!("Iterating directory: {:?}", args.dirs);
|
||||
|
||||
|
|
@ -257,8 +253,7 @@ fn main() {
|
|||
}
|
||||
|
||||
if results.is_empty() {
|
||||
info!("All files have valid extensions!");
|
||||
exit(0);
|
||||
info!("All files have valid extensions!")
|
||||
}
|
||||
|
||||
match args.output_format {
|
||||
|
|
|
|||
147
src/mimedb.rs
147
src/mimedb.rs
|
|
@ -1,4 +1,6 @@
|
|||
use cfg_if::cfg_if;
|
||||
#[cfg(feature = "infer-backend")]
|
||||
use std::str::FromStr;
|
||||
|
||||
use mime_guess::Mime;
|
||||
|
||||
pub trait MimeDb {
|
||||
|
|
@ -6,94 +8,75 @@ pub trait MimeDb {
|
|||
fn get_type(&self, data: &[u8]) -> Option<Mime>;
|
||||
}
|
||||
|
||||
cfg_if! {
|
||||
if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
|
||||
use std::str::FromStr;
|
||||
#[cfg(feature = "infer-backend")]
|
||||
pub struct InferDb {
|
||||
db: infer::Infer,
|
||||
}
|
||||
|
||||
pub struct InferDb {
|
||||
db: infer::Infer,
|
||||
}
|
||||
#[cfg(feature = "infer-backend")]
|
||||
impl MimeDb for InferDb {
|
||||
fn init() -> Self {
|
||||
let mut info = infer::Infer::new();
|
||||
|
||||
fn open_document_check(buf: &[u8], kind: &str) -> bool {
|
||||
let mime = format!("application/vnd.oasis.opendocument.{}", kind);
|
||||
let mime = mime.as_bytes();
|
||||
// jpeg2000 support because why the stinch not
|
||||
info.add("image/jpeg2000", ".jp2", |buf| {
|
||||
buf.len() > 23 && buf[..23] == b"\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A\x6A\x70\x32\x20"[..]
|
||||
});
|
||||
|
||||
buf.len() > 38 + mime.len() && buf.starts_with(b"PK\x03\x04") && buf[38..mime.len() + 38] == mime[..]
|
||||
}
|
||||
info.add("image/svg+xml", "svg", |buf| {
|
||||
// before doing the moderately expensive SVG check, we should make sure that the input is actually SGML-ish
|
||||
// by "SGML-ish", i mean starts with anywhere from zero to ∞-1 whitespace characters, and then a less than sign,
|
||||
// and then there's some other stuff we don't care about right now
|
||||
|
||||
impl MimeDb for InferDb {
|
||||
fn init() -> Self {
|
||||
let mut info = infer::Infer::new();
|
||||
|
||||
// jpeg2000 support because why the stinch not
|
||||
info.add("image/jpeg2000", ".jp2", |buf| {
|
||||
buf.len() > 23 && buf[..23] == b"\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A\x6A\x70\x32\x20"[..]
|
||||
});
|
||||
|
||||
info.add("application/vnd.oasis.opendocument.text", "odt", |buf| {
|
||||
open_document_check(buf, "text")
|
||||
});
|
||||
|
||||
info.add("application/vnd.oasis.opendocument.spreadsheet", "ods", |buf| {
|
||||
open_document_check(buf, "spreadsheet")
|
||||
});
|
||||
|
||||
info.add("application/vnd.oasis.opendocument.presentation", "odp", |buf| {
|
||||
open_document_check(buf, "presentation")
|
||||
});
|
||||
|
||||
info.add("image/svg+xml", "svg", |buf| {
|
||||
// before doing the moderately expensive SVG check, we should make sure that the input is actually SGML-ish
|
||||
// by "SGML-ish", i mean starts with anywhere from zero to ∞-1 whitespace characters, and then a less than sign,
|
||||
// and then there's some other stuff we don't care about right now
|
||||
|
||||
// so, here comes our fancy pants """""SGML-ish validator"""""
|
||||
for c in buf {
|
||||
match c {
|
||||
// whitespace (according to https://www.w3.org/TR/xml/#NT-S)
|
||||
b'\t' | b'\r' | b'\n' | b'\x20' => continue,
|
||||
b'<' => break,
|
||||
_ => return false,
|
||||
}
|
||||
}
|
||||
|
||||
// finally, to check whether or not the file is an SVG:
|
||||
// - split the buffer up into chunks separated by the less than sign
|
||||
// - check to see if this chunk starts with any of these identifiers:
|
||||
let identifiers: Vec<&[u8]> = vec![b"svg", b"SVG", b"!DOCTYPE svg", b"!DOCTYPE SVG"];
|
||||
// - if it does, the nested `any` will short circuit and immediately return true, causing the parent `any` to do
|
||||
// the same
|
||||
// - and finally, if none of the chunks match, we'll return false
|
||||
|
||||
// TODO: this is kind of messy, i'd like to clean it up somehow :(
|
||||
buf
|
||||
.split(|c| *c == b'<')
|
||||
.any(|buf| identifiers.iter().any(|id| buf.starts_with(id)))
|
||||
});
|
||||
|
||||
// unmut
|
||||
let info = info;
|
||||
|
||||
Self { db: info }
|
||||
// so, here comes our fancy pants """""SGML-ish validator"""""
|
||||
for c in buf {
|
||||
match c {
|
||||
// whitespace (according to https://www.w3.org/TR/xml/#NT-S)
|
||||
b'\t' | b'\r' | b'\n' | b'\x20' => continue,
|
||||
b'<' => break,
|
||||
_ => return false,
|
||||
}
|
||||
}
|
||||
|
||||
fn get_type(&self, data: &[u8]) -> Option<Mime> {
|
||||
self.db.get(data).map(|f| Mime::from_str(f.mime_type()).unwrap())
|
||||
}
|
||||
}
|
||||
} else {
|
||||
pub struct XdgDb {
|
||||
db: xdg_mime::SharedMimeInfo,
|
||||
}
|
||||
// finally, to check whether or not the file is an SVG:
|
||||
// - split the buffer up into chunks separated by the less than sign
|
||||
// - check to see if this chunk starts with any of these identifiers:
|
||||
let identifiers: Vec<&[u8]> = vec![b"svg", b"SVG", b"!DOCTYPE svg", b"!DOCTYPE SVG"];
|
||||
// - if it does, the nested `any` will short circuit and immediately return true, causing the parent `any` to do
|
||||
// the same
|
||||
// - and finally, if none of the chunks match, we'll return false
|
||||
|
||||
impl MimeDb for XdgDb {
|
||||
fn init() -> Self {
|
||||
Self { db: xdg_mime::SharedMimeInfo::new() }
|
||||
}
|
||||
// TODO: this is kind of messy, i'd like to clean it up somehow :(
|
||||
buf
|
||||
.split(|c| *c == b'<')
|
||||
.any(|buf| identifiers.iter().any(|id| buf.starts_with(id)))
|
||||
});
|
||||
|
||||
fn get_type(&self, data: &[u8]) -> Option<Mime> {
|
||||
self.db.get_mime_type_for_data(data).map(|m| m.0)
|
||||
}
|
||||
}
|
||||
// unmut
|
||||
let info = info;
|
||||
|
||||
Self { db: info }
|
||||
}
|
||||
|
||||
fn get_type(&self, data: &[u8]) -> Option<Mime> {
|
||||
self.db.get(data).map(|f| Mime::from_str(f.mime_type()).unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "xdg-mime-backend")]
|
||||
pub struct XdgDb {
|
||||
db: xdg_mime::SharedMimeInfo,
|
||||
}
|
||||
|
||||
#[cfg(feature = "xdg-mime-backend")]
|
||||
impl MimeDb for XdgDb {
|
||||
fn init() -> Self {
|
||||
Self {
|
||||
db: xdg_mime::SharedMimeInfo::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_type(&self, data: &[u8]) -> Option<Mime> {
|
||||
self.db.get_mime_type_for_data(&data).map(|m| m.0)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ pub enum OutputFormat {
|
|||
}
|
||||
|
||||
#[derive(Clap, Debug)]
|
||||
#[clap(version = option_env!("CARGO_PKG_VERSION").unwrap_or("???"))]
|
||||
pub struct Parameters {
|
||||
/// Only examine files with these extensions (Comma-separated list)
|
||||
#[clap(
|
||||
|
|
|
|||
Loading…
Reference in a new issue