lynnesbian/fif
2
1
Fork 0
Code Releases Activity

use cfg_if to make the code more readable

Browse source
This commit is contained in:
Lynne Megido 2021-02-27 12:13:57 +10:00
parent b141b85ea7
commit ebc2bcfb41
Signed by: lynnesbian
GPG key ID: F0A184B5213D9F90
4 changed files with 134 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Cargo.lock generated
View file

@ -176,7 +176,7 @@ checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193"
[[package]] [[package]]
name = "fif" name = "fif"
version = "0.2.4" version = "0.2.5"
dependencies = [ dependencies = [
"cached", "cached",
"cfg-if", "cfg-if",

2
Cargo.toml
View file

@ -1,7 +1,7 @@
[package] [package]
name = "fif" name = "fif"
description = "A command-line tool for detecting and optionally correcting files with incorrect extensions." description = "A command-line tool for detecting and optionally correcting files with incorrect extensions."
version = "0.2.4" version = "0.2.5"
authors = ["Lynnesbian <lynne@bune.city>"] authors = ["Lynnesbian <lynne@bune.city>"]
edition = "2018" edition = "2018"
license = "GPL-3.0-or-later" license = "GPL-3.0-or-later"

107
src/main.rs
View file

@ -17,6 +17,7 @@
use std::io::{stdout, BufWriter}; use std::io::{stdout, BufWriter};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use cfg_if::cfg_if;
use clap::Clap; use clap::Clap;
use log::{debug, error, info, trace, warn}; use log::{debug, error, info, trace, warn};
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
@ -40,29 +41,33 @@ mod mimedb;
mod parameters; mod parameters;
mod scanerror; mod scanerror;
#[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] cfg_if! {
static MIMEDB: OnceCell<mimedb::InferDb> = OnceCell::new(); if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
static MIMEDB: OnceCell<mimedb::InferDb> = OnceCell::new();
#[cfg(any(all(target_os = "linux", not(feature = "infer-backend")), all(not(target_os = "linux"), not(feature = "xdg-mime-backend"))))] } else {
static MIMEDB: OnceCell<mimedb::XdgDb> = OnceCell::new(); static MIMEDB: OnceCell<mimedb::XdgDb> = OnceCell::new();
}
// TODO: test if this actually works on a windows machine
#[cfg(windows)]
fn is_hidden(entry: &DirEntry) -> bool {
use std::os::windows::prelude::*;
std::fs::metadata(entry.path()) // try to get metadata for file
.map_or(
false, // if getting metadata/attributes fails, assume it's not hidden
|f| f.file_attributes() & 0x2 > 0, // flag for hidden - https://docs.microsoft.com/windows/win32/fileio/file-attribute-constants
)
} }
#[cfg(not(windows))] cfg_if! {
fn is_hidden(entry: &DirEntry) -> bool { // TODO: test if this actually works on a windows machine
entry if #[cfg(windows)] {
.file_name() fn is_hidden(entry: &DirEntry) -> bool {
.to_str() use std::os::windows::prelude::*;
.map_or(false, |f| f.starts_with('.') && f != ".") std::fs::metadata(entry.path()) // try to get metadata for file
.map_or(
false, // if getting metadata/attributes fails, assume it's not hidden
|f| f.file_attributes() & 0x2 > 0, // flag for hidden - https://docs.microsoft.com/windows/win32/fileio/file-attribute-constants
)
}
} else {
fn is_hidden(entry: &DirEntry) -> bool {
entry
.file_name()
.to_str()
.map_or(false, |f| f.starts_with('.') && f != ".")
}
}
} }
fn wanted_file(args: &parameters::Parameters, exts: &[&str], entry: &DirEntry) -> bool { fn wanted_file(args: &parameters::Parameters, exts: &[&str], entry: &DirEntry) -> bool {
@ -129,25 +134,23 @@ fn scan_file(entry: &DirEntry) -> Result<Findings, (ScanError, PathBuf)> {
} }
fn scan_from_walkdir(entries: &[DirEntry]) -> Vec<Result<Findings, (ScanError, PathBuf)>> { fn scan_from_walkdir(entries: &[DirEntry]) -> Vec<Result<Findings, (ScanError, PathBuf)>> {
#[cfg(feature = "multi-threaded")] cfg_if! {
{ if #[cfg(feature = "multi-threaded")] {
// rather than using a standard par_iter, split the entries into chunks of 32 first. // rather than using a standard par_iter, split the entries into chunks of 32 first.
// this allows each spawned thread to handle 32 files before before closing, rather than creating a new thread for // this allows each spawned thread to handle 32 files before before closing, rather than creating a new thread for
// each file. this leads to a pretty substantial speedup that i'm pretty substantially happy about 0u0 // each file. this leads to a pretty substantial speedup that i'm pretty substantially happy about 0u0
entries entries
.par_chunks(32) // split into chunks of 32 .par_chunks(32) // split into chunks of 32
.flat_map(|chunk| { .flat_map(|chunk| {
chunk // return Vec<...> instead of Chunk<Vec<...>> chunk // return Vec<...> instead of Chunk<Vec<...>>
.iter() // iter over the chunk, which is a slice of DirEntry structs .iter() // iter over the chunk, which is a slice of DirEntry structs
.map(|entry| scan_file(entry)) .map(|entry| scan_file(entry))
.collect::<Vec<_>>() .collect::<Vec<_>>()
}) })
.collect() .collect()
} } else {
entries.iter().map(|entry: &DirEntry| scan_file(entry)).collect()
#[cfg(not(feature = "multi-threaded"))] }
{
entries.iter().map(|entry: &DirEntry| scan_file(entry)).collect()
} }
} }
@ -162,17 +165,19 @@ fn main() {
// .target(env_logger::Target::Stdout) // log to stdout rather than stderr // .target(env_logger::Target::Stdout) // log to stdout rather than stderr
.init(); .init();
#[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] cfg_if! {
MIMEDB if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
.set(mimedb::InferDb::init()) MIMEDB
.or(Err("Failed to initialise Infer backend!")) .set(mimedb::InferDb::init())
.unwrap(); .or(Err("Failed to initialise Infer backend!"))
.unwrap();
#[cfg(any(all(target_os = "linux", not(feature = "infer-backend")), all(not(target_os = "linux"), not(feature = "xdg-mime-backend"))))] } else {
MIMEDB MIMEDB
.set(mimedb::XdgDb::init()) .set(mimedb::XdgDb::init())
.or(Err("Failed to initialise XDG Mime backend!")) .or(Err("Failed to initialise XDG Mime backend!"))
.unwrap(); .unwrap();
}
}
debug!("Iterating directory: {:?}", args.dirs); debug!("Iterating directory: {:?}", args.dirs);

155
src/mimedb.rs
View file

@ -1,6 +1,4 @@
#[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] use cfg_if::cfg_if;
use std::str::FromStr;
use mime_guess::Mime; use mime_guess::Mime;
pub trait MimeDb { pub trait MimeDb {
@ -8,95 +6,94 @@ pub trait MimeDb {
fn get_type(&self, data: &[u8]) -> Option<Mime>; fn get_type(&self, data: &[u8]) -> Option<Mime>;
} }
#[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] cfg_if! {
pub struct InferDb { if #[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] {
db: infer::Infer, use std::str::FromStr;
}
#[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] pub struct InferDb {
fn open_document_check(buf: &[u8], kind: &str) -> bool { db: infer::Infer,
let mime = format!("application/vnd.oasis.opendocument.{}", kind); }
let mime = mime.as_bytes();
buf.len() > 38 + mime.len() && buf.starts_with(b"PK\x03\x04") && buf[38..mime.len() + 38] == mime[..] fn open_document_check(buf: &[u8], kind: &str) -> bool {
} let mime = format!("application/vnd.oasis.opendocument.{}", kind);
let mime = mime.as_bytes();
#[cfg(any(all(not(target_os = "linux"), not(feature = "xdg-mime-backend")), all(target_os = "linux", feature = "infer-backend")))] buf.len() > 38 + mime.len() && buf.starts_with(b"PK\x03\x04") && buf[38..mime.len() + 38] == mime[..]
impl MimeDb for InferDb { }
fn init() -> Self {
let mut info = infer::Infer::new();
// jpeg2000 support because why the stinch not impl MimeDb for InferDb {
info.add("image/jpeg2000", ".jp2", |buf| { fn init() -> Self {
buf.len() > 23 && buf[..23] == b"\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A\x6A\x70\x32\x20"[..] let mut info = infer::Infer::new();
});
info.add("application/vnd.oasis.opendocument.text", "odt", |buf| { // jpeg2000 support because why the stinch not
open_document_check(buf, "text") info.add("image/jpeg2000", ".jp2", |buf| {
}); buf.len() > 23 && buf[..23] == b"\x00\x00\x00\x0C\x6A\x50\x20\x20\x0D\x0A\x87\x0A\x6A\x70\x32\x20"[..]
});
info.add("application/vnd.oasis.opendocument.spreadsheet", "ods", |buf| { info.add("application/vnd.oasis.opendocument.text", "odt", |buf| {
open_document_check(buf, "spreadsheet") open_document_check(buf, "text")
}); });
info.add("application/vnd.oasis.opendocument.presentation", "odp", |buf| { info.add("application/vnd.oasis.opendocument.spreadsheet", "ods", |buf| {
open_document_check(buf, "presentation") open_document_check(buf, "spreadsheet")
}); });
info.add("image/svg+xml", "svg", |buf| { info.add("application/vnd.oasis.opendocument.presentation", "odp", |buf| {
// before doing the moderately expensive SVG check, we should make sure that the input is actually SGML-ish open_document_check(buf, "presentation")
// by "SGML-ish", i mean starts with anywhere from zero to ∞-1 whitespace characters, and then a less than sign, });
// and then there's some other stuff we don't care about right now
// so, here comes our fancy pants """""SGML-ish validator""""" info.add("image/svg+xml", "svg", |buf| {
for c in buf { // before doing the moderately expensive SVG check, we should make sure that the input is actually SGML-ish
match c { // by "SGML-ish", i mean starts with anywhere from zero to ∞-1 whitespace characters, and then a less than sign,
// whitespace (according to https://www.w3.org/TR/xml/#NT-S) // and then there's some other stuff we don't care about right now
b'\t' | b'\r' | b'\n' | b'\x20' => continue,
b'<' => break, // so, here comes our fancy pants """""SGML-ish validator"""""
_ => return false, for c in buf {
} match c {
// whitespace (according to https://www.w3.org/TR/xml/#NT-S)
b'\t' | b'\r' | b'\n' | b'\x20' => continue,
b'<' => break,
_ => return false,
}
}
// finally, to check whether or not the file is an SVG:
// - split the buffer up into chunks separated by the less than sign
// - check to see if this chunk starts with any of these identifiers:
let identifiers: Vec<&[u8]> = vec![b"svg", b"SVG", b"!DOCTYPE svg", b"!DOCTYPE SVG"];
// - if it does, the nested `any` will short circuit and immediately return true, causing the parent `any` to do
// the same
// - and finally, if none of the chunks match, we'll return false
// TODO: this is kind of messy, i'd like to clean it up somehow :(
buf
.split(|c| *c == b'<')
.any(|buf| identifiers.iter().any(|id| buf.starts_with(id)))
});
// unmut
let info = info;
Self { db: info }
} }
// finally, to check whether or not the file is an SVG: fn get_type(&self, data: &[u8]) -> Option<Mime> {
// - split the buffer up into chunks separated by the less than sign self.db.get(data).map(|f| Mime::from_str(f.mime_type()).unwrap())
// - check to see if this chunk starts with any of these identifiers: }
let identifiers: Vec<&[u8]> = vec![b"svg", b"SVG", b"!DOCTYPE svg", b"!DOCTYPE SVG"]; }
// - if it does, the nested `any` will short circuit and immediately return true, causing the parent `any` to do } else {
// the same pub struct XdgDb {
// - and finally, if none of the chunks match, we'll return false db: xdg_mime::SharedMimeInfo,
}
// TODO: this is kind of messy, i'd like to clean it up somehow :( impl MimeDb for XdgDb {
buf fn init() -> Self {
.split(|c| *c == b'<') Self { db: xdg_mime::SharedMimeInfo::new() }
.any(|buf| identifiers.iter().any(|id| buf.starts_with(id))) }
});
// unmut fn get_type(&self, data: &[u8]) -> Option<Mime> {
let info = info; self.db.get_mime_type_for_data(&data).map(|m| m.0)
}
Self { db: info }
}
fn get_type(&self, data: &[u8]) -> Option<Mime> {
self.db.get(data).map(|f| Mime::from_str(f.mime_type()).unwrap())
}
}
#[cfg(any(all(target_os = "linux", not(feature = "infer-backend")), all(not(target_os = "linux"), not(feature = "xdg-mime-backend"))))]
pub struct XdgDb {
db: xdg_mime::SharedMimeInfo,
}
#[cfg(any(all(target_os = "linux", not(feature = "infer-backend")), all(not(target_os = "linux"), not(feature = "xdg-mime-backend"))))]
impl MimeDb for XdgDb {
fn init() -> Self {
Self {
db: xdg_mime::SharedMimeInfo::new(),
} }
} }
fn get_type(&self, data: &[u8]) -> Option<Mime> {
self.db.get_mime_type_for_data(&data).map(|m| m.0)
}
} }