lynnesbian
/
fif
2
1
Fork 0
Code Releases Activity

Merge branch 'master' of https://git.bune.city/lynnesbian/fif 

 Conflicts:
	Cargo.lock
This commit is contained in:
petra-fied 2021-04-14 19:08:52 +10:00
parent 59a3042ebc 7f87f59670
commit 48632a471a
7 changed files with 65 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
CHANGELOG.md
View File

@ -3,14 +3,23 @@ Dates are given in YYYY-MM-DD format.
## v0.2
### v0.2.12 (2021-???)
- Much better README.md
- Added Apple iWork document formats to documents extension set
- Cleaned up and properly documented tests
- Renamed `Script` (in `formats.rs`) to `Shell`, in line with renaming in `paramaters.rs`
### v0.2.12 (2021-04-14)
#### Features
- Added Text extension set
- Better documentation for command line arguments
#### Bugfixes
- Fixed a very minor output bug relating to scanning symlinked directories
- Better detection for pre-OOXML Office files
#### Other
- Much better README.md
- Added more stuff to test.py
- PKGBUILD for Arch-based distros
- Added Text extension set
- More test coverage
- Fixed a very minor output bug relating to scanning symlinked directories
- Better detection for a specific formats (pre-OOXML Office, EXE, DLL)
- Doubled BUF_SIZE
### v0.2.11 (2021-04-04)
#### Features

4
Cargo.toml
View File

@ -1,7 +1,7 @@
[package]
name = "fif"
description = "A command-line tool for detecting and optionally correcting files with incorrect extensions."
version = "0.2.11"
version = "0.2.12"
authors = ["Lynnesbian <lynne@bune.city>"]
edition = "2018"
license = "GPL-3.0-or-later"
@ -29,7 +29,7 @@ log = "0.4.14"
mime_guess = "2.0.3"
snailquote = "0.3.0"
once_cell = "1.7.2"
infer = "0.3.6"
infer = "0.4.0"
rayon = { version = "1.5.0", optional = true }
exitcode = "1.1.2"
cfg-if = "1.0.0"

1
src/extension_set.rs
View File

@ -37,6 +37,7 @@ impl ExtensionSet {
.concat(),
Self::Documents => vec![
"pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "csv", "tsv", "odt", "ods", "odp", "oda", "rtf", "ps",
"pages", "key", "numbers",
],
Self::Text => mime_guess::get_mime_extensions_str("text/*").unwrap().to_vec(),
// many compressed file types follow the name scheme "application/x.+compressed.*" - maybe this can be used

4
src/formats.rs
View File

@ -110,9 +110,9 @@ pub trait Format {
}
/// Bourne-Shell compatible script.
pub struct Script {}
pub struct Shell {}
impl Format for Script {
impl Format for Shell {
fn new() -> Self {
Self {}
}

9
src/inspectors.rs
View File

@ -20,16 +20,14 @@ use crate::string_type::String;
pub const INITIAL_BUF_SIZE: usize = 128;
/// The number of bytes to read if the file couldn't be identified from its first [`INITIAL_BUF_SIZE`] bytes.
pub const BUF_SIZE: usize = 4096;
pub const BUF_SIZE: usize = 8192;
/// Tries to identify the mimetype of a file from a given path.
pub fn mime_type<T: MimeDb>(db: &T, path: &Path) -> io::Result<Option<Mime>> {
let mut buffer = [0; INITIAL_BUF_SIZE];
let mut file = File::open(path)?;
// this lint can be ignored: it's okay if the file isn't long enough to fill the buffer, as we only care about the
// first few bytes for the purpose of mime sniffing
#[allow(clippy::unused_io_amount)]
// read a small amount to start with
file.read(&mut buffer)?;
let r = db.get_type(&buffer).filter(|mime|
@ -43,8 +41,7 @@ pub fn mime_type<T: MimeDb>(db: &T, path: &Path) -> io::Result<Option<Mime>> {
// doc/ppt/xls files are a subset of what's known as an "OLE2 compound document storage", at least according to
// shared-mime-info. if a pre-OOXML era MS office file is scanned and identified as x-ole-storage, reading further
// will allow it to be detected correctly as the appropriate filetype.
&& mime != &Mime::from_str("application/x-ole-storage").unwrap()
);
&& mime != &Mime::from_str("application/x-ole-storage").unwrap());
if r.is_some() {
return Ok(r);

4
src/main.rs
View File

@ -29,7 +29,7 @@ use rayon::prelude::*;
use walkdir::{DirEntry, WalkDir};
use crate::findings::Findings;
use crate::formats::{Format, PowerShell, Script};
use crate::formats::{Format, PowerShell, Shell};
use crate::mime_db::MimeDb;
use crate::parameters::{OutputFormat, ScanOpts};
use crate::scan_error::ScanError;
@ -133,7 +133,7 @@ fn main() {
let mut buffered_stdout = BufWriter::new(stdout());
let result = match args.output_format {
OutputFormat::Sh => Script::new().write_all(&results, &mut buffered_stdout),
OutputFormat::Sh => Shell::new().write_all(&results, &mut buffered_stdout),
OutputFormat::PowerShell | OutputFormat::Powershell => PowerShell::new().write_all(&results, &mut buffered_stdout),
OutputFormat::Text => todo!(),
};

68
src/tests/mod.rs
View File

@ -1,19 +1,15 @@
use crate::findings::Findings;
use crate::formats::{Format, Shell};
use crate::inspectors::{mime_extension_lookup, BUF_SIZE};
use crate::{extension_from_path, init_db, scan_directory, scan_from_walkdir};
use crate::mime_db::MimeDb;
use crate::parameters::{Parameters, ScanOpts};
use crate::string_type::String;
use cfg_if::cfg_if;
use crate::{extension_from_path, scan_directory, scan_from_walkdir};
use mime_guess::mime::{APPLICATION_OCTET_STREAM, APPLICATION_PDF, IMAGE_JPEG, IMAGE_PNG};
use mime_guess::Mime;
use crate::findings::Findings;
use crate::formats::{Format, Script};
use std::borrow::Borrow;
use std::collections::HashMap;
use std::ffi::OsStr;
use std::io::Read;
use std::path::Path;
const JPEG_BYTES: &[u8] = b"\xFF\xD8\xFF";
@ -21,7 +17,7 @@ const PNG_BYTES: &[u8] = b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A";
const PDF_BYTES: &[u8] = b"%PDF-";
const ZIP_BYTES: &[u8] = b"PK\x03\x04";
cfg_if! {
cfg_if::cfg_if! {
if #[cfg(any(all(unix, feature = "infer-backend"), all(not(unix), not(feature = "xdg-mime-backend"))))] {
fn get_mime_db() -> crate::mime_db::InferDb {
crate::mime_db::InferDb::init()
@ -39,6 +35,7 @@ fn application_zip() -> Mime {
}
#[test]
/// Ensure that `extension_from_path` successfully returns the extension from a set of paths.
fn get_ext() {
let mut ext_checks: HashMap<_, Option<&OsStr>> = HashMap::new();
ext_checks.insert(Path::new("test.txt"), Some(OsStr::new("txt")));
@ -54,6 +51,7 @@ fn get_ext() {
}
#[test]
/// Ensure that the mime types for JPEG, PNG, PDF, and ZIP are detected from their magic numbers.
fn detect_type() {
let db = get_mime_db();
assert_eq!(db.get_type(JPEG_BYTES), Some(IMAGE_JPEG));
@ -63,6 +61,8 @@ fn detect_type() {
}
#[test]
/// Ensure that `mime_extension_lookup` works as expected, and that the set of extensions for JPEG, PNG, PDF, and ZIP
/// contain "jpg", "png", "pdf", and "zip", respectively.
fn recommend_ext() {
assert!(mime_extension_lookup(IMAGE_JPEG)
.unwrap()
@ -77,11 +77,17 @@ fn recommend_ext() {
}
#[test]
/// Create a simple directory with some files, run `scan_directory` on it, and ensure that the files have their
/// associated mime types correctly deduced.
fn simple_directory() {
use crate::parameters::ScanOpts;
use std::borrow::Borrow;
use std::fs::File;
use std::io::Write;
use tempfile::tempdir;
// set of files to scan. all but the last files have magic numbers corresponding to their extension, except for
// "wrong.jpg", which is actually a png.
let mut files = HashMap::new();
files.insert("test.jpg", JPEG_BYTES);
files.insert("test.jpeg", JPEG_BYTES);
@ -107,41 +113,39 @@ fn simple_directory() {
follow_symlinks: false,
};
let entries = scan_directory(
&dir.path().to_path_buf(),
Some(&vec!["jpg", "jpeg", "png", "pdf", "zip"]),
&scan_opts,
)
.expect("Directory scan failed.");
let entries = scan_directory(&dir.path().to_path_buf(), None, &scan_opts).expect("Directory scan failed.");
assert_eq!(entries.len(), files.len());
// initialise global mime DB
init_db();
// initialise global mime DB - this is needed because `scan_from_walkdir` expects it to be present.
crate::init_db();
let results = scan_from_walkdir(&entries);
for result in results {
// there should be no IO errors during this test. any IO errors encountered are outside the scope of this test.
let result = result.expect("Error while scanning file");
if !result.valid {
// this should be "wrong.jpg", which is a misnamed png file
// 1. ensure extension is "png"
// the only invalid file detected should be "wrong.jpg", which is a misnamed png file
// 1. ensure detected extension is "jpg"
assert_eq!(extension_from_path(&*result.file).unwrap(), OsStr::new("jpg"));
// 2. ensure mime type detected is IMAGE_PNG
// 2. ensure detected mime type is IMAGE_PNG
assert_eq!(result.mime, IMAGE_PNG);
// 3. ensure recommended extension is in the list of known extensions for PNG files
assert!(mime_extension_lookup(IMAGE_PNG)
.unwrap()
.contains(&result.recommended_extension().unwrap()));
// 3. ensure the recommended extension for "wrong.jpg" is "png"
assert_eq!(&result.recommended_extension().unwrap(), &String::from("png"));
continue;
}
// check if the recommended extension for this file is in the list of known extensions for its mimetype
// check if the recommended extension for this file is in the list of known extensions for its mimetype - for
// example, if the file is determined to be an IMAGE_PNG, its recommended extension should be one of the extensions
// returned by `mime_extension_lookup(IMAGE_PNG)`.
assert!(mime_extension_lookup(result.mime.clone())
.unwrap()
.contains(&result.recommended_extension().unwrap()));
// make sure the guessed mimetype is correct based on the extension of the scanned file
// because we already know that the extensions match the mimetype (as we created these files ourselves earlier in
// the test), all files with the "jpg" extension should be IMAGE_JPEGs, etc.
let ext = extension_from_path(result.file);
assert!(ext.is_some());
assert_eq!(
@ -161,7 +165,10 @@ fn simple_directory() {
}
#[test]
/// Ensure that command line argument parsing works correctly - flags are interpreted, booleans are set, and so on.
fn argument_parsing() {
use crate::parameters::{Parameters, ScanOpts};
use clap::Clap;
// pass `-f`, which enables following symlinks, and `-E images`, which scans files with image extensions
@ -192,7 +199,10 @@ fn argument_parsing() {
}
#[test]
/// Ensure that badly formed command line arguments are rejected.
fn rejects_bad_args() {
use crate::parameters::Parameters;
use clap::Clap;
let tests = [
// Non-existent flags:
@ -213,6 +223,8 @@ fn rejects_bad_args() {
}
#[test]
/// Generate random series of bytes and try to identify them. This test makes no assertions and can only fail if the
/// mime database somehow panics or hangs.
fn identify_random_bytes() {
let db = get_mime_db();
let rng = fastrand::Rng::new();
@ -233,7 +245,11 @@ fn identify_random_bytes() {
}
#[test]
/// Ensure that, for a given file "wrong.bad", which should have extension "good", the shell output contains something
/// like "mv wrong.bad wrong.good".
fn outputs_move_commands() {
use std::io::Read;
// create an example finding stating that "misnamed_file.png" has been identified as a jpeg file
let entries = vec![Ok(Findings {
file: Path::new("misnamed_file.png"),
@ -244,7 +260,7 @@ fn outputs_move_commands() {
let mut cursor = std::io::Cursor::new(Vec::new());
let mut contents = std::string::String::new();
Script::new()
Shell::new()
.write_all(&entries, &mut cursor)
.expect("Failed to write to cursor");
cursor.set_position(0);