Compare commits
10 commits
7e3efbed5c
...
534101db8b
Author | SHA1 | Date | |
---|---|---|---|
534101db8b | |||
cb6e111f16 | |||
4f5914ed75 | |||
d5d58e1830 | |||
af3d51fcda | |||
d578efa7a4 | |||
22f1f280d7 | |||
b3ce5d3d46 | |||
b12c42adbb | |||
fc51f8128a |
9 changed files with 264 additions and 96 deletions
|
@ -10,24 +10,70 @@ cache:
|
||||||
|
|
||||||
default:
|
default:
|
||||||
before_script:
|
before_script:
|
||||||
- rustc --version
|
- rustc --version
|
||||||
|
- cargo version
|
||||||
|
|
||||||
stages:
|
stages:
|
||||||
|
- lint
|
||||||
- build
|
- build
|
||||||
- test
|
- test
|
||||||
|
- version
|
||||||
|
|
||||||
build:
|
# TEMPLATE
|
||||||
|
.cargo-build:
|
||||||
stage: build
|
stage: build
|
||||||
|
parallel:
|
||||||
|
matrix:
|
||||||
|
- FEATURES: [ 'xdg-mime-backend', 'infer-backend', 'multi-threaded xdg-mime-backend',
|
||||||
|
'multi-threaded infer-backend' ]
|
||||||
script:
|
script:
|
||||||
- cargo build --verbose --locked
|
cargo build --no-default-features --locked --features="$FEATURES"
|
||||||
|
|
||||||
cargo-test:
|
.cargo-test:
|
||||||
stage: test
|
stage: test
|
||||||
|
parallel:
|
||||||
|
matrix:
|
||||||
|
- FEATURES: [ 'xdg-mime-backend', 'infer-backend', 'multi-threaded xdg-mime-backend',
|
||||||
|
'multi-threaded infer-backend' ]
|
||||||
script:
|
script:
|
||||||
cargo test --verbose --locked
|
cargo test --no-default-features --locked --verbose --features="$FEATURES"
|
||||||
|
|
||||||
clippy:
|
clippy:
|
||||||
stage: test
|
stage: lint
|
||||||
script:
|
script:
|
||||||
- rustup component add clippy
|
- rustup component add clippy
|
||||||
- ./clippy.sh
|
- cargo clippy --version
|
||||||
|
- ./clippy.sh ci
|
||||||
|
|
||||||
|
# BUILD
|
||||||
|
|
||||||
|
build-stable:
|
||||||
|
extends: .cargo-build
|
||||||
|
|
||||||
|
build-msrv:
|
||||||
|
extends: build-stable
|
||||||
|
image: "rust:1.43.0"
|
||||||
|
|
||||||
|
build-nightly:
|
||||||
|
extends: build-stable
|
||||||
|
image: "rustlang/rust:nightly"
|
||||||
|
|
||||||
|
# TEST
|
||||||
|
|
||||||
|
test-stable:
|
||||||
|
extends: .cargo-test
|
||||||
|
|
||||||
|
test-msrv:
|
||||||
|
extends: test-stable
|
||||||
|
image: "rust:1.43.0"
|
||||||
|
|
||||||
|
test-nightly:
|
||||||
|
extends: test-stable
|
||||||
|
image: "rustlang/rust:nightly"
|
||||||
|
|
||||||
|
# VERSION
|
||||||
|
|
||||||
|
fif-version:
|
||||||
|
stage: version
|
||||||
|
script:
|
||||||
|
cargo run -- -V
|
17
CHANGELOG.md
17
CHANGELOG.md
|
@ -4,7 +4,19 @@ Dates are given in YYYY-MM-DD format.
|
||||||
## v0.2
|
## v0.2
|
||||||
### v0.2.14 (2021-xx-yy)
|
### v0.2.14 (2021-xx-yy)
|
||||||
#### Features
|
#### Features
|
||||||
- Added `-x`/`--exclude` flag for excluding file extensions (overrides `-e` or `-E`)
|
- Added `-x`/`--exclude` flag for excluding file extensions (overrides `-e` or `-E` - `-E images -x jpg` scans all image
|
||||||
|
files, except ".jpg" files)
|
||||||
|
- Added `-X`/`--exclude-set` flag for excluding sets of files, with the same syntax and sets as `-E`
|
||||||
|
- In addition to supplying included extensions as a comma separated list (like `-e jpg,png`), it is now possible to
|
||||||
|
supply them through multiple uses of the `-e` flag (like `-e jpg -e png`). This also applies to `-x`
|
||||||
|
- `-e` and `-E` no longer conflict with each other, and can now be used together. For example, `-E images -e mp3`
|
||||||
|
will scan all images *and* all MP3 files
|
||||||
|
- It is now possible to specify multiple extension sets at once: `-E images,system` will scan all images and archives
|
||||||
|
#### Other
|
||||||
|
- Published my fork of ['mime_guess'] as ['new_mime_guess'], allowing it to be used properly with
|
||||||
|
[crates.io](https://crates.io)
|
||||||
|
- The `videos` extension set has been renamed to `video`, in line with `audio`. `fif --help` has actually mistakenly
|
||||||
|
referred to the set as `video` since v0.2.12! 0uo
|
||||||
|
|
||||||
### v0.2.13 (2021-04-26)
|
### v0.2.13 (2021-04-26)
|
||||||
#### Features
|
#### Features
|
||||||
|
@ -148,4 +160,5 @@ Initial commit!
|
||||||
[`clap`]: https://crates.io/crates/clap
|
[`clap`]: https://crates.io/crates/clap
|
||||||
[`infer`]: https://crates.io/crates/infer
|
[`infer`]: https://crates.io/crates/infer
|
||||||
[`mime_guess`]: https://crates.io/crates/mime_guess
|
[`mime_guess`]: https://crates.io/crates/mime_guess
|
||||||
[`snailquote`]: https://crates.io/crates/snailquote
|
[`new_mime_guess`]: https://crates.io/crates/new_mime_guess
|
||||||
|
[`snailquote`]: https://crates.io/crates/snailquote
|
||||||
|
|
13
Cargo.lock
generated
13
Cargo.lock
generated
|
@ -204,7 +204,7 @@ dependencies = [
|
||||||
"infer",
|
"infer",
|
||||||
"itertools",
|
"itertools",
|
||||||
"log",
|
"log",
|
||||||
"mime_guess",
|
"new_mime_guess",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"rayon",
|
"rayon",
|
||||||
"smartstring",
|
"smartstring",
|
||||||
|
@ -348,9 +348,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
|
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mime_guess"
|
name = "new_mime_guess"
|
||||||
version = "2.0.4"
|
version = "2.1.0"
|
||||||
source = "git+https://github.com/Lynnesbian/mime_guess#5432b3c1991372291a5e67457cc9307c85f77bd9"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e714f72c691c7d2b344ec8dd57d7f52b59651f46b9de477fb68363f097d694ae"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"mime",
|
"mime",
|
||||||
"unicase",
|
"unicase",
|
||||||
|
@ -578,9 +579,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "syn"
|
name = "syn"
|
||||||
version = "1.0.70"
|
version = "1.0.71"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b9505f307c872bab8eb46f77ae357c8eba1fdacead58ee5a850116b1d7f82883"
|
checksum = "ad184cc9470f9117b2ac6817bfe297307418819ba40552f9b3846f05c33d5373"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
|
|
|
@ -26,7 +26,7 @@ xdg-mime-backend = []
|
||||||
[dependencies]
|
[dependencies]
|
||||||
walkdir = "2.3.2"
|
walkdir = "2.3.2"
|
||||||
log = "0.4.14"
|
log = "0.4.14"
|
||||||
mime_guess = "2.0.3"
|
mime_guess = { package = "new_mime_guess", version = "2.1.0" }
|
||||||
snailquote = "0.3.0"
|
snailquote = "0.3.0"
|
||||||
once_cell = "1.7.2"
|
once_cell = "1.7.2"
|
||||||
infer = "0.4.0"
|
infer = "0.4.0"
|
||||||
|
@ -41,10 +41,6 @@ xdg-mime = "0.3.3"
|
||||||
[target.'cfg(not(all(target_endian = "big", target_pointer_width = "32")))'.dependencies]
|
[target.'cfg(not(all(target_endian = "big", target_pointer_width = "32")))'.dependencies]
|
||||||
smartstring = "0.2.6"
|
smartstring = "0.2.6"
|
||||||
|
|
||||||
[patch.crates-io]
|
|
||||||
# forked version with many more mime types
|
|
||||||
mime_guess = { git = "https://github.com/Lynnesbian/mime_guess", version = "2.0.4" }
|
|
||||||
|
|
||||||
[dependencies.clap]
|
[dependencies.clap]
|
||||||
version = "3.0.0-beta.2"
|
version = "3.0.0-beta.2"
|
||||||
default-features = false
|
default-features = false
|
||||||
|
|
15
clippy.sh
15
clippy.sh
|
@ -1,5 +1,15 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
fd -e rs -x touch {}
|
set -e
|
||||||
|
|
||||||
|
_extra=""
|
||||||
|
if [ "$1" == "ci" ]; then
|
||||||
|
# deny on warnings when running in CI
|
||||||
|
_extra="-Dwarnings"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# allow find to fail
|
||||||
|
find . -name '*.rs' -exec touch "{}" \; || true
|
||||||
|
|
||||||
cargo clippy --all-features --tests -- \
|
cargo clippy --all-features --tests -- \
|
||||||
-W clippy::nursery \
|
-W clippy::nursery \
|
||||||
-W clippy::perf \
|
-W clippy::perf \
|
||||||
|
@ -14,7 +24,8 @@ cargo clippy --all-features --tests -- \
|
||||||
-A clippy::unused_io_amount \
|
-A clippy::unused_io_amount \
|
||||||
-A clippy::redundant_closure_for_method_calls \
|
-A clippy::redundant_closure_for_method_calls \
|
||||||
-A clippy::shadow_unrelated \
|
-A clippy::shadow_unrelated \
|
||||||
-A clippy::option_if_let_else
|
-A clippy::option_if_let_else \
|
||||||
|
"$_extra"
|
||||||
|
|
||||||
# ALLOWS:
|
# ALLOWS:
|
||||||
# unused_io_amount: there are two places where i want to read up to X bytes and i'm fine with getting less than that
|
# unused_io_amount: there are two places where i want to read up to X bytes and i'm fine with getting less than that
|
||||||
|
|
|
@ -101,14 +101,17 @@ cached! {
|
||||||
// add "jpg" to the start of the possible_exts list, ensuring that it will be the extension suggested by fif.
|
// add "jpg" to the start of the possible_exts list, ensuring that it will be the extension suggested by fif.
|
||||||
[vec![String::from("jpg")], possible_exts].concat()
|
[vec![String::from("jpg")], possible_exts].concat()
|
||||||
|
|
||||||
} else if mime == mime_guess::mime::TEXT_XML {
|
} else if mime == mime_guess::mime::TEXT_XML || mime == Mime::from_str("application/xml").unwrap() {
|
||||||
// a somewhat similar case arises with XML files - the first suggested extension is "addin", when it should
|
// a somewhat similar case arises with XML files - the first suggested extension is "asa", when it should
|
||||||
// (in my opinion) be "xml".
|
// (in my opinion) be "xml".
|
||||||
// there's also another problem: SVG files can easily be misidentified as XML files, because they usually
|
// there's also another problem: SVG files can easily be misidentified as XML files, because they usually
|
||||||
// *are* valid XML - the more whitespace and comments an SVG file begins with, the more bytes must be read
|
// *are* valid XML - the more whitespace and comments an SVG file begins with, the more bytes must be read
|
||||||
// before it's possible to determine that it's an SVG rather than an XML file. to "fix" this, we can add "svg"
|
// before it's possible to determine that it's an SVG rather than an XML file. to "fix" this, we can add "svg"
|
||||||
// as a valid extension for XML files, ensuring that SVG files misidentified as XML will still be considered
|
// as a valid extension for XML files, ensuring that SVG files misidentified as XML will still be considered
|
||||||
// to have valid extensions.
|
// to have valid extensions.
|
||||||
|
// TODO: if a file is detected as application/xml, but it has an extension like "xht" which corresponds to
|
||||||
|
// "application/xhtml+xml", let it through - in other words, if it's identified as application/xml, but its
|
||||||
|
// extension is classes as application/*+xml, consider it OK
|
||||||
[vec![String::from("xml"), String::from("svg")], possible_exts].concat()
|
[vec![String::from("xml"), String::from("svg")], possible_exts].concat()
|
||||||
|
|
||||||
} else if mime == Mime::from_str("application/msword").unwrap() {
|
} else if mime == Mime::from_str("application/msword").unwrap() {
|
||||||
|
|
24
src/main.rs
24
src/main.rs
|
@ -34,6 +34,7 @@ use crate::formats::{Format, PowerShell, Shell};
|
||||||
use crate::mime_db::MimeDb;
|
use crate::mime_db::MimeDb;
|
||||||
use crate::parameters::{OutputFormat, ScanOpts};
|
use crate::parameters::{OutputFormat, ScanOpts};
|
||||||
use crate::scan_error::ScanError;
|
use crate::scan_error::ScanError;
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
|
||||||
mod findings;
|
mod findings;
|
||||||
mod formats;
|
mod formats;
|
||||||
|
@ -88,7 +89,12 @@ fn main() {
|
||||||
debug!("Checking files regardless of extensions");
|
debug!("Checking files regardless of extensions");
|
||||||
}
|
}
|
||||||
|
|
||||||
let entries = scan_directory(&args.dirs, extensions.as_ref(), excludes.as_ref(), &args.get_scan_opts());
|
let entries = scan_directory(
|
||||||
|
&args.dirs,
|
||||||
|
extensions.as_ref(),
|
||||||
|
excludes.as_ref(),
|
||||||
|
&args.get_scan_opts(),
|
||||||
|
);
|
||||||
|
|
||||||
if entries.is_none() {
|
if entries.is_none() {
|
||||||
// no need to log anything for fatal errors - fif will already have printed something obvious like
|
// no need to log anything for fatal errors - fif will already have printed something obvious like
|
||||||
|
@ -138,7 +144,7 @@ fn main() {
|
||||||
|
|
||||||
let result = match args.output_format {
|
let result = match args.output_format {
|
||||||
OutputFormat::Sh => Shell::new().write_all(&results, &mut buffered_stdout),
|
OutputFormat::Sh => Shell::new().write_all(&results, &mut buffered_stdout),
|
||||||
OutputFormat::PowerShell | OutputFormat::Powershell => PowerShell::new().write_all(&results, &mut buffered_stdout),
|
OutputFormat::PowerShell => PowerShell::new().write_all(&results, &mut buffered_stdout),
|
||||||
OutputFormat::Text => todo!(),
|
OutputFormat::Text => todo!(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -179,7 +185,12 @@ cfg_if! {
|
||||||
|
|
||||||
/// Returns `true` if a file matches the given criteria. This means checking whether the file's extension appears in
|
/// Returns `true` if a file matches the given criteria. This means checking whether the file's extension appears in
|
||||||
/// `exts` (if specified), potentially skipping over hidden files, and so on.
|
/// `exts` (if specified), potentially skipping over hidden files, and so on.
|
||||||
fn wanted_file(entry: &DirEntry, exts: Option<&Vec<&str>>, exclude: Option<&Vec<&str>>, scan_opts: &ScanOpts) -> bool {
|
fn wanted_file(
|
||||||
|
entry: &DirEntry,
|
||||||
|
exts: Option<&BTreeSet<&str>>,
|
||||||
|
exclude: Option<&BTreeSet<&str>>,
|
||||||
|
scan_opts: &ScanOpts,
|
||||||
|
) -> bool {
|
||||||
if entry.depth() == 0 {
|
if entry.depth() == 0 {
|
||||||
// the root directory should always be scanned.
|
// the root directory should always be scanned.
|
||||||
return true;
|
return true;
|
||||||
|
@ -285,7 +296,12 @@ fn scan_from_walkdir(entries: &[DirEntry]) -> Vec<Result<Findings, ScanError>> {
|
||||||
|
|
||||||
/// Scans a given directory with [`WalkDir`], filters with [`wanted_file`], checks for errors, and returns a vector of
|
/// Scans a given directory with [`WalkDir`], filters with [`wanted_file`], checks for errors, and returns a vector of
|
||||||
/// [DirEntry]s.
|
/// [DirEntry]s.
|
||||||
fn scan_directory(dirs: &Path, exts: Option<&Vec<&str>>, exclude: Option<&Vec<&str>>, scan_opts: &ScanOpts) -> Option<Vec<DirEntry>> {
|
fn scan_directory(
|
||||||
|
dirs: &Path,
|
||||||
|
exts: Option<&BTreeSet<&str>>,
|
||||||
|
exclude: Option<&BTreeSet<&str>>,
|
||||||
|
scan_opts: &ScanOpts,
|
||||||
|
) -> Option<Vec<DirEntry>> {
|
||||||
let stepper = WalkDir::new(dirs).follow_links(scan_opts.follow_symlinks).into_iter();
|
let stepper = WalkDir::new(dirs).follow_links(scan_opts.follow_symlinks).into_iter();
|
||||||
let mut probably_fatal_error = false;
|
let mut probably_fatal_error = false;
|
||||||
let entries: Vec<DirEntry> = stepper
|
let entries: Vec<DirEntry> = stepper
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
use crate::string_type::String as StringType;
|
use crate::string_type::String as StringType;
|
||||||
use cfg_if::cfg_if;
|
use cfg_if::cfg_if;
|
||||||
use clap::{AppSettings, Clap};
|
use clap::{AppSettings, Clap};
|
||||||
|
use std::collections::BTreeSet;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
cfg_if! {
|
cfg_if! {
|
||||||
|
@ -18,9 +19,8 @@ pub enum OutputFormat {
|
||||||
/// A Bourne shell compatible script.
|
/// A Bourne shell compatible script.
|
||||||
Sh,
|
Sh,
|
||||||
/// A PowerShell script.
|
/// A PowerShell script.
|
||||||
|
#[clap(alias = "powershell")]
|
||||||
PowerShell,
|
PowerShell,
|
||||||
/// Also a PowerShell script, with different casing to allow for `fif -o powershell`.
|
|
||||||
Powershell,
|
|
||||||
/// Plain text.
|
/// Plain text.
|
||||||
Text,
|
Text,
|
||||||
}
|
}
|
||||||
|
@ -40,26 +40,32 @@ pub enum OutputFormat {
|
||||||
setting(AppSettings::ColoredHelp)
|
setting(AppSettings::ColoredHelp)
|
||||||
)]
|
)]
|
||||||
pub struct Parameters {
|
pub struct Parameters {
|
||||||
/// Only examine files with these extensions (comma-separated list).
|
// NOTE: clap's comma-separated argument parser makes it impossible to specify extensions with commas in their name -
|
||||||
/// This argument conflicts with `-E`.
|
// `-e sil\,ly` is treated as ["sil", "ly"] rather than as ["silly"], no matter how i escape the comma (in bash,
|
||||||
#[clap(short, long, use_delimiter = true, require_delimiter = true, group = "extensions")]
|
// anyway). is this really an issue? it does technically exclude some perfectly valid extensions, but i've never seen
|
||||||
|
// a file extension with a comma in its name before.
|
||||||
|
/// Only examine files with these extensions.
|
||||||
|
/// Multiple extensions can be specified by either using the flag multiple times (`-e jpg -e png -e gif`), or by
|
||||||
|
/// separating them with commas (`-e jpg,png,gif`).
|
||||||
|
#[clap(short, long, use_delimiter = true, require_delimiter = true)]
|
||||||
pub exts: Option<Vec<StringType>>,
|
pub exts: Option<Vec<StringType>>,
|
||||||
|
|
||||||
/// Use a preset list of extensions as the search filter.
|
/// Use these preset lists of extensions as the search filter (comma-separated list).
|
||||||
/// `media` includes all extensions from the `audio`, `video`, and `images` sets. This argument conflicts with `-e`.
|
/// `media` includes all extensions from the `audio`, `video`, and `images` sets, making `-E media` equivalent to
|
||||||
#[clap(short = 'E', long, arg_enum, group = "extensions")]
|
/// `-E audio,video,images`.
|
||||||
pub ext_set: Option<ExtensionSet>,
|
#[clap(short = 'E', long, arg_enum, use_delimiter = true, require_delimiter = true)]
|
||||||
|
pub ext_set: Vec<ExtensionSet>,
|
||||||
|
|
||||||
/// Don't scan files with these extensions (comma-separated list).
|
/// Don't scan files with these extensions.
|
||||||
/// This option takes preference over files specified with -e or -E.
|
/// This option takes precedence over extensions specified with `-e` or `-E`.
|
||||||
#[clap(
|
#[clap(short = 'x', long, use_delimiter = true, require_delimiter = true)]
|
||||||
short = 'x',
|
|
||||||
long,
|
|
||||||
use_delimiter = true,
|
|
||||||
require_delimiter = true,
|
|
||||||
)]
|
|
||||||
pub exclude: Option<Vec<StringType>>,
|
pub exclude: Option<Vec<StringType>>,
|
||||||
|
|
||||||
|
/// Exclude files using a preset list of extensions.
|
||||||
|
/// This option takes precedence over extensions specified with `-e` or `-E`.
|
||||||
|
#[clap(short = 'X', long, arg_enum, use_delimiter = true, require_delimiter = true)]
|
||||||
|
pub exclude_set: Vec<ExtensionSet>,
|
||||||
|
|
||||||
/// Don't skip hidden files and directories.
|
/// Don't skip hidden files and directories.
|
||||||
/// Even if this flag is not present, fif will still recurse into a hidden root directory - for example, `fif
|
/// Even if this flag is not present, fif will still recurse into a hidden root directory - for example, `fif
|
||||||
/// ~/.hidden` will recurse into `~/.hidden` regardless of whether or not -s was passed as an argument.
|
/// ~/.hidden` will recurse into `~/.hidden` regardless of whether or not -s was passed as an argument.
|
||||||
|
@ -105,38 +111,62 @@ pub struct ScanOpts {
|
||||||
|
|
||||||
impl Parameters {
|
impl Parameters {
|
||||||
/// Returns an optional vec of the extensions to be scanned - i.e., extensions specified via the `-e` or `-E` flag,
|
/// Returns an optional vec of the extensions to be scanned - i.e., extensions specified via the `-e` or `-E` flag,
|
||||||
/// minus the extensions excluded with the `-x` flag.
|
/// minus the extensions excluded with the `-x` flag; i.e., the difference between the included and excluded sets.
|
||||||
pub fn extensions(&self) -> Option<Vec<&str>> {
|
pub fn extensions(&self) -> Option<BTreeSet<&str>> {
|
||||||
let empty_vec = vec![];
|
if let Some(included) = self.included_extensions() {
|
||||||
let exclude = &self.excluded_extensions().unwrap_or(empty_vec);
|
if let Some(excluded) = self.excluded_extensions() {
|
||||||
|
// return included extensions without excluded extensions
|
||||||
// TODO: bleugh
|
// ...maybe i should have called them "suffixes" instead of extensions...
|
||||||
if let Some(exts) = &self.exts {
|
Some(included.into_iter().filter(|ext| !excluded.contains(ext)).collect())
|
||||||
// extensions supplied like "-e png,jpg,jpeg"
|
} else {
|
||||||
Some(
|
// no extensions excluded - just return all included
|
||||||
exts
|
Some(included)
|
||||||
.iter()
|
}
|
||||||
.map(|ext| ext.as_str())
|
|
||||||
.filter(|ext| !exclude.contains(ext))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
} else if let Some(exts) = &self.ext_set {
|
|
||||||
// extensions supplied like "-E images"
|
|
||||||
Some(
|
|
||||||
exts
|
|
||||||
.extensions()
|
|
||||||
.into_iter()
|
|
||||||
.filter(|ext| !exclude.contains(ext))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
// neither -E nor -e was passed
|
// no extensions included - return none
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn excluded_extensions(&self) -> Option<Vec<&str>> {
|
/// Returns an optional vec of extensions that were specified by `-e` or `-E`. Note that this doesn't account for
|
||||||
self.exclude.as_ref().map(|exclude| exclude.iter().map(|ext| ext.as_str()).collect())
|
/// extensions excluded by the exclusion flags.
|
||||||
|
pub fn included_extensions(&self) -> Option<BTreeSet<&str>> {
|
||||||
|
let mut included = BTreeSet::new();
|
||||||
|
if let Some(exts) = self.exts.as_ref() {
|
||||||
|
// -e
|
||||||
|
included.extend(exts.iter().map(|ext| ext.as_str()));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !&self.ext_set.is_empty() {
|
||||||
|
// -E
|
||||||
|
included.extend(self.ext_set.iter().flat_map(|set| set.extensions()));
|
||||||
|
}
|
||||||
|
|
||||||
|
match included {
|
||||||
|
x if x.is_empty() => None,
|
||||||
|
x => Some(x),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns an optional vec of extensions that were specified by `-x` or `-X`.
|
||||||
|
pub fn excluded_extensions(&self) -> Option<BTreeSet<&str>> {
|
||||||
|
let mut excluded = BTreeSet::new();
|
||||||
|
if let Some(exclude) = self.exclude.as_ref() {
|
||||||
|
// -x
|
||||||
|
excluded.extend(exclude.iter().map(|ext| ext.as_str()));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !&self.exclude_set.is_empty() {
|
||||||
|
// -X
|
||||||
|
excluded.extend(self.exclude_set.iter().flat_map(|set| set.extensions()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// excluded doesn't sound like a word anymore
|
||||||
|
// tongue twister: enter X-options' excellent extension exclusion
|
||||||
|
match excluded {
|
||||||
|
x if x.is_empty() => None,
|
||||||
|
x => Some(x),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub const fn get_scan_opts(&self) -> ScanOpts {
|
pub const fn get_scan_opts(&self) -> ScanOpts {
|
||||||
|
@ -168,7 +198,7 @@ pub enum ExtensionSet {
|
||||||
/// Extensions used for audio file formats, such as `mp3`, `ogg`, `flac`, etc.
|
/// Extensions used for audio file formats, such as `mp3`, `ogg`, `flac`, etc.
|
||||||
Audio,
|
Audio,
|
||||||
/// Extensions used for video file formats, such as `mkv`, `mp4`, `mov`, etc.
|
/// Extensions used for video file formats, such as `mkv`, `mp4`, `mov`, etc.
|
||||||
Videos,
|
Video,
|
||||||
/// Extensions used for media file formats. This acts as a combination of the [Images](ExtensionSet::Images),
|
/// Extensions used for media file formats. This acts as a combination of the [Images](ExtensionSet::Images),
|
||||||
/// [Audio](ExtensionSet::Audio) and [Videos](ExtensionSet::Videos) variants.
|
/// [Audio](ExtensionSet::Audio) and [Videos](ExtensionSet::Videos) variants.
|
||||||
Media,
|
Media,
|
||||||
|
@ -188,24 +218,27 @@ impl ExtensionSet {
|
||||||
match self {
|
match self {
|
||||||
Self::Images => mime_guess::get_mime_extensions_str("image/*").unwrap().to_vec(),
|
Self::Images => mime_guess::get_mime_extensions_str("image/*").unwrap().to_vec(),
|
||||||
Self::Audio => mime_guess::get_mime_extensions_str("audio/*").unwrap().to_vec(),
|
Self::Audio => mime_guess::get_mime_extensions_str("audio/*").unwrap().to_vec(),
|
||||||
Self::Videos => mime_guess::get_mime_extensions_str("video/*").unwrap().to_vec(),
|
Self::Video => mime_guess::get_mime_extensions_str("video/*").unwrap().to_vec(),
|
||||||
Self::Media => [
|
Self::Media => [
|
||||||
Self::Images.extensions(),
|
Self::Images.extensions(),
|
||||||
Self::Audio.extensions(),
|
Self::Audio.extensions(),
|
||||||
Self::Videos.extensions(),
|
Self::Video.extensions(),
|
||||||
]
|
]
|
||||||
.concat(),
|
.concat(),
|
||||||
Self::Documents => vec![
|
Self::Documents => vec![
|
||||||
"pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "csv", "tsv", "odt", "ods", "odp", "oda", "rtf", "ps",
|
"pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "csv", "tsv", "odt", "ods", "odp", "oda", "rtf", "ps",
|
||||||
"pages", "key", "numbers",
|
"pages", "key", "numbers",
|
||||||
],
|
],
|
||||||
Self::Text => mime_guess::get_mime_extensions_str("text/*").unwrap().to_vec(),
|
Self::Text => [mime_guess::get_mime_extensions_str("text/*").unwrap(), &["js", "pl", "csh", "sh", "bash", "zsh", "fish", "bat", "php"]].concat(),
|
||||||
// many compressed file types follow the name scheme "application/x.+compressed.*" - maybe this can be used
|
// many compressed file types follow the name scheme "application/x.+compressed.*" - maybe this can be used
|
||||||
// somehow to extract extensions for compressed files from mime_guess?
|
// somehow to extract extensions for compressed files from mime_guess?
|
||||||
Self::Archives => vec!["zip", "tar", "gz", "zst", "xz", "rar", "7z", "bz", "bz2", "tgz", "rpa"],
|
Self::Archives => vec![
|
||||||
|
"zip", "tar", "gz", "zst", "xz", "rar", "7z", "bz", "bz2", "tgz", "rpa", "txz", "tz2", "sea", "sitx", "z",
|
||||||
|
"cpio",
|
||||||
|
],
|
||||||
Self::System => vec![
|
Self::System => vec![
|
||||||
"com", "dll", "exe", "sys", "reg", "nt", "cpl", "msi", "efi", "bio", "rcv", "mbr", "sbf", "grub", "ko",
|
"com", "dll", "exe", "sys", "reg", "nt", "cpl", "msi", "efi", "bio", "rcv", "mbr", "sbf", "grub", "ko",
|
||||||
"dylib", "pdb", "hdmp", "crash",
|
"dylib", "pdb", "hdmp", "crash", "cab",
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,12 +5,15 @@ use crate::mime_db::MimeDb;
|
||||||
use crate::string_type::String;
|
use crate::string_type::String;
|
||||||
use crate::{extension_from_path, scan_directory, scan_from_walkdir};
|
use crate::{extension_from_path, scan_directory, scan_from_walkdir};
|
||||||
|
|
||||||
|
use crate::parameters::Parameters;
|
||||||
|
use clap::Clap;
|
||||||
use mime_guess::mime::{APPLICATION_OCTET_STREAM, APPLICATION_PDF, IMAGE_JPEG, IMAGE_PNG};
|
use mime_guess::mime::{APPLICATION_OCTET_STREAM, APPLICATION_PDF, IMAGE_JPEG, IMAGE_PNG};
|
||||||
use mime_guess::Mime;
|
use mime_guess::Mime;
|
||||||
|
|
||||||
|
use crate::parameters::ExtensionSet;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::ffi::OsStr;
|
use std::ffi::OsStr;
|
||||||
use std::path::Path;
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
const JPEG_BYTES: &[u8] = b"\xFF\xD8\xFF";
|
const JPEG_BYTES: &[u8] = b"\xFF\xD8\xFF";
|
||||||
const PNG_BYTES: &[u8] = b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A";
|
const PNG_BYTES: &[u8] = b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A";
|
||||||
|
@ -167,17 +170,17 @@ fn simple_directory() {
|
||||||
#[test]
|
#[test]
|
||||||
/// Ensure that command line argument parsing works correctly - flags are interpreted, booleans are set, and so on.
|
/// Ensure that command line argument parsing works correctly - flags are interpreted, booleans are set, and so on.
|
||||||
fn argument_parsing() {
|
fn argument_parsing() {
|
||||||
use crate::parameters::{Parameters, ScanOpts};
|
use crate::parameters::ScanOpts;
|
||||||
use clap::Clap;
|
|
||||||
|
|
||||||
// pass `-f`, which enables following symlinks, and `-E images`, which scans files with image extensions
|
// pass `-f`, which enables following symlinks, and `-E images`, which scans files with image extensions
|
||||||
let args: Parameters = Parameters::parse_from(vec!["fif", "-f", "-E", "images"]);
|
let args: Parameters = Parameters::parse_from(vec!["fif", "-f", "-E", "images"]);
|
||||||
|
|
||||||
// check if "jpg" is in the list of extensions to be scanned
|
// check if "jpg" is in the list of extensions to be scanned
|
||||||
assert!(args
|
assert!(
|
||||||
.extensions()
|
args
|
||||||
.expect("args.extensions() should be Some(_)!")
|
.extensions()
|
||||||
.contains(&"jpg"),
|
.expect("args.extensions() should be Some(_)!")
|
||||||
|
.contains(&"jpg"),
|
||||||
"args.extensions() should contain the `images` set!"
|
"args.extensions() should contain the `images` set!"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -203,11 +206,19 @@ fn argument_parsing() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
/// Ensure exclude overrides `-e` and `-E`.
|
/// Ensure that `fif -e jpg dir` is interpreted as "scan for jpg files in dir" and not "scan for jpg and dir files"
|
||||||
fn exclude_overrides() {
|
fn positional_args() {
|
||||||
use crate::parameters::{Parameters};
|
for flag in &["-x", "-e", "-X", "-E"] {
|
||||||
use clap::Clap;
|
assert_eq!(
|
||||||
|
Parameters::parse_from(vec!["fif", flag, "images", "directory"]).dirs,
|
||||||
|
PathBuf::from("directory")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
/// Ensure the `exclude` flag (`-x`) overrides `-e` and `-E`.
|
||||||
|
fn exclude_overrides() {
|
||||||
// pass `-E images`, which includes many image extensions, and `-x jpg,png`, which should remove "jpg" and "png" from
|
// pass `-E images`, which includes many image extensions, and `-x jpg,png`, which should remove "jpg" and "png" from
|
||||||
// the extensions list
|
// the extensions list
|
||||||
let args: Parameters = Parameters::parse_from(vec!["fif", "-x", "jpg,png", "-E", "images"]);
|
let args: Parameters = Parameters::parse_from(vec!["fif", "-x", "jpg,png", "-E", "images"]);
|
||||||
|
@ -231,13 +242,46 @@ fn exclude_overrides() {
|
||||||
assert!(extensions.contains(&"jkl"));
|
assert!(extensions.contains(&"jkl"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
/// Ensure the `exclude_set` flag (`-X`) overrides `-e`.
|
||||||
|
fn exclude_set_overrides_includes() {
|
||||||
|
// pass `-e jpg,flac` and `-X images` -- which should produce the equivalent of `-e flag`
|
||||||
|
let args: Parameters = Parameters::parse_from(vec!["fif", "-e", "jpg,flac", "-X", "images"]);
|
||||||
|
let extensions = args.extensions();
|
||||||
|
assert!(extensions.is_some(), "Extensions should be set!");
|
||||||
|
let mut extensions = extensions.unwrap().into_iter();
|
||||||
|
|
||||||
|
assert_eq!(extensions.next(), Some("flac"), "Extensions should contain flac!");
|
||||||
|
assert_eq!(extensions.next(), None, "Too many extensions!");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
/// Ensure the `exclude_set` flag (`-X`) overrides `-E`.
|
||||||
|
fn exclude_set_overrides_include_set() {
|
||||||
|
// pass `-E media` and `-X images` -- which should produce the equivalent of `-E audio,video`
|
||||||
|
let args: Parameters = Parameters::parse_from(vec!["fif", "-E", "media", "-X", "images"]);
|
||||||
|
let extensions = args.extensions();
|
||||||
|
assert!(extensions.is_some(), "Extensions should be set!");
|
||||||
|
let extensions = extensions.unwrap();
|
||||||
|
|
||||||
|
// ensure all of audio and video's extensions are here
|
||||||
|
for &ext in ExtensionSet::Audio
|
||||||
|
.extensions()
|
||||||
|
.iter()
|
||||||
|
.chain(ExtensionSet::Video.extensions().iter())
|
||||||
|
{
|
||||||
|
assert!(extensions.contains(&ext), "Extensions should contain {}!", ext)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensure all of images' extensions are excluded
|
||||||
|
for ext in ExtensionSet::Images.extensions() {
|
||||||
|
assert!(!extensions.contains(&ext), "Extensions should not contain {}!", ext)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
/// Ensure that badly formed command line arguments are rejected.
|
/// Ensure that badly formed command line arguments are rejected.
|
||||||
fn rejects_bad_args() {
|
fn rejects_bad_args() {
|
||||||
use crate::parameters::Parameters;
|
|
||||||
|
|
||||||
use clap::Clap;
|
|
||||||
let tests = [
|
let tests = [
|
||||||
// Non-existent flags:
|
// Non-existent flags:
|
||||||
vec!["fif", "-abcdefghijklmnopqrstuvwxyz"],
|
vec!["fif", "-abcdefghijklmnopqrstuvwxyz"],
|
||||||
|
@ -245,8 +289,8 @@ fn rejects_bad_args() {
|
||||||
vec!["fif", "-E"],
|
vec!["fif", "-E"],
|
||||||
// `-E` with an invalid set:
|
// `-E` with an invalid set:
|
||||||
vec!["fif", "-E", "pebis"],
|
vec!["fif", "-E", "pebis"],
|
||||||
// `-E` and `-e`:
|
// `-X` with an invalid set:
|
||||||
vec!["fif", "-E", "media", "-e", "jpg"],
|
vec!["fif", "-X", "pebis"],
|
||||||
// `-e` with nothing but commas:
|
// `-e` with nothing but commas:
|
||||||
vec!["fif", "-e", ",,,,,"],
|
vec!["fif", "-e", ",,,,,"],
|
||||||
];
|
];
|
||||||
|
@ -312,14 +356,19 @@ fn outputs_move_commands() {
|
||||||
#[test]
|
#[test]
|
||||||
/// Ensure that the Media extension set contains all (is a superset) of Audio, Video, and Images.
|
/// Ensure that the Media extension set contains all (is a superset) of Audio, Video, and Images.
|
||||||
fn media_contains_audio_video_images() {
|
fn media_contains_audio_video_images() {
|
||||||
use crate::parameters::ExtensionSet::{Audio, Images, Media, Videos};
|
use crate::parameters::ExtensionSet::{Audio, Images, Media, Video};
|
||||||
let media_exts = Media.extensions();
|
let media_exts = Media.extensions();
|
||||||
|
|
||||||
// assert every extension in the audio/video/image sets is contained in the media set
|
// assert every extension in the audio/video/image sets is contained in the media set
|
||||||
[Audio.extensions(), Videos.extensions(), Images.extensions()]
|
[Audio.extensions(), Video.extensions(), Images.extensions()]
|
||||||
.concat()
|
.concat()
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.for_each(|ext| assert!(media_exts.contains(&ext)));
|
.for_each(|ext| assert!(media_exts.contains(&ext)));
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
Parameters::parse_from(&["fif", "-E", "media"]).extensions(),
|
||||||
|
Parameters::parse_from(&["fif", "-E", "audio,video,images"]).extensions()
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
Loading…
Reference in a new issue