feat: add infer and tree_magic_mini meta plugins, make zstd internal by default
- Add infer crate as meta plugin for MIME type detection - Add tree_magic_mini crate as alternative meta plugin for MIME type detection - Add zstd, infer, tree_magic_mini to default features - Fix static build script to use musl target instead of glibc+crt-static - Remove hardcoded shell list from --generate-completion help text - Fix update() in both new plugins to emit MIME metadata when buffer fills
This commit is contained in:
69
Cargo.lock
generated
69
Cargo.lock
generated
@@ -378,6 +378,17 @@ dependencies = [
|
|||||||
"shlex",
|
"shlex",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cfb"
|
||||||
|
version = "0.7.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
"fnv",
|
||||||
|
"uuid",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cfg-if"
|
name = "cfg-if"
|
||||||
version = "1.0.4"
|
version = "1.0.4"
|
||||||
@@ -1020,6 +1031,12 @@ version = "0.1.9"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fixedbitset"
|
||||||
|
version = "0.5.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "flate2"
|
name = "flate2"
|
||||||
version = "1.1.9"
|
version = "1.1.9"
|
||||||
@@ -1553,6 +1570,15 @@ dependencies = [
|
|||||||
"serde_core",
|
"serde_core",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "infer"
|
||||||
|
version = "0.19.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7"
|
||||||
|
dependencies = [
|
||||||
|
"cfb",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "inventory"
|
name = "inventory"
|
||||||
version = "0.3.22"
|
version = "0.3.22"
|
||||||
@@ -1686,6 +1712,7 @@ dependencies = [
|
|||||||
"http-body-util",
|
"http-body-util",
|
||||||
"humansize",
|
"humansize",
|
||||||
"hyper",
|
"hyper",
|
||||||
|
"infer",
|
||||||
"inventory",
|
"inventory",
|
||||||
"is-terminal",
|
"is-terminal",
|
||||||
"jsonwebtoken",
|
"jsonwebtoken",
|
||||||
@@ -1726,6 +1753,7 @@ dependencies = [
|
|||||||
"tokio-util",
|
"tokio-util",
|
||||||
"tower",
|
"tower",
|
||||||
"tower-http",
|
"tower-http",
|
||||||
|
"tree_magic_mini",
|
||||||
"ureq",
|
"ureq",
|
||||||
"utoipa",
|
"utoipa",
|
||||||
"utoipa-swagger-ui",
|
"utoipa-swagger-ui",
|
||||||
@@ -1975,6 +2003,15 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nom"
|
||||||
|
version = "8.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-bigint"
|
name = "num-bigint"
|
||||||
version = "0.4.6"
|
version = "0.4.6"
|
||||||
@@ -2141,6 +2178,17 @@ dependencies = [
|
|||||||
"sha2 0.10.9",
|
"sha2 0.10.9",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "petgraph"
|
||||||
|
version = "0.8.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
|
||||||
|
dependencies = [
|
||||||
|
"fixedbitset",
|
||||||
|
"hashbrown 0.15.5",
|
||||||
|
"indexmap",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pin-project-lite"
|
name = "pin-project-lite"
|
||||||
version = "0.2.17"
|
version = "0.2.17"
|
||||||
@@ -3184,6 +3232,17 @@ dependencies = [
|
|||||||
"once_cell",
|
"once_cell",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tree_magic_mini"
|
||||||
|
version = "3.2.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b8765b90061cba6c22b5831f675da109ae5561588290f9fa2317adab2714d5a6"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
"nom",
|
||||||
|
"petgraph",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "try-lock"
|
name = "try-lock"
|
||||||
version = "0.2.5"
|
version = "0.2.5"
|
||||||
@@ -3372,6 +3431,16 @@ dependencies = [
|
|||||||
"zip",
|
"zip",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "uuid"
|
||||||
|
version = "1.22.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
|
||||||
|
dependencies = [
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "uzers"
|
name = "uzers"
|
||||||
version = "0.12.2"
|
version = "0.12.2"
|
||||||
|
|||||||
20
Cargo.toml
20
Cargo.toml
@@ -41,6 +41,8 @@ log = "0.4"
|
|||||||
lz4_flex = { version = "0.12", optional = true }
|
lz4_flex = { version = "0.12", optional = true }
|
||||||
zstd = { version = "0.13", optional = true }
|
zstd = { version = "0.13", optional = true }
|
||||||
magic = { version = "0.13", optional = true }
|
magic = { version = "0.13", optional = true }
|
||||||
|
infer = { version = "0.19", optional = true }
|
||||||
|
tree_magic_mini = { version = "3.2", optional = true }
|
||||||
nix = { version = "0.30", features = ["fs", "process"] }
|
nix = { version = "0.30", features = ["fs", "process"] }
|
||||||
once_cell = "1.21"
|
once_cell = "1.21"
|
||||||
comfy-table = "7.2"
|
comfy-table = "7.2"
|
||||||
@@ -83,11 +85,15 @@ tiktoken-rs = { version = "0.9", optional = true }
|
|||||||
|
|
||||||
[features]
|
[features]
|
||||||
# Default features include core compression engines and swagger UI
|
# Default features include core compression engines and swagger UI
|
||||||
default = ["magic", "lz4", "gzip", "client", "tokens"]
|
default = [
|
||||||
|
"client",
|
||||||
# Full
|
"gzip",
|
||||||
#default = ["server", "magic", "lz4", "swagger"]
|
"infer",
|
||||||
|
"lz4",
|
||||||
|
"tokens",
|
||||||
|
"tree_magic_mini",
|
||||||
|
"zstd"
|
||||||
|
]
|
||||||
|
|
||||||
# Server feature (includes axum and related dependencies)
|
# Server feature (includes axum and related dependencies)
|
||||||
server = ["dep:axum", "dep:tower", "dep:tower-http", "dep:utoipa", "dep:jsonwebtoken"]
|
server = ["dep:axum", "dep:tower", "dep:tower-http", "dep:utoipa", "dep:jsonwebtoken"]
|
||||||
@@ -100,11 +106,13 @@ xz = []
|
|||||||
zstd = ["dep:zstd"]
|
zstd = ["dep:zstd"]
|
||||||
|
|
||||||
# Plugin features (meta and filter)
|
# Plugin features (meta and filter)
|
||||||
all-meta-plugins = ["dep:magic"]
|
all-meta-plugins = ["dep:magic", "dep:infer", "dep:tree_magic_mini"]
|
||||||
all-filter-plugins = []
|
all-filter-plugins = []
|
||||||
|
|
||||||
# Individual plugin features
|
# Individual plugin features
|
||||||
magic = ["dep:magic"]
|
magic = ["dep:magic"]
|
||||||
|
infer = ["dep:infer"]
|
||||||
|
tree_magic_mini = ["dep:tree_magic_mini"]
|
||||||
|
|
||||||
# Swagger UI feature
|
# Swagger UI feature
|
||||||
swagger = ["dep:utoipa-swagger-ui"]
|
swagger = ["dep:utoipa-swagger-ui"]
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
|
|
||||||
set -ex
|
set -ex
|
||||||
|
|
||||||
export RUSTFLAGS='-C target-feature=+crt-static'
|
cargo build --release --target x86_64-unknown-linux-musl
|
||||||
cargo build --release --target x86_64-unknown-linux-gnu
|
|
||||||
mkdir -p bin
|
mkdir -p bin
|
||||||
cp target/x86_64-unknown-linux-gnu/release/keep ./bin/
|
cp target/x86_64-unknown-linux-musl/release/keep ./bin/
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ pub struct ModeArgs {
|
|||||||
pub generate_config: bool,
|
pub generate_config: bool,
|
||||||
|
|
||||||
#[arg(help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "generate_config", "export", "import"]))]
|
#[arg(help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "generate_config", "export", "import"]))]
|
||||||
#[arg(help("Generate shell completion script (bash, zsh, fish, elvish, powershell)"))]
|
#[arg(help("Generate shell completion script"))]
|
||||||
pub generate_completion: Option<Shell>,
|
pub generate_completion: Option<Shell>,
|
||||||
|
|
||||||
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_ADDRESS"))]
|
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_ADDRESS"))]
|
||||||
|
|||||||
@@ -11,12 +11,12 @@ use std::io::{Read, Write};
|
|||||||
#[cfg(feature = "gzip")]
|
#[cfg(feature = "gzip")]
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
#[cfg(feature = "gzip")]
|
||||||
|
use flate2::Compression;
|
||||||
#[cfg(feature = "gzip")]
|
#[cfg(feature = "gzip")]
|
||||||
use flate2::read::GzDecoder;
|
use flate2::read::GzDecoder;
|
||||||
#[cfg(feature = "gzip")]
|
#[cfg(feature = "gzip")]
|
||||||
use flate2::write::GzEncoder;
|
use flate2::write::GzEncoder;
|
||||||
#[cfg(feature = "gzip")]
|
|
||||||
use flate2::Compression;
|
|
||||||
|
|
||||||
#[cfg(feature = "gzip")]
|
#[cfg(feature = "gzip")]
|
||||||
use crate::compression_engine::CompressionEngine;
|
use crate::compression_engine::CompressionEngine;
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use anyhow::{anyhow, Result};
|
use anyhow::{Result, anyhow};
|
||||||
use std::io;
|
use std::io;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use anyhow::{anyhow, Context, Result};
|
use anyhow::{Context, Result, anyhow};
|
||||||
use log::*;
|
use log::*;
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
|
|||||||
@@ -71,6 +71,14 @@ use crate::meta_plugin::magic_file;
|
|||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
use crate::meta_plugin::tokens;
|
use crate::meta_plugin::tokens;
|
||||||
|
|
||||||
|
#[cfg(feature = "infer")]
|
||||||
|
#[allow(unused_imports)]
|
||||||
|
use crate::meta_plugin::infer_plugin;
|
||||||
|
|
||||||
|
#[cfg(feature = "tree_magic_mini")]
|
||||||
|
#[allow(unused_imports)]
|
||||||
|
use crate::meta_plugin::tree_magic_mini;
|
||||||
|
|
||||||
/// Initializes plugins at library load time.
|
/// Initializes plugins at library load time.
|
||||||
///
|
///
|
||||||
/// Plugin registration happens automatically via `#[ctor]` constructors
|
/// Plugin registration happens automatically via `#[ctor]` constructors
|
||||||
|
|||||||
177
src/meta_plugin/infer_plugin.rs
Normal file
177
src/meta_plugin/infer_plugin.rs
Normal file
@@ -0,0 +1,177 @@
|
|||||||
|
use crate::common::PIPESIZE;
|
||||||
|
use crate::meta_plugin::{
|
||||||
|
process_metadata_outputs, register_meta_plugin, BaseMetaPlugin, MetaPlugin, MetaPluginResponse,
|
||||||
|
MetaPluginType,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct InferMetaPlugin {
|
||||||
|
buffer: Vec<u8>,
|
||||||
|
max_buffer_size: usize,
|
||||||
|
is_finalized: bool,
|
||||||
|
base: BaseMetaPlugin,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl InferMetaPlugin {
|
||||||
|
pub fn new(
|
||||||
|
options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
|
||||||
|
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
|
||||||
|
) -> InferMetaPlugin {
|
||||||
|
let mut base = BaseMetaPlugin::new();
|
||||||
|
|
||||||
|
if let Some(opts) = options {
|
||||||
|
for (key, value) in opts {
|
||||||
|
base.options.insert(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let max_buffer_size = base
|
||||||
|
.options
|
||||||
|
.get("max_buffer_size")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.unwrap_or(PIPESIZE as u64) as usize;
|
||||||
|
|
||||||
|
base.outputs.insert(
|
||||||
|
"infer_mime_type".to_string(),
|
||||||
|
serde_yaml::Value::String("infer_mime_type".to_string()),
|
||||||
|
);
|
||||||
|
|
||||||
|
if let Some(outs) = outputs {
|
||||||
|
for (key, value) in outs {
|
||||||
|
base.outputs.insert(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
InferMetaPlugin {
|
||||||
|
buffer: Vec::new(),
|
||||||
|
max_buffer_size,
|
||||||
|
is_finalized: false,
|
||||||
|
base,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MetaPlugin for InferMetaPlugin {
|
||||||
|
fn meta_type(&self) -> MetaPluginType {
|
||||||
|
MetaPluginType::Infer
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_finalized(&self) -> bool {
|
||||||
|
self.is_finalized
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_finalized(&mut self, finalized: bool) {
|
||||||
|
self.is_finalized = finalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_save_meta(&mut self, save_meta: crate::meta_plugin::SaveMetaFn) {
|
||||||
|
self.base.set_save_meta(save_meta);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn save_meta(&self, name: &str, value: &str) {
|
||||||
|
self.base.save_meta(name, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
|
||||||
|
if self.is_finalized {
|
||||||
|
return MetaPluginResponse {
|
||||||
|
metadata: Vec::new(),
|
||||||
|
is_finalized: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let remaining = self.max_buffer_size.saturating_sub(self.buffer.len());
|
||||||
|
let to_add = &data[..data.len().min(remaining)];
|
||||||
|
self.buffer.extend_from_slice(to_add);
|
||||||
|
|
||||||
|
if self.buffer.len() >= self.max_buffer_size {
|
||||||
|
let mime_type = infer::get(&self.buffer)
|
||||||
|
.map(|kind| kind.mime_type().to_string())
|
||||||
|
.unwrap_or_else(|| "application/octet-stream".to_string());
|
||||||
|
|
||||||
|
self.is_finalized = true;
|
||||||
|
|
||||||
|
let metadata = process_metadata_outputs(
|
||||||
|
"infer_mime_type",
|
||||||
|
serde_yaml::Value::String(mime_type),
|
||||||
|
self.base.outputs(),
|
||||||
|
)
|
||||||
|
.map(|m| vec![m])
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
return MetaPluginResponse {
|
||||||
|
metadata,
|
||||||
|
is_finalized: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
MetaPluginResponse {
|
||||||
|
metadata: Vec::new(),
|
||||||
|
is_finalized: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finalize(&mut self) -> MetaPluginResponse {
|
||||||
|
if self.is_finalized {
|
||||||
|
return MetaPluginResponse {
|
||||||
|
metadata: Vec::new(),
|
||||||
|
is_finalized: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let mime_type = infer::get(&self.buffer)
|
||||||
|
.map(|kind| kind.mime_type().to_string())
|
||||||
|
.unwrap_or_else(|| "application/octet-stream".to_string());
|
||||||
|
|
||||||
|
self.is_finalized = true;
|
||||||
|
|
||||||
|
let metadata = process_metadata_outputs(
|
||||||
|
"infer_mime_type",
|
||||||
|
serde_yaml::Value::String(mime_type),
|
||||||
|
self.base.outputs(),
|
||||||
|
)
|
||||||
|
.map(|m| vec![m])
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
MetaPluginResponse {
|
||||||
|
metadata,
|
||||||
|
is_finalized: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
|
||||||
|
self.base.outputs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn outputs_mut(
|
||||||
|
&mut self,
|
||||||
|
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
|
||||||
|
Ok(self.base.outputs_mut())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_outputs(&self) -> Vec<String> {
|
||||||
|
vec!["infer_mime_type".to_string()]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
|
||||||
|
self.base.options()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn options_mut(
|
||||||
|
&mut self,
|
||||||
|
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
|
||||||
|
Ok(self.base.options_mut())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parallel_safe(&self) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[ctor::ctor]
|
||||||
|
fn register_infer_plugin() {
|
||||||
|
register_meta_plugin(MetaPluginType::Infer, |options, outputs| {
|
||||||
|
Box::new(InferMetaPlugin::new(options, outputs))
|
||||||
|
})
|
||||||
|
.expect("Failed to register InferMetaPlugin");
|
||||||
|
}
|
||||||
@@ -9,6 +9,8 @@ pub mod digest;
|
|||||||
pub mod env;
|
pub mod env;
|
||||||
pub mod exec;
|
pub mod exec;
|
||||||
pub mod hostname;
|
pub mod hostname;
|
||||||
|
#[cfg(feature = "infer")]
|
||||||
|
pub mod infer_plugin;
|
||||||
pub mod keep_pid;
|
pub mod keep_pid;
|
||||||
pub mod magic_file;
|
pub mod magic_file;
|
||||||
pub mod read_rate;
|
pub mod read_rate;
|
||||||
@@ -18,6 +20,8 @@ pub mod shell_pid;
|
|||||||
pub mod text;
|
pub mod text;
|
||||||
#[cfg(feature = "tokens")]
|
#[cfg(feature = "tokens")]
|
||||||
pub mod tokens;
|
pub mod tokens;
|
||||||
|
#[cfg(feature = "tree_magic_mini")]
|
||||||
|
pub mod tree_magic_mini;
|
||||||
pub mod user;
|
pub mod user;
|
||||||
|
|
||||||
pub use digest::DigestMetaPlugin;
|
pub use digest::DigestMetaPlugin;
|
||||||
@@ -28,11 +32,15 @@ pub use magic_file::MagicFileMetaPlugin;
|
|||||||
pub use cwd::CwdMetaPlugin;
|
pub use cwd::CwdMetaPlugin;
|
||||||
pub use env::EnvMetaPlugin;
|
pub use env::EnvMetaPlugin;
|
||||||
pub use hostname::HostnameMetaPlugin;
|
pub use hostname::HostnameMetaPlugin;
|
||||||
|
#[cfg(feature = "infer")]
|
||||||
|
pub use infer_plugin::InferMetaPlugin;
|
||||||
pub use keep_pid::KeepPidMetaPlugin;
|
pub use keep_pid::KeepPidMetaPlugin;
|
||||||
pub use read_rate::ReadRateMetaPlugin;
|
pub use read_rate::ReadRateMetaPlugin;
|
||||||
pub use read_time::ReadTimeMetaPlugin;
|
pub use read_time::ReadTimeMetaPlugin;
|
||||||
pub use shell::ShellMetaPlugin;
|
pub use shell::ShellMetaPlugin;
|
||||||
pub use shell_pid::ShellPidMetaPlugin;
|
pub use shell_pid::ShellPidMetaPlugin;
|
||||||
|
#[cfg(feature = "tree_magic_mini")]
|
||||||
|
pub use tree_magic_mini::TreeMagicMiniMetaPlugin;
|
||||||
pub use user::UserMetaPlugin;
|
pub use user::UserMetaPlugin;
|
||||||
|
|
||||||
#[cfg(not(feature = "magic"))]
|
#[cfg(not(feature = "magic"))]
|
||||||
@@ -263,6 +271,8 @@ pub enum MetaPluginType {
|
|||||||
Exec,
|
Exec,
|
||||||
Env,
|
Env,
|
||||||
Tokens,
|
Tokens,
|
||||||
|
TreeMagicMini,
|
||||||
|
Infer,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Central function to handle metadata output with name mapping.
|
/// Central function to handle metadata output with name mapping.
|
||||||
|
|||||||
173
src/meta_plugin/tree_magic_mini.rs
Normal file
173
src/meta_plugin/tree_magic_mini.rs
Normal file
@@ -0,0 +1,173 @@
|
|||||||
|
use crate::common::PIPESIZE;
|
||||||
|
use crate::meta_plugin::{
|
||||||
|
process_metadata_outputs, register_meta_plugin, BaseMetaPlugin, MetaPlugin, MetaPluginResponse,
|
||||||
|
MetaPluginType,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct TreeMagicMiniMetaPlugin {
|
||||||
|
buffer: Vec<u8>,
|
||||||
|
max_buffer_size: usize,
|
||||||
|
is_finalized: bool,
|
||||||
|
base: BaseMetaPlugin,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TreeMagicMiniMetaPlugin {
|
||||||
|
pub fn new(
|
||||||
|
options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
|
||||||
|
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
|
||||||
|
) -> TreeMagicMiniMetaPlugin {
|
||||||
|
let mut base = BaseMetaPlugin::new();
|
||||||
|
|
||||||
|
if let Some(opts) = options {
|
||||||
|
for (key, value) in opts {
|
||||||
|
base.options.insert(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let max_buffer_size = base
|
||||||
|
.options
|
||||||
|
.get("max_buffer_size")
|
||||||
|
.and_then(|v| v.as_u64())
|
||||||
|
.unwrap_or(PIPESIZE as u64) as usize;
|
||||||
|
|
||||||
|
base.outputs.insert(
|
||||||
|
"tree_magic_mime_type".to_string(),
|
||||||
|
serde_yaml::Value::String("tree_magic_mime_type".to_string()),
|
||||||
|
);
|
||||||
|
|
||||||
|
if let Some(outs) = outputs {
|
||||||
|
for (key, value) in outs {
|
||||||
|
base.outputs.insert(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TreeMagicMiniMetaPlugin {
|
||||||
|
buffer: Vec::new(),
|
||||||
|
max_buffer_size,
|
||||||
|
is_finalized: false,
|
||||||
|
base,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MetaPlugin for TreeMagicMiniMetaPlugin {
|
||||||
|
fn meta_type(&self) -> MetaPluginType {
|
||||||
|
MetaPluginType::TreeMagicMini
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_finalized(&self) -> bool {
|
||||||
|
self.is_finalized
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_finalized(&mut self, finalized: bool) {
|
||||||
|
self.is_finalized = finalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_save_meta(&mut self, save_meta: crate::meta_plugin::SaveMetaFn) {
|
||||||
|
self.base.set_save_meta(save_meta);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn save_meta(&self, name: &str, value: &str) {
|
||||||
|
self.base.save_meta(name, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
|
||||||
|
if self.is_finalized {
|
||||||
|
return MetaPluginResponse {
|
||||||
|
metadata: Vec::new(),
|
||||||
|
is_finalized: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let remaining = self.max_buffer_size.saturating_sub(self.buffer.len());
|
||||||
|
let to_add = &data[..data.len().min(remaining)];
|
||||||
|
self.buffer.extend_from_slice(to_add);
|
||||||
|
|
||||||
|
if self.buffer.len() >= self.max_buffer_size {
|
||||||
|
let mime_type = tree_magic_mini::from_u8(&self.buffer);
|
||||||
|
|
||||||
|
self.is_finalized = true;
|
||||||
|
|
||||||
|
let metadata = process_metadata_outputs(
|
||||||
|
"tree_magic_mime_type",
|
||||||
|
serde_yaml::Value::String(mime_type.to_string()),
|
||||||
|
self.base.outputs(),
|
||||||
|
)
|
||||||
|
.map(|m| vec![m])
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
return MetaPluginResponse {
|
||||||
|
metadata,
|
||||||
|
is_finalized: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
MetaPluginResponse {
|
||||||
|
metadata: Vec::new(),
|
||||||
|
is_finalized: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn finalize(&mut self) -> MetaPluginResponse {
|
||||||
|
if self.is_finalized {
|
||||||
|
return MetaPluginResponse {
|
||||||
|
metadata: Vec::new(),
|
||||||
|
is_finalized: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let mime_type = tree_magic_mini::from_u8(&self.buffer);
|
||||||
|
|
||||||
|
self.is_finalized = true;
|
||||||
|
|
||||||
|
let metadata = process_metadata_outputs(
|
||||||
|
"tree_magic_mime_type",
|
||||||
|
serde_yaml::Value::String(mime_type.to_string()),
|
||||||
|
self.base.outputs(),
|
||||||
|
)
|
||||||
|
.map(|m| vec![m])
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
MetaPluginResponse {
|
||||||
|
metadata,
|
||||||
|
is_finalized: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
|
||||||
|
self.base.outputs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn outputs_mut(
|
||||||
|
&mut self,
|
||||||
|
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
|
||||||
|
Ok(self.base.outputs_mut())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_outputs(&self) -> Vec<String> {
|
||||||
|
vec!["tree_magic_mime_type".to_string()]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
|
||||||
|
self.base.options()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn options_mut(
|
||||||
|
&mut self,
|
||||||
|
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
|
||||||
|
Ok(self.base.options_mut())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parallel_safe(&self) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[ctor::ctor]
|
||||||
|
fn register_tree_magic_mini_plugin() {
|
||||||
|
register_meta_plugin(MetaPluginType::TreeMagicMini, |options, outputs| {
|
||||||
|
Box::new(TreeMagicMiniMetaPlugin::new(options, outputs))
|
||||||
|
})
|
||||||
|
.expect("Failed to register TreeMagicMiniMetaPlugin");
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
use crate::compression_engine::{get_compression_engine, CompressionType};
|
use crate::compression_engine::{CompressionType, get_compression_engine};
|
||||||
use crate::services::error::CoreError;
|
use crate::services::error::CoreError;
|
||||||
use anyhow::anyhow;
|
use anyhow::anyhow;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -187,8 +187,8 @@ impl CompressionService {
|
|||||||
) -> Box<dyn Write> {
|
) -> Box<dyn Write> {
|
||||||
match compression {
|
match compression {
|
||||||
CompressionType::GZip => {
|
CompressionType::GZip => {
|
||||||
use flate2::write::GzEncoder;
|
|
||||||
use flate2::Compression;
|
use flate2::Compression;
|
||||||
|
use flate2::write::GzEncoder;
|
||||||
Box::new(GzEncoder::new(writer, Compression::default()))
|
Box::new(GzEncoder::new(writer, Compression::default()))
|
||||||
}
|
}
|
||||||
CompressionType::LZ4 => Box::new(lz4_flex::frame::FrameEncoder::new(writer)),
|
CompressionType::LZ4 => Box::new(lz4_flex::frame::FrameEncoder::new(writer)),
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use crate::compression_engine::gzip::CompressionEngineGZip;
|
|
||||||
use crate::compression_engine::CompressionEngine;
|
use crate::compression_engine::CompressionEngine;
|
||||||
|
use crate::compression_engine::gzip::CompressionEngineGZip;
|
||||||
use crate::tests::common::test_helpers::test_compression_engine;
|
use crate::tests::common::test_helpers::test_compression_engine;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
33
src/tests/meta_plugin/infer_tests.rs
Normal file
33
src/tests/meta_plugin/infer_tests.rs
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::meta_plugin::MetaPlugin;
|
||||||
|
use crate::meta_plugin::infer_plugin::InferMetaPlugin;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_infer_meta_plugin() {
|
||||||
|
let plugin = InferMetaPlugin::new(None, None);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
plugin.meta_type(),
|
||||||
|
crate::meta_plugin::MetaPluginType::Infer
|
||||||
|
);
|
||||||
|
assert!(plugin.is_internal());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_infer_png_detection() {
|
||||||
|
let png_header: &[u8] = &[
|
||||||
|
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48,
|
||||||
|
0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00,
|
||||||
|
0x00, 0x90, 0x77, 0x53, 0xDE,
|
||||||
|
];
|
||||||
|
let mut plugin = InferMetaPlugin::new(None, None);
|
||||||
|
plugin.update(png_header);
|
||||||
|
let response = plugin.finalize();
|
||||||
|
|
||||||
|
assert!(response.is_finalized);
|
||||||
|
assert!(!response.metadata.is_empty());
|
||||||
|
assert_eq!(response.metadata[0].name, "infer_mime_type");
|
||||||
|
assert_eq!(response.metadata[0].value, "image/png");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,3 +2,11 @@
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub mod digest_tests;
|
pub mod digest_tests;
|
||||||
|
|
||||||
|
#[cfg(feature = "infer")]
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod infer_tests;
|
||||||
|
|
||||||
|
#[cfg(feature = "tree_magic_mini")]
|
||||||
|
#[cfg(test)]
|
||||||
|
pub mod tree_magic_mini_tests;
|
||||||
|
|||||||
33
src/tests/meta_plugin/tree_magic_mini_tests.rs
Normal file
33
src/tests/meta_plugin/tree_magic_mini_tests.rs
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::meta_plugin::MetaPlugin;
|
||||||
|
use crate::meta_plugin::tree_magic_mini::TreeMagicMiniMetaPlugin;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tree_magic_mini_meta_plugin() {
|
||||||
|
let plugin = TreeMagicMiniMetaPlugin::new(None, None);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
plugin.meta_type(),
|
||||||
|
crate::meta_plugin::MetaPluginType::TreeMagicMini
|
||||||
|
);
|
||||||
|
assert!(plugin.is_internal());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tree_magic_mini_png_detection() {
|
||||||
|
let png_header: &[u8] = &[
|
||||||
|
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48,
|
||||||
|
0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00,
|
||||||
|
0x00, 0x90, 0x77, 0x53, 0xDE,
|
||||||
|
];
|
||||||
|
let mut plugin = TreeMagicMiniMetaPlugin::new(None, None);
|
||||||
|
plugin.update(png_header);
|
||||||
|
let response = plugin.finalize();
|
||||||
|
|
||||||
|
assert!(response.is_finalized);
|
||||||
|
assert!(!response.metadata.is_empty());
|
||||||
|
assert_eq!(response.metadata[0].name, "tree_magic_mime_type");
|
||||||
|
assert_eq!(response.metadata[0].value, "image/png");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user