feat: add infer and tree_magic_mini meta plugins, make zstd internal by default

- Add infer crate as meta plugin for MIME type detection
- Add tree_magic_mini crate as alternative meta plugin for MIME type detection
- Add zstd, infer, tree_magic_mini to default features
- Fix static build script to use musl target instead of glibc+crt-static
- Remove hardcoded shell list from --generate-completion help text
- Fix update() in both new plugins to emit MIME metadata when buffer fills
This commit is contained in:
2026-03-17 14:46:51 -03:00
parent a90c19efc1
commit a8759c4b83
17 changed files with 535 additions and 17 deletions

69
Cargo.lock generated
View File

@@ -378,6 +378,17 @@ dependencies = [
"shlex",
]
[[package]]
name = "cfb"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f"
dependencies = [
"byteorder",
"fnv",
"uuid",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
@@ -1020,6 +1031,12 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "fixedbitset"
version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "flate2"
version = "1.1.9"
@@ -1553,6 +1570,15 @@ dependencies = [
"serde_core",
]
[[package]]
name = "infer"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7"
dependencies = [
"cfb",
]
[[package]]
name = "inventory"
version = "0.3.22"
@@ -1686,6 +1712,7 @@ dependencies = [
"http-body-util",
"humansize",
"hyper",
"infer",
"inventory",
"is-terminal",
"jsonwebtoken",
@@ -1726,6 +1753,7 @@ dependencies = [
"tokio-util",
"tower",
"tower-http",
"tree_magic_mini",
"ureq",
"utoipa",
"utoipa-swagger-ui",
@@ -1975,6 +2003,15 @@ dependencies = [
"libc",
]
[[package]]
name = "nom"
version = "8.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
dependencies = [
"memchr",
]
[[package]]
name = "num-bigint"
version = "0.4.6"
@@ -2141,6 +2178,17 @@ dependencies = [
"sha2 0.10.9",
]
[[package]]
name = "petgraph"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
dependencies = [
"fixedbitset",
"hashbrown 0.15.5",
"indexmap",
]
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -3184,6 +3232,17 @@ dependencies = [
"once_cell",
]
[[package]]
name = "tree_magic_mini"
version = "3.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8765b90061cba6c22b5831f675da109ae5561588290f9fa2317adab2714d5a6"
dependencies = [
"memchr",
"nom",
"petgraph",
]
[[package]]
name = "try-lock"
version = "0.2.5"
@@ -3372,6 +3431,16 @@ dependencies = [
"zip",
]
[[package]]
name = "uuid"
version = "1.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "uzers"
version = "0.12.2"

View File

@@ -41,6 +41,8 @@ log = "0.4"
lz4_flex = { version = "0.12", optional = true }
zstd = { version = "0.13", optional = true }
magic = { version = "0.13", optional = true }
infer = { version = "0.19", optional = true }
tree_magic_mini = { version = "3.2", optional = true }
nix = { version = "0.30", features = ["fs", "process"] }
once_cell = "1.21"
comfy-table = "7.2"
@@ -83,11 +85,15 @@ tiktoken-rs = { version = "0.9", optional = true }
[features]
# Default features include core compression engines and swagger UI
default = ["magic", "lz4", "gzip", "client", "tokens"]
# Full
#default = ["server", "magic", "lz4", "swagger"]
default = [
"client",
"gzip",
"infer",
"lz4",
"tokens",
"tree_magic_mini",
"zstd"
]
# Server feature (includes axum and related dependencies)
server = ["dep:axum", "dep:tower", "dep:tower-http", "dep:utoipa", "dep:jsonwebtoken"]
@@ -100,11 +106,13 @@ xz = []
zstd = ["dep:zstd"]
# Plugin features (meta and filter)
all-meta-plugins = ["dep:magic"]
all-meta-plugins = ["dep:magic", "dep:infer", "dep:tree_magic_mini"]
all-filter-plugins = []
# Individual plugin features
magic = ["dep:magic"]
infer = ["dep:infer"]
tree_magic_mini = ["dep:tree_magic_mini"]
# Swagger UI feature
swagger = ["dep:utoipa-swagger-ui"]

BIN
bin/keep Executable file

Binary file not shown.

View File

@@ -2,7 +2,6 @@
set -ex
export RUSTFLAGS='-C target-feature=+crt-static'
cargo build --release --target x86_64-unknown-linux-gnu
cargo build --release --target x86_64-unknown-linux-musl
mkdir -p bin
cp target/x86_64-unknown-linux-gnu/release/keep ./bin/
cp target/x86_64-unknown-linux-musl/release/keep ./bin/

View File

@@ -82,7 +82,7 @@ pub struct ModeArgs {
pub generate_config: bool,
#[arg(help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "generate_config", "export", "import"]))]
#[arg(help("Generate shell completion script (bash, zsh, fish, elvish, powershell)"))]
#[arg(help("Generate shell completion script"))]
pub generate_completion: Option<Shell>,
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_ADDRESS"))]

View File

@@ -11,12 +11,12 @@ use std::io::{Read, Write};
#[cfg(feature = "gzip")]
use std::path::PathBuf;
#[cfg(feature = "gzip")]
use flate2::Compression;
#[cfg(feature = "gzip")]
use flate2::read::GzDecoder;
#[cfg(feature = "gzip")]
use flate2::write::GzEncoder;
#[cfg(feature = "gzip")]
use flate2::Compression;
#[cfg(feature = "gzip")]
use crate::compression_engine::CompressionEngine;

View File

@@ -1,4 +1,4 @@
use anyhow::{anyhow, Result};
use anyhow::{Result, anyhow};
use std::io;
use std::io::{Read, Write};
use std::path::PathBuf;

View File

@@ -1,4 +1,4 @@
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use log::*;
use std::fs::File;
use std::io::{Read, Write};

View File

@@ -71,6 +71,14 @@ use crate::meta_plugin::magic_file;
#[allow(unused_imports)]
use crate::meta_plugin::tokens;
#[cfg(feature = "infer")]
#[allow(unused_imports)]
use crate::meta_plugin::infer_plugin;
#[cfg(feature = "tree_magic_mini")]
#[allow(unused_imports)]
use crate::meta_plugin::tree_magic_mini;
/// Initializes plugins at library load time.
///
/// Plugin registration happens automatically via `#[ctor]` constructors

View File

@@ -0,0 +1,177 @@
use crate::common::PIPESIZE;
use crate::meta_plugin::{
process_metadata_outputs, register_meta_plugin, BaseMetaPlugin, MetaPlugin, MetaPluginResponse,
MetaPluginType,
};
#[derive(Debug, Default)]
pub struct InferMetaPlugin {
buffer: Vec<u8>,
max_buffer_size: usize,
is_finalized: bool,
base: BaseMetaPlugin,
}
impl InferMetaPlugin {
pub fn new(
options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> InferMetaPlugin {
let mut base = BaseMetaPlugin::new();
if let Some(opts) = options {
for (key, value) in opts {
base.options.insert(key, value);
}
}
let max_buffer_size = base
.options
.get("max_buffer_size")
.and_then(|v| v.as_u64())
.unwrap_or(PIPESIZE as u64) as usize;
base.outputs.insert(
"infer_mime_type".to_string(),
serde_yaml::Value::String("infer_mime_type".to_string()),
);
if let Some(outs) = outputs {
for (key, value) in outs {
base.outputs.insert(key, value);
}
}
InferMetaPlugin {
buffer: Vec::new(),
max_buffer_size,
is_finalized: false,
base,
}
}
}
impl MetaPlugin for InferMetaPlugin {
fn meta_type(&self) -> MetaPluginType {
MetaPluginType::Infer
}
fn is_finalized(&self) -> bool {
self.is_finalized
}
fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized;
}
fn set_save_meta(&mut self, save_meta: crate::meta_plugin::SaveMetaFn) {
self.base.set_save_meta(save_meta);
}
fn save_meta(&self, name: &str, value: &str) {
self.base.save_meta(name, value);
}
fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
if self.is_finalized {
return MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
let remaining = self.max_buffer_size.saturating_sub(self.buffer.len());
let to_add = &data[..data.len().min(remaining)];
self.buffer.extend_from_slice(to_add);
if self.buffer.len() >= self.max_buffer_size {
let mime_type = infer::get(&self.buffer)
.map(|kind| kind.mime_type().to_string())
.unwrap_or_else(|| "application/octet-stream".to_string());
self.is_finalized = true;
let metadata = process_metadata_outputs(
"infer_mime_type",
serde_yaml::Value::String(mime_type),
self.base.outputs(),
)
.map(|m| vec![m])
.unwrap_or_default();
return MetaPluginResponse {
metadata,
is_finalized: true,
};
}
MetaPluginResponse {
metadata: Vec::new(),
is_finalized: false,
}
}
fn finalize(&mut self) -> MetaPluginResponse {
if self.is_finalized {
return MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
let mime_type = infer::get(&self.buffer)
.map(|kind| kind.mime_type().to_string())
.unwrap_or_else(|| "application/octet-stream".to_string());
self.is_finalized = true;
let metadata = process_metadata_outputs(
"infer_mime_type",
serde_yaml::Value::String(mime_type),
self.base.outputs(),
)
.map(|m| vec![m])
.unwrap_or_default();
MetaPluginResponse {
metadata,
is_finalized: true,
}
}
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs()
}
fn outputs_mut(
&mut self,
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
Ok(self.base.outputs_mut())
}
fn default_outputs(&self) -> Vec<String> {
vec!["infer_mime_type".to_string()]
}
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options()
}
fn options_mut(
&mut self,
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
Ok(self.base.options_mut())
}
fn parallel_safe(&self) -> bool {
true
}
}
#[ctor::ctor]
fn register_infer_plugin() {
register_meta_plugin(MetaPluginType::Infer, |options, outputs| {
Box::new(InferMetaPlugin::new(options, outputs))
})
.expect("Failed to register InferMetaPlugin");
}

View File

@@ -9,6 +9,8 @@ pub mod digest;
pub mod env;
pub mod exec;
pub mod hostname;
#[cfg(feature = "infer")]
pub mod infer_plugin;
pub mod keep_pid;
pub mod magic_file;
pub mod read_rate;
@@ -18,6 +20,8 @@ pub mod shell_pid;
pub mod text;
#[cfg(feature = "tokens")]
pub mod tokens;
#[cfg(feature = "tree_magic_mini")]
pub mod tree_magic_mini;
pub mod user;
pub use digest::DigestMetaPlugin;
@@ -28,11 +32,15 @@ pub use magic_file::MagicFileMetaPlugin;
pub use cwd::CwdMetaPlugin;
pub use env::EnvMetaPlugin;
pub use hostname::HostnameMetaPlugin;
#[cfg(feature = "infer")]
pub use infer_plugin::InferMetaPlugin;
pub use keep_pid::KeepPidMetaPlugin;
pub use read_rate::ReadRateMetaPlugin;
pub use read_time::ReadTimeMetaPlugin;
pub use shell::ShellMetaPlugin;
pub use shell_pid::ShellPidMetaPlugin;
#[cfg(feature = "tree_magic_mini")]
pub use tree_magic_mini::TreeMagicMiniMetaPlugin;
pub use user::UserMetaPlugin;
#[cfg(not(feature = "magic"))]
@@ -263,6 +271,8 @@ pub enum MetaPluginType {
Exec,
Env,
Tokens,
TreeMagicMini,
Infer,
}
/// Central function to handle metadata output with name mapping.

View File

@@ -0,0 +1,173 @@
use crate::common::PIPESIZE;
use crate::meta_plugin::{
process_metadata_outputs, register_meta_plugin, BaseMetaPlugin, MetaPlugin, MetaPluginResponse,
MetaPluginType,
};
#[derive(Debug, Default)]
pub struct TreeMagicMiniMetaPlugin {
buffer: Vec<u8>,
max_buffer_size: usize,
is_finalized: bool,
base: BaseMetaPlugin,
}
impl TreeMagicMiniMetaPlugin {
pub fn new(
options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> TreeMagicMiniMetaPlugin {
let mut base = BaseMetaPlugin::new();
if let Some(opts) = options {
for (key, value) in opts {
base.options.insert(key, value);
}
}
let max_buffer_size = base
.options
.get("max_buffer_size")
.and_then(|v| v.as_u64())
.unwrap_or(PIPESIZE as u64) as usize;
base.outputs.insert(
"tree_magic_mime_type".to_string(),
serde_yaml::Value::String("tree_magic_mime_type".to_string()),
);
if let Some(outs) = outputs {
for (key, value) in outs {
base.outputs.insert(key, value);
}
}
TreeMagicMiniMetaPlugin {
buffer: Vec::new(),
max_buffer_size,
is_finalized: false,
base,
}
}
}
impl MetaPlugin for TreeMagicMiniMetaPlugin {
fn meta_type(&self) -> MetaPluginType {
MetaPluginType::TreeMagicMini
}
fn is_finalized(&self) -> bool {
self.is_finalized
}
fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized;
}
fn set_save_meta(&mut self, save_meta: crate::meta_plugin::SaveMetaFn) {
self.base.set_save_meta(save_meta);
}
fn save_meta(&self, name: &str, value: &str) {
self.base.save_meta(name, value);
}
fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
if self.is_finalized {
return MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
let remaining = self.max_buffer_size.saturating_sub(self.buffer.len());
let to_add = &data[..data.len().min(remaining)];
self.buffer.extend_from_slice(to_add);
if self.buffer.len() >= self.max_buffer_size {
let mime_type = tree_magic_mini::from_u8(&self.buffer);
self.is_finalized = true;
let metadata = process_metadata_outputs(
"tree_magic_mime_type",
serde_yaml::Value::String(mime_type.to_string()),
self.base.outputs(),
)
.map(|m| vec![m])
.unwrap_or_default();
return MetaPluginResponse {
metadata,
is_finalized: true,
};
}
MetaPluginResponse {
metadata: Vec::new(),
is_finalized: false,
}
}
fn finalize(&mut self) -> MetaPluginResponse {
if self.is_finalized {
return MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
let mime_type = tree_magic_mini::from_u8(&self.buffer);
self.is_finalized = true;
let metadata = process_metadata_outputs(
"tree_magic_mime_type",
serde_yaml::Value::String(mime_type.to_string()),
self.base.outputs(),
)
.map(|m| vec![m])
.unwrap_or_default();
MetaPluginResponse {
metadata,
is_finalized: true,
}
}
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs()
}
fn outputs_mut(
&mut self,
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
Ok(self.base.outputs_mut())
}
fn default_outputs(&self) -> Vec<String> {
vec!["tree_magic_mime_type".to_string()]
}
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options()
}
fn options_mut(
&mut self,
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
Ok(self.base.options_mut())
}
fn parallel_safe(&self) -> bool {
true
}
}
#[ctor::ctor]
fn register_tree_magic_mini_plugin() {
register_meta_plugin(MetaPluginType::TreeMagicMini, |options, outputs| {
Box::new(TreeMagicMiniMetaPlugin::new(options, outputs))
})
.expect("Failed to register TreeMagicMiniMetaPlugin");
}

View File

@@ -1,4 +1,4 @@
use crate::compression_engine::{get_compression_engine, CompressionType};
use crate::compression_engine::{CompressionType, get_compression_engine};
use crate::services::error::CoreError;
use anyhow::anyhow;
use std::io::{Read, Write};
@@ -187,8 +187,8 @@ impl CompressionService {
) -> Box<dyn Write> {
match compression {
CompressionType::GZip => {
use flate2::write::GzEncoder;
use flate2::Compression;
use flate2::write::GzEncoder;
Box::new(GzEncoder::new(writer, Compression::default()))
}
CompressionType::LZ4 => Box::new(lz4_flex::frame::FrameEncoder::new(writer)),

View File

@@ -1,7 +1,7 @@
#[cfg(test)]
mod tests {
use crate::compression_engine::gzip::CompressionEngineGZip;
use crate::compression_engine::CompressionEngine;
use crate::compression_engine::gzip::CompressionEngineGZip;
use crate::tests::common::test_helpers::test_compression_engine;
#[test]

View File

@@ -0,0 +1,33 @@
#[cfg(test)]
mod tests {
use crate::meta_plugin::MetaPlugin;
use crate::meta_plugin::infer_plugin::InferMetaPlugin;
#[test]
fn test_infer_meta_plugin() {
let plugin = InferMetaPlugin::new(None, None);
assert_eq!(
plugin.meta_type(),
crate::meta_plugin::MetaPluginType::Infer
);
assert!(plugin.is_internal());
}
#[test]
fn test_infer_png_detection() {
let png_header: &[u8] = &[
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48,
0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00,
0x00, 0x90, 0x77, 0x53, 0xDE,
];
let mut plugin = InferMetaPlugin::new(None, None);
plugin.update(png_header);
let response = plugin.finalize();
assert!(response.is_finalized);
assert!(!response.metadata.is_empty());
assert_eq!(response.metadata[0].name, "infer_mime_type");
assert_eq!(response.metadata[0].value, "image/png");
}
}

View File

@@ -2,3 +2,11 @@
#[cfg(test)]
pub mod digest_tests;
#[cfg(feature = "infer")]
#[cfg(test)]
pub mod infer_tests;
#[cfg(feature = "tree_magic_mini")]
#[cfg(test)]
pub mod tree_magic_mini_tests;

View File

@@ -0,0 +1,33 @@
#[cfg(test)]
mod tests {
use crate::meta_plugin::MetaPlugin;
use crate::meta_plugin::tree_magic_mini::TreeMagicMiniMetaPlugin;
#[test]
fn test_tree_magic_mini_meta_plugin() {
let plugin = TreeMagicMiniMetaPlugin::new(None, None);
assert_eq!(
plugin.meta_type(),
crate::meta_plugin::MetaPluginType::TreeMagicMini
);
assert!(plugin.is_internal());
}
#[test]
fn test_tree_magic_mini_png_detection() {
let png_header: &[u8] = &[
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48,
0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00,
0x00, 0x90, 0x77, 0x53, 0xDE,
];
let mut plugin = TreeMagicMiniMetaPlugin::new(None, None);
plugin.update(png_header);
let response = plugin.finalize();
assert!(response.is_finalized);
assert!(!response.metadata.is_empty());
assert_eq!(response.metadata[0].name, "tree_magic_mime_type");
assert_eq!(response.metadata[0].value, "image/png");
}
}