feat: add infer and tree_magic_mini meta plugins, make zstd internal by default

- Add infer crate as meta plugin for MIME type detection
- Add tree_magic_mini crate as alternative meta plugin for MIME type detection
- Add zstd, infer, tree_magic_mini to default features
- Fix static build script to use musl target instead of glibc+crt-static
- Remove hardcoded shell list from --generate-completion help text
- Fix update() in both new plugins to emit MIME metadata when buffer fills

src/args.rs

@@ -82,7 +82,7 @@ pub struct ModeArgs {
     pub generate_config: bool,
 
     #[arg(help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "generate_config", "export", "import"]))]
-    #[arg(help("Generate shell completion script (bash, zsh, fish, elvish, powershell)"))]
+    #[arg(help("Generate shell completion script"))]
     pub generate_completion: Option<Shell>,
 
     #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_ADDRESS"))]

src/compression_engine/gzip.rs

@@ -11,12 +11,12 @@ use std::io::{Read, Write};
 #[cfg(feature = "gzip")]
 use std::path::PathBuf;
 
+#[cfg(feature = "gzip")]
+use flate2::Compression;
 #[cfg(feature = "gzip")]
 use flate2::read::GzDecoder;
 #[cfg(feature = "gzip")]
 use flate2::write::GzEncoder;
-#[cfg(feature = "gzip")]
-use flate2::Compression;
 
 #[cfg(feature = "gzip")]
 use crate::compression_engine::CompressionEngine;

src/compression_engine/mod.rs

@@ -1,4 +1,4 @@
-use anyhow::{anyhow, Result};
+use anyhow::{Result, anyhow};
 use std::io;
 use std::io::{Read, Write};
 use std::path::PathBuf;

src/compression_engine/program.rs

@@ -1,4 +1,4 @@
-use anyhow::{anyhow, Context, Result};
+use anyhow::{Context, Result, anyhow};
 use log::*;
 use std::fs::File;
 use std::io::{Read, Write};

src/lib.rs

@@ -71,6 +71,14 @@ use crate::meta_plugin::magic_file;
 #[allow(unused_imports)]
 use crate::meta_plugin::tokens;
 
+#[cfg(feature = "infer")]
+#[allow(unused_imports)]
+use crate::meta_plugin::infer_plugin;
+
+#[cfg(feature = "tree_magic_mini")]
+#[allow(unused_imports)]
+use crate::meta_plugin::tree_magic_mini;
+
 /// Initializes plugins at library load time.
 ///
 /// Plugin registration happens automatically via `#[ctor]` constructors

src/meta_plugin/infer_plugin.rs

@@ -0,0 +1,177 @@
+use crate::common::PIPESIZE;
+use crate::meta_plugin::{
+    process_metadata_outputs, register_meta_plugin, BaseMetaPlugin, MetaPlugin, MetaPluginResponse,
+    MetaPluginType,
+};
+
+#[derive(Debug, Default)]
+pub struct InferMetaPlugin {
+    buffer: Vec<u8>,
+    max_buffer_size: usize,
+    is_finalized: bool,
+    base: BaseMetaPlugin,
+}
+
+impl InferMetaPlugin {
+    pub fn new(
+        options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
+        outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
+    ) -> InferMetaPlugin {
+        let mut base = BaseMetaPlugin::new();
+
+        if let Some(opts) = options {
+            for (key, value) in opts {
+                base.options.insert(key, value);
+            }
+        }
+
+        let max_buffer_size = base
+            .options
+            .get("max_buffer_size")
+            .and_then(|v| v.as_u64())
+            .unwrap_or(PIPESIZE as u64) as usize;
+
+        base.outputs.insert(
+            "infer_mime_type".to_string(),
+            serde_yaml::Value::String("infer_mime_type".to_string()),
+        );
+
+        if let Some(outs) = outputs {
+            for (key, value) in outs {
+                base.outputs.insert(key, value);
+            }
+        }
+
+        InferMetaPlugin {
+            buffer: Vec::new(),
+            max_buffer_size,
+            is_finalized: false,
+            base,
+        }
+    }
+}
+
+impl MetaPlugin for InferMetaPlugin {
+    fn meta_type(&self) -> MetaPluginType {
+        MetaPluginType::Infer
+    }
+
+    fn is_finalized(&self) -> bool {
+        self.is_finalized
+    }
+
+    fn set_finalized(&mut self, finalized: bool) {
+        self.is_finalized = finalized;
+    }
+
+    fn set_save_meta(&mut self, save_meta: crate::meta_plugin::SaveMetaFn) {
+        self.base.set_save_meta(save_meta);
+    }
+
+    fn save_meta(&self, name: &str, value: &str) {
+        self.base.save_meta(name, value);
+    }
+
+    fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
+        if self.is_finalized {
+            return MetaPluginResponse {
+                metadata: Vec::new(),
+                is_finalized: true,
+            };
+        }
+
+        let remaining = self.max_buffer_size.saturating_sub(self.buffer.len());
+        let to_add = &data[..data.len().min(remaining)];
+        self.buffer.extend_from_slice(to_add);
+
+        if self.buffer.len() >= self.max_buffer_size {
+            let mime_type = infer::get(&self.buffer)
+                .map(|kind| kind.mime_type().to_string())
+                .unwrap_or_else(|| "application/octet-stream".to_string());
+
+            self.is_finalized = true;
+
+            let metadata = process_metadata_outputs(
+                "infer_mime_type",
+                serde_yaml::Value::String(mime_type),
+                self.base.outputs(),
+            )
+            .map(|m| vec![m])
+            .unwrap_or_default();
+
+            return MetaPluginResponse {
+                metadata,
+                is_finalized: true,
+            };
+        }
+
+        MetaPluginResponse {
+            metadata: Vec::new(),
+            is_finalized: false,
+        }
+    }
+
+    fn finalize(&mut self) -> MetaPluginResponse {
+        if self.is_finalized {
+            return MetaPluginResponse {
+                metadata: Vec::new(),
+                is_finalized: true,
+            };
+        }
+
+        let mime_type = infer::get(&self.buffer)
+            .map(|kind| kind.mime_type().to_string())
+            .unwrap_or_else(|| "application/octet-stream".to_string());
+
+        self.is_finalized = true;
+
+        let metadata = process_metadata_outputs(
+            "infer_mime_type",
+            serde_yaml::Value::String(mime_type),
+            self.base.outputs(),
+        )
+        .map(|m| vec![m])
+        .unwrap_or_default();
+
+        MetaPluginResponse {
+            metadata,
+            is_finalized: true,
+        }
+    }
+
+    fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
+        self.base.outputs()
+    }
+
+    fn outputs_mut(
+        &mut self,
+    ) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
+        Ok(self.base.outputs_mut())
+    }
+
+    fn default_outputs(&self) -> Vec<String> {
+        vec!["infer_mime_type".to_string()]
+    }
+
+    fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
+        self.base.options()
+    }
+
+    fn options_mut(
+        &mut self,
+    ) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
+        Ok(self.base.options_mut())
+    }
+
+    fn parallel_safe(&self) -> bool {
+        true
+    }
+}
+
+#[ctor::ctor]
+fn register_infer_plugin() {
+    register_meta_plugin(MetaPluginType::Infer, |options, outputs| {
+        Box::new(InferMetaPlugin::new(options, outputs))
+    })
+    .expect("Failed to register InferMetaPlugin");
+}

src/meta_plugin/mod.rs

@@ -9,6 +9,8 @@ pub mod digest;
 pub mod env;
 pub mod exec;
 pub mod hostname;
+#[cfg(feature = "infer")]
+pub mod infer_plugin;
 pub mod keep_pid;
 pub mod magic_file;
 pub mod read_rate;
@@ -18,6 +20,8 @@ pub mod shell_pid;
 pub mod text;
 #[cfg(feature = "tokens")]
 pub mod tokens;
+#[cfg(feature = "tree_magic_mini")]
+pub mod tree_magic_mini;
 pub mod user;
 
 pub use digest::DigestMetaPlugin;
@@ -28,11 +32,15 @@ pub use magic_file::MagicFileMetaPlugin;
 pub use cwd::CwdMetaPlugin;
 pub use env::EnvMetaPlugin;
 pub use hostname::HostnameMetaPlugin;
+#[cfg(feature = "infer")]
+pub use infer_plugin::InferMetaPlugin;
 pub use keep_pid::KeepPidMetaPlugin;
 pub use read_rate::ReadRateMetaPlugin;
 pub use read_time::ReadTimeMetaPlugin;
 pub use shell::ShellMetaPlugin;
 pub use shell_pid::ShellPidMetaPlugin;
+#[cfg(feature = "tree_magic_mini")]
+pub use tree_magic_mini::TreeMagicMiniMetaPlugin;
 pub use user::UserMetaPlugin;
 
 #[cfg(not(feature = "magic"))]
@@ -263,6 +271,8 @@ pub enum MetaPluginType {
     Exec,
     Env,
     Tokens,
+    TreeMagicMini,
+    Infer,
 }
 
 /// Central function to handle metadata output with name mapping.

src/meta_plugin/tree_magic_mini.rs

@@ -0,0 +1,173 @@
+use crate::common::PIPESIZE;
+use crate::meta_plugin::{
+    process_metadata_outputs, register_meta_plugin, BaseMetaPlugin, MetaPlugin, MetaPluginResponse,
+    MetaPluginType,
+};
+
+#[derive(Debug, Default)]
+pub struct TreeMagicMiniMetaPlugin {
+    buffer: Vec<u8>,
+    max_buffer_size: usize,
+    is_finalized: bool,
+    base: BaseMetaPlugin,
+}
+
+impl TreeMagicMiniMetaPlugin {
+    pub fn new(
+        options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
+        outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
+    ) -> TreeMagicMiniMetaPlugin {
+        let mut base = BaseMetaPlugin::new();
+
+        if let Some(opts) = options {
+            for (key, value) in opts {
+                base.options.insert(key, value);
+            }
+        }
+
+        let max_buffer_size = base
+            .options
+            .get("max_buffer_size")
+            .and_then(|v| v.as_u64())
+            .unwrap_or(PIPESIZE as u64) as usize;
+
+        base.outputs.insert(
+            "tree_magic_mime_type".to_string(),
+            serde_yaml::Value::String("tree_magic_mime_type".to_string()),
+        );
+
+        if let Some(outs) = outputs {
+            for (key, value) in outs {
+                base.outputs.insert(key, value);
+            }
+        }
+
+        TreeMagicMiniMetaPlugin {
+            buffer: Vec::new(),
+            max_buffer_size,
+            is_finalized: false,
+            base,
+        }
+    }
+}
+
+impl MetaPlugin for TreeMagicMiniMetaPlugin {
+    fn meta_type(&self) -> MetaPluginType {
+        MetaPluginType::TreeMagicMini
+    }
+
+    fn is_finalized(&self) -> bool {
+        self.is_finalized
+    }
+
+    fn set_finalized(&mut self, finalized: bool) {
+        self.is_finalized = finalized;
+    }
+
+    fn set_save_meta(&mut self, save_meta: crate::meta_plugin::SaveMetaFn) {
+        self.base.set_save_meta(save_meta);
+    }
+
+    fn save_meta(&self, name: &str, value: &str) {
+        self.base.save_meta(name, value);
+    }
+
+    fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
+        if self.is_finalized {
+            return MetaPluginResponse {
+                metadata: Vec::new(),
+                is_finalized: true,
+            };
+        }
+
+        let remaining = self.max_buffer_size.saturating_sub(self.buffer.len());
+        let to_add = &data[..data.len().min(remaining)];
+        self.buffer.extend_from_slice(to_add);
+
+        if self.buffer.len() >= self.max_buffer_size {
+            let mime_type = tree_magic_mini::from_u8(&self.buffer);
+
+            self.is_finalized = true;
+
+            let metadata = process_metadata_outputs(
+                "tree_magic_mime_type",
+                serde_yaml::Value::String(mime_type.to_string()),
+                self.base.outputs(),
+            )
+            .map(|m| vec![m])
+            .unwrap_or_default();
+
+            return MetaPluginResponse {
+                metadata,
+                is_finalized: true,
+            };
+        }
+
+        MetaPluginResponse {
+            metadata: Vec::new(),
+            is_finalized: false,
+        }
+    }
+
+    fn finalize(&mut self) -> MetaPluginResponse {
+        if self.is_finalized {
+            return MetaPluginResponse {
+                metadata: Vec::new(),
+                is_finalized: true,
+            };
+        }
+
+        let mime_type = tree_magic_mini::from_u8(&self.buffer);
+
+        self.is_finalized = true;
+
+        let metadata = process_metadata_outputs(
+            "tree_magic_mime_type",
+            serde_yaml::Value::String(mime_type.to_string()),
+            self.base.outputs(),
+        )
+        .map(|m| vec![m])
+        .unwrap_or_default();
+
+        MetaPluginResponse {
+            metadata,
+            is_finalized: true,
+        }
+    }
+
+    fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
+        self.base.outputs()
+    }
+
+    fn outputs_mut(
+        &mut self,
+    ) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
+        Ok(self.base.outputs_mut())
+    }
+
+    fn default_outputs(&self) -> Vec<String> {
+        vec!["tree_magic_mime_type".to_string()]
+    }
+
+    fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
+        self.base.options()
+    }
+
+    fn options_mut(
+        &mut self,
+    ) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
+        Ok(self.base.options_mut())
+    }
+
+    fn parallel_safe(&self) -> bool {
+        true
+    }
+}
+
+#[ctor::ctor]
+fn register_tree_magic_mini_plugin() {
+    register_meta_plugin(MetaPluginType::TreeMagicMini, |options, outputs| {
+        Box::new(TreeMagicMiniMetaPlugin::new(options, outputs))
+    })
+    .expect("Failed to register TreeMagicMiniMetaPlugin");
+}

src/services/compression_service.rs

@@ -1,4 +1,4 @@
-use crate::compression_engine::{get_compression_engine, CompressionType};
+use crate::compression_engine::{CompressionType, get_compression_engine};
 use crate::services::error::CoreError;
 use anyhow::anyhow;
 use std::io::{Read, Write};
@@ -187,8 +187,8 @@ impl CompressionService {
     ) -> Box<dyn Write> {
         match compression {
             CompressionType::GZip => {
-                use flate2::write::GzEncoder;
                 use flate2::Compression;
+                use flate2::write::GzEncoder;
                 Box::new(GzEncoder::new(writer, Compression::default()))
             }
             CompressionType::LZ4 => Box::new(lz4_flex::frame::FrameEncoder::new(writer)),

src/tests/compression/gzip_tests.rs

@@ -1,7 +1,7 @@
 #[cfg(test)]
 mod tests {
-    use crate::compression_engine::gzip::CompressionEngineGZip;
     use crate::compression_engine::CompressionEngine;
+    use crate::compression_engine::gzip::CompressionEngineGZip;
     use crate::tests::common::test_helpers::test_compression_engine;
 
     #[test]

src/tests/meta_plugin/infer_tests.rs

@@ -0,0 +1,33 @@
+#[cfg(test)]
+mod tests {
+    use crate::meta_plugin::MetaPlugin;
+    use crate::meta_plugin::infer_plugin::InferMetaPlugin;
+
+    #[test]
+    fn test_infer_meta_plugin() {
+        let plugin = InferMetaPlugin::new(None, None);
+
+        assert_eq!(
+            plugin.meta_type(),
+            crate::meta_plugin::MetaPluginType::Infer
+        );
+        assert!(plugin.is_internal());
+    }
+
+    #[test]
+    fn test_infer_png_detection() {
+        let png_header: &[u8] = &[
+            0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48,
+            0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00,
+            0x00, 0x90, 0x77, 0x53, 0xDE,
+        ];
+        let mut plugin = InferMetaPlugin::new(None, None);
+        plugin.update(png_header);
+        let response = plugin.finalize();
+
+        assert!(response.is_finalized);
+        assert!(!response.metadata.is_empty());
+        assert_eq!(response.metadata[0].name, "infer_mime_type");
+        assert_eq!(response.metadata[0].value, "image/png");
+    }
+}

src/tests/meta_plugin/mod.rs

@@ -2,3 +2,11 @@
 
 #[cfg(test)]
 pub mod digest_tests;
+
+#[cfg(feature = "infer")]
+#[cfg(test)]
+pub mod infer_tests;
+
+#[cfg(feature = "tree_magic_mini")]
+#[cfg(test)]
+pub mod tree_magic_mini_tests;

src/tests/meta_plugin/tree_magic_mini_tests.rs

@@ -0,0 +1,33 @@
+#[cfg(test)]
+mod tests {
+    use crate::meta_plugin::MetaPlugin;
+    use crate::meta_plugin::tree_magic_mini::TreeMagicMiniMetaPlugin;
+
+    #[test]
+    fn test_tree_magic_mini_meta_plugin() {
+        let plugin = TreeMagicMiniMetaPlugin::new(None, None);
+
+        assert_eq!(
+            plugin.meta_type(),
+            crate::meta_plugin::MetaPluginType::TreeMagicMini
+        );
+        assert!(plugin.is_internal());
+    }
+
+    #[test]
+    fn test_tree_magic_mini_png_detection() {
+        let png_header: &[u8] = &[
+            0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48,
+            0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00,
+            0x00, 0x90, 0x77, 0x53, 0xDE,
+        ];
+        let mut plugin = TreeMagicMiniMetaPlugin::new(None, None);
+        plugin.update(png_header);
+        let response = plugin.finalize();
+
+        assert!(response.is_finalized);
+        assert!(!response.metadata.is_empty());
+        assert_eq!(response.metadata[0].name, "tree_magic_mime_type");
+        assert_eq!(response.metadata[0].value, "image/png");
+    }
+}