feat: Add type and module reorganization for Services, Modes, Meta and Filter Plugins

Co-authored-by: aider (openai/andrew/openrouter/sonoma-sky-alpha) <aider@aider.chat>

Cargo.toml

@@ -23,7 +23,7 @@ ctor = "0.2"
 directories = "6.0.0"
 dns-lookup = "2.0.2"
 enum-map = "2.6.1"
-flate2 = { version = "1.0.27", features = ["zlib-ng-compat"] }
+flate2 = { version = "1.0.27", features = ["zlib-ng-compat"], optional = true }
 futures = "0.3"
 gethostname = "1.0.2"
 humansize = "2.1.3"
@@ -34,8 +34,8 @@ lazy_static = "1.4.0"
 libc = "0.2.147"
 local-ip-address = "0.6.5"
 log = "0.4.19"
-lz4_flex = "0.11.1"
-magic = "0.13.0"
+lz4_flex = { version = "0.11.1", optional = true }
+magic = { version = "0.13.0", optional = true }
 nix = "0.30.1"
 once_cell = "1.19.0"
 comfy-table = "7.2.0"
@@ -67,7 +67,24 @@ strip-ansi-escapes = "0.2.1"
 pest = "2.8.1"
 pest_derive = "2.8.1"
 
+[features]
+# Default features include core compression engines
+default = ["gzip", "lz4"]
+
+# Compression features
+gzip = ["flate2"]
+lz4 = ["lz4_flex"]
+bzip2 = []
+xz = []
+zstd = []
+
+# Plugin features (meta and filter)
+all-meta-plugins = ["magic"]
+all-filter-plugins = []
+
+# Individual plugin features
+magic = ["magic"]
+
 [dev-dependencies]
 tempfile = "3.3.0"
 rand = "0.8.5"
-

README.md

@@ -0,0 +1,16 @@
+# Keep - Temporary File Management with Compression and Metadata
+
+Keep is a command-line tool for managing temporary files with automatic compression, metadata generation, and querying capabilities. It supports various compression algorithms and metadata plugins for rich item inspection.
+
+## Features
+
+- **Store and Retrieve**: Save content with automatic compression and retrieve by ID or tags.
+- **Compression Support**: Built-in support for LZ4, GZip, and more via external programs (BZip2, XZ, ZStd).
+- **Metadata Plugins**: Automatic extraction of file type, digests, hostname, user info, and custom metadata.
+- **Filtering**: Apply filters (head, tail, grep, etc.) when retrieving content.
+- **Querying**: List, search, and diff items with flexible formatting.
+- **REST API Server**: Optional HTTP server for programmatic access.
+- **Modular Design**: Extensible via plugins for compression, metadata, and filtering.
+
+## Installation
+

src/filter_plugin/mod.rs

@@ -11,6 +11,12 @@ pub mod utils;
 
 use std::collections::HashMap;
 
+pub use head::{HeadBytesFilter, HeadLinesFilter};
+pub use tail::{TailBytesFilter, TailLinesFilter};
+pub use grep::GrepFilter;
+pub use skip::{SkipBytesFilter, SkipLinesFilter};
+pub use strip_ansi::StripAnsiFilter;
+
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
 pub struct FilterOption {
     pub name: String,

src/lib.rs

@@ -9,6 +9,8 @@ pub mod filter_plugin;
 pub mod modes;
 pub mod plugins;
 pub mod args;
+pub mod parser;
+pub mod utils;
 
 // Re-export Args struct for library usage
 pub use args::Args;
@@ -28,5 +30,11 @@ use meta_plugin::{
     read_time, read_rate, hostname, exec, env
 };
 
+// Initialize plugins at library load time
+pub fn init_plugins() {
+    // This will be expanded in Step 3 implementation
+    // For now, the ctors handle registration
+}
+
 #[cfg(test)]
 mod tests;

src/meta_plugin/mod.rs

@@ -5,7 +5,6 @@ use std::sync::Mutex;
 use once_cell::sync::Lazy;
 
 pub mod exec;
-use crate::meta_plugin::exec::MetaPluginExec;
 pub mod digest;
 pub mod magic;
 pub mod text;
@@ -19,6 +18,20 @@ pub mod shell_pid;
 pub mod keep_pid;
 pub mod env;
 
+pub use exec::MetaPluginExec;
+pub use digest::DigestMetaPlugin;
+pub use magic::MagicFileMetaPlugin;
+pub use text::TextMetaPlugin;
+pub use read_time::ReadTimeMetaPlugin;
+pub use read_rate::ReadRateMetaPlugin;
+pub use hostname::HostnameMetaPlugin;
+pub use cwd::CwdMetaPlugin;
+pub use user::UserMetaPlugin;
+pub use shell::ShellMetaPlugin;
+pub use shell_pid::ShellPidMetaPlugin;
+pub use keep_pid::KeepPidMetaPlugin;
+pub use env::EnvMetaPlugin;
+
 /// Represents metadata to be stored
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MetaData {
@@ -323,7 +336,7 @@ pub fn get_meta_plugin(
         }
         
         return Box::new(MetaPluginExec::new(&program_name, 
-                                    args.iter().map(|s: &String| s.as_str()).collect(), 
+                                    args.iter().map(|s| s.as_str()).collect(), 
                                     meta_name, 
                                     split_whitespace, 
                                     options, 

src/modes/mod.rs

@@ -9,3 +9,15 @@ pub mod save;
 pub mod server;
 pub mod status;
 pub mod status_plugins;
+
+pub use common::{ColumnType, OutputFormat, format_size, settings_output_format};
+pub use delete::mode_delete;
+pub use diff::mode_diff;
+pub use generate_config::mode_generate_config;
+pub use get::mode_get;
+pub use info::mode_info;
+pub use list::mode_list;
+pub use save::mode_save;
+pub use server::mode_server;
+pub use status::mode_status;
+pub use status_plugins::mode_status_plugins;

src/parser/filter_parser.rs

@@ -0,0 +1,124 @@
+use pest::Parser;
+use pest_derive::Parser;
+use std::collections::HashMap;
+
+#[derive(Parser)]
+#[grammar = "filter.pest"]
+pub struct FilterParser;
+
+#[derive(Debug)]
+pub struct Filter {
+    pub name: String,
+    pub options: HashMap<String, serde_json::Value>,
+}
+
+pub fn parse_filter_string(input: &str) -> Result<Vec<Filter>, Box<dyn std::error::Error>> {
+    let mut filters = Vec::new();
+    let pairs = FilterParser::parse(Rule::filters, input)?;
+    
+    for pair in pairs {
+        if pair.as_rule() == Rule::filter {
+            let mut name = String::new();
+            let mut options = HashMap::new();
+            
+            for inner_pair in pair.into_inner() {
+                match inner_pair.as_rule() {
+                    Rule::filter_name => {
+                        name = inner_pair.as_str().to_string();
+                    }
+                    Rule::options => {
+                        for option_pair in inner_pair.into_inner() {
+                            if option_pair.as_rule() == Rule::option {
+                                let mut option_name = None;
+                                let mut option_value = None;
+                                
+                                for option_inner in option_pair.into_inner() {
+                                    match option_inner.as_rule() {
+                                        Rule::option_name => {
+                                            option_name = Some(option_inner.as_str().to_string());
+                                        }
+                                        Rule::option_value => {
+                                            option_value = Some(parse_option_value(option_inner.as_str())?);
+                                        }
+                                        _ => {}
+                                    }
+                                }
+                                
+                                if let Some(value) = option_value {
+                                    // If no name is provided, use the filter name as the key
+                                    let key = option_name.unwrap_or_else(|| name.clone());
+                                    options.insert(key, value);
+                                }
+                            }
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            
+            filters.push(Filter { name, options });
+        }
+    }
+    
+    Ok(filters)
+}
+
+fn parse_option_value(input: &str) -> Result<serde_json::Value, Box<dyn std::error::Error>> {
+    // Try to parse as number
+    if let Ok(num) = input.parse::<i64>() {
+        return Ok(serde_json::Value::Number(num.into()));
+    }
+    if let Ok(num) = input.parse::<f64>() {
+        if let Some(number) = serde_json::Number::from_f64(num) {
+            return Ok(serde_json::Value::Number(number));
+        }
+    }
+    
+    // Try to parse as boolean
+    if let Ok(boolean) = input.parse::<bool>() {
+        return Ok(serde_json::Value::Bool(boolean));
+    }
+    
+    // Treat as string (remove quotes if present)
+    let value = if input.starts_with('"') && input.ends_with('"') {
+        input[1..input.len()-1].to_string()
+    } else if input.starts_with('\'') && input.ends_with('\'') {
+        input[1..input.len()-1].to_string()
+    } else {
+        input.to_string()
+    };
+    
+    Ok(serde_json::Value::String(value))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    
+    #[test]
+    fn test_parse_simple_filter() {
+        let result = parse_filter_string("grep").unwrap();
+        assert_eq!(result.len(), 1);
+        assert_eq!(result[0].name, "grep");
+        assert!(result[0].options.is_empty());
+    }
+    
+    #[test]
+    fn test_parse_filter_with_options() {
+        let result = parse_filter_string("head_lines(10)").unwrap();
+        assert_eq!(result.len(), 1);
+        assert_eq!(result[0].name, "head_lines");
+        assert_eq!(result[0].options["head_lines"], 10);
+    }
+    
+    #[test]
+    fn test_parse_filter_with_named_options() {
+        let result = parse_filter_string("grep(pattern=\"error\")").unwrap();
+        assert_eq!(result.len(), 1);
+        assert_eq!(result[0].name, "grep");
+        assert_eq!(result[0].options["pattern"], "error");
+    }
+    
+    #[test]
+    fn test_parse_multiple_filters() {
+        let result =

src/services/mod.rs

@@ -6,3 +6,12 @@ pub mod item_service;
 pub mod meta_service;
 pub mod status_service;
 pub mod types;
+
+pub use async_item_service::AsyncItemService;
+pub use compression_service::CompressionService;
+pub use error::CoreError;
+pub use filter_service::{FilterService, register_filter_plugin};
+pub use item_service::ItemService;
+pub use meta_service::MetaService;
+pub use status_service::StatusService;
+pub use types::{ItemWithContent, ItemWithMeta};