From 832330f31bdd21405bf96899326349c091e5d17f Mon Sep 17 00:00:00 2001 From: Andrew Phillips Date: Wed, 10 Sep 2025 09:39:22 -0300 Subject: [PATCH] feat: Add type and module reorganization for Services, Modes, Meta and Filter Plugins Co-authored-by: aider (openai/andrew/openrouter/sonoma-sky-alpha) --- Cargo.toml | 25 ++++++-- README.md | 16 +++++ src/filter_plugin/mod.rs | 6 ++ src/lib.rs | 8 +++ src/meta_plugin/mod.rs | 17 ++++- src/modes/mod.rs | 12 ++++ src/parser/filter_parser.rs | 124 ++++++++++++++++++++++++++++++++++++ src/services/mod.rs | 9 +++ 8 files changed, 211 insertions(+), 6 deletions(-) create mode 100644 src/parser/filter_parser.rs diff --git a/Cargo.toml b/Cargo.toml index c5955f9..66477c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ ctor = "0.2" directories = "6.0.0" dns-lookup = "2.0.2" enum-map = "2.6.1" -flate2 = { version = "1.0.27", features = ["zlib-ng-compat"] } +flate2 = { version = "1.0.27", features = ["zlib-ng-compat"], optional = true } futures = "0.3" gethostname = "1.0.2" humansize = "2.1.3" @@ -34,8 +34,8 @@ lazy_static = "1.4.0" libc = "0.2.147" local-ip-address = "0.6.5" log = "0.4.19" -lz4_flex = "0.11.1" -magic = "0.13.0" +lz4_flex = { version = "0.11.1", optional = true } +magic = { version = "0.13.0", optional = true } nix = "0.30.1" once_cell = "1.19.0" comfy-table = "7.2.0" @@ -67,7 +67,24 @@ strip-ansi-escapes = "0.2.1" pest = "2.8.1" pest_derive = "2.8.1" +[features] +# Default features include core compression engines +default = ["gzip", "lz4"] + +# Compression features +gzip = ["flate2"] +lz4 = ["lz4_flex"] +bzip2 = [] +xz = [] +zstd = [] + +# Plugin features (meta and filter) +all-meta-plugins = ["magic"] +all-filter-plugins = [] + +# Individual plugin features +magic = ["magic"] + [dev-dependencies] tempfile = "3.3.0" rand = "0.8.5" - diff --git a/README.md b/README.md index e69de29..f2e3a6e 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,16 @@ +# Keep - Temporary File Management with Compression and Metadata + +Keep is a command-line tool for managing temporary files with automatic compression, metadata generation, and querying capabilities. It supports various compression algorithms and metadata plugins for rich item inspection. + +## Features + +- **Store and Retrieve**: Save content with automatic compression and retrieve by ID or tags. +- **Compression Support**: Built-in support for LZ4, GZip, and more via external programs (BZip2, XZ, ZStd). +- **Metadata Plugins**: Automatic extraction of file type, digests, hostname, user info, and custom metadata. +- **Filtering**: Apply filters (head, tail, grep, etc.) when retrieving content. +- **Querying**: List, search, and diff items with flexible formatting. +- **REST API Server**: Optional HTTP server for programmatic access. +- **Modular Design**: Extensible via plugins for compression, metadata, and filtering. + +## Installation + diff --git a/src/filter_plugin/mod.rs b/src/filter_plugin/mod.rs index 1787052..4f984af 100644 --- a/src/filter_plugin/mod.rs +++ b/src/filter_plugin/mod.rs @@ -11,6 +11,12 @@ pub mod utils; use std::collections::HashMap; +pub use head::{HeadBytesFilter, HeadLinesFilter}; +pub use tail::{TailBytesFilter, TailLinesFilter}; +pub use grep::GrepFilter; +pub use skip::{SkipBytesFilter, SkipLinesFilter}; +pub use strip_ansi::StripAnsiFilter; + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] pub struct FilterOption { pub name: String, diff --git a/src/lib.rs b/src/lib.rs index c1e253f..25e6443 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,8 @@ pub mod filter_plugin; pub mod modes; pub mod plugins; pub mod args; +pub mod parser; +pub mod utils; // Re-export Args struct for library usage pub use args::Args; @@ -28,5 +30,11 @@ use meta_plugin::{ read_time, read_rate, hostname, exec, env }; +// Initialize plugins at library load time +pub fn init_plugins() { + // This will be expanded in Step 3 implementation + // For now, the ctors handle registration +} + #[cfg(test)] mod tests; diff --git a/src/meta_plugin/mod.rs b/src/meta_plugin/mod.rs index d17d38e..54d9f60 100644 --- a/src/meta_plugin/mod.rs +++ b/src/meta_plugin/mod.rs @@ -5,7 +5,6 @@ use std::sync::Mutex; use once_cell::sync::Lazy; pub mod exec; -use crate::meta_plugin::exec::MetaPluginExec; pub mod digest; pub mod magic; pub mod text; @@ -19,6 +18,20 @@ pub mod shell_pid; pub mod keep_pid; pub mod env; +pub use exec::MetaPluginExec; +pub use digest::DigestMetaPlugin; +pub use magic::MagicFileMetaPlugin; +pub use text::TextMetaPlugin; +pub use read_time::ReadTimeMetaPlugin; +pub use read_rate::ReadRateMetaPlugin; +pub use hostname::HostnameMetaPlugin; +pub use cwd::CwdMetaPlugin; +pub use user::UserMetaPlugin; +pub use shell::ShellMetaPlugin; +pub use shell_pid::ShellPidMetaPlugin; +pub use keep_pid::KeepPidMetaPlugin; +pub use env::EnvMetaPlugin; + /// Represents metadata to be stored #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MetaData { @@ -323,7 +336,7 @@ pub fn get_meta_plugin( } return Box::new(MetaPluginExec::new(&program_name, - args.iter().map(|s: &String| s.as_str()).collect(), + args.iter().map(|s| s.as_str()).collect(), meta_name, split_whitespace, options, diff --git a/src/modes/mod.rs b/src/modes/mod.rs index 5fac735..f6aee9a 100644 --- a/src/modes/mod.rs +++ b/src/modes/mod.rs @@ -9,3 +9,15 @@ pub mod save; pub mod server; pub mod status; pub mod status_plugins; + +pub use common::{ColumnType, OutputFormat, format_size, settings_output_format}; +pub use delete::mode_delete; +pub use diff::mode_diff; +pub use generate_config::mode_generate_config; +pub use get::mode_get; +pub use info::mode_info; +pub use list::mode_list; +pub use save::mode_save; +pub use server::mode_server; +pub use status::mode_status; +pub use status_plugins::mode_status_plugins; diff --git a/src/parser/filter_parser.rs b/src/parser/filter_parser.rs new file mode 100644 index 0000000..25636b7 --- /dev/null +++ b/src/parser/filter_parser.rs @@ -0,0 +1,124 @@ +use pest::Parser; +use pest_derive::Parser; +use std::collections::HashMap; + +#[derive(Parser)] +#[grammar = "filter.pest"] +pub struct FilterParser; + +#[derive(Debug)] +pub struct Filter { + pub name: String, + pub options: HashMap, +} + +pub fn parse_filter_string(input: &str) -> Result, Box> { + let mut filters = Vec::new(); + let pairs = FilterParser::parse(Rule::filters, input)?; + + for pair in pairs { + if pair.as_rule() == Rule::filter { + let mut name = String::new(); + let mut options = HashMap::new(); + + for inner_pair in pair.into_inner() { + match inner_pair.as_rule() { + Rule::filter_name => { + name = inner_pair.as_str().to_string(); + } + Rule::options => { + for option_pair in inner_pair.into_inner() { + if option_pair.as_rule() == Rule::option { + let mut option_name = None; + let mut option_value = None; + + for option_inner in option_pair.into_inner() { + match option_inner.as_rule() { + Rule::option_name => { + option_name = Some(option_inner.as_str().to_string()); + } + Rule::option_value => { + option_value = Some(parse_option_value(option_inner.as_str())?); + } + _ => {} + } + } + + if let Some(value) = option_value { + // If no name is provided, use the filter name as the key + let key = option_name.unwrap_or_else(|| name.clone()); + options.insert(key, value); + } + } + } + } + _ => {} + } + } + + filters.push(Filter { name, options }); + } + } + + Ok(filters) +} + +fn parse_option_value(input: &str) -> Result> { + // Try to parse as number + if let Ok(num) = input.parse::() { + return Ok(serde_json::Value::Number(num.into())); + } + if let Ok(num) = input.parse::() { + if let Some(number) = serde_json::Number::from_f64(num) { + return Ok(serde_json::Value::Number(number)); + } + } + + // Try to parse as boolean + if let Ok(boolean) = input.parse::() { + return Ok(serde_json::Value::Bool(boolean)); + } + + // Treat as string (remove quotes if present) + let value = if input.starts_with('"') && input.ends_with('"') { + input[1..input.len()-1].to_string() + } else if input.starts_with('\'') && input.ends_with('\'') { + input[1..input.len()-1].to_string() + } else { + input.to_string() + }; + + Ok(serde_json::Value::String(value)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_simple_filter() { + let result = parse_filter_string("grep").unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].name, "grep"); + assert!(result[0].options.is_empty()); + } + + #[test] + fn test_parse_filter_with_options() { + let result = parse_filter_string("head_lines(10)").unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].name, "head_lines"); + assert_eq!(result[0].options["head_lines"], 10); + } + + #[test] + fn test_parse_filter_with_named_options() { + let result = parse_filter_string("grep(pattern=\"error\")").unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].name, "grep"); + assert_eq!(result[0].options["pattern"], "error"); + } + + #[test] + fn test_parse_multiple_filters() { + let result = \ No newline at end of file diff --git a/src/services/mod.rs b/src/services/mod.rs index 058646a..434f8af 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -6,3 +6,12 @@ pub mod item_service; pub mod meta_service; pub mod status_service; pub mod types; + +pub use async_item_service::AsyncItemService; +pub use compression_service::CompressionService; +pub use error::CoreError; +pub use filter_service::{FilterService, register_filter_plugin}; +pub use item_service::ItemService; +pub use meta_service::MetaService; +pub use status_service::StatusService; +pub use types::{ItemWithContent, ItemWithMeta};