Compare commits

..

5 Commits

Author SHA1 Message Date
8379ae2136 refactor: rename plugin features with type prefix for consistency
- Plugin features now use type_ prefix (meta_magic, filter_grep, etc.)
- Added meta_all_musl and filter_all_musl for MUSL-compatible builds
- grep filter plugin made optional via filter_grep feature flag
- Removed regex crate from grep-related code, uses strip_prefix instead
- Updated CHANGELOG.md with breaking change documentation
2026-03-21 17:36:29 -03:00
12de215527 feat: feature-gate CLI args by server/client features
- CLI now shows only relevant options: --server and --server-* args
  hidden when built without 'server' feature; --client-* args hidden
  without 'client' feature. Run --help only displays applicable options.
- Removed verbose 'conflicts_with_all' from all mode args — clap's
  implicit group("mode") already enforces mutual exclusivity.
- 'server' feature now includes TLS/HTTPS by default (axum-server);
  'tls' feature removed. rustls already available via client/ureq.
- Gated KeepModes::Server, server mode detection, and server-password
  validation in main.rs.
- Gated server arg reads in config.rs.
- Removed redundant #[cfg(feature = "tls")] guards from server/mod.rs.
- Gated resolve_item_id/resolve_item_ids helpers in common.rs.
- All 4 feature combinations (server+client, server-only, client-only,
  neither) compile and pass tests.
2026-03-21 16:26:27 -03:00
e2cb36d2a8 feat(server): add file_size to API ItemInfo response 2026-03-21 14:03:58 -03:00
0004324301 perf: pre-allocate status info collections with known capacities 2026-03-21 13:54:37 -03:00
b3edfe7de6 chore: code review cleanup — fixes, deps, docs
Fixed:
- CLI help typo: "metatdata" -> "metadata"
- Filter buffer OOM: check size before loading into memory

Changed:
- #[inline] on HTML escape helpers for hot path performance
- Replaced once_cell and lazy_static with std::sync::LazyLock
- Removed unused once_cell and lazy_static crate dependencies

Refactored:
- Added module-level doc to services/ module

Documentation:
- README.md: zstd is native not external, "none" -> "raw"
- DESIGN.md: current schema and meta plugins section
- CHANGELOG.md: Unreleased section populated
2026-03-21 11:44:37 -03:00
24 changed files with 381 additions and 268 deletions

View File

@@ -7,6 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
### Added
- New `filter_grep` feature to optionally include the grep filter plugin (regex-based line filtering). Disabling this feature removes the `regex` crate and its ~800 KiB dependency stack from the binary.
- New `meta_all_musl` feature for all MUSL-compatible meta plugins (excludes `meta_magic` which requires libmagic)
- New `filter_all_musl` feature for all MUSL-compatible filter plugins
- Database index on `items(ts)` column for faster ORDER BY sorting
- Server API `ItemInfo` now includes `file_size` — actual filesystem-reported size of the item data file
### Changed
- CLI args now feature-gated: `--server` and related options hidden when built without `server` feature; `--client-*` options hidden when built without `client` feature. Run `--help` only shows relevant options.
- `server` Cargo feature now includes TLS support by default (`axum-server`); `tls` feature removed
- Clap `conflicts_with_all` removed from all mode args — exclusivity now handled by implicit `group("mode")`
- Filter plugins check size before loading content into memory (prevents OOM on large inputs)
- Status page pre-allocates collections with known capacities (meta plugins, compression info)
- `#[inline]` on HTML escape helper functions (`esc`, `esc_attr`) for hot path performance
- Removed `once_cell` crate (replaced with `std::sync::LazyLock` from Rust 1.80)
- Removed `lazy_static` crate (replaced with `std::sync::LazyLock`)
### Breaking
- Plugin feature flags renamed with type prefix for consistency:
- `magic``meta_magic`
- `infer``meta_infer`
- `tree_magic_mini``meta_tree_magic_mini`
- `tokens``meta_tokens`
- `grep``filter_grep`
- `all-meta-plugins``meta_all`
- `all-filter-plugins``filter_all`
### Fixed
- CLI help text typo: "metatdata" → "metadata" in `--get` and `--info` descriptions
### Refactored
- Added module-level documentation to `services/` module
### Documentation
- README.md: Fixed compression table — zstd is native (not external), "none" renamed to "raw"
- DESIGN.md: Updated schema to reflect current `items` table columns and meta plugin inventory
## [0.1.0] - 2026-03-21 ## [0.1.0] - 2026-03-21
### Added ### Added

2
Cargo.lock generated
View File

@@ -1727,7 +1727,6 @@ dependencies = [
"inventory", "inventory",
"is-terminal", "is-terminal",
"jsonwebtoken", "jsonwebtoken",
"lazy_static",
"libc", "libc",
"local-ip-address", "local-ip-address",
"log", "log",
@@ -1735,7 +1734,6 @@ dependencies = [
"magic", "magic",
"md5", "md5",
"nix", "nix",
"once_cell",
"os_pipe", "os_pipe",
"pest", "pest",
"pest_derive", "pest_derive",

View File

@@ -35,7 +35,6 @@ hyper = { version = "1.0", features = ["full"] }
http-body-util = "0.1" http-body-util = "0.1"
inventory = "0.3" inventory = "0.3"
is-terminal = "0.4" is-terminal = "0.4"
lazy_static = "1.5"
libc = "0.2" libc = "0.2"
local-ip-address = "0.6" local-ip-address = "0.6"
log = "0.4" log = "0.4"
@@ -45,10 +44,9 @@ magic = { version = "0.13", optional = true }
infer = { version = "0.19", optional = true } infer = { version = "0.19", optional = true }
tree_magic_mini = { version = "3.2", optional = true } tree_magic_mini = { version = "3.2", optional = true }
nix = { version = "0.30", features = ["fs", "process"] } nix = { version = "0.30", features = ["fs", "process"] }
once_cell = "1.21"
comfy-table = "7.2" comfy-table = "7.2"
pwhash = "1.0" pwhash = "1.0"
regex = "1.10" regex = { version = "1.10", optional = true }
ringbuf = "0.4" ringbuf = "0.4"
rusqlite = { version = "0.37", features = ["bundled", "array", "chrono"] } rusqlite = { version = "0.37", features = ["bundled", "array", "chrono"] }
rusqlite_migration = "2.3" rusqlite_migration = "2.3"
@@ -87,19 +85,20 @@ tiktoken-rs = { version = "0.9", optional = true }
tempfile = "3.3" tempfile = "3.3"
[features] [features]
# Default features include core compression engines and swagger UI # Default features include core compression engines plugins that support MUSL
default = [ default = [
"client", "client",
"gzip", "gzip",
"infer", "filter_grep",
"meta_infer",
"lz4", "lz4",
"tokens", "meta_tokens",
"tree_magic_mini", "meta_tree_magic_mini",
"zstd" "zstd"
] ]
# Server feature (includes axum and related dependencies) # Server feature (includes axum and TLS/HTTPS via axum-server; rustls already available via client/ureq)
server = ["dep:axum", "dep:tower", "dep:tower-http", "dep:utoipa", "dep:jsonwebtoken"] server = ["dep:axum", "dep:tower", "dep:tower-http", "dep:utoipa", "dep:jsonwebtoken", "dep:axum-server"]
# Compression features # Compression features
gzip = ["flate2"] gzip = ["flate2"]
@@ -108,14 +107,18 @@ bzip2 = []
xz = [] xz = []
zstd = ["dep:zstd"] zstd = ["dep:zstd"]
# Plugin features (meta and filter) # Meta plugin features
all-meta-plugins = ["dep:magic", "dep:infer", "dep:tree_magic_mini"] meta_magic = ["dep:magic"]
all-filter-plugins = [] meta_infer = ["dep:infer"]
meta_tree_magic_mini = ["dep:tree_magic_mini"]
meta_tokens = ["dep:tiktoken-rs"]
meta_all = ["meta_magic", "meta_infer", "meta_tree_magic_mini", "meta_tokens"]
meta_all_musl = ["meta_infer", "meta_tree_magic_mini", "meta_tokens"]
# Individual plugin features # Filter plugin features
magic = ["dep:magic"] filter_grep = ["dep:regex"]
infer = ["dep:infer"] filter_all = ["filter_grep"]
tree_magic_mini = ["dep:tree_magic_mini"] filter_all_musl = ["filter_grep"]
# Swagger UI feature # Swagger UI feature
swagger = ["dep:utoipa-swagger-ui"] swagger = ["dep:utoipa-swagger-ui"]
@@ -123,11 +126,5 @@ swagger = ["dep:utoipa-swagger-ui"]
# Client feature (HTTP client for remote server) # Client feature (HTTP client for remote server)
client = ["dep:ureq", "dep:os_pipe"] client = ["dep:ureq", "dep:os_pipe"]
# TLS feature (HTTPS server support)
tls = ["dep:axum-server"]
# Token counting feature (LLM token support via tiktoken)
tokens = ["dep:tiktoken-rs"]
[dev-dependencies] [dev-dependencies]
rand = "0.9" rand = "0.9"

View File

@@ -117,7 +117,7 @@
## Data Storage ## Data Storage
### Database Schema ### Database Schema
- `items` table: id (primary key), ts (timestamp), size (optional), compression - `items` table: id (primary key), ts (timestamp), uncompressed_size (optional), compressed_size (optional), closed (boolean), compression
- `tags` table: id (foreign key to items), name (tag name) - `tags` table: id (foreign key to items), name (tag name)
- `metas` table: id (foreign key to items), name (meta key), value (meta value) - `metas` table: id (foreign key to items), name (meta key), value (meta value)
- Indexes on tag names and meta names for faster queries - Indexes on tag names and meta names for faster queries
@@ -178,26 +178,25 @@
- None (no compression) - None (no compression)
## Supported Meta Plugins ## Supported Meta Plugins
- FileMagic - File type detection using file command
- FileMime - MIME type detection using file command Meta plugins collect metadata during item save. Each plugin produces one or more key-value pairs:
- FileEncoding - File encoding detection using file command
- LineCount - Line count using wc command - `magic_file` - File type detection using libmagic (when `magic` feature enabled)
- WordCount - Word count using wc command - `infer` - MIME type detection using infer crate (when `infer` feature enabled)
- Cwd - Current working directory - `tree_magic_mini` - MIME type detection using tree_magic_mini (when `tree_magic_mini` feature enabled)
- Binary - Binary file detection - `tokens` - LLM token counting using tiktoken (when `tokens` feature enabled)
- Uid - Current user ID - `text` - Text analysis: line count, word count, char count, line average length
- User - Current username - `digest` - SHA-256 and MD5 checksums
- Gid - Current group ID - `hostname` - System hostname (full and short)
- Group - Current group name - `cwd` - Current working directory
- Shell - Shell path from SHELL environment variable - `user` - Current username and UID
- ShellPid - Shell process ID from PPID environment variable - `shell` - Shell path from SHELL environment variable
- KeepPid - Keep process ID - `shell_pid` - Shell process ID from PPID
- DigestSha256 - SHA-256 digest - `keep_pid` - Keep process ID
- DigestMd5 - MD5 digest using md5sum command - `env` - Arbitrary environment variables (via `KEEP_META_ENV_*` prefix)
- ReadTime - Time taken to read data - `exec` - Execute external commands for custom metadata
- ReadRate - Rate of data reading - `read_time` - Time taken to read content
- Hostname - System hostname - `read_rate` - Content read rate (bytes/second)
- FullHostname - Fully qualified domain name
## Testing Strategy ## Testing Strategy
- Unit tests for each module in `src/tests/` - Unit tests for each module in `src/tests/`

View File

@@ -345,8 +345,8 @@ Items are compressed automatically on save. Default: LZ4.
| `gzip` | Internal | Fast | Good | | `gzip` | Internal | Fast | Good |
| `bzip2` | External | Slow | Better | | `bzip2` | External | Slow | Better |
| `xz` | External | Slowest | Best | | `xz` | External | Slowest | Best |
| `zstd` | External | Fast | Good | | `zstd` | Internal | Fast | Good |
| `none` | Internal | N/A | N/A | | `raw` | Internal | N/A | N/A |
```sh ```sh
# Specify compression per item # Specify compression per item

View File

@@ -24,81 +24,80 @@ pub struct Args {
/// Struct for mode-specific arguments, defining CLI flags for different operations. /// Struct for mode-specific arguments, defining CLI flags for different operations.
#[derive(Parser, Debug, Clone)] #[derive(Parser, Debug, Clone)]
pub struct ModeArgs { pub struct ModeArgs {
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["get", "diff", "list", "delete", "info", "update", "status", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help("Save an item using any tags or metadata provided"))] #[arg(help("Save an item using any tags or metadata provided"))]
pub save: bool, pub save: bool,
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "diff", "list", "delete", "info", "update", "status", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help( #[arg(help("Get an item either by its ID or by a combination of matching tags and metadata"))]
"Get an item either by it's ID or by a combination of matching tags and metatdata"
))]
pub get: bool, pub get: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "list", "delete", "info", "update", "status", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Show a diff between two items by ID"))] #[arg(help("Show a diff between two items by ID"))]
pub diff: bool, pub diff: bool,
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "diff", "delete", "info", "update", "status", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help("List items, filtering on tags or metadata if given"))] #[arg(help("List items, filtering on tags or metadata if given"))]
pub list: bool, pub list: bool,
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "diff", "list", "info", "update", "status", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help("Delete items either by ID or by matching tags"))] #[arg(help("Delete items either by ID or by matching tags"))]
#[arg(requires = "ids_or_tags")] #[arg(requires = "ids_or_tags")]
pub delete: bool, pub delete: bool,
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "diff", "list", "delete", "update", "status", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help( #[arg(help("Get an item either by its ID or by a combination of matching tags and metadata"))]
"Get an item either by it's ID or by a combination of matching tags and metatdata"
))]
pub info: bool, pub info: bool,
#[arg(group("mode"), help_heading("Mode Options"), short('u'), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "status", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), short('u'), long)]
#[arg(help("Update an item's tags and metadata by ID"))] #[arg(help("Update an item's tags and metadata by ID"))]
pub update: bool, pub update: bool,
#[arg(group("mode"), help_heading("Mode Options"), short('S'), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "server", "status_plugins", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), short('S'), long)]
#[arg(help("Show status of directories and supported compression algorithms"))] #[arg(help("Show status of directories and supported compression algorithms"))]
pub status: bool, pub status: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Show available plugins and their configurations"))] #[arg(help("Show available plugins and their configurations"))]
pub status_plugins: bool, pub status_plugins: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Export items to a .keep.tar archive (requires IDs or tags)"))] #[arg(help("Export items to a .keep.tar archive (requires IDs or tags)"))]
pub export: bool, pub export: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, value_name("FILE"), conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "export"]))] #[arg(group("mode"), help_heading("Mode Options"), long, value_name("FILE"))]
#[arg(help("Import items from a .keep.tar archive or legacy .meta.yml file"))] #[arg(help("Import items from a .keep.tar archive or legacy .meta.yml file"))]
pub import: Option<String>, pub import: Option<String>,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status"]))] #[cfg(feature = "server")]
#[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Start REST HTTP server"))] #[arg(help("Start REST HTTP server"))]
pub server: bool, pub server: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "export", "import"]))] #[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Generate default configuration and output to stdout"))] #[arg(help("Generate default configuration and output to stdout"))]
pub generate_config: bool, pub generate_config: bool,
#[arg(help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "generate_config", "export", "import"]))] #[arg(help_heading("Mode Options"), long)]
#[arg(help("Generate shell completion script"))] #[arg(help("Generate shell completion script"))]
pub generate_completion: Option<Shell>, pub generate_completion: Option<Shell>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_ADDRESS"))] #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_ADDRESS"))]
#[arg(help("Server address to bind to"))] #[arg(help("Server address to bind to"))]
pub server_address: Option<String>, pub server_address: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PORT"))] #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PORT"))]
#[arg(help("Server port to bind to"))] #[arg(help("Server port to bind to"))]
pub server_port: Option<u16>, pub server_port: Option<u16>,
#[cfg(feature = "tls")] #[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_CERT"))] #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_CERT"))]
#[arg(help("Path to TLS certificate file (PEM) for HTTPS"))] #[arg(help("Path to TLS certificate file (PEM) for HTTPS"))]
pub server_cert: Option<PathBuf>, pub server_cert: Option<PathBuf>,
#[cfg(feature = "tls")] #[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_KEY"))] #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_KEY"))]
#[arg(help("Path to TLS private key file (PEM) for HTTPS"))] #[arg(help("Path to TLS private key file (PEM) for HTTPS"))]
pub server_key: Option<PathBuf>, pub server_key: Option<PathBuf>,
@@ -253,24 +252,29 @@ pub struct OptionsArgs {
#[arg(help("Output format (only works with --info, --status, --list)"))] #[arg(help("Output format (only works with --info, --status, --list)"))]
pub output_format: Option<String>, pub output_format: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PASSWORD"))] #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PASSWORD"))]
#[arg(help("Password for server authentication (requires --server)"))] #[arg(help("Password for server authentication (requires --server)"))]
pub server_password: Option<String>, pub server_password: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PASSWORD_HASH"))] #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PASSWORD_HASH"))]
#[arg(help("Password hash for server authentication (requires --server)"))] #[arg(help("Password hash for server authentication (requires --server)"))]
pub server_password_hash: Option<String>, pub server_password_hash: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_USERNAME"))] #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_USERNAME"))]
#[arg(help( #[arg(help(
"Username for server Basic authentication (requires --server, defaults to 'keep')" "Username for server Basic authentication (requires --server, defaults to 'keep')"
))] ))]
pub server_username: Option<String>, pub server_username: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_JWT_SECRET"))] #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_JWT_SECRET"))]
#[arg(help("JWT secret for token-based authentication (requires --server)"))] #[arg(help("JWT secret for token-based authentication (requires --server)"))]
pub server_jwt_secret: Option<String>, pub server_jwt_secret: Option<String>,
#[cfg(feature = "server")]
#[arg( #[arg(
help_heading("Server Options"), help_heading("Server Options"),
long, long,
@@ -279,6 +283,7 @@ pub struct OptionsArgs {
#[arg(help("Path to file containing JWT secret (requires --server)"))] #[arg(help("Path to file containing JWT secret (requires --server)"))]
pub server_jwt_secret_file: Option<PathBuf>, pub server_jwt_secret_file: Option<PathBuf>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_MAX_BODY_SIZE"))] #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_MAX_BODY_SIZE"))]
#[arg(help("Maximum request body size in bytes (requires --server, default: unlimited)"))] #[arg(help("Maximum request body size in bytes (requires --server, default: unlimited)"))]
pub server_max_body_size: Option<u64>, pub server_max_body_size: Option<u64>,

View File

@@ -89,7 +89,7 @@ pub fn generate_status_info(
}; };
let _default_type = crate::compression_engine::default_compression_type(); let _default_type = crate::compression_engine::default_compression_type();
let mut compression_info = Vec::new(); let mut compression_info = Vec::with_capacity(CompressionType::iter().count());
// Sort compression types by their string representation // Sort compression types by their string representation
let mut sorted_compression_types: Vec<CompressionType> = CompressionType::iter().collect(); let mut sorted_compression_types: Vec<CompressionType> = CompressionType::iter().collect();
@@ -141,7 +141,8 @@ pub fn generate_status_info(
}); });
} }
let mut meta_plugins_map = std::collections::HashMap::new(); let mut meta_plugins_map =
std::collections::HashMap::with_capacity(MetaPluginType::iter().count());
let mut enabled_meta_plugins_vec = Vec::new(); let mut enabled_meta_plugins_vec = Vec::new();
// Sort meta plugin types by their string representation to avoid creating plugins just for sorting // Sort meta plugin types by their string representation to avoid creating plugins just for sorting

View File

@@ -7,8 +7,6 @@ use strum::{Display, EnumIter, EnumString};
use log::*; use log::*;
use lazy_static::lazy_static;
extern crate enum_map; extern crate enum_map;
use enum_map::enum_map; use enum_map::enum_map;
use enum_map::{Enum, EnumMap}; use enum_map::{Enum, EnumMap};
@@ -180,63 +178,65 @@ impl Clone for Box<dyn CompressionEngine> {
} }
} }
lazy_static! { fn init_compression_engines() -> EnumMap<CompressionType, Box<dyn CompressionEngine>> {
static ref COMPRESSION_ENGINES: EnumMap<CompressionType, Box<dyn CompressionEngine>> = { #[allow(unused_mut)]
#[allow(unused_mut)] // mut needed when gzip/lz4 features are enabled let mut em: EnumMap<CompressionType, Box<dyn CompressionEngine>> = enum_map! {
let mut em = enum_map! { CompressionType::LZ4 => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
CompressionType::LZ4 => Box::new(crate::compression_engine::program::CompressionEngineProgram::new( "lz4",
"lz4", vec!["-c"],
vec!["-c"], vec!["-d", "-c"]
vec!["-d", "-c"] )) as Box<dyn CompressionEngine>,
)) as Box<dyn CompressionEngine>, CompressionType::GZip => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
CompressionType::GZip => Box::new(crate::compression_engine::program::CompressionEngineProgram::new( "gzip",
"gzip", vec!["-c"],
vec!["-c"], vec!["-d", "-c"]
vec!["-d", "-c"] )) as Box<dyn CompressionEngine>,
)) as Box<dyn CompressionEngine>, CompressionType::BZip2 => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
CompressionType::BZip2 => Box::new(crate::compression_engine::program::CompressionEngineProgram::new( "bzip2",
"bzip2", vec!["-c"],
vec!["-c"], vec!["-d", "-c"]
vec!["-d", "-c"] )) as Box<dyn CompressionEngine>,
)) as Box<dyn CompressionEngine>, CompressionType::XZ => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
CompressionType::XZ => Box::new(crate::compression_engine::program::CompressionEngineProgram::new( "xz",
"xz", vec!["-c"],
vec!["-c"], vec!["-d", "-c"]
vec!["-d", "-c"] )) as Box<dyn CompressionEngine>,
)) as Box<dyn CompressionEngine>, CompressionType::ZStd => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
CompressionType::ZStd => Box::new(crate::compression_engine::program::CompressionEngineProgram::new( "zstd",
"zstd", vec!["-c"],
vec!["-c"], vec!["-d", "-c"]
vec!["-d", "-c"] )) as Box<dyn CompressionEngine>,
)) as Box<dyn CompressionEngine>, CompressionType::Raw => Box::new(crate::compression_engine::raw::CompressionEngineRaw::new()) as Box<dyn CompressionEngine>
CompressionType::Raw => Box::new(crate::compression_engine::raw::CompressionEngineRaw::new()) as Box<dyn CompressionEngine>
};
#[cfg(feature = "gzip")]
{
em[CompressionType::GZip] =
Box::new(crate::compression_engine::gzip::CompressionEngineGZip::new())
as Box<dyn CompressionEngine>;
}
#[cfg(feature = "lz4")]
{
em[CompressionType::LZ4] =
Box::new(crate::compression_engine::lz4::CompressionEngineLZ4::new())
as Box<dyn CompressionEngine>;
}
#[cfg(feature = "zstd")]
{
em[CompressionType::ZStd] =
Box::new(crate::compression_engine::zstd::CompressionEngineZstd::new())
as Box<dyn CompressionEngine>;
}
em
}; };
#[cfg(feature = "gzip")]
{
em[CompressionType::GZip] =
Box::new(crate::compression_engine::gzip::CompressionEngineGZip::new())
as Box<dyn CompressionEngine>;
}
#[cfg(feature = "lz4")]
{
em[CompressionType::LZ4] =
Box::new(crate::compression_engine::lz4::CompressionEngineLZ4::new())
as Box<dyn CompressionEngine>;
}
#[cfg(feature = "zstd")]
{
em[CompressionType::ZStd] =
Box::new(crate::compression_engine::zstd::CompressionEngineZstd::new())
as Box<dyn CompressionEngine>;
}
em
} }
static COMPRESSION_ENGINES: std::sync::LazyLock<
EnumMap<CompressionType, Box<dyn CompressionEngine>>,
> = std::sync::LazyLock::new(init_compression_engines);
pub fn default_compression_type() -> CompressionType { pub fn default_compression_type() -> CompressionType {
CompressionType::LZ4 CompressionType::LZ4
} }

View File

@@ -301,42 +301,48 @@ impl Settings {
config_builder = config_builder.set_override("force", true)?; config_builder = config_builder.set_override("force", true)?;
} }
#[cfg(feature = "server")]
if let Some(server_password) = &args.options.server_password { if let Some(server_password) = &args.options.server_password {
config_builder = config_builder =
config_builder.set_override("server.password", server_password.as_str())?; config_builder.set_override("server.password", server_password.as_str())?;
} }
#[cfg(feature = "server")]
if let Some(server_password_hash) = &args.options.server_password_hash { if let Some(server_password_hash) = &args.options.server_password_hash {
config_builder = config_builder config_builder = config_builder
.set_override("server.password_hash", server_password_hash.as_str())?; .set_override("server.password_hash", server_password_hash.as_str())?;
} }
#[cfg(feature = "server")]
if let Some(server_username) = &args.options.server_username { if let Some(server_username) = &args.options.server_username {
config_builder = config_builder =
config_builder.set_override("server.username", server_username.as_str())?; config_builder.set_override("server.username", server_username.as_str())?;
} }
#[cfg(feature = "server")]
if let Some(server_address) = &args.mode.server_address { if let Some(server_address) = &args.mode.server_address {
config_builder = config_builder =
config_builder.set_override("server.address", server_address.as_str())?; config_builder.set_override("server.address", server_address.as_str())?;
} }
#[cfg(feature = "server")]
if let Some(server_port) = args.mode.server_port { if let Some(server_port) = args.mode.server_port {
config_builder = config_builder.set_override("server.port", server_port)?; config_builder = config_builder.set_override("server.port", server_port)?;
} }
#[cfg(feature = "tls")] #[cfg(feature = "server")]
if let Some(server_cert) = &args.mode.server_cert { if let Some(server_cert) = &args.mode.server_cert {
config_builder = config_builder config_builder = config_builder
.set_override("server.cert_file", server_cert.to_string_lossy().as_ref())?; .set_override("server.cert_file", server_cert.to_string_lossy().as_ref())?;
} }
#[cfg(feature = "tls")] #[cfg(feature = "server")]
if let Some(server_key) = &args.mode.server_key { if let Some(server_key) = &args.mode.server_key {
config_builder = config_builder config_builder = config_builder
.set_override("server.key_file", server_key.to_string_lossy().as_ref())?; .set_override("server.key_file", server_key.to_string_lossy().as_ref())?;
} }
#[cfg(feature = "server")]
if let Some(max_body_size) = args.options.server_max_body_size { if let Some(max_body_size) = args.options.server_max_body_size {
config_builder = config_builder.set_override("server.max_body_size", max_body_size)?; config_builder = config_builder.set_override("server.max_body_size", max_body_size)?;
} }

View File

@@ -1,6 +1,5 @@
use anyhow::{Context, Error, Result, anyhow}; use anyhow::{Context, Error, Result, anyhow};
use chrono::prelude::*; use chrono::prelude::*;
use lazy_static::lazy_static;
use log::*; use log::*;
use rusqlite::{Connection, OpenFlags, Row, params}; use rusqlite::{Connection, OpenFlags, Row, params};
use rusqlite_migration::{M, Migrations}; use rusqlite_migration::{M, Migrations};
@@ -47,25 +46,21 @@ let id = db::insert_item(&conn, item)?;
``` ```
*/ */
lazy_static! { static MIGRATIONS: std::sync::LazyLock<Migrations<'static>> = std::sync::LazyLock::new(|| {
// Database schema migrations for the Keep application. Migrations::new(vec![
//
// Defines the sequence of migrations to create and update the schema.
// Applied automatically when opening a database connection.
static ref MIGRATIONS: Migrations<'static> = Migrations::new(vec![
M::up( M::up(
"CREATE TABLE items( "CREATE TABLE items(
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
ts TEXT NOT NULL, ts TEXT NOT NULL,
size INTEGER NULL, size INTEGER NULL,
compression TEXT NOT NULL)" compression TEXT NOT NULL)",
), ),
M::up( M::up(
"CREATE TABLE tags ( "CREATE TABLE tags (
id INTEGER NOT NULL, id INTEGER NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE, FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY(id, name));" PRIMARY KEY(id, name));",
), ),
M::up( M::up(
"CREATE TABLE metas ( "CREATE TABLE metas (
@@ -73,16 +68,17 @@ lazy_static! {
name TEXT NOT NULL, name TEXT NOT NULL,
value TEXT NOT NULL, value TEXT NOT NULL,
FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE, FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY(id, name));" PRIMARY KEY(id, name));",
), ),
M::up("CREATE INDEX idx_tags_name ON tags(name)"), M::up("CREATE INDEX idx_tags_name ON tags(name)"),
M::up("CREATE INDEX idx_metas_name ON metas(name)"), M::up("CREATE INDEX idx_metas_name ON metas(name)"),
M::up("CREATE INDEX idx_items_ts ON items(ts)"),
M::up("UPDATE items SET compression = 'raw' WHERE compression = 'none'"), M::up("UPDATE items SET compression = 'raw' WHERE compression = 'none'"),
M::up("ALTER TABLE items RENAME COLUMN size TO uncompressed_size"), M::up("ALTER TABLE items RENAME COLUMN size TO uncompressed_size"),
M::up("ALTER TABLE items ADD COLUMN compressed_size INTEGER NULL"), M::up("ALTER TABLE items ADD COLUMN compressed_size INTEGER NULL"),
M::up("ALTER TABLE items ADD COLUMN closed BOOLEAN NOT NULL DEFAULT 1"), M::up("ALTER TABLE items ADD COLUMN closed BOOLEAN NOT NULL DEFAULT 1"),
]); ])
} });
/// Represents an item stored in the database. /// Represents an item stored in the database.
/// ///

View File

@@ -2,6 +2,7 @@ use std::io::{Read, Result, Write};
use std::str::FromStr; use std::str::FromStr;
use strum::EnumString; use strum::EnumString;
#[cfg(feature = "filter_grep")]
pub mod grep; pub mod grep;
/// Filter plugin module for processing input streams. /// Filter plugin module for processing input streams.
/// ///
@@ -16,7 +17,7 @@ pub mod grep;
/// ``` /// ```
/// # use std::io::{Read, Write}; /// # use std::io::{Read, Write};
/// # use keep::filter_plugin::parse_filter_string; /// # use keep::filter_plugin::parse_filter_string;
/// let mut chain = parse_filter_string("head_lines(10)|grep(pattern=error)")?; /// let mut chain = parse_filter_string("head_lines(10)|tail_lines(5)")?;
/// # let mut reader: &mut dyn Read = &mut std::io::empty(); /// # let mut reader: &mut dyn Read = &mut std::io::empty();
/// # let mut writer: Vec<u8> = Vec::new(); /// # let mut writer: Vec<u8> = Vec::new();
/// # chain.filter(&mut reader, &mut writer)?; /// # chain.filter(&mut reader, &mut writer)?;
@@ -26,12 +27,13 @@ pub mod head;
pub mod skip; pub mod skip;
pub mod strip_ansi; pub mod strip_ansi;
pub mod tail; pub mod tail;
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
pub mod tokens; pub mod tokens;
pub mod utils; pub mod utils;
use std::collections::HashMap; use std::collections::HashMap;
#[cfg(feature = "filter_grep")]
pub use grep::GrepFilter; pub use grep::GrepFilter;
pub use head::{HeadBytesFilter, HeadLinesFilter}; pub use head::{HeadBytesFilter, HeadLinesFilter};
pub use skip::{SkipBytesFilter, SkipLinesFilter}; pub use skip::{SkipBytesFilter, SkipLinesFilter};
@@ -199,13 +201,14 @@ pub enum FilterType {
TailLines, TailLines,
SkipBytes, SkipBytes,
SkipLines, SkipLines,
#[cfg(feature = "filter_grep")]
Grep, Grep,
StripAnsi, StripAnsi,
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
HeadTokens, HeadTokens,
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
SkipTokens, SkipTokens,
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
TailTokens, TailTokens,
} }
@@ -213,6 +216,44 @@ pub enum FilterType {
/// Prevents OOM on large files by rejecting inputs that exceed this limit. /// Prevents OOM on large files by rejecting inputs that exceed this limit.
const MAX_FILTER_BUFFER_SIZE: usize = 256 * 1024 * 1024; const MAX_FILTER_BUFFER_SIZE: usize = 256 * 1024 * 1024;
struct BoundedVecWriter {
data: Vec<u8>,
limit: usize,
}
impl BoundedVecWriter {
fn new(limit: usize) -> Self {
Self {
data: Vec::new(),
limit,
}
}
fn into_inner(self) -> Vec<u8> {
self.data
}
}
impl std::io::Write for BoundedVecWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if self.data.len() + buf.len() > self.limit {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!(
"Input size exceeds maximum filter buffer size ({} bytes)",
MAX_FILTER_BUFFER_SIZE
),
));
}
self.data.write_all(buf)?;
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
/// A chain of filter plugins applied sequentially. /// A chain of filter plugins applied sequentially.
/// ///
/// Chains multiple filters, applying them in order to the input stream. /// Chains multiple filters, applying them in order to the input stream.
@@ -318,9 +359,8 @@ impl FilterChain {
/// # Examples /// # Examples
/// ///
/// ``` /// ```
/// # use keep::filter_plugin::{FilterChain, GrepFilter}; /// # use keep::filter_plugin::FilterChain;
/// let mut chain = FilterChain::new(); /// let mut chain = FilterChain::new();
/// chain.add_plugin(Box::new(GrepFilter::new("error".to_string()).unwrap()));
/// ``` /// ```
pub fn add_plugin(&mut self, plugin: Box<dyn FilterPlugin>) { pub fn add_plugin(&mut self, plugin: Box<dyn FilterPlugin>) {
self.plugins.push(plugin); self.plugins.push(plugin);
@@ -360,21 +400,10 @@ impl FilterChain {
} }
// For multiple plugins, we need to chain them together // For multiple plugins, we need to chain them together
// We'll use a temporary buffer to hold intermediate results // We'll use a bounded buffer to hold intermediate results
let mut current_data = Vec::new(); let mut bounded_writer = BoundedVecWriter::new(MAX_FILTER_BUFFER_SIZE);
std::io::copy(reader, &mut current_data)?; std::io::copy(reader, &mut bounded_writer)?;
let mut current_data = bounded_writer.into_inner();
if current_data.len() > MAX_FILTER_BUFFER_SIZE {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!(
"Input size ({} bytes) exceeds maximum filter buffer size ({} bytes). \
Consider using fewer filter plugins or smaller inputs.",
current_data.len(),
MAX_FILTER_BUFFER_SIZE
),
));
}
// Store the plugins length to avoid borrowing issues // Store the plugins length to avoid borrowing issues
let plugins_len = self.plugins.len(); let plugins_len = self.plugins.len();
@@ -508,6 +537,7 @@ fn create_filter_with_options(
// Get the default options for this filter type by creating a temporary instance // Get the default options for this filter type by creating a temporary instance
// To do this, we need to create a default instance of the appropriate filter // To do this, we need to create a default instance of the appropriate filter
let option_defs = match filter_type { let option_defs = match filter_type {
#[cfg(feature = "filter_grep")]
FilterType::Grep => grep::GrepFilter::new("".to_string())?.options(), FilterType::Grep => grep::GrepFilter::new("".to_string())?.options(),
FilterType::HeadBytes => head::HeadBytesFilter::new(0).options(), FilterType::HeadBytes => head::HeadBytesFilter::new(0).options(),
FilterType::HeadLines => head::HeadLinesFilter::new(0).options(), FilterType::HeadLines => head::HeadLinesFilter::new(0).options(),
@@ -516,11 +546,11 @@ fn create_filter_with_options(
FilterType::SkipBytes => skip::SkipBytesFilter::new(0).options(), FilterType::SkipBytes => skip::SkipBytesFilter::new(0).options(),
FilterType::SkipLines => skip::SkipLinesFilter::new(0).options(), FilterType::SkipLines => skip::SkipLinesFilter::new(0).options(),
FilterType::StripAnsi => strip_ansi::StripAnsiFilter::new().options(), FilterType::StripAnsi => strip_ansi::StripAnsiFilter::new().options(),
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
FilterType::HeadTokens => tokens::HeadTokensFilter::new(0).options(), FilterType::HeadTokens => tokens::HeadTokensFilter::new(0).options(),
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
FilterType::SkipTokens => tokens::SkipTokensFilter::new(0).options(), FilterType::SkipTokens => tokens::SkipTokensFilter::new(0).options(),
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
FilterType::TailTokens => tokens::TailTokensFilter::new(0).options(), FilterType::TailTokens => tokens::TailTokensFilter::new(0).options(),
}; };
@@ -590,6 +620,7 @@ fn create_specific_filter(
options: &HashMap<String, serde_json::Value>, options: &HashMap<String, serde_json::Value>,
) -> Result<Box<dyn FilterPlugin>> { ) -> Result<Box<dyn FilterPlugin>> {
match filter_type { match filter_type {
#[cfg(feature = "filter_grep")]
FilterType::Grep => { FilterType::Grep => {
let pattern = options let pattern = options
.get("pattern") .get("pattern")
@@ -690,7 +721,7 @@ fn create_specific_filter(
} }
Ok(Box::new(strip_ansi::StripAnsiFilter::new())) Ok(Box::new(strip_ansi::StripAnsiFilter::new()))
} }
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
FilterType::HeadTokens => { FilterType::HeadTokens => {
let count = options let count = options
.get("count") .get("count")
@@ -708,7 +739,7 @@ fn create_specific_filter(
f.encoding = encoding; f.encoding = encoding;
Ok(Box::new(f)) Ok(Box::new(f))
} }
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
FilterType::SkipTokens => { FilterType::SkipTokens => {
let count = options let count = options
.get("count") .get("count")
@@ -726,7 +757,7 @@ fn create_specific_filter(
f.encoding = encoding; f.encoding = encoding;
Ok(Box::new(f)) Ok(Box::new(f))
} }
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
FilterType::TailTokens => { FilterType::TailTokens => {
let count = options let count = options
.get("count") .get("count")
@@ -747,7 +778,7 @@ fn create_specific_filter(
} }
} }
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
fn parse_encoding_option( fn parse_encoding_option(
options: &std::collections::HashMap<String, serde_json::Value>, options: &std::collections::HashMap<String, serde_json::Value>,
) -> (crate::tokenizer::TokenEncoding, crate::tokenizer::Tokenizer) { ) -> (crate::tokenizer::TokenEncoding, crate::tokenizer::Tokenizer) {

View File

@@ -45,7 +45,7 @@ pub mod services;
#[cfg(feature = "client")] #[cfg(feature = "client")]
pub mod client; pub mod client;
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
pub mod tokenizer; pub mod tokenizer;
// Re-export Args struct for library usage // Re-export Args struct for library usage
@@ -56,9 +56,12 @@ pub use services::CoreError;
// Import all filter plugins to ensure they register themselves // Import all filter plugins to ensure they register themselves
#[allow(unused_imports)] #[allow(unused_imports)]
use filter_plugin::{grep, head, skip, strip_ansi, tail}; #[cfg(feature = "filter_grep")]
use filter_plugin::grep;
#[allow(unused_imports)]
use filter_plugin::{head, skip, strip_ansi, tail};
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
#[allow(unused_imports)] #[allow(unused_imports)]
use filter_plugin::tokens as token_filters; use filter_plugin::tokens as token_filters;
@@ -66,19 +69,19 @@ use crate::meta_plugin::{
cwd, digest, env, exec, hostname, keep_pid, read_rate, read_time, shell, shell_pid, user, cwd, digest, env, exec, hostname, keep_pid, read_rate, read_time, shell, shell_pid, user,
}; };
#[cfg(feature = "magic")] #[cfg(feature = "meta_magic")]
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::meta_plugin::magic_file; use crate::meta_plugin::magic_file;
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::meta_plugin::tokens; use crate::meta_plugin::tokens;
#[cfg(feature = "infer")] #[cfg(feature = "meta_infer")]
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::meta_plugin::infer_plugin; use crate::meta_plugin::infer_plugin;
#[cfg(feature = "tree_magic_mini")] #[cfg(feature = "meta_tree_magic_mini")]
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::meta_plugin::tree_magic_mini; use crate::meta_plugin::tree_magic_mini;

View File

@@ -122,6 +122,7 @@ fn main() -> Result<(), Error> {
Import, Import,
Status, Status,
StatusPlugins, StatusPlugins,
#[cfg(feature = "server")]
Server, Server,
GenerateConfig, GenerateConfig,
} }
@@ -150,9 +151,14 @@ fn main() -> Result<(), Error> {
mode = KeepModes::Status; mode = KeepModes::Status;
} else if args.mode.status_plugins { } else if args.mode.status_plugins {
mode = KeepModes::StatusPlugins; mode = KeepModes::StatusPlugins;
} else if args.mode.server { }
mode = KeepModes::Server; #[cfg(feature = "server")]
} else if args.mode.generate_config { {
if args.mode.server {
mode = KeepModes::Server;
}
}
if args.mode.generate_config {
mode = KeepModes::GenerateConfig; mode = KeepModes::GenerateConfig;
} }
@@ -188,6 +194,7 @@ fn main() -> Result<(), Error> {
} }
// Validate server password usage // Validate server password usage
#[cfg(feature = "server")]
if settings.server_password().is_some() && mode != KeepModes::Server { if settings.server_password().is_some() && mode != KeepModes::Server {
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
@@ -355,19 +362,8 @@ fn main() -> Result<(), Error> {
KeepModes::StatusPlugins => { KeepModes::StatusPlugins => {
modes::status_plugins::mode_status_plugins(&mut cmd, &settings, data_path, db_path) modes::status_plugins::mode_status_plugins(&mut cmd, &settings, data_path, db_path)
} }
KeepModes::Server => { #[cfg(feature = "server")]
#[cfg(feature = "server")] KeepModes::Server => modes::server::mode_server(&mut cmd, &settings, &mut conn, data_path),
{
modes::server::mode_server(&mut cmd, &settings, &mut conn, data_path)
}
#[cfg(not(feature = "server"))]
{
cmd.error(
ErrorKind::MissingRequiredArgument,
"This binary was not compiled with server support. Recompile with --features server"
).exit();
}
}
KeepModes::GenerateConfig => { KeepModes::GenerateConfig => {
modes::generate_config::mode_generate_config(&mut cmd, &settings) modes::generate_config::mode_generate_config(&mut cmd, &settings)
} }

View File

@@ -1,6 +1,6 @@
#[cfg(feature = "magic")] #[cfg(feature = "meta_magic")]
use magic::{Cookie, CookieFlags}; use magic::{Cookie, CookieFlags};
#[cfg(not(feature = "magic"))] #[cfg(not(feature = "meta_magic"))]
use std::process::{Command, Stdio}; use std::process::{Command, Stdio};
use std::io::{self, Write}; use std::io::{self, Write};
@@ -16,12 +16,12 @@ use crate::meta_plugin::{
// separate cookies can be used from different threads concurrently without // separate cookies can be used from different threads concurrently without
// synchronization. Using thread_local! avoids unsafe impl Send since the // synchronization. Using thread_local! avoids unsafe impl Send since the
// storage is inherently !Send. // storage is inherently !Send.
#[cfg(feature = "magic")] #[cfg(feature = "meta_magic")]
thread_local! { thread_local! {
static MAGIC_COOKIE: std::cell::RefCell<Option<Cookie>> = const { std::cell::RefCell::new(None) }; static MAGIC_COOKIE: std::cell::RefCell<Option<Cookie>> = const { std::cell::RefCell::new(None) };
} }
#[cfg(feature = "magic")] #[cfg(feature = "meta_magic")]
#[derive(Debug)] #[derive(Debug)]
pub struct MagicFileMetaPluginImpl { pub struct MagicFileMetaPluginImpl {
buffer: Vec<u8>, buffer: Vec<u8>,
@@ -30,7 +30,7 @@ pub struct MagicFileMetaPluginImpl {
base: BaseMetaPlugin, base: BaseMetaPlugin,
} }
#[cfg(feature = "magic")] #[cfg(feature = "meta_magic")]
impl MagicFileMetaPluginImpl { impl MagicFileMetaPluginImpl {
pub fn new( pub fn new(
options: Option<std::collections::HashMap<String, serde_yaml::Value>>, options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
@@ -113,7 +113,7 @@ impl MagicFileMetaPluginImpl {
} }
} }
#[cfg(feature = "magic")] #[cfg(feature = "meta_magic")]
impl MetaPlugin for MagicFileMetaPluginImpl { impl MetaPlugin for MagicFileMetaPluginImpl {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
@@ -222,10 +222,10 @@ impl MetaPlugin for MagicFileMetaPluginImpl {
} }
} }
#[cfg(feature = "magic")] #[cfg(feature = "meta_magic")]
pub use MagicFileMetaPluginImpl as MagicFileMetaPlugin; pub use MagicFileMetaPluginImpl as MagicFileMetaPlugin;
#[cfg(not(feature = "magic"))] #[cfg(not(feature = "meta_magic"))]
#[derive(Debug)] #[derive(Debug)]
pub struct FallbackMagicFileMetaPlugin { pub struct FallbackMagicFileMetaPlugin {
buffer: Vec<u8>, buffer: Vec<u8>,
@@ -234,7 +234,7 @@ pub struct FallbackMagicFileMetaPlugin {
base: BaseMetaPlugin, base: BaseMetaPlugin,
} }
#[cfg(not(feature = "magic"))] #[cfg(not(feature = "meta_magic"))]
impl FallbackMagicFileMetaPlugin { impl FallbackMagicFileMetaPlugin {
pub fn new( pub fn new(
options: Option<std::collections::HashMap<String, serde_yaml::Value>>, options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
@@ -336,7 +336,7 @@ impl FallbackMagicFileMetaPlugin {
} }
} }
#[cfg(not(feature = "magic"))] #[cfg(not(feature = "meta_magic"))]
impl MetaPlugin for FallbackMagicFileMetaPlugin { impl MetaPlugin for FallbackMagicFileMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
@@ -441,7 +441,7 @@ impl MetaPlugin for FallbackMagicFileMetaPlugin {
} }
} }
#[cfg(not(feature = "magic"))] #[cfg(not(feature = "meta_magic"))]
pub use FallbackMagicFileMetaPlugin as MagicFileMetaPlugin; pub use FallbackMagicFileMetaPlugin as MagicFileMetaPlugin;
use crate::meta_plugin::register_meta_plugin; use crate::meta_plugin::register_meta_plugin;

View File

@@ -1,5 +1,4 @@
use log::{debug, warn}; use log::{debug, warn};
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
@@ -9,7 +8,7 @@ pub mod digest;
pub mod env; pub mod env;
pub mod exec; pub mod exec;
pub mod hostname; pub mod hostname;
#[cfg(feature = "infer")] #[cfg(feature = "meta_infer")]
pub mod infer_plugin; pub mod infer_plugin;
pub mod keep_pid; pub mod keep_pid;
pub mod magic_file; pub mod magic_file;
@@ -18,32 +17,32 @@ pub mod read_time;
pub mod shell; pub mod shell;
pub mod shell_pid; pub mod shell_pid;
pub mod text; pub mod text;
#[cfg(feature = "tokens")] #[cfg(feature = "meta_tokens")]
pub mod tokens; pub mod tokens;
#[cfg(feature = "tree_magic_mini")] #[cfg(feature = "meta_tree_magic_mini")]
pub mod tree_magic_mini; pub mod tree_magic_mini;
pub mod user; pub mod user;
pub use digest::DigestMetaPlugin; pub use digest::DigestMetaPlugin;
pub use exec::MetaPluginExec; pub use exec::MetaPluginExec;
#[cfg(feature = "magic")] #[cfg(feature = "meta_magic")]
pub use magic_file::MagicFileMetaPlugin; pub use magic_file::MagicFileMetaPlugin;
// pub use text::TextMetaPlugin; // Removed duplicate // pub use text::TextMetaPlugin; // Removed duplicate
pub use cwd::CwdMetaPlugin; pub use cwd::CwdMetaPlugin;
pub use env::EnvMetaPlugin; pub use env::EnvMetaPlugin;
pub use hostname::HostnameMetaPlugin; pub use hostname::HostnameMetaPlugin;
#[cfg(feature = "infer")] #[cfg(feature = "meta_infer")]
pub use infer_plugin::InferMetaPlugin; pub use infer_plugin::InferMetaPlugin;
pub use keep_pid::KeepPidMetaPlugin; pub use keep_pid::KeepPidMetaPlugin;
pub use read_rate::ReadRateMetaPlugin; pub use read_rate::ReadRateMetaPlugin;
pub use read_time::ReadTimeMetaPlugin; pub use read_time::ReadTimeMetaPlugin;
pub use shell::ShellMetaPlugin; pub use shell::ShellMetaPlugin;
pub use shell_pid::ShellPidMetaPlugin; pub use shell_pid::ShellPidMetaPlugin;
#[cfg(feature = "tree_magic_mini")] #[cfg(feature = "meta_tree_magic_mini")]
pub use tree_magic_mini::TreeMagicMiniMetaPlugin; pub use tree_magic_mini::TreeMagicMiniMetaPlugin;
pub use user::UserMetaPlugin; pub use user::UserMetaPlugin;
#[cfg(not(feature = "magic"))] #[cfg(not(feature = "meta_magic"))]
pub use magic_file::FallbackMagicFileMetaPlugin as MagicFileMetaPlugin; pub use magic_file::FallbackMagicFileMetaPlugin as MagicFileMetaPlugin;
type PluginConstructor = fn( type PluginConstructor = fn(
@@ -444,9 +443,9 @@ where
/// ///
/// An empty `HashMap` (default implementation). /// An empty `HashMap` (default implementation).
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
use once_cell::sync::Lazy; use std::sync::LazyLock;
static EMPTY: Lazy<std::collections::HashMap<String, serde_yaml::Value>> = static EMPTY: LazyLock<std::collections::HashMap<String, serde_yaml::Value>> =
Lazy::new(std::collections::HashMap::new); LazyLock::new(std::collections::HashMap::new);
&EMPTY &EMPTY
} }
@@ -471,9 +470,9 @@ where
/// ///
/// An empty `HashMap` (default implementation). /// An empty `HashMap` (default implementation).
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
use once_cell::sync::Lazy; use std::sync::LazyLock;
static EMPTY: Lazy<std::collections::HashMap<String, serde_yaml::Value>> = static EMPTY: LazyLock<std::collections::HashMap<String, serde_yaml::Value>> =
Lazy::new(std::collections::HashMap::new); LazyLock::new(std::collections::HashMap::new);
&EMPTY &EMPTY
} }
@@ -602,8 +601,9 @@ where
} }
/// Global registry for meta plugins. /// Global registry for meta plugins.
static META_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<MetaPluginType, PluginConstructor>>> = static META_PLUGIN_REGISTRY: std::sync::LazyLock<
Lazy::new(|| Mutex::new(HashMap::new())); Mutex<HashMap<MetaPluginType, PluginConstructor>>,
> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
/// Register a meta plugin with the global registry. /// Register a meta plugin with the global registry.
/// ///

View File

@@ -21,9 +21,7 @@ use chrono::{DateTime, Utc};
use clap::Command; use clap::Command;
use clap::error::ErrorKind; use clap::error::ErrorKind;
use comfy_table::{Attribute, Cell, ContentArrangement, Table}; use comfy_table::{Attribute, Cell, ContentArrangement, Table};
use lazy_static::lazy_static;
use log::debug; use log::debug;
use regex::Regex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
use std::env; use std::env;
@@ -57,24 +55,18 @@ pub enum OutputFormat {
Yaml, Yaml,
} }
lazy_static! {
static ref KEEP_META_RE: Regex = Regex::new(r"^KEEP_META_(.+)$").unwrap();
}
pub const IMPORT_FORMAT_ERROR: &str = pub const IMPORT_FORMAT_ERROR: &str =
"Unsupported import format: {} (expected .keep.tar or .meta.yml)"; "Unsupported import format: {} (expected .keep.tar or .meta.yml)";
pub fn get_meta_from_env() -> HashMap<String, String> { pub fn get_meta_from_env() -> HashMap<String, String> {
debug!("COMMON: Getting meta from KEEP_META_*"); debug!("COMMON: Getting meta from KEEP_META_*");
let mut meta_env: HashMap<String, String> = HashMap::new(); let mut meta_env: HashMap<String, String> = HashMap::new();
const PREFIX: &str = "KEEP_META_";
for (key, value) in env::vars() { for (key, value) in env::vars() {
if let Some(meta_name_caps) = KEEP_META_RE.captures(key.as_str()) { if let Some(name) = key.strip_prefix(PREFIX) {
let name = meta_name_caps.get(1).map(|m| m.as_str().to_string()); if !name.is_empty() && name != "PLUGINS" {
if let Some(name) = name { debug!("COMMON: Found meta: {}={}", name, value);
if name != "PLUGINS" { meta_env.insert(name.to_string(), value);
debug!("COMMON: Found meta: {}={}", name, value);
meta_env.insert(name, value);
}
} }
} }
} }
@@ -643,6 +635,7 @@ pub struct ImportMeta {
/// ///
/// Returns the first ID if provided, the newest item matching tags, /// Returns the first ID if provided, the newest item matching tags,
/// or the newest item overall if neither is specified. /// or the newest item overall if neither is specified.
#[cfg(feature = "client")]
pub fn resolve_item_id( pub fn resolve_item_id(
client: &crate::client::KeepClient, client: &crate::client::KeepClient,
ids: &[i64], ids: &[i64],
@@ -666,6 +659,7 @@ pub fn resolve_item_id(
} }
/// Resolve item IDs from explicit IDs or tags (multi-item variant). /// Resolve item IDs from explicit IDs or tags (multi-item variant).
#[cfg(feature = "client")]
pub fn resolve_item_ids( pub fn resolve_item_ids(
client: &crate::client::KeepClient, client: &crate::client::KeepClient,
ids: &[i64], ids: &[i64],

View File

@@ -178,6 +178,7 @@ pub async fn handle_list_items(
let item_infos: Vec<ItemInfo> = items_with_meta let item_infos: Vec<ItemInfo> = items_with_meta
.into_iter() .into_iter()
.filter_map(|iwm| ItemInfo::try_from(iwm).ok()) .filter_map(|iwm| ItemInfo::try_from(iwm).ok())
.map(|info| info.with_file_size(&state.data_dir))
.collect(); .collect();
ResponseBuilder::json(ApiResponse::ok(item_infos)) ResponseBuilder::json(ApiResponse::ok(item_infos))
@@ -339,6 +340,7 @@ pub async fn handle_post_item(
let db = state.db.clone(); let db = state.db.clone();
let item_service = state.item_service.clone(); let item_service = state.item_service.clone();
let settings = state.settings.clone(); let settings = state.settings.clone();
let data_dir = state.data_dir.clone();
// Parse tags from query parameter // Parse tags from query parameter
let tags: Vec<String> = params let tags: Vec<String> = params
@@ -472,10 +474,12 @@ pub async fn handle_post_item(
return Err(StatusCode::PAYLOAD_TOO_LARGE); return Err(StatusCode::PAYLOAD_TOO_LARGE);
} }
let item_info = ItemInfo::try_from(item_with_meta).map_err(|e| { let item_info = ItemInfo::try_from(item_with_meta)
warn!("Item conversion failed: {e}"); .map(|info| info.with_file_size(&data_dir))
StatusCode::INTERNAL_SERVER_ERROR .map_err(|e| {
})?; warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(ApiResponse::ok(item_info))) Ok(Json(ApiResponse::ok(item_info)))
} }
@@ -1092,6 +1096,7 @@ pub async fn handle_delete_item(
compression: deleted_item.compression, compression: deleted_item.compression,
tags: vec![], tags: vec![],
metadata: HashMap::new(), metadata: HashMap::new(),
file_size: None,
}; };
Ok(Json(ApiResponse::ok(item_info))) Ok(Json(ApiResponse::ok(item_info)))
@@ -1124,6 +1129,7 @@ pub async fn handle_get_item_info(
let db = state.db.clone(); let db = state.db.clone();
let item_service = state.item_service.clone(); let item_service = state.item_service.clone();
let data_dir = state.data_dir.clone();
let item_with_meta = task::spawn_blocking(move || { let item_with_meta = task::spawn_blocking(move || {
let conn = db.blocking_lock(); let conn = db.blocking_lock();
@@ -1136,10 +1142,12 @@ pub async fn handle_get_item_info(
})? })?
.map_err(handle_item_error)?; .map_err(handle_item_error)?;
let item_info = ItemInfo::try_from(item_with_meta).map_err(|e| { let item_info = ItemInfo::try_from(item_with_meta)
warn!("Item conversion failed: {e}"); .map(|info| info.with_file_size(&data_dir))
StatusCode::INTERNAL_SERVER_ERROR .map_err(|e| {
})?; warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(ApiResponse::ok(item_info))) Ok(Json(ApiResponse::ok(item_info)))
} }
@@ -1352,6 +1360,7 @@ pub async fn handle_update_item(
let db = state.db.clone(); let db = state.db.clone();
let item_service = state.item_service.clone(); let item_service = state.item_service.clone();
let settings = state.settings.clone(); let settings = state.settings.clone();
let data_dir = state.data_dir.clone();
let size_param = params.uncompressed_size; let size_param = params.uncompressed_size;
let item_info = task::spawn_blocking(move || { let item_info = task::spawn_blocking(move || {
@@ -1369,10 +1378,12 @@ pub async fn handle_update_item(
return Err(StatusCode::INTERNAL_SERVER_ERROR); return Err(StatusCode::INTERNAL_SERVER_ERROR);
} }
match item_service.get_item(&conn, item_id) { match item_service.get_item(&conn, item_id) {
Ok(iwm) => ItemInfo::try_from(iwm).map_err(|e| { Ok(iwm) => ItemInfo::try_from(iwm)
warn!("Item conversion failed: {e}"); .map(|info| info.with_file_size(&data_dir))
StatusCode::INTERNAL_SERVER_ERROR .map_err(|e| {
}), warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
}),
Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR), Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR),
} }
} }
@@ -1392,10 +1403,12 @@ pub async fn handle_update_item(
); );
match result { match result {
Ok(item_with_meta) => ItemInfo::try_from(item_with_meta).map_err(|e| { Ok(item_with_meta) => ItemInfo::try_from(item_with_meta)
warn!("Item conversion failed: {e}"); .map(|info| info.with_file_size(&data_dir))
StatusCode::INTERNAL_SERVER_ERROR .map_err(|e| {
}), warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
}),
Err(CoreError::ItemNotFound(_)) => Err(StatusCode::NOT_FOUND), Err(CoreError::ItemNotFound(_)) => Err(StatusCode::NOT_FOUND),
Err(e) => { Err(e) => {
warn!("Failed to update item {item_id}: {e}"); warn!("Failed to update item {item_id}: {e}");

View File

@@ -366,10 +366,13 @@ pub struct StatusInfoResponse {
/// let item_info = ItemInfo { /// let item_info = ItemInfo {
/// id: 42, /// id: 42,
/// ts: "2023-12-01T15:30:45Z".to_string(), /// ts: "2023-12-01T15:30:45Z".to_string(),
/// size: Some(1024), /// uncompressed_size: Some(1024),
/// compressed_size: Some(512),
/// closed: true,
/// compression: "gzip".to_string(), /// compression: "gzip".to_string(),
/// tags: vec!["important".to_string()], /// tags: vec!["important".to_string()],
/// metadata: HashMap::from([("mime_type".to_string(), "text/plain".to_string())]), /// metadata: HashMap::from([("mime_type".to_string(), "text/plain".to_string())]),
/// file_size: Some(512),
/// }; /// };
/// ``` /// ```
#[derive(Serialize, Deserialize, ToSchema)] #[derive(Serialize, Deserialize, ToSchema)]
@@ -413,6 +416,33 @@ pub struct ItemInfo {
/// Key-value pairs containing additional metadata about the item. /// Key-value pairs containing additional metadata about the item.
#[schema(example = json!({"mime_type": "text/plain", "mime_encoding": "utf-8", "line_count": "42"}))] #[schema(example = json!({"mime_type": "text/plain", "mime_encoding": "utf-8", "line_count": "42"}))]
pub metadata: HashMap<String, String>, pub metadata: HashMap<String, String>,
/// Actual file size in bytes.
///
/// The filesystem-reported size of the item's data file. This may differ from
/// `compressed_size` if the file was written and the database hasn't been updated.
/// None if the file cannot be read (e.g., file not found, permission denied).
#[schema(example = 512)]
pub file_size: Option<i64>,
}
impl ItemInfo {
/// Enriches this `ItemInfo` with the actual filesystem-reported size.
///
/// Reads the size of the item's data file from disk and sets `file_size`.
/// If the file cannot be read, `file_size` is left as None.
///
/// # Arguments
///
/// * `data_dir` - The data directory path containing item files.
///
/// # Returns
///
/// A new `ItemInfo` with `file_size` populated from the filesystem.
pub fn with_file_size(mut self, data_dir: &std::path::Path) -> Self {
let item_path = data_dir.join(self.id.to_string());
self.file_size = std::fs::metadata(&item_path).map(|m| m.len() as i64).ok();
self
}
} }
impl TryFrom<ItemWithMeta> for ItemInfo { impl TryFrom<ItemWithMeta> for ItemInfo {
@@ -433,6 +463,7 @@ impl TryFrom<ItemWithMeta> for ItemInfo {
compression: item_with_meta.item.compression, compression: item_with_meta.item.compression,
tags, tags,
metadata, metadata,
file_size: None,
}) })
} }
} }
@@ -501,6 +532,7 @@ pub struct TagsQuery {
/// ```rust /// ```rust
/// use keep::modes::server::common::ListItemsQuery; /// use keep::modes::server::common::ListItemsQuery;
/// let query = ListItemsQuery { /// let query = ListItemsQuery {
/// ids: None,
/// tags: Some("important".to_string()), /// tags: Some("important".to_string()),
/// order: Some("newest".to_string()), /// order: Some("newest".to_string()),
/// start: Some(0), /// start: Some(0),

View File

@@ -179,24 +179,18 @@ async fn run_server(
let addr: SocketAddr = bind_address.parse()?; let addr: SocketAddr = bind_address.parse()?;
// Warn if authentication is enabled without TLS // Warn if authentication is enabled without TLS
if config.password.is_some() || config.password_hash.is_some() || config.jwt_secret.is_some() { if (config.password.is_some() || config.password_hash.is_some() || config.jwt_secret.is_some())
#[cfg(not(feature = "tls"))] && (config.cert_file.is_none() || config.key_file.is_none())
{
log::warn!( log::warn!(
"SECURITY: Authentication enabled but TLS support is not compiled in. Credentials will be transmitted in plain text!" "SECURITY: Authentication enabled but TLS is not configured. Credentials will be transmitted in plain text!"
); );
#[cfg(feature = "tls")]
if config.cert_file.is_none() || config.key_file.is_none() {
log::warn!(
"SECURITY: Authentication enabled but TLS is not configured. Credentials will be transmitted in plain text!"
);
}
} }
// Build the app into a service // Build the app into a service
let service = app.into_make_service_with_connect_info::<SocketAddr>(); let service = app.into_make_service_with_connect_info::<SocketAddr>();
// Use TLS if both cert and key files are provided // Use TLS if both cert and key files are provided
#[cfg(feature = "tls")]
if let (Some(cert_file), Some(key_file)) = (&config.cert_file, &config.key_file) { if let (Some(cert_file), Some(key_file)) = (&config.cert_file, &config.key_file) {
info!("SERVER: HTTPS server listening on {addr}"); info!("SERVER: HTTPS server listening on {addr}");

View File

@@ -13,11 +13,13 @@ use serde::Deserialize;
use std::collections::HashMap; use std::collections::HashMap;
/// Escape text content for safe HTML insertion. /// Escape text content for safe HTML insertion.
#[inline]
fn esc(s: &str) -> String { fn esc(s: &str) -> String {
encode_text(s).to_string() encode_text(s).to_string()
} }
/// Escape attribute values for safe HTML attribute insertion. /// Escape attribute values for safe HTML attribute insertion.
#[inline]
fn esc_attr(s: &str) -> String { fn esc_attr(s: &str) -> String {
encode_double_quoted_attribute(s).to_string() encode_double_quoted_attribute(s).to_string()
} }

View File

@@ -1,5 +1,4 @@
use crate::filter_plugin::{FilterChain, parse_filter_string}; use crate::filter_plugin::{FilterChain, parse_filter_string};
use once_cell::sync::Lazy;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::{Read, Result, Write}; use std::io::{Read, Result, Write};
use std::sync::Mutex; use std::sync::Mutex;
@@ -166,8 +165,8 @@ impl FilterService {
/// # Panics /// # Panics
/// ///
/// Lock acquisition failures (rare) cause panics in accessors. /// Lock acquisition failures (rare) cause panics in accessors.
static FILTER_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<String, FilterConstructor>>> = static FILTER_PLUGIN_REGISTRY: std::sync::LazyLock<Mutex<HashMap<String, FilterConstructor>>> =
Lazy::new(|| Mutex::new(HashMap::new())); std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
/// Registers a filter plugin in the global registry. /// Registers a filter plugin in the global registry.
/// ///

View File

@@ -1,3 +1,8 @@
/// Business logic services for the Keep application.
///
/// This module provides the core service layer that orchestrates item storage,
/// compression, metadata collection, and filtering. Services are used by both
/// local CLI modes and the HTTP server.
pub mod compression_service; pub mod compression_service;
pub mod error; pub mod error;
pub mod filter_service; pub mod filter_service;

View File

@@ -3,10 +3,10 @@
#[cfg(test)] #[cfg(test)]
pub mod digest_tests; pub mod digest_tests;
#[cfg(feature = "infer")] #[cfg(feature = "meta_infer")]
#[cfg(test)] #[cfg(test)]
pub mod infer_tests; pub mod infer_tests;
#[cfg(feature = "tree_magic_mini")] #[cfg(feature = "meta_tree_magic_mini")]
#[cfg(test)] #[cfg(test)]
pub mod tree_magic_mini_tests; pub mod tree_magic_mini_tests;

View File

@@ -1,5 +1,4 @@
use anyhow::{Result, bail}; use anyhow::{Result, bail};
use once_cell::sync::Lazy;
/// Supported LLM token encodings. /// Supported LLM token encodings.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -48,10 +47,10 @@ impl std::fmt::Debug for Tokenizer {
} }
/// Static tokenizer instances — loaded once per process, shared across all plugins. /// Static tokenizer instances — loaded once per process, shared across all plugins.
static CL100K: Lazy<Tokenizer> = Lazy::new(|| { static CL100K: std::sync::LazyLock<Tokenizer> = std::sync::LazyLock::new(|| {
Tokenizer::new(TokenEncoding::Cl100kBase).expect("Failed to create cl100k_base tokenizer") Tokenizer::new(TokenEncoding::Cl100kBase).expect("Failed to create cl100k_base tokenizer")
}); });
static O200K: Lazy<Tokenizer> = Lazy::new(|| { static O200K: std::sync::LazyLock<Tokenizer> = std::sync::LazyLock::new(|| {
Tokenizer::new(TokenEncoding::O200kBase).expect("Failed to create o200k_base tokenizer") Tokenizer::new(TokenEncoding::O200kBase).expect("Failed to create o200k_base tokenizer")
}); });