Compare commits

...

5 Commits

Author SHA1 Message Date
8379ae2136 refactor: rename plugin features with type prefix for consistency
- Plugin features now use type_ prefix (meta_magic, filter_grep, etc.)
- Added meta_all_musl and filter_all_musl for MUSL-compatible builds
- grep filter plugin made optional via filter_grep feature flag
- Removed regex crate from grep-related code, uses strip_prefix instead
- Updated CHANGELOG.md with breaking change documentation
2026-03-21 17:36:29 -03:00
12de215527 feat: feature-gate CLI args by server/client features
- CLI now shows only relevant options: --server and --server-* args
  hidden when built without 'server' feature; --client-* args hidden
  without 'client' feature. Run --help only displays applicable options.
- Removed verbose 'conflicts_with_all' from all mode args — clap's
  implicit group("mode") already enforces mutual exclusivity.
- 'server' feature now includes TLS/HTTPS by default (axum-server);
  'tls' feature removed. rustls already available via client/ureq.
- Gated KeepModes::Server, server mode detection, and server-password
  validation in main.rs.
- Gated server arg reads in config.rs.
- Removed redundant #[cfg(feature = "tls")] guards from server/mod.rs.
- Gated resolve_item_id/resolve_item_ids helpers in common.rs.
- All 4 feature combinations (server+client, server-only, client-only,
  neither) compile and pass tests.
2026-03-21 16:26:27 -03:00
e2cb36d2a8 feat(server): add file_size to API ItemInfo response 2026-03-21 14:03:58 -03:00
0004324301 perf: pre-allocate status info collections with known capacities 2026-03-21 13:54:37 -03:00
b3edfe7de6 chore: code review cleanup — fixes, deps, docs
Fixed:
- CLI help typo: "metatdata" -> "metadata"
- Filter buffer OOM: check size before loading into memory

Changed:
- #[inline] on HTML escape helpers for hot path performance
- Replaced once_cell and lazy_static with std::sync::LazyLock
- Removed unused once_cell and lazy_static crate dependencies

Refactored:
- Added module-level doc to services/ module

Documentation:
- README.md: zstd is native not external, "none" -> "raw"
- DESIGN.md: current schema and meta plugins section
- CHANGELOG.md: Unreleased section populated
2026-03-21 11:44:37 -03:00
24 changed files with 381 additions and 268 deletions

View File

@@ -7,6 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- New `filter_grep` feature to optionally include the grep filter plugin (regex-based line filtering). Disabling this feature removes the `regex` crate and its ~800 KiB dependency stack from the binary.
- New `meta_all_musl` feature for all MUSL-compatible meta plugins (excludes `meta_magic` which requires libmagic)
- New `filter_all_musl` feature for all MUSL-compatible filter plugins
- Database index on `items(ts)` column for faster ORDER BY sorting
- Server API `ItemInfo` now includes `file_size` — actual filesystem-reported size of the item data file
### Changed
- CLI args now feature-gated: `--server` and related options hidden when built without `server` feature; `--client-*` options hidden when built without `client` feature. Run `--help` only shows relevant options.
- `server` Cargo feature now includes TLS support by default (`axum-server`); `tls` feature removed
- Clap `conflicts_with_all` removed from all mode args — exclusivity now handled by implicit `group("mode")`
- Filter plugins check size before loading content into memory (prevents OOM on large inputs)
- Status page pre-allocates collections with known capacities (meta plugins, compression info)
- `#[inline]` on HTML escape helper functions (`esc`, `esc_attr`) for hot path performance
- Removed `once_cell` crate (replaced with `std::sync::LazyLock` from Rust 1.80)
- Removed `lazy_static` crate (replaced with `std::sync::LazyLock`)
### Breaking
- Plugin feature flags renamed with type prefix for consistency:
- `magic``meta_magic`
- `infer``meta_infer`
- `tree_magic_mini``meta_tree_magic_mini`
- `tokens``meta_tokens`
- `grep``filter_grep`
- `all-meta-plugins``meta_all`
- `all-filter-plugins``filter_all`
### Fixed
- CLI help text typo: "metatdata" → "metadata" in `--get` and `--info` descriptions
### Refactored
- Added module-level documentation to `services/` module
### Documentation
- README.md: Fixed compression table — zstd is native (not external), "none" renamed to "raw"
- DESIGN.md: Updated schema to reflect current `items` table columns and meta plugin inventory
## [0.1.0] - 2026-03-21
### Added

2
Cargo.lock generated
View File

@@ -1727,7 +1727,6 @@ dependencies = [
"inventory",
"is-terminal",
"jsonwebtoken",
"lazy_static",
"libc",
"local-ip-address",
"log",
@@ -1735,7 +1734,6 @@ dependencies = [
"magic",
"md5",
"nix",
"once_cell",
"os_pipe",
"pest",
"pest_derive",

View File

@@ -35,7 +35,6 @@ hyper = { version = "1.0", features = ["full"] }
http-body-util = "0.1"
inventory = "0.3"
is-terminal = "0.4"
lazy_static = "1.5"
libc = "0.2"
local-ip-address = "0.6"
log = "0.4"
@@ -45,10 +44,9 @@ magic = { version = "0.13", optional = true }
infer = { version = "0.19", optional = true }
tree_magic_mini = { version = "3.2", optional = true }
nix = { version = "0.30", features = ["fs", "process"] }
once_cell = "1.21"
comfy-table = "7.2"
pwhash = "1.0"
regex = "1.10"
regex = { version = "1.10", optional = true }
ringbuf = "0.4"
rusqlite = { version = "0.37", features = ["bundled", "array", "chrono"] }
rusqlite_migration = "2.3"
@@ -87,19 +85,20 @@ tiktoken-rs = { version = "0.9", optional = true }
tempfile = "3.3"
[features]
# Default features include core compression engines and swagger UI
# Default features include core compression engines plugins that support MUSL
default = [
"client",
"gzip",
"infer",
"filter_grep",
"meta_infer",
"lz4",
"tokens",
"tree_magic_mini",
"meta_tokens",
"meta_tree_magic_mini",
"zstd"
]
# Server feature (includes axum and related dependencies)
server = ["dep:axum", "dep:tower", "dep:tower-http", "dep:utoipa", "dep:jsonwebtoken"]
# Server feature (includes axum and TLS/HTTPS via axum-server; rustls already available via client/ureq)
server = ["dep:axum", "dep:tower", "dep:tower-http", "dep:utoipa", "dep:jsonwebtoken", "dep:axum-server"]
# Compression features
gzip = ["flate2"]
@@ -108,14 +107,18 @@ bzip2 = []
xz = []
zstd = ["dep:zstd"]
# Plugin features (meta and filter)
all-meta-plugins = ["dep:magic", "dep:infer", "dep:tree_magic_mini"]
all-filter-plugins = []
# Meta plugin features
meta_magic = ["dep:magic"]
meta_infer = ["dep:infer"]
meta_tree_magic_mini = ["dep:tree_magic_mini"]
meta_tokens = ["dep:tiktoken-rs"]
meta_all = ["meta_magic", "meta_infer", "meta_tree_magic_mini", "meta_tokens"]
meta_all_musl = ["meta_infer", "meta_tree_magic_mini", "meta_tokens"]
# Individual plugin features
magic = ["dep:magic"]
infer = ["dep:infer"]
tree_magic_mini = ["dep:tree_magic_mini"]
# Filter plugin features
filter_grep = ["dep:regex"]
filter_all = ["filter_grep"]
filter_all_musl = ["filter_grep"]
# Swagger UI feature
swagger = ["dep:utoipa-swagger-ui"]
@@ -123,11 +126,5 @@ swagger = ["dep:utoipa-swagger-ui"]
# Client feature (HTTP client for remote server)
client = ["dep:ureq", "dep:os_pipe"]
# TLS feature (HTTPS server support)
tls = ["dep:axum-server"]
# Token counting feature (LLM token support via tiktoken)
tokens = ["dep:tiktoken-rs"]
[dev-dependencies]
rand = "0.9"

View File

@@ -117,7 +117,7 @@
## Data Storage
### Database Schema
- `items` table: id (primary key), ts (timestamp), size (optional), compression
- `items` table: id (primary key), ts (timestamp), uncompressed_size (optional), compressed_size (optional), closed (boolean), compression
- `tags` table: id (foreign key to items), name (tag name)
- `metas` table: id (foreign key to items), name (meta key), value (meta value)
- Indexes on tag names and meta names for faster queries
@@ -178,26 +178,25 @@
- None (no compression)
## Supported Meta Plugins
- FileMagic - File type detection using file command
- FileMime - MIME type detection using file command
- FileEncoding - File encoding detection using file command
- LineCount - Line count using wc command
- WordCount - Word count using wc command
- Cwd - Current working directory
- Binary - Binary file detection
- Uid - Current user ID
- User - Current username
- Gid - Current group ID
- Group - Current group name
- Shell - Shell path from SHELL environment variable
- ShellPid - Shell process ID from PPID environment variable
- KeepPid - Keep process ID
- DigestSha256 - SHA-256 digest
- DigestMd5 - MD5 digest using md5sum command
- ReadTime - Time taken to read data
- ReadRate - Rate of data reading
- Hostname - System hostname
- FullHostname - Fully qualified domain name
Meta plugins collect metadata during item save. Each plugin produces one or more key-value pairs:
- `magic_file` - File type detection using libmagic (when `magic` feature enabled)
- `infer` - MIME type detection using infer crate (when `infer` feature enabled)
- `tree_magic_mini` - MIME type detection using tree_magic_mini (when `tree_magic_mini` feature enabled)
- `tokens` - LLM token counting using tiktoken (when `tokens` feature enabled)
- `text` - Text analysis: line count, word count, char count, line average length
- `digest` - SHA-256 and MD5 checksums
- `hostname` - System hostname (full and short)
- `cwd` - Current working directory
- `user` - Current username and UID
- `shell` - Shell path from SHELL environment variable
- `shell_pid` - Shell process ID from PPID
- `keep_pid` - Keep process ID
- `env` - Arbitrary environment variables (via `KEEP_META_ENV_*` prefix)
- `exec` - Execute external commands for custom metadata
- `read_time` - Time taken to read content
- `read_rate` - Content read rate (bytes/second)
## Testing Strategy
- Unit tests for each module in `src/tests/`

View File

@@ -345,8 +345,8 @@ Items are compressed automatically on save. Default: LZ4.
| `gzip` | Internal | Fast | Good |
| `bzip2` | External | Slow | Better |
| `xz` | External | Slowest | Best |
| `zstd` | External | Fast | Good |
| `none` | Internal | N/A | N/A |
| `zstd` | Internal | Fast | Good |
| `raw` | Internal | N/A | N/A |
```sh
# Specify compression per item

View File

@@ -24,81 +24,80 @@ pub struct Args {
/// Struct for mode-specific arguments, defining CLI flags for different operations.
#[derive(Parser, Debug, Clone)]
pub struct ModeArgs {
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["get", "diff", "list", "delete", "info", "update", "status", "export", "import"]))]
#[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help("Save an item using any tags or metadata provided"))]
pub save: bool,
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "diff", "list", "delete", "info", "update", "status", "export", "import"]))]
#[arg(help(
"Get an item either by it's ID or by a combination of matching tags and metatdata"
))]
#[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help("Get an item either by its ID or by a combination of matching tags and metadata"))]
pub get: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "list", "delete", "info", "update", "status", "export", "import"]))]
#[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Show a diff between two items by ID"))]
pub diff: bool,
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "diff", "delete", "info", "update", "status", "export", "import"]))]
#[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help("List items, filtering on tags or metadata if given"))]
pub list: bool,
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "diff", "list", "info", "update", "status", "export", "import"]))]
#[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help("Delete items either by ID or by matching tags"))]
#[arg(requires = "ids_or_tags")]
pub delete: bool,
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "diff", "list", "delete", "update", "status", "export", "import"]))]
#[arg(help(
"Get an item either by it's ID or by a combination of matching tags and metatdata"
))]
#[arg(group("mode"), help_heading("Mode Options"), short, long)]
#[arg(help("Get an item either by its ID or by a combination of matching tags and metadata"))]
pub info: bool,
#[arg(group("mode"), help_heading("Mode Options"), short('u'), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "status", "export", "import"]))]
#[arg(group("mode"), help_heading("Mode Options"), short('u'), long)]
#[arg(help("Update an item's tags and metadata by ID"))]
pub update: bool,
#[arg(group("mode"), help_heading("Mode Options"), short('S'), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "server", "status_plugins", "export", "import"]))]
#[arg(group("mode"), help_heading("Mode Options"), short('S'), long)]
#[arg(help("Show status of directories and supported compression algorithms"))]
pub status: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "export", "import"]))]
#[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Show available plugins and their configurations"))]
pub status_plugins: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "import"]))]
#[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Export items to a .keep.tar archive (requires IDs or tags)"))]
pub export: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, value_name("FILE"), conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "export"]))]
#[arg(group("mode"), help_heading("Mode Options"), long, value_name("FILE"))]
#[arg(help("Import items from a .keep.tar archive or legacy .meta.yml file"))]
pub import: Option<String>,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status"]))]
#[cfg(feature = "server")]
#[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Start REST HTTP server"))]
pub server: bool,
#[arg(group("mode"), help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "export", "import"]))]
#[arg(group("mode"), help_heading("Mode Options"), long)]
#[arg(help("Generate default configuration and output to stdout"))]
pub generate_config: bool,
#[arg(help_heading("Mode Options"), long, conflicts_with_all(["save", "get", "diff", "list", "delete", "info", "update", "status", "server", "generate_config", "export", "import"]))]
#[arg(help_heading("Mode Options"), long)]
#[arg(help("Generate shell completion script"))]
pub generate_completion: Option<Shell>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_ADDRESS"))]
#[arg(help("Server address to bind to"))]
pub server_address: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PORT"))]
#[arg(help("Server port to bind to"))]
pub server_port: Option<u16>,
#[cfg(feature = "tls")]
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_CERT"))]
#[arg(help("Path to TLS certificate file (PEM) for HTTPS"))]
pub server_cert: Option<PathBuf>,
#[cfg(feature = "tls")]
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_KEY"))]
#[arg(help("Path to TLS private key file (PEM) for HTTPS"))]
pub server_key: Option<PathBuf>,
@@ -253,24 +252,29 @@ pub struct OptionsArgs {
#[arg(help("Output format (only works with --info, --status, --list)"))]
pub output_format: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PASSWORD"))]
#[arg(help("Password for server authentication (requires --server)"))]
pub server_password: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PASSWORD_HASH"))]
#[arg(help("Password hash for server authentication (requires --server)"))]
pub server_password_hash: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_USERNAME"))]
#[arg(help(
"Username for server Basic authentication (requires --server, defaults to 'keep')"
))]
pub server_username: Option<String>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_JWT_SECRET"))]
#[arg(help("JWT secret for token-based authentication (requires --server)"))]
pub server_jwt_secret: Option<String>,
#[cfg(feature = "server")]
#[arg(
help_heading("Server Options"),
long,
@@ -279,6 +283,7 @@ pub struct OptionsArgs {
#[arg(help("Path to file containing JWT secret (requires --server)"))]
pub server_jwt_secret_file: Option<PathBuf>,
#[cfg(feature = "server")]
#[arg(help_heading("Server Options"), long, env("KEEP_SERVER_MAX_BODY_SIZE"))]
#[arg(help("Maximum request body size in bytes (requires --server, default: unlimited)"))]
pub server_max_body_size: Option<u64>,

View File

@@ -89,7 +89,7 @@ pub fn generate_status_info(
};
let _default_type = crate::compression_engine::default_compression_type();
let mut compression_info = Vec::new();
let mut compression_info = Vec::with_capacity(CompressionType::iter().count());
// Sort compression types by their string representation
let mut sorted_compression_types: Vec<CompressionType> = CompressionType::iter().collect();
@@ -141,7 +141,8 @@ pub fn generate_status_info(
});
}
let mut meta_plugins_map = std::collections::HashMap::new();
let mut meta_plugins_map =
std::collections::HashMap::with_capacity(MetaPluginType::iter().count());
let mut enabled_meta_plugins_vec = Vec::new();
// Sort meta plugin types by their string representation to avoid creating plugins just for sorting

View File

@@ -7,8 +7,6 @@ use strum::{Display, EnumIter, EnumString};
use log::*;
use lazy_static::lazy_static;
extern crate enum_map;
use enum_map::enum_map;
use enum_map::{Enum, EnumMap};
@@ -180,63 +178,65 @@ impl Clone for Box<dyn CompressionEngine> {
}
}
lazy_static! {
static ref COMPRESSION_ENGINES: EnumMap<CompressionType, Box<dyn CompressionEngine>> = {
#[allow(unused_mut)] // mut needed when gzip/lz4 features are enabled
let mut em = enum_map! {
CompressionType::LZ4 => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"lz4",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::GZip => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"gzip",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::BZip2 => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"bzip2",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::XZ => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"xz",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::ZStd => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"zstd",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::Raw => Box::new(crate::compression_engine::raw::CompressionEngineRaw::new()) as Box<dyn CompressionEngine>
};
#[cfg(feature = "gzip")]
{
em[CompressionType::GZip] =
Box::new(crate::compression_engine::gzip::CompressionEngineGZip::new())
as Box<dyn CompressionEngine>;
}
#[cfg(feature = "lz4")]
{
em[CompressionType::LZ4] =
Box::new(crate::compression_engine::lz4::CompressionEngineLZ4::new())
as Box<dyn CompressionEngine>;
}
#[cfg(feature = "zstd")]
{
em[CompressionType::ZStd] =
Box::new(crate::compression_engine::zstd::CompressionEngineZstd::new())
as Box<dyn CompressionEngine>;
}
em
fn init_compression_engines() -> EnumMap<CompressionType, Box<dyn CompressionEngine>> {
#[allow(unused_mut)]
let mut em: EnumMap<CompressionType, Box<dyn CompressionEngine>> = enum_map! {
CompressionType::LZ4 => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"lz4",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::GZip => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"gzip",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::BZip2 => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"bzip2",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::XZ => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"xz",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::ZStd => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"zstd",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::Raw => Box::new(crate::compression_engine::raw::CompressionEngineRaw::new()) as Box<dyn CompressionEngine>
};
#[cfg(feature = "gzip")]
{
em[CompressionType::GZip] =
Box::new(crate::compression_engine::gzip::CompressionEngineGZip::new())
as Box<dyn CompressionEngine>;
}
#[cfg(feature = "lz4")]
{
em[CompressionType::LZ4] =
Box::new(crate::compression_engine::lz4::CompressionEngineLZ4::new())
as Box<dyn CompressionEngine>;
}
#[cfg(feature = "zstd")]
{
em[CompressionType::ZStd] =
Box::new(crate::compression_engine::zstd::CompressionEngineZstd::new())
as Box<dyn CompressionEngine>;
}
em
}
static COMPRESSION_ENGINES: std::sync::LazyLock<
EnumMap<CompressionType, Box<dyn CompressionEngine>>,
> = std::sync::LazyLock::new(init_compression_engines);
pub fn default_compression_type() -> CompressionType {
CompressionType::LZ4
}

View File

@@ -301,42 +301,48 @@ impl Settings {
config_builder = config_builder.set_override("force", true)?;
}
#[cfg(feature = "server")]
if let Some(server_password) = &args.options.server_password {
config_builder =
config_builder.set_override("server.password", server_password.as_str())?;
}
#[cfg(feature = "server")]
if let Some(server_password_hash) = &args.options.server_password_hash {
config_builder = config_builder
.set_override("server.password_hash", server_password_hash.as_str())?;
}
#[cfg(feature = "server")]
if let Some(server_username) = &args.options.server_username {
config_builder =
config_builder.set_override("server.username", server_username.as_str())?;
}
#[cfg(feature = "server")]
if let Some(server_address) = &args.mode.server_address {
config_builder =
config_builder.set_override("server.address", server_address.as_str())?;
}
#[cfg(feature = "server")]
if let Some(server_port) = args.mode.server_port {
config_builder = config_builder.set_override("server.port", server_port)?;
}
#[cfg(feature = "tls")]
#[cfg(feature = "server")]
if let Some(server_cert) = &args.mode.server_cert {
config_builder = config_builder
.set_override("server.cert_file", server_cert.to_string_lossy().as_ref())?;
}
#[cfg(feature = "tls")]
#[cfg(feature = "server")]
if let Some(server_key) = &args.mode.server_key {
config_builder = config_builder
.set_override("server.key_file", server_key.to_string_lossy().as_ref())?;
}
#[cfg(feature = "server")]
if let Some(max_body_size) = args.options.server_max_body_size {
config_builder = config_builder.set_override("server.max_body_size", max_body_size)?;
}

View File

@@ -1,6 +1,5 @@
use anyhow::{Context, Error, Result, anyhow};
use chrono::prelude::*;
use lazy_static::lazy_static;
use log::*;
use rusqlite::{Connection, OpenFlags, Row, params};
use rusqlite_migration::{M, Migrations};
@@ -47,25 +46,21 @@ let id = db::insert_item(&conn, item)?;
```
*/
lazy_static! {
// Database schema migrations for the Keep application.
//
// Defines the sequence of migrations to create and update the schema.
// Applied automatically when opening a database connection.
static ref MIGRATIONS: Migrations<'static> = Migrations::new(vec![
static MIGRATIONS: std::sync::LazyLock<Migrations<'static>> = std::sync::LazyLock::new(|| {
Migrations::new(vec![
M::up(
"CREATE TABLE items(
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
ts TEXT NOT NULL,
size INTEGER NULL,
compression TEXT NOT NULL)"
compression TEXT NOT NULL)",
),
M::up(
"CREATE TABLE tags (
id INTEGER NOT NULL,
name TEXT NOT NULL,
FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY(id, name));"
PRIMARY KEY(id, name));",
),
M::up(
"CREATE TABLE metas (
@@ -73,16 +68,17 @@ lazy_static! {
name TEXT NOT NULL,
value TEXT NOT NULL,
FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY(id, name));"
PRIMARY KEY(id, name));",
),
M::up("CREATE INDEX idx_tags_name ON tags(name)"),
M::up("CREATE INDEX idx_metas_name ON metas(name)"),
M::up("CREATE INDEX idx_items_ts ON items(ts)"),
M::up("UPDATE items SET compression = 'raw' WHERE compression = 'none'"),
M::up("ALTER TABLE items RENAME COLUMN size TO uncompressed_size"),
M::up("ALTER TABLE items ADD COLUMN compressed_size INTEGER NULL"),
M::up("ALTER TABLE items ADD COLUMN closed BOOLEAN NOT NULL DEFAULT 1"),
]);
}
])
});
/// Represents an item stored in the database.
///

View File

@@ -2,6 +2,7 @@ use std::io::{Read, Result, Write};
use std::str::FromStr;
use strum::EnumString;
#[cfg(feature = "filter_grep")]
pub mod grep;
/// Filter plugin module for processing input streams.
///
@@ -16,7 +17,7 @@ pub mod grep;
/// ```
/// # use std::io::{Read, Write};
/// # use keep::filter_plugin::parse_filter_string;
/// let mut chain = parse_filter_string("head_lines(10)|grep(pattern=error)")?;
/// let mut chain = parse_filter_string("head_lines(10)|tail_lines(5)")?;
/// # let mut reader: &mut dyn Read = &mut std::io::empty();
/// # let mut writer: Vec<u8> = Vec::new();
/// # chain.filter(&mut reader, &mut writer)?;
@@ -26,12 +27,13 @@ pub mod head;
pub mod skip;
pub mod strip_ansi;
pub mod tail;
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
pub mod tokens;
pub mod utils;
use std::collections::HashMap;
#[cfg(feature = "filter_grep")]
pub use grep::GrepFilter;
pub use head::{HeadBytesFilter, HeadLinesFilter};
pub use skip::{SkipBytesFilter, SkipLinesFilter};
@@ -199,13 +201,14 @@ pub enum FilterType {
TailLines,
SkipBytes,
SkipLines,
#[cfg(feature = "filter_grep")]
Grep,
StripAnsi,
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
HeadTokens,
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
SkipTokens,
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
TailTokens,
}
@@ -213,6 +216,44 @@ pub enum FilterType {
/// Prevents OOM on large files by rejecting inputs that exceed this limit.
const MAX_FILTER_BUFFER_SIZE: usize = 256 * 1024 * 1024;
struct BoundedVecWriter {
data: Vec<u8>,
limit: usize,
}
impl BoundedVecWriter {
fn new(limit: usize) -> Self {
Self {
data: Vec::new(),
limit,
}
}
fn into_inner(self) -> Vec<u8> {
self.data
}
}
impl std::io::Write for BoundedVecWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if self.data.len() + buf.len() > self.limit {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!(
"Input size exceeds maximum filter buffer size ({} bytes)",
MAX_FILTER_BUFFER_SIZE
),
));
}
self.data.write_all(buf)?;
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
/// A chain of filter plugins applied sequentially.
///
/// Chains multiple filters, applying them in order to the input stream.
@@ -318,9 +359,8 @@ impl FilterChain {
/// # Examples
///
/// ```
/// # use keep::filter_plugin::{FilterChain, GrepFilter};
/// # use keep::filter_plugin::FilterChain;
/// let mut chain = FilterChain::new();
/// chain.add_plugin(Box::new(GrepFilter::new("error".to_string()).unwrap()));
/// ```
pub fn add_plugin(&mut self, plugin: Box<dyn FilterPlugin>) {
self.plugins.push(plugin);
@@ -360,21 +400,10 @@ impl FilterChain {
}
// For multiple plugins, we need to chain them together
// We'll use a temporary buffer to hold intermediate results
let mut current_data = Vec::new();
std::io::copy(reader, &mut current_data)?;
if current_data.len() > MAX_FILTER_BUFFER_SIZE {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!(
"Input size ({} bytes) exceeds maximum filter buffer size ({} bytes). \
Consider using fewer filter plugins or smaller inputs.",
current_data.len(),
MAX_FILTER_BUFFER_SIZE
),
));
}
// We'll use a bounded buffer to hold intermediate results
let mut bounded_writer = BoundedVecWriter::new(MAX_FILTER_BUFFER_SIZE);
std::io::copy(reader, &mut bounded_writer)?;
let mut current_data = bounded_writer.into_inner();
// Store the plugins length to avoid borrowing issues
let plugins_len = self.plugins.len();
@@ -508,6 +537,7 @@ fn create_filter_with_options(
// Get the default options for this filter type by creating a temporary instance
// To do this, we need to create a default instance of the appropriate filter
let option_defs = match filter_type {
#[cfg(feature = "filter_grep")]
FilterType::Grep => grep::GrepFilter::new("".to_string())?.options(),
FilterType::HeadBytes => head::HeadBytesFilter::new(0).options(),
FilterType::HeadLines => head::HeadLinesFilter::new(0).options(),
@@ -516,11 +546,11 @@ fn create_filter_with_options(
FilterType::SkipBytes => skip::SkipBytesFilter::new(0).options(),
FilterType::SkipLines => skip::SkipLinesFilter::new(0).options(),
FilterType::StripAnsi => strip_ansi::StripAnsiFilter::new().options(),
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
FilterType::HeadTokens => tokens::HeadTokensFilter::new(0).options(),
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
FilterType::SkipTokens => tokens::SkipTokensFilter::new(0).options(),
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
FilterType::TailTokens => tokens::TailTokensFilter::new(0).options(),
};
@@ -590,6 +620,7 @@ fn create_specific_filter(
options: &HashMap<String, serde_json::Value>,
) -> Result<Box<dyn FilterPlugin>> {
match filter_type {
#[cfg(feature = "filter_grep")]
FilterType::Grep => {
let pattern = options
.get("pattern")
@@ -690,7 +721,7 @@ fn create_specific_filter(
}
Ok(Box::new(strip_ansi::StripAnsiFilter::new()))
}
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
FilterType::HeadTokens => {
let count = options
.get("count")
@@ -708,7 +739,7 @@ fn create_specific_filter(
f.encoding = encoding;
Ok(Box::new(f))
}
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
FilterType::SkipTokens => {
let count = options
.get("count")
@@ -726,7 +757,7 @@ fn create_specific_filter(
f.encoding = encoding;
Ok(Box::new(f))
}
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
FilterType::TailTokens => {
let count = options
.get("count")
@@ -747,7 +778,7 @@ fn create_specific_filter(
}
}
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
fn parse_encoding_option(
options: &std::collections::HashMap<String, serde_json::Value>,
) -> (crate::tokenizer::TokenEncoding, crate::tokenizer::Tokenizer) {

View File

@@ -45,7 +45,7 @@ pub mod services;
#[cfg(feature = "client")]
pub mod client;
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
pub mod tokenizer;
// Re-export Args struct for library usage
@@ -56,9 +56,12 @@ pub use services::CoreError;
// Import all filter plugins to ensure they register themselves
#[allow(unused_imports)]
use filter_plugin::{grep, head, skip, strip_ansi, tail};
#[cfg(feature = "filter_grep")]
use filter_plugin::grep;
#[allow(unused_imports)]
use filter_plugin::{head, skip, strip_ansi, tail};
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
#[allow(unused_imports)]
use filter_plugin::tokens as token_filters;
@@ -66,19 +69,19 @@ use crate::meta_plugin::{
cwd, digest, env, exec, hostname, keep_pid, read_rate, read_time, shell, shell_pid, user,
};
#[cfg(feature = "magic")]
#[cfg(feature = "meta_magic")]
#[allow(unused_imports)]
use crate::meta_plugin::magic_file;
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
#[allow(unused_imports)]
use crate::meta_plugin::tokens;
#[cfg(feature = "infer")]
#[cfg(feature = "meta_infer")]
#[allow(unused_imports)]
use crate::meta_plugin::infer_plugin;
#[cfg(feature = "tree_magic_mini")]
#[cfg(feature = "meta_tree_magic_mini")]
#[allow(unused_imports)]
use crate::meta_plugin::tree_magic_mini;

View File

@@ -122,6 +122,7 @@ fn main() -> Result<(), Error> {
Import,
Status,
StatusPlugins,
#[cfg(feature = "server")]
Server,
GenerateConfig,
}
@@ -150,9 +151,14 @@ fn main() -> Result<(), Error> {
mode = KeepModes::Status;
} else if args.mode.status_plugins {
mode = KeepModes::StatusPlugins;
} else if args.mode.server {
mode = KeepModes::Server;
} else if args.mode.generate_config {
}
#[cfg(feature = "server")]
{
if args.mode.server {
mode = KeepModes::Server;
}
}
if args.mode.generate_config {
mode = KeepModes::GenerateConfig;
}
@@ -188,6 +194,7 @@ fn main() -> Result<(), Error> {
}
// Validate server password usage
#[cfg(feature = "server")]
if settings.server_password().is_some() && mode != KeepModes::Server {
cmd.error(
ErrorKind::InvalidValue,
@@ -355,19 +362,8 @@ fn main() -> Result<(), Error> {
KeepModes::StatusPlugins => {
modes::status_plugins::mode_status_plugins(&mut cmd, &settings, data_path, db_path)
}
KeepModes::Server => {
#[cfg(feature = "server")]
{
modes::server::mode_server(&mut cmd, &settings, &mut conn, data_path)
}
#[cfg(not(feature = "server"))]
{
cmd.error(
ErrorKind::MissingRequiredArgument,
"This binary was not compiled with server support. Recompile with --features server"
).exit();
}
}
#[cfg(feature = "server")]
KeepModes::Server => modes::server::mode_server(&mut cmd, &settings, &mut conn, data_path),
KeepModes::GenerateConfig => {
modes::generate_config::mode_generate_config(&mut cmd, &settings)
}

View File

@@ -1,6 +1,6 @@
#[cfg(feature = "magic")]
#[cfg(feature = "meta_magic")]
use magic::{Cookie, CookieFlags};
#[cfg(not(feature = "magic"))]
#[cfg(not(feature = "meta_magic"))]
use std::process::{Command, Stdio};
use std::io::{self, Write};
@@ -16,12 +16,12 @@ use crate::meta_plugin::{
// separate cookies can be used from different threads concurrently without
// synchronization. Using thread_local! avoids unsafe impl Send since the
// storage is inherently !Send.
#[cfg(feature = "magic")]
#[cfg(feature = "meta_magic")]
thread_local! {
static MAGIC_COOKIE: std::cell::RefCell<Option<Cookie>> = const { std::cell::RefCell::new(None) };
}
#[cfg(feature = "magic")]
#[cfg(feature = "meta_magic")]
#[derive(Debug)]
pub struct MagicFileMetaPluginImpl {
buffer: Vec<u8>,
@@ -30,7 +30,7 @@ pub struct MagicFileMetaPluginImpl {
base: BaseMetaPlugin,
}
#[cfg(feature = "magic")]
#[cfg(feature = "meta_magic")]
impl MagicFileMetaPluginImpl {
pub fn new(
options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
@@ -113,7 +113,7 @@ impl MagicFileMetaPluginImpl {
}
}
#[cfg(feature = "magic")]
#[cfg(feature = "meta_magic")]
impl MetaPlugin for MagicFileMetaPluginImpl {
fn is_finalized(&self) -> bool {
self.is_finalized
@@ -222,10 +222,10 @@ impl MetaPlugin for MagicFileMetaPluginImpl {
}
}
#[cfg(feature = "magic")]
#[cfg(feature = "meta_magic")]
pub use MagicFileMetaPluginImpl as MagicFileMetaPlugin;
#[cfg(not(feature = "magic"))]
#[cfg(not(feature = "meta_magic"))]
#[derive(Debug)]
pub struct FallbackMagicFileMetaPlugin {
buffer: Vec<u8>,
@@ -234,7 +234,7 @@ pub struct FallbackMagicFileMetaPlugin {
base: BaseMetaPlugin,
}
#[cfg(not(feature = "magic"))]
#[cfg(not(feature = "meta_magic"))]
impl FallbackMagicFileMetaPlugin {
pub fn new(
options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
@@ -336,7 +336,7 @@ impl FallbackMagicFileMetaPlugin {
}
}
#[cfg(not(feature = "magic"))]
#[cfg(not(feature = "meta_magic"))]
impl MetaPlugin for FallbackMagicFileMetaPlugin {
fn is_finalized(&self) -> bool {
self.is_finalized
@@ -441,7 +441,7 @@ impl MetaPlugin for FallbackMagicFileMetaPlugin {
}
}
#[cfg(not(feature = "magic"))]
#[cfg(not(feature = "meta_magic"))]
pub use FallbackMagicFileMetaPlugin as MagicFileMetaPlugin;
use crate::meta_plugin::register_meta_plugin;

View File

@@ -1,5 +1,4 @@
use log::{debug, warn};
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
@@ -9,7 +8,7 @@ pub mod digest;
pub mod env;
pub mod exec;
pub mod hostname;
#[cfg(feature = "infer")]
#[cfg(feature = "meta_infer")]
pub mod infer_plugin;
pub mod keep_pid;
pub mod magic_file;
@@ -18,32 +17,32 @@ pub mod read_time;
pub mod shell;
pub mod shell_pid;
pub mod text;
#[cfg(feature = "tokens")]
#[cfg(feature = "meta_tokens")]
pub mod tokens;
#[cfg(feature = "tree_magic_mini")]
#[cfg(feature = "meta_tree_magic_mini")]
pub mod tree_magic_mini;
pub mod user;
pub use digest::DigestMetaPlugin;
pub use exec::MetaPluginExec;
#[cfg(feature = "magic")]
#[cfg(feature = "meta_magic")]
pub use magic_file::MagicFileMetaPlugin;
// pub use text::TextMetaPlugin; // Removed duplicate
pub use cwd::CwdMetaPlugin;
pub use env::EnvMetaPlugin;
pub use hostname::HostnameMetaPlugin;
#[cfg(feature = "infer")]
#[cfg(feature = "meta_infer")]
pub use infer_plugin::InferMetaPlugin;
pub use keep_pid::KeepPidMetaPlugin;
pub use read_rate::ReadRateMetaPlugin;
pub use read_time::ReadTimeMetaPlugin;
pub use shell::ShellMetaPlugin;
pub use shell_pid::ShellPidMetaPlugin;
#[cfg(feature = "tree_magic_mini")]
#[cfg(feature = "meta_tree_magic_mini")]
pub use tree_magic_mini::TreeMagicMiniMetaPlugin;
pub use user::UserMetaPlugin;
#[cfg(not(feature = "magic"))]
#[cfg(not(feature = "meta_magic"))]
pub use magic_file::FallbackMagicFileMetaPlugin as MagicFileMetaPlugin;
type PluginConstructor = fn(
@@ -444,9 +443,9 @@ where
///
/// An empty `HashMap` (default implementation).
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
use once_cell::sync::Lazy;
static EMPTY: Lazy<std::collections::HashMap<String, serde_yaml::Value>> =
Lazy::new(std::collections::HashMap::new);
use std::sync::LazyLock;
static EMPTY: LazyLock<std::collections::HashMap<String, serde_yaml::Value>> =
LazyLock::new(std::collections::HashMap::new);
&EMPTY
}
@@ -471,9 +470,9 @@ where
///
/// An empty `HashMap` (default implementation).
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
use once_cell::sync::Lazy;
static EMPTY: Lazy<std::collections::HashMap<String, serde_yaml::Value>> =
Lazy::new(std::collections::HashMap::new);
use std::sync::LazyLock;
static EMPTY: LazyLock<std::collections::HashMap<String, serde_yaml::Value>> =
LazyLock::new(std::collections::HashMap::new);
&EMPTY
}
@@ -602,8 +601,9 @@ where
}
/// Global registry for meta plugins.
static META_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<MetaPluginType, PluginConstructor>>> =
Lazy::new(|| Mutex::new(HashMap::new()));
static META_PLUGIN_REGISTRY: std::sync::LazyLock<
Mutex<HashMap<MetaPluginType, PluginConstructor>>,
> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
/// Register a meta plugin with the global registry.
///

View File

@@ -21,9 +21,7 @@ use chrono::{DateTime, Utc};
use clap::Command;
use clap::error::ErrorKind;
use comfy_table::{Attribute, Cell, ContentArrangement, Table};
use lazy_static::lazy_static;
use log::debug;
use regex::Regex;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::env;
@@ -57,24 +55,18 @@ pub enum OutputFormat {
Yaml,
}
lazy_static! {
static ref KEEP_META_RE: Regex = Regex::new(r"^KEEP_META_(.+)$").unwrap();
}
pub const IMPORT_FORMAT_ERROR: &str =
"Unsupported import format: {} (expected .keep.tar or .meta.yml)";
pub fn get_meta_from_env() -> HashMap<String, String> {
debug!("COMMON: Getting meta from KEEP_META_*");
let mut meta_env: HashMap<String, String> = HashMap::new();
const PREFIX: &str = "KEEP_META_";
for (key, value) in env::vars() {
if let Some(meta_name_caps) = KEEP_META_RE.captures(key.as_str()) {
let name = meta_name_caps.get(1).map(|m| m.as_str().to_string());
if let Some(name) = name {
if name != "PLUGINS" {
debug!("COMMON: Found meta: {}={}", name, value);
meta_env.insert(name, value);
}
if let Some(name) = key.strip_prefix(PREFIX) {
if !name.is_empty() && name != "PLUGINS" {
debug!("COMMON: Found meta: {}={}", name, value);
meta_env.insert(name.to_string(), value);
}
}
}
@@ -643,6 +635,7 @@ pub struct ImportMeta {
///
/// Returns the first ID if provided, the newest item matching tags,
/// or the newest item overall if neither is specified.
#[cfg(feature = "client")]
pub fn resolve_item_id(
client: &crate::client::KeepClient,
ids: &[i64],
@@ -666,6 +659,7 @@ pub fn resolve_item_id(
}
/// Resolve item IDs from explicit IDs or tags (multi-item variant).
#[cfg(feature = "client")]
pub fn resolve_item_ids(
client: &crate::client::KeepClient,
ids: &[i64],

View File

@@ -178,6 +178,7 @@ pub async fn handle_list_items(
let item_infos: Vec<ItemInfo> = items_with_meta
.into_iter()
.filter_map(|iwm| ItemInfo::try_from(iwm).ok())
.map(|info| info.with_file_size(&state.data_dir))
.collect();
ResponseBuilder::json(ApiResponse::ok(item_infos))
@@ -339,6 +340,7 @@ pub async fn handle_post_item(
let db = state.db.clone();
let item_service = state.item_service.clone();
let settings = state.settings.clone();
let data_dir = state.data_dir.clone();
// Parse tags from query parameter
let tags: Vec<String> = params
@@ -472,10 +474,12 @@ pub async fn handle_post_item(
return Err(StatusCode::PAYLOAD_TOO_LARGE);
}
let item_info = ItemInfo::try_from(item_with_meta).map_err(|e| {
warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let item_info = ItemInfo::try_from(item_with_meta)
.map(|info| info.with_file_size(&data_dir))
.map_err(|e| {
warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(ApiResponse::ok(item_info)))
}
@@ -1092,6 +1096,7 @@ pub async fn handle_delete_item(
compression: deleted_item.compression,
tags: vec![],
metadata: HashMap::new(),
file_size: None,
};
Ok(Json(ApiResponse::ok(item_info)))
@@ -1124,6 +1129,7 @@ pub async fn handle_get_item_info(
let db = state.db.clone();
let item_service = state.item_service.clone();
let data_dir = state.data_dir.clone();
let item_with_meta = task::spawn_blocking(move || {
let conn = db.blocking_lock();
@@ -1136,10 +1142,12 @@ pub async fn handle_get_item_info(
})?
.map_err(handle_item_error)?;
let item_info = ItemInfo::try_from(item_with_meta).map_err(|e| {
warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
let item_info = ItemInfo::try_from(item_with_meta)
.map(|info| info.with_file_size(&data_dir))
.map_err(|e| {
warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
})?;
Ok(Json(ApiResponse::ok(item_info)))
}
@@ -1352,6 +1360,7 @@ pub async fn handle_update_item(
let db = state.db.clone();
let item_service = state.item_service.clone();
let settings = state.settings.clone();
let data_dir = state.data_dir.clone();
let size_param = params.uncompressed_size;
let item_info = task::spawn_blocking(move || {
@@ -1369,10 +1378,12 @@ pub async fn handle_update_item(
return Err(StatusCode::INTERNAL_SERVER_ERROR);
}
match item_service.get_item(&conn, item_id) {
Ok(iwm) => ItemInfo::try_from(iwm).map_err(|e| {
warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
}),
Ok(iwm) => ItemInfo::try_from(iwm)
.map(|info| info.with_file_size(&data_dir))
.map_err(|e| {
warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
}),
Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR),
}
}
@@ -1392,10 +1403,12 @@ pub async fn handle_update_item(
);
match result {
Ok(item_with_meta) => ItemInfo::try_from(item_with_meta).map_err(|e| {
warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
}),
Ok(item_with_meta) => ItemInfo::try_from(item_with_meta)
.map(|info| info.with_file_size(&data_dir))
.map_err(|e| {
warn!("Item conversion failed: {e}");
StatusCode::INTERNAL_SERVER_ERROR
}),
Err(CoreError::ItemNotFound(_)) => Err(StatusCode::NOT_FOUND),
Err(e) => {
warn!("Failed to update item {item_id}: {e}");

View File

@@ -366,10 +366,13 @@ pub struct StatusInfoResponse {
/// let item_info = ItemInfo {
/// id: 42,
/// ts: "2023-12-01T15:30:45Z".to_string(),
/// size: Some(1024),
/// uncompressed_size: Some(1024),
/// compressed_size: Some(512),
/// closed: true,
/// compression: "gzip".to_string(),
/// tags: vec!["important".to_string()],
/// metadata: HashMap::from([("mime_type".to_string(), "text/plain".to_string())]),
/// file_size: Some(512),
/// };
/// ```
#[derive(Serialize, Deserialize, ToSchema)]
@@ -413,6 +416,33 @@ pub struct ItemInfo {
/// Key-value pairs containing additional metadata about the item.
#[schema(example = json!({"mime_type": "text/plain", "mime_encoding": "utf-8", "line_count": "42"}))]
pub metadata: HashMap<String, String>,
/// Actual file size in bytes.
///
/// The filesystem-reported size of the item's data file. This may differ from
/// `compressed_size` if the file was written and the database hasn't been updated.
/// None if the file cannot be read (e.g., file not found, permission denied).
#[schema(example = 512)]
pub file_size: Option<i64>,
}
impl ItemInfo {
/// Enriches this `ItemInfo` with the actual filesystem-reported size.
///
/// Reads the size of the item's data file from disk and sets `file_size`.
/// If the file cannot be read, `file_size` is left as None.
///
/// # Arguments
///
/// * `data_dir` - The data directory path containing item files.
///
/// # Returns
///
/// A new `ItemInfo` with `file_size` populated from the filesystem.
pub fn with_file_size(mut self, data_dir: &std::path::Path) -> Self {
let item_path = data_dir.join(self.id.to_string());
self.file_size = std::fs::metadata(&item_path).map(|m| m.len() as i64).ok();
self
}
}
impl TryFrom<ItemWithMeta> for ItemInfo {
@@ -433,6 +463,7 @@ impl TryFrom<ItemWithMeta> for ItemInfo {
compression: item_with_meta.item.compression,
tags,
metadata,
file_size: None,
})
}
}
@@ -501,6 +532,7 @@ pub struct TagsQuery {
/// ```rust
/// use keep::modes::server::common::ListItemsQuery;
/// let query = ListItemsQuery {
/// ids: None,
/// tags: Some("important".to_string()),
/// order: Some("newest".to_string()),
/// start: Some(0),

View File

@@ -179,24 +179,18 @@ async fn run_server(
let addr: SocketAddr = bind_address.parse()?;
// Warn if authentication is enabled without TLS
if config.password.is_some() || config.password_hash.is_some() || config.jwt_secret.is_some() {
#[cfg(not(feature = "tls"))]
if (config.password.is_some() || config.password_hash.is_some() || config.jwt_secret.is_some())
&& (config.cert_file.is_none() || config.key_file.is_none())
{
log::warn!(
"SECURITY: Authentication enabled but TLS support is not compiled in. Credentials will be transmitted in plain text!"
"SECURITY: Authentication enabled but TLS is not configured. Credentials will be transmitted in plain text!"
);
#[cfg(feature = "tls")]
if config.cert_file.is_none() || config.key_file.is_none() {
log::warn!(
"SECURITY: Authentication enabled but TLS is not configured. Credentials will be transmitted in plain text!"
);
}
}
// Build the app into a service
let service = app.into_make_service_with_connect_info::<SocketAddr>();
// Use TLS if both cert and key files are provided
#[cfg(feature = "tls")]
if let (Some(cert_file), Some(key_file)) = (&config.cert_file, &config.key_file) {
info!("SERVER: HTTPS server listening on {addr}");

View File

@@ -13,11 +13,13 @@ use serde::Deserialize;
use std::collections::HashMap;
/// Escape text content for safe HTML insertion.
#[inline]
fn esc(s: &str) -> String {
encode_text(s).to_string()
}
/// Escape attribute values for safe HTML attribute insertion.
#[inline]
fn esc_attr(s: &str) -> String {
encode_double_quoted_attribute(s).to_string()
}

View File

@@ -1,5 +1,4 @@
use crate::filter_plugin::{FilterChain, parse_filter_string};
use once_cell::sync::Lazy;
use std::collections::HashMap;
use std::io::{Read, Result, Write};
use std::sync::Mutex;
@@ -166,8 +165,8 @@ impl FilterService {
/// # Panics
///
/// Lock acquisition failures (rare) cause panics in accessors.
static FILTER_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<String, FilterConstructor>>> =
Lazy::new(|| Mutex::new(HashMap::new()));
static FILTER_PLUGIN_REGISTRY: std::sync::LazyLock<Mutex<HashMap<String, FilterConstructor>>> =
std::sync::LazyLock::new(|| Mutex::new(HashMap::new()));
/// Registers a filter plugin in the global registry.
///

View File

@@ -1,3 +1,8 @@
/// Business logic services for the Keep application.
///
/// This module provides the core service layer that orchestrates item storage,
/// compression, metadata collection, and filtering. Services are used by both
/// local CLI modes and the HTTP server.
pub mod compression_service;
pub mod error;
pub mod filter_service;

View File

@@ -3,10 +3,10 @@
#[cfg(test)]
pub mod digest_tests;
#[cfg(feature = "infer")]
#[cfg(feature = "meta_infer")]
#[cfg(test)]
pub mod infer_tests;
#[cfg(feature = "tree_magic_mini")]
#[cfg(feature = "meta_tree_magic_mini")]
#[cfg(test)]
pub mod tree_magic_mini_tests;

View File

@@ -1,5 +1,4 @@
use anyhow::{Result, bail};
use once_cell::sync::Lazy;
/// Supported LLM token encodings.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -48,10 +47,10 @@ impl std::fmt::Debug for Tokenizer {
}
/// Static tokenizer instances — loaded once per process, shared across all plugins.
static CL100K: Lazy<Tokenizer> = Lazy::new(|| {
static CL100K: std::sync::LazyLock<Tokenizer> = std::sync::LazyLock::new(|| {
Tokenizer::new(TokenEncoding::Cl100kBase).expect("Failed to create cl100k_base tokenizer")
});
static O200K: Lazy<Tokenizer> = Lazy::new(|| {
static O200K: std::sync::LazyLock<Tokenizer> = std::sync::LazyLock::new(|| {
Tokenizer::new(TokenEncoding::O200kBase).expect("Failed to create o200k_base tokenizer")
});