diff --git a/Cargo.lock b/Cargo.lock index 4c25343..1f90f25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,6 +124,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +dependencies = [ + "rustversion", +] + [[package]] name = "arraydeque" version = "0.5.1" @@ -175,6 +184,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.8.4" @@ -229,6 +260,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-server" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1df331683d982a0b9492b38127151e6453639cd34926eb9c07d4cd8c6d22bfc" +dependencies = [ + "arc-swap", + "bytes", + "either", + "fs-err", + "http", + "http-body", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.75" @@ -324,6 +377,8 @@ version = "1.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -771,6 +826,12 @@ dependencies = [ "litrs", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -906,6 +967,22 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +dependencies = [ + "autocfg", + "tokio", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -1185,13 +1262,14 @@ dependencies = [ [[package]] name = "hyper" -version = "1.6.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ + "atomic-waker", "bytes", "futures-channel", - "futures-util", + "futures-core", "h2", "http", "http-body", @@ -1199,6 +1277,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", + "pin-utils", "smallvec", "tokio", "want", @@ -1206,12 +1285,11 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.16" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "bytes", - "futures-core", "http", "http-body", "hyper", @@ -1411,6 +1489,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -1439,6 +1527,7 @@ dependencies = [ "anyhow", "async-stream", "axum", + "axum-server", "base64 0.22.1", "chrono", "clap", @@ -2235,6 +2324,7 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -2259,6 +2349,7 @@ version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2761,6 +2852,16 @@ dependencies = [ "syn 2.0.105", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" diff --git a/Cargo.toml b/Cargo.toml index 8541261..7763ecb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ dirs = "6.0.0" similar = { version = "2.7.0", default-features = false, features = ["text"] } ureq = { version = "3", features = ["json"], optional = true } os_pipe = { version = "1", optional = true } +axum-server = { version = "0.8", features = ["tls-rustls"], optional = true } [features] # Default features include core compression engines and swagger UI @@ -107,6 +108,9 @@ swagger = ["dep:utoipa-swagger-ui"] # Client feature (HTTP client for remote server) client = ["dep:ureq", "dep:os_pipe"] +# TLS feature (HTTPS server support) +tls = ["dep:axum-server"] + [dev-dependencies] tempfile = "3.3.0" rand = "0.8.5" diff --git a/DESIGN.md b/DESIGN.md index 2b64afc..7fe5b44 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -31,7 +31,8 @@ - `modes/info.rs` - Show detailed item information - `modes/diff.rs` - Compare two items - `modes/status.rs` - Show system status and capabilities -- `modes/server.rs` - REST HTTP server mode with OpenAPI documentation +- `modes/server.rs` - REST HTTP/HTTPS server mode with OpenAPI documentation +- `modes/client.rs` - Client mode for remote server (streaming save, local decompression) - `modes/common.rs` - Shared utilities for all modes ### Database Module @@ -57,6 +58,16 @@ - `common/is_binary.rs` - Binary file detection utilities - `common/status.rs` - Status information generation +### Client Module +- `client.rs` - HTTP client wrapper (ureq-based, supports streaming POST) +- `modes/client/save.rs` - 3-thread streaming save (stdin → tee → compress → pipe → HTTP POST) +- `modes/client/get.rs` - Get with server-side raw fetch + local decompression +- `modes/client/list.rs` - List delegation to server +- `modes/client/info.rs` - Info delegation to server +- `modes/client/delete.rs` - Delete delegation to server +- `modes/client/diff.rs` - Diff delegation to server +- `modes/client/status.rs` - Status delegation to server + ### Utility Modules - `plugins.rs` - Shared plugin utilities - `args.rs` - CLI argument definitions @@ -88,8 +99,14 @@ - `--quiet` - Do not show any messages - `--output-format ` - Output format for info, status, and list modes - `--server-password ` - Password for server authentication +- `--server-cert ` - TLS certificate file (PEM) for HTTPS server +- `--server-key ` - TLS private key file (PEM) for HTTPS server - `--force` - Force output even when binary data would be sent to a TTY +### Client Options (requires `client` feature) +- `--client-url ` - Remote keep server URL +- `--client-password ` - Remote server password + ## Data Storage ### Database Schema @@ -107,17 +124,31 @@ ### Status Operations - `GET /api/status` - Get system status information +- `GET /api/plugins/status` - Get plugin status information ### Item Operations -- `GET /api/item/` - Get a list of items as JSON. Optional params: `order=newest|oldest`, `start=0`, `count=100`, `tags[]=tag1&tags[]=tag2` -- `POST /api/item/` - Add a new item +- `GET /api/item/` - Get a list of items as JSON. Optional params: `order=newest|oldest`, `start=0`, `count=100`, `tags=tag1,tag2` +- `POST /api/item/` - Add a new item (body: raw content). Query params: `tags`, `metadata` (JSON), `compress=true|false`, `meta=true|false` +- `POST /api/item/<#>/meta` - Add metadata to an existing item (body: JSON object) - `DELETE /api/item/<#>` - Delete an item -- `GET /api/item/latest` - Return the latest item as JSON. Optional params: `tags[]=tag1&tags[]=tag2`, `allow_binary=true|false` -- `GET /api/item/latest/meta` - Return the latest item metadata as JSON. Optional params: `tags[]=tag1&tags[]=tag2` -- `GET /api/item/latest/content` - Return the raw content of the latest item. Optional params: `tags[]=tag1&tags[]=tag2` +- `GET /api/item/latest` - Return the latest item as JSON. Optional params: `tags=tag1,tag2`, `allow_binary=true|false` +- `GET /api/item/latest/meta` - Return the latest item metadata as JSON. Optional params: `tags=tag1,tag2` +- `GET /api/item/latest/content` - Return the raw content of the latest item. Optional params: `tags=tag1,tag2`, `decompress=true|false` - `GET /api/item/<#>` - Return the item as JSON. Optional params: `allow_binary=true|false` - `GET /api/item/<#>/meta` - Return the item metadata as JSON -- `GET /api/item/<#>/content` - Return the raw content of the item +- `GET /api/item/<#>/content` - Return the raw content of the item. Optional params: `decompress=true|false` +- `GET /api/diff` - Diff two items. Params: `id_a`, `id_b` + +### Server Modes +- **Plain HTTP** (default): `tokio::net::TcpListener` + `axum::serve()` +- **HTTPS** (with `tls` feature): `axum_server::bind_rustls()` with rustls when `--server-cert` and `--server-key` are provided +- Conditional selection at startup: cert+key present → HTTPS, otherwise → HTTP + +### Client/Server Protocol +- Smart clients (keep CLI) set `compress=false` and `meta=false` on POST, handling compression/metadata locally +- Dumb clients (curl) use defaults (`compress=true`, `meta=true`), server handles everything +- GET responses include `X-Keep-Compression` header when `decompress=false` +- Streaming save uses chunked transfer encoding for constant memory usage ### Authentication - Bearer token authentication: `Authorization: Bearer ` @@ -173,5 +204,19 @@ - File permissions are restricted to user only (umask 077) - Input validation for item IDs to prevent path traversal - Authentication for server mode with bearer or basic auth +- TLS/HTTPS support via rustls when certificate and key are provided - Proper resource cleanup using RAII patterns - Safe handling of external processes with proper stdin/stdout management + +## Feature Flags +- `server` - HTTP REST API server (axum-based) +- `tls` - HTTPS/TLS support for server (axum-server + rustls) +- `client` - HTTP client for remote server (ureq-based, includes streaming save) +- `mcp` - Model Context Protocol for AI assistant integration +- `swagger` - OpenAPI/Swagger UI documentation +- `magic` - File type detection via libmagic +- `lz4` - LZ4 compression (internal) +- `gzip` - GZip compression (internal) +- `bzip2` - BZip2 compression (external) +- `xz` - XZ compression (external) +- `zstd` - ZStd compression (external) diff --git a/README.md b/README.md index dcf1a20..48508d5 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,53 @@ # Keep -A command-line utility for managing temporary files with automatic compression, metadata generation, and querying. Instead of redirecting output to temporary files, pipe it into `keep` for organized storage and retrieval. +A command-line utility for storing and retrieving temporary data with automatic compression, metadata extraction, and querying. Pipe any output into `keep` for organized storage — no more losing data in `/tmp` files with cryptic names. ```sh # Instead of this: curl -s https://api.example.com/data > /tmp/api-data.json -cat /tmp/api-data.json # Do this: -curl -s https://api.example.com/data | keep api-data +curl -s https://api.example.com/data | keep --save api-data keep --get api-data ``` +## Table of Contents + +- [Features](#features) +- [Installation](#installation) +- [Quick Start](#quick-start) +- [Usage](#usage) + - [Save Mode](#save-mode) + - [Get Mode](#get-mode) + - [List Mode](#list-mode) + - [Info Mode](#info-mode) + - [Update Mode](#update-mode) + - [Delete Mode](#delete-mode) + - [Diff Mode](#diff-mode) + - [Status Mode](#status-mode) +- [Filters](#filters) +- [Compression](#compression) +- [Meta Plugins](#meta-plugins) +- [Configuration](#configuration) +- [Client/Server Mode](#clientserver-mode) + - [Server Mode](#server-mode) + - [Client Mode](#client-mode) + - [API Endpoints](#api-endpoints) +- [MCP (Model Context Protocol)](#mcp-model-context-protocol) +- [Shell Integration](#shell-integration) +- [Feature Flags](#feature-flags) +- [License](#license) + ## Features -- **Store and Retrieve** - Save content with tags, retrieve by ID or tag -- **Automatic Compression** - LZ4, GZip, BZip2, XZ, ZStd support -- **Metadata Plugins** - Auto-extract file type, digests, hostname, user info, and more -- **Filters** - Apply transformations (head, tail, grep, strip ANSI) on retrieval -- **Querying** - List, search, diff items with flexible formatting -- **REST API Server** - Optional HTTP server for programmatic access -- **MCP Support** - Model Context Protocol integration for AI assistants -- **Modular Design** - Extensible plugin system for compression, metadata, and filtering +- **Store and retrieve** — Save content with tags, retrieve by ID or tag +- **Automatic compression** — LZ4, GZip, BZip2, XZ, ZStd support +- **Metadata plugins** — Auto-extract file type, digests, hostname, user info, and more +- **Filters** — Apply transformations (head, tail, grep, strip ANSI) on retrieval +- **Querying** — List, search, diff items with flexible formatting +- **Client/server architecture** — Optional HTTP server with streaming support +- **MCP support** — Model Context Protocol integration for AI assistants +- **Modular design** — Extensible plugin system for compression, metadata, and filtering ## Installation @@ -31,6 +57,11 @@ Requires Rust and Cargo. ```sh cargo build --release +``` + +### Install via Cargo + +```sh cargo install --path . ``` @@ -41,22 +72,29 @@ cargo install --path . # Binary at bin/keep ``` -### Using Cargo +### Build with Server/Client Features ```sh -cargo install --path . +# Server only +cargo build --release --features server + +# Client only (for connecting to a remote keep server) +cargo build --release --features client + +# Server + client + all optional features +cargo build --release --features server,tls,client,swagger,mcp ``` ## Quick Start ```sh -# Save content with tags +# Save content with a tag echo "Hello, world!" | keep --save greeting # Retrieve by tag keep --get greeting -# List all items +# List all stored items keep --list # Get item details @@ -66,6 +104,28 @@ keep --info greeting keep --delete greeting ``` +### Real-World Examples + +```sh +# Save API response +curl -s https://api.github.com/repos/user/repo | keep --save repo-info + +# Save test output with metadata +npm test 2>&1 | keep --save test-results --meta project=myapp --meta env=staging + +# Chain commands: process and store +cat data.csv | sort | uniq | keep --save cleaned-data + +# Diff two versions +keep --diff 1 5 + +# Get first 20 lines of an item +keep --get 1 --filters "head_lines(20)" + +# List items from a specific project +keep --list --meta project=myapp +``` + ## Usage ### Save Mode @@ -73,22 +133,24 @@ keep --delete greeting Save stdin content with tags and metadata. ```sh -# Basic save (auto-assigned ID) +# Save (auto-assigned ID, no tag) echo "data" | keep --save -# Save with tags +# Save with a tag echo "data" | keep --save my-tag # Save with multiple tags and metadata cat report.pdf | keep --save report --meta project=alpha --meta env=prod -# Specify compression and digest +# Specify compression and digest algorithm echo "data" | keep --save my-tag --compression gzip --digest sha256 ``` +Tags and metadata make items easy to find later. Tags are simple identifiers; metadata is key-value pairs. + ### Get Mode -Retrieve items by ID or tags. Default mode when IDs are provided. +Retrieve items by ID or tags. This is the default mode when IDs are provided. ```sh # Get by ID @@ -99,13 +161,13 @@ keep 1 keep --get my-tag keep my-tag -# Get with filters +# Get with filters applied keep --get 1 --filters "head_lines(10)" -# Get with metadata filter +# Get by metadata filter keep --get --meta project=alpha -# Force binary output to TTY +# Force binary output to TTY (override safety check) keep --get 1 --force ``` @@ -120,16 +182,16 @@ keep --list # List by tag keep --list my-tag -# List with metadata filter +# Filter by metadata keep --list --meta env=prod # Custom column format keep --list --list-format "id,time,size,tags" -# JSON output +# JSON output for scripting keep --list --output-format json -# Human-readable sizes +# Human-readable file sizes keep --list --human-readable ``` @@ -154,7 +216,7 @@ keep --update 1 new-tag # Update metadata keep --update 1 --meta key=newvalue -# Remove metadata (no value) +# Remove a metadata key keep --update 1 --meta key ``` @@ -193,7 +255,7 @@ Apply transformations to item content during retrieval. Filters are chained with # First 10 lines keep --get 1 --filters "head_lines(10)" -# Skip first 5 lines, then grep +# Skip first 5 lines, then grep for errors keep --get 1 --filters "skip_lines(5)|grep(pattern=error)" # Strip ANSI escape codes @@ -201,6 +263,9 @@ keep --get 1 --filters "strip_ansi" # Last 100 bytes keep --get 1 --filters "tail_bytes(100)" + +# Complex chain +keep --get 1 --filters "skip_lines(10)|grep(pattern=TODO)|head_lines(5)" ``` ### Available Filters @@ -216,7 +281,61 @@ keep --get 1 --filters "tail_bytes(100)" | `grep(pattern)` | Filter matching lines | `pattern` (regex) | | `strip_ansi` | Remove ANSI escape codes | none | -Set the `KEEP_FILTERS` environment variable to apply filters by default. +Set `KEEP_FILTERS` to apply a default filter chain to all retrievals. + +## Compression + +Items are compressed automatically on save. Default: LZ4. + +| Algorithm | Type | Speed | Ratio | +|-----------|------|-------|-------| +| `lz4` | Internal | Fastest | Lower | +| `gzip` | Internal | Fast | Good | +| `bzip2` | External | Slow | Better | +| `xz` | External | Slowest | Best | +| `zstd` | External | Fast | Good | +| `none` | Internal | N/A | N/A | + +```sh +# Specify compression per item +echo "data" | keep --save my-tag --compression zstd + +# Set default via environment +export KEEP_COMPRESSION=gzip +``` + +External compression programs (`bzip2`, `xz`, `zstd`) must be installed on the system. + +## Meta Plugins + +Metadata is automatically extracted when saving items. + +| Plugin | Key | Description | +|--------|-----|-------------| +| `env` | `*` | Capture `KEEP_META_*` environment variables | +| `magic_file` | `file_type` | File type detection (requires `magic` feature) | +| `text` | `text_line_count`, `text_word_count` | Line and word counts | +| `user` | `uid`, `user`, `gid`, `group` | Current user info | +| `shell` | `shell` | Current shell path | +| `shell_pid` | `shell_pid` | Shell process ID | +| `keep_pid` | `keep_pid` | Keep process ID | +| `digest` | `digest_sha256`, `digest_md5` | Content digests | +| `read_time` | `read_time` | Time to read content | +| `read_rate` | `read_rate` | Data read rate | +| `hostname` | `hostname`, `hostname_short` | System hostname | +| `exec` | Custom | Run external commands for metadata | +| `cwd` | `cwd` | Current working directory | + +```sh +# Use specific plugins +echo "data" | keep --save tag --meta-plugins "digest,text,user" + +# Capture custom metadata via environment +KEEP_META_project=alpha echo "data" | keep --save tag + +# Combine environment and CLI metadata +KEEP_META_build=1234 echo "data" | keep --save tag --meta env=staging +``` ## Configuration @@ -234,11 +353,13 @@ Set the `KEEP_FILTERS` environment variable to apply filters by default. | `KEEP_SERVER_PORT` | Server port | `21080` | | `KEEP_SERVER_PASSWORD` | Server password | none | | `KEEP_SERVER_PASSWORD_HASH` | Server password hash | none | +| `KEEP_SERVER_CERT` | TLS certificate file path (PEM) | none | +| `KEEP_SERVER_KEY` | TLS private key file path (PEM) | none | +| `KEEP_CLIENT_URL` | Remote keep server URL | none | +| `KEEP_CLIENT_PASSWORD` | Remote server password | none | Any config setting can be overridden with `KEEP__` environment variables (double underscore separator). -Capture shell commands automatically by setting `KEEP_META_command` in your shell profile (see Shell Integration). - ### Configuration File Default location: `~/.config/keep/config.yml` @@ -289,18 +410,35 @@ server: address: "127.0.0.1" port: 21080 password: "secret" + # TLS (requires tls feature) + # cert_file: /path/to/cert.pem + # key_file: /path/to/key.pem + +# Client settings +client: + url: "http://localhost:21080" + password: "secret" human_readable: true quiet: false force: false ``` -## Server Mode +## Client/Server Mode -Start an HTTP REST API server for programmatic access. +Keep supports a client/server architecture where one machine runs a keep server and other machines connect as clients. This is useful for: + +- Centralizing stored data across multiple machines +- Sharing items between team members +- Offloading storage to a dedicated server +- Piping data from long-running processes without local storage + +### Server Mode + +Start an HTTP REST API server: ```sh -# Start server on default address (127.0.0.1:21080) +# Default: 127.0.0.1:21080 keep --server # Custom address and port @@ -310,24 +448,165 @@ keep --server --server-address 0.0.0.0 --server-port 8080 keep --server --server-password mypassword ``` +#### HTTPS / TLS + +Build with the `tls` feature to enable HTTPS: + +```sh +cargo build --release --features server,tls +``` + +Provide a TLS certificate and private key (both PEM format): + +```sh +# Via CLI flags +keep --server \ + --server-cert /path/to/cert.pem \ + --server-key /path/to/key.pem + +# Via environment variables +export KEEP_SERVER_CERT=/path/to/cert.pem +export KEEP_SERVER_KEY=/path/to/key.pem +keep --server + +# Via config file (config.yml) +server: + cert_file: /path/to/cert.pem + key_file: /path/to/key.pem +``` + +When cert and key are provided, the server listens with HTTPS. Without them, it falls back to plain HTTP. The port is controlled by `--server-port` (default: 21080). + +**Self-signed certificates** (for development): + +```sh +# Generate a self-signed cert +openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem \ + -days 365 -nodes -subj "/CN=localhost" + +# Start server with self-signed cert +keep --server --server-cert cert.pem --server-key key.pem + +# Connect client with HTTPS +keep --client-url https://localhost:21080 --save my-tag +``` + +The server accepts data from both dumb clients (raw HTTP/curl) and smart clients (the keep CLI). + +#### Server Query Parameters + +The server supports query parameters that control processing: + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `tags` | none | Comma-separated tags | +| `metadata` | none | JSON-encoded metadata | +| `compress` | `true` | `false` = client already compressed, store as-is | +| `meta` | `true` | `false` = client handles metadata, skip server-side plugins | +| `decompress` | `true` | `false` = return raw compressed bytes on GET | + +When using a smart client, these are set automatically. For curl, the server handles everything by default. + +#### Example: Curl as a Dumb Client + +```sh +# Save (server handles compression and metadata) +curl -X POST -d "my data" http://localhost:21080/api/item/?tags=my-tag + +# Retrieve (server decompresses) +curl http://localhost:21080/api/item/1/content + +# Save compressed (client handles compression, server skips) +gzip -c data.txt | curl -X POST -d @- "http://localhost:21080/api/item/?compress=false&tags=my-tag" +``` + +### Client Mode + +The keep CLI can connect to a remote server as a smart client. Build with the `client` feature: + +```sh +cargo build --release --features client +``` + +```sh +# Set server URL via flag or environment +keep --client-url http://server:21080 --save my-tag +export KEEP_CLIENT_URL=http://server:21080 + +# With authentication +keep --client-url http://server:21080 --client-password mypassword --save my-tag +export KEEP_CLIENT_PASSWORD=mypassword +``` + +#### How Client Mode Works + +Client mode uses **local plugins** and **remote storage**: + +1. **Save**: Local compression and metadata plugins run on the client; compressed data streams to the server +2. **Get**: Server sends raw compressed data; client decompresses locally and applies filters +3. **Other operations** (list, info, delete, diff): Delegated directly to the server + +This means client behavior is consistent with local mode — the same compression settings and filters apply. + +#### Streaming Architecture + +Client save uses a 3-thread streaming pipeline for constant memory usage regardless of data size: + +``` +┌──────────────┐ OS pipe ┌────────────────┐ +│ Reader thread ├──────────────────┤ Streamer thread│ +│ │ (compressed │ │ +│ stdin → tee │ bytes) │ pipe → POST │ +│ → hash │ │ (chunked) │ +│ → compress│ │ │ +└──────────────┘ └────────────────┘ + │ │ + ▼ ▼ + stdout + Server stores blob + SHA-256 digest +``` + +- **Reader thread**: Reads stdin, tees output to stdout, computes SHA-256, compresses data, writes to OS pipe +- **Streamer thread**: Reads compressed bytes from pipe, streams to server via chunked HTTP POST +- **Main thread**: After streaming completes, sends computed metadata (digest, hostname, size) to server + +Memory usage is O(PIPESIZE) — typically 64KB — regardless of how much data is being stored. + +#### Example: Remote Pipeline + +```sh +# On a build server, pipe logs to a central keep server +make build 2>&1 | keep --client-url http://logserver:21080 \ + --save build-logs \ + --meta project=myapp \ + --meta branch=$(git branch --show-current) + +# Retrieve from any machine +keep --client-url http://logserver:21080 --get build-logs + +# List recent builds from a specific project +keep --client-url http://logserver:21080 --list --meta project=myapp +``` + ### API Endpoints | Method | Path | Description | |--------|------|-------------| | `GET` | `/api/status` | System status | -| `GET` | `/api/plugins/status` | Plugins status | -| `GET` | `/api/item/` | List items (supports `tags`, `order`, `start`, `count` params) | -| `POST` | `/api/item/` | Create item (body: raw content, query: `tags`, `metadata`) | +| `GET` | `/api/plugins/status` | Plugin status | +| `GET` | `/api/item/` | List items (`tags`, `order`, `start`, `count` params) | +| `POST` | `/api/item/` | Create item (body: raw content, params: `tags`, `metadata`, `compress`, `meta`) | | `GET` | `/api/item/latest/content` | Latest item content | | `GET` | `/api/item/latest/meta` | Latest item metadata | | `GET` | `/api/item/{id}` | Item info by ID | | `GET` | `/api/item/{id}/content` | Item content by ID | | `GET` | `/api/item/{id}/meta` | Item metadata by ID | | `GET` | `/api/item/{id}/info` | Item info by ID | +| `POST` | `/api/item/{id}/meta` | Add metadata to existing item (body: JSON object) | | `DELETE` | `/api/item/{id}` | Delete item by ID | | `GET` | `/api/diff` | Diff two items (`id_a`, `id_b` params) | -### Authentication +#### Authentication ```sh # Bearer token @@ -339,7 +618,7 @@ curl -u keep:mypassword http://localhost:21080/api/status When no password is configured, authentication is disabled. -### Swagger UI +#### Swagger UI Build with the `swagger` feature to enable OpenAPI documentation: @@ -369,28 +648,6 @@ MCP endpoint available at `/mcp/sse` when the server is running. | `list_items` | List items | `tags[]`, `limit`, `offset` | | `search_items` | Search items | `tags[]`, `metadata{}` | -## Feature Flags - -| Feature | Default | Description | -|---------|---------|-------------| -| `magic` | Yes | File type detection via libmagic | -| `lz4` | Yes | LZ4 compression (internal) | -| `gzip` | Yes | GZip compression (internal) | -| `server` | No | HTTP REST API server | -| `mcp` | No | Model Context Protocol support | -| `swagger` | No | Swagger UI for API docs | -| `bzip2` | No | BZip2 compression (external program) | -| `xz` | No | XZ compression (external program) | -| `zstd` | No | ZStd compression (external program) | - -```sh -# Build with server and swagger -cargo build --features server,swagger - -# Build with all features -cargo build --features server,mcp,swagger,magic -``` - ## Shell Integration Source `profile.bash` to enable shell integration: @@ -401,9 +658,9 @@ source /path/to/keep/profile.bash This provides: -- **`keep` function** - Captures the current command in metadata automatically -- **`@` alias** - Shorthand for `keep --save` -- **`@@` alias** - Shorthand for `keep --get` +- **`keep` function** — Captures the current command in metadata automatically +- **`@` alias** — Shorthand for `keep --save` +- **`@@` alias** — Shorthand for `keep --get` ```sh # Save with automatic command capture @@ -413,55 +670,34 @@ curl -s api.example.com | @ api-response @@ api-response ``` -## Compression +## Feature Flags -Items are compressed automatically on save. Default: LZ4. - -| Algorithm | Type | Speed | Ratio | -|-----------|------|-------|-------| -| `lz4` | Internal | Fastest | Lower | -| `gzip` | Internal | Fast | Good | -| `bzip2` | External | Slow | Better | -| `xz` | External | Slowest | Best | -| `zstd` | External | Fast | Good | -| `none` | Internal | N/A | N/A | +| Feature | Default | Description | +|---------|---------|-------------| +| `magic` | Yes | File type detection via libmagic | +| `lz4` | Yes | LZ4 compression (internal) | +| `gzip` | Yes | GZip compression (internal) | +| `server` | No | HTTP REST API server | +| `tls` | No | HTTPS/TLS server support (requires `server`) | +| `client` | No | HTTP client for remote server | +| `mcp` | No | Model Context Protocol support | +| `swagger` | No | Swagger UI for API docs | +| `bzip2` | No | BZip2 compression (external program) | +| `xz` | No | XZ compression (external program) | +| `zstd` | No | ZStd compression (external program) | ```sh -# Specify compression per item -echo "data" | keep --save my-tag --compression zstd +# Server with Swagger UI +cargo build --features server,swagger -# Set default via environment -export KEEP_COMPRESSION=gzip -``` +# Server with HTTPS +cargo build --features server,tls -External compression programs (`bzip2`, `xz`, `zstd`) must be installed on the system. +# Client only +cargo build --features client -## Meta Plugins - -Metadata is automatically extracted when saving items. - -| Plugin | Key | Description | -|--------|-----|-------------| -| `env` | `*` | Capture `KEEP_META_*` environment variables | -| `magic_file` | `file_type` | File type detection (requires `magic` feature) | -| `text` | `text_line_count`, `text_word_count` | Line and word counts | -| `user` | `uid`, `user`, `gid`, `group` | Current user info | -| `shell` | `shell` | Current shell path | -| `shell_pid` | `shell_pid` | Shell process ID | -| `keep_pid` | `keep_pid` | Keep process ID | -| `digest` | `digest_sha256`, `digest_md5` | Content digests | -| `read_time` | `read_time` | Time to read content | -| `read_rate` | `read_rate` | Data read rate | -| `hostname` | `hostname`, `hostname_short` | System hostname | -| `exec` | Custom | Run external commands for metadata | -| `cwd` | `cwd` | Current working directory | - -```sh -# Use specific plugins -echo "data" | keep --save tag --meta-plugins "digest,text,user" - -# Capture custom metadata via environment -KEEP_META_project=alpha echo "data" | keep --save tag +# Everything +cargo build --features server,tls,client,mcp,swagger,magic ``` ## License diff --git a/src/args.rs b/src/args.rs index d994a4b..b53679a 100644 --- a/src/args.rs +++ b/src/args.rs @@ -75,6 +75,16 @@ pub struct ModeArgs { #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_PORT"))] #[arg(help("Server port to bind to"))] pub server_port: Option, + + #[cfg(feature = "tls")] + #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_CERT"))] + #[arg(help("Path to TLS certificate file (PEM) for HTTPS"))] + pub server_cert: Option, + + #[cfg(feature = "tls")] + #[arg(help_heading("Server Options"), long, env("KEEP_SERVER_KEY"))] + #[arg(help("Path to TLS private key file (PEM) for HTTPS"))] + pub server_key: Option, } /// Struct for item-specific arguments, such as compression and plugins. diff --git a/src/config.rs b/src/config.rs index 7faee68..6ec7c52 100644 --- a/src/config.rs +++ b/src/config.rs @@ -146,6 +146,8 @@ pub struct ServerConfig { pub password_file: Option, pub password: Option, pub password_hash: Option, + pub cert_file: Option, + pub key_file: Option, } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -287,6 +289,18 @@ impl Settings { config_builder = config_builder.set_override("server.port", server_port)?; } + #[cfg(feature = "tls")] + if let Some(server_cert) = &args.mode.server_cert { + config_builder = config_builder + .set_override("server.cert_file", server_cert.to_string_lossy().as_ref())?; + } + + #[cfg(feature = "tls")] + if let Some(server_key) = &args.mode.server_key { + config_builder = config_builder + .set_override("server.key_file", server_key.to_string_lossy().as_ref())?; + } + if let Some(compression) = &args.item.compression { config_builder = config_builder.set_override("compression_plugin.name", compression.as_str())?; @@ -480,6 +494,14 @@ impl Settings { self.server.as_ref().and_then(|s| s.port) } + pub fn server_cert_file(&self) -> Option { + self.server.as_ref().and_then(|s| s.cert_file.clone()) + } + + pub fn server_key_file(&self) -> Option { + self.server.as_ref().and_then(|s| s.key_file.clone()) + } + pub fn compression(&self) -> Option { self.compression_plugin.as_ref().map(|c| c.name.clone()) } diff --git a/src/modes/server/common.rs b/src/modes/server/common.rs index 1551f50..77d0c84 100644 --- a/src/modes/server/common.rs +++ b/src/modes/server/common.rs @@ -67,6 +67,14 @@ pub struct ServerConfig { /// Pre-hashed password (Unix crypt format) for secure authentication. Preferred /// over plain text password for production use. pub password_hash: Option, + /// Optional path to TLS certificate file (PEM). + /// + /// When both cert_file and key_file are set, the server uses HTTPS. + pub cert_file: Option, + /// Optional path to TLS private key file (PEM). + /// + /// When both cert_file and key_file are set, the server uses HTTPS. + pub key_file: Option, } /// Application state shared across all routes. diff --git a/src/modes/server/mod.rs b/src/modes/server/mod.rs index a693165..416a25b 100644 --- a/src/modes/server/mod.rs +++ b/src/modes/server/mod.rs @@ -52,6 +52,8 @@ pub fn mode_server( port: Some(server_port), password: settings.server_password(), password_hash: settings.server_password_hash(), + cert_file: settings.server_cert_file(), + key_file: settings.server_key_file(), }; // Create ItemService once @@ -138,14 +140,31 @@ async fn run_server( let addr: SocketAddr = bind_address.parse()?; + // Build the app into a service + let service = app.into_make_service_with_connect_info::(); + + // Use TLS if both cert and key files are provided + #[cfg(feature = "tls")] + if let (Some(cert_file), Some(key_file)) = (&config.cert_file, &config.key_file) { + info!("SERVER: HTTPS server listening on {addr}"); + + use axum_server::tls_rustls::RustlsConfig; + + let tls_config = RustlsConfig::from_pem_file(cert_file, key_file) + .await + .map_err(|e| anyhow::anyhow!("Failed to load TLS config: {e}"))?; + + axum_server::bind_rustls(addr, tls_config) + .serve(service) + .await?; + + return Ok(()); + } + info!("SERVER: HTTP server listening on {addr}"); let listener = tokio::net::TcpListener::bind(addr).await?; - axum::serve( - listener, - app.into_make_service_with_connect_info::(), - ) - .await?; + axum::serve(listener, service).await?; Ok(()) }