feat: add client mode with streaming support

Add client mode enabling the keep CLI to connect to a remote keep
server over HTTP. Local plugins (compression, meta, filters) run on
the client; the server stores/retrieves binary blobs.

Architecture:
- Client save uses 3-thread streaming pipeline: reader thread (stdin
  → tee/stdout → hash → compress), OS pipe, streamer thread (pipe →
  chunked HTTP POST). Memory usage is O(PIPESIZE) regardless of data
  size.
- Server accepts compress=false, meta=false, decompress=false query
  params for granular control of server-side processing.
- Streaming body handling on server via async channel → sync reader
  bridge (ChannelReader).

Key additions:
- src/client.rs: KeepClient with post_stream() for chunked upload
- src/modes/client/: save, get, list, info, delete, diff, status
- --client-url / KEEP_CLIENT_URL configuration
- --client-password / KEEP_CLIENT_PASSWORD for auth
- os_pipe dependency for zero-copy pipe streaming

Co-Authored-By: andrew/openrouter/hunter-alpha <noreply@opencode.ai>
This commit is contained in:
2026-03-12 18:01:36 -03:00
parent d2581358e9
commit c5529bedbf
16 changed files with 1105 additions and 20 deletions

View File

@@ -0,0 +1,21 @@
use crate::client::KeepClient;
use clap::Command;
use log::debug;
pub fn mode(
client: &KeepClient,
_cmd: &mut Command,
settings: &crate::config::Settings,
ids: &[i64],
) -> Result<(), anyhow::Error> {
debug!("CLIENT_DELETE: Deleting items via remote server");
for &id in ids {
client.delete_item(id)?;
if !settings.quiet {
eprintln!("Deleted item {id}");
}
}
Ok(())
}

24
src/modes/client/diff.rs Normal file
View File

@@ -0,0 +1,24 @@
use crate::client::KeepClient;
use clap::Command;
use log::debug;
pub fn mode(
client: &KeepClient,
_cmd: &mut Command,
_settings: &crate::config::Settings,
ids: &[i64],
) -> Result<(), anyhow::Error> {
debug!("CLIENT_DIFF: Getting diff via remote server");
if ids.len() != 2 {
return Err(anyhow::anyhow!("Diff requires exactly 2 item IDs"));
}
let diff_lines = client.diff_items(ids[0], ids[1])?;
for line in &diff_lines {
println!("{line}");
}
Ok(())
}

95
src/modes/client/get.rs Normal file
View File

@@ -0,0 +1,95 @@
use crate::client::KeepClient;
use crate::compression_engine::CompressionType;
use crate::filter_plugin::FilterChain;
use anyhow::Result;
use clap::Command;
use is_terminal::IsTerminal;
use log::debug;
use std::io::{Read, Write};
use std::str::FromStr;
pub fn mode(
client: &KeepClient,
_cmd: &mut Command,
settings: &crate::config::Settings,
ids: &[i64],
tags: &[String],
filter_chain: Option<FilterChain>,
) -> Result<(), anyhow::Error> {
debug!("CLIENT_GET: Getting item via remote server");
// Find the item ID
let item_id = if !ids.is_empty() {
ids[0]
} else if !tags.is_empty() {
// Find item by tags
let items = client.list_items(tags, "newest", 0, 1)?;
if items.is_empty() {
return Err(anyhow::anyhow!("No items found matching tags: {:?}", tags));
}
items[0].id
} else {
// Get latest item
let items = client.list_items(&[], "newest", 0, 1)?;
if items.is_empty() {
return Err(anyhow::anyhow!("No items found"));
}
items[0].id
};
// Get item info to determine compression type
let item_info = client.get_item_info(item_id)?;
// Get raw content from server
let (raw_bytes, compression) = client.get_item_content_raw(item_id)?;
// Check if binary content would be sent to TTY
let is_text = item_info
.metadata
.get("text")
.map(|v| v == "true")
.unwrap_or(false);
if std::io::stdout().is_terminal() && !is_text && !settings.force {
// Check if content is binary
let sample_len = std::cmp::min(raw_bytes.len(), 8192);
if crate::common::is_binary::is_binary(&raw_bytes[..sample_len]) {
return Err(anyhow::anyhow!(
"Refusing to output binary data to a terminal. Use --force to override."
));
}
}
// Decompress locally
let compression_type = CompressionType::from_str(&compression).unwrap_or(CompressionType::None);
let decompressed = match compression_type {
CompressionType::GZip => {
use flate2::read::GzDecoder;
let mut decoder = GzDecoder::new(&raw_bytes[..]);
let mut content = Vec::new();
decoder.read_to_end(&mut content)?;
content
}
CompressionType::LZ4 => lz4_flex::decompress_size_prepended(&raw_bytes)
.map_err(|e| anyhow::anyhow!("LZ4 decompression failed: {}", e))?,
_ => raw_bytes,
};
// Apply filters if present
let output = if let Some(mut chain) = filter_chain {
let mut filtered = Vec::new();
chain.filter(&mut &decompressed[..], &mut filtered)?;
filtered
} else {
decompressed
};
// Stream to stdout
let stdout = std::io::stdout();
let mut stdout = stdout.lock();
stdout.write_all(&output)?;
stdout.flush()?;
Ok(())
}

65
src/modes/client/info.rs Normal file
View File

@@ -0,0 +1,65 @@
use crate::client::KeepClient;
use crate::modes::common::{OutputFormat, format_size, settings_output_format};
use clap::Command;
use log::debug;
pub fn mode(
client: &KeepClient,
_cmd: &mut Command,
settings: &crate::config::Settings,
ids: &[i64],
tags: &[String],
) -> Result<(), anyhow::Error> {
debug!("CLIENT_INFO: Getting item info via remote server");
let output_format = settings_output_format(settings);
// If tags provided, find matching item first
let item_ids: Vec<i64> = if !tags.is_empty() {
let items = client.list_items(tags, "newest", 0, 1)?;
if items.is_empty() {
return Err(anyhow::anyhow!("No items found matching tags: {:?}", tags));
}
items.into_iter().map(|i| i.id).collect()
} else {
ids.to_vec()
};
for &id in &item_ids {
let item = client.get_item_info(id)?;
match output_format {
OutputFormat::Json => {
println!("{}", serde_json::to_string_pretty(&item)?);
}
OutputFormat::Yaml => {
println!("{}", serde_yaml::to_string(&item)?);
}
OutputFormat::Table => {
use comfy_table::{Table, presets::UTF8_FULL};
let mut table = Table::new();
table.load_preset(UTF8_FULL);
let size_str = item
.size
.map(|s| format_size(s as u64, settings.human_readable))
.unwrap_or_else(|| "N/A".to_string());
table.add_row(vec!["ID".to_string(), item.id.to_string()]);
table.add_row(vec!["Time".to_string(), item.ts.clone()]);
table.add_row(vec!["Size".to_string(), size_str]);
table.add_row(vec!["Compression".to_string(), item.compression.clone()]);
table.add_row(vec!["Tags".to_string(), item.tags.join(", ")]);
for (key, value) in &item.metadata {
table.add_row(vec![format!("Meta: {}", key), value.clone()]);
}
println!("{table}");
}
}
}
Ok(())
}

55
src/modes/client/list.rs Normal file
View File

@@ -0,0 +1,55 @@
use crate::client::KeepClient;
use crate::modes::common::{OutputFormat, format_size, settings_output_format};
use clap::Command;
use log::debug;
pub fn mode(
client: &KeepClient,
_cmd: &mut Command,
settings: &crate::config::Settings,
tags: &[String],
) -> Result<(), anyhow::Error> {
debug!("CLIENT_LIST: Listing items via remote server");
let items = client.list_items(tags, "newest", 0, 100)?;
let output_format = settings_output_format(settings);
match output_format {
OutputFormat::Json => {
println!("{}", serde_json::to_string_pretty(&items)?);
}
OutputFormat::Yaml => {
println!("{}", serde_yaml::to_string(&items)?);
}
OutputFormat::Table => {
use comfy_table::{Table, presets::UTF8_FULL};
let mut table = Table::new();
table.load_preset(UTF8_FULL);
// Header
let headers = ["ID", "Time", "Size", "Compression", "Tags"];
table.set_header(headers.iter().map(|h| h.to_string()).collect::<Vec<_>>());
for item in &items {
let size_str = item
.size
.map(|s| format_size(s as u64, settings.human_readable))
.unwrap_or_default();
table.add_row(vec![
item.id.to_string(),
item.ts.clone(),
size_str,
item.compression.clone(),
item.tags.join(", "),
]);
}
println!("{table}");
}
}
Ok(())
}

7
src/modes/client/mod.rs Normal file
View File

@@ -0,0 +1,7 @@
pub mod delete;
pub mod diff;
pub mod get;
pub mod info;
pub mod list;
pub mod save;
pub mod status;

171
src/modes/client/save.rs Normal file
View File

@@ -0,0 +1,171 @@
use crate::client::{ItemInfo, KeepClient};
use crate::compression_engine::CompressionType;
use crate::config::Settings;
use crate::modes::common::settings_compression_type;
use anyhow::Result;
use clap::Command;
use is_terminal::IsTerminal;
use log::debug;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::io::{Read, Write};
use std::sync::{Arc, Mutex};
/// Streaming save mode for client.
///
/// Uses three threads for true streaming with constant memory:
/// - Reader thread: reads stdin, tees to stdout, computes SHA-256,
/// compresses data, writes to OS pipe
/// - Pipe: zero-copy transfer of compressed bytes between threads
/// - Streamer thread: reads from pipe, streams to server via chunked HTTP
///
/// Memory usage is O(PIPESIZE) regardless of data size.
pub fn mode(
client: &KeepClient,
cmd: &mut Command,
settings: &Settings,
tags: &mut Vec<String>,
metadata: HashMap<String, String>,
) -> Result<(), anyhow::Error> {
debug!("CLIENT_SAVE: Saving item via remote server (streaming)");
if tags.is_empty() {
tags.push("none".to_string());
}
// Determine compression type from settings
let compression_type = settings_compression_type(cmd, settings);
let server_compress = matches!(compression_type, CompressionType::None);
// Create OS pipe for streaming compressed bytes between threads
let (pipe_reader, pipe_writer) = os_pipe::pipe()?;
// Shared state for reader thread results
let shared = Arc::new(Mutex::new((0u64, String::new())));
let shared_reader = Arc::clone(&shared);
// Reader thread: stdin → tee(stdout) → hash → compress → pipe
let compression_type_clone = compression_type.clone();
let reader_handle = std::thread::spawn(move || -> Result<(u64, String)> {
let stdin = std::io::stdin();
let stdout = std::io::stdout();
let mut stdin_lock = stdin.lock();
let mut stdout_lock = stdout.lock();
let mut hasher = Sha256::new();
let mut total_bytes = 0u64;
let mut buffer = [0u8; 8192];
// Wrap pipe writer with appropriate compression
let mut compressor: Box<dyn Write> = match compression_type_clone {
CompressionType::GZip => {
use flate2::Compression;
use flate2::write::GzEncoder;
Box::new(GzEncoder::new(pipe_writer, Compression::default()))
}
CompressionType::LZ4 => Box::new(lz4_flex::frame::FrameEncoder::new(pipe_writer)),
_ => Box::new(pipe_writer),
};
loop {
let n = stdin_lock.read(&mut buffer)?;
if n == 0 {
break;
}
// Tee to stdout
stdout_lock.write_all(&buffer[..n])?;
// Update hash
hasher.update(&buffer[..n]);
total_bytes += n as u64;
// Compress and write to pipe
compressor.write_all(&buffer[..n])?;
}
// Finalize compression (flushes any buffered compressed data)
drop(compressor);
// Pipe writer is now dropped (inside compressor), signaling EOF to streamer
let digest = format!("{:x}", hasher.finalize());
// Set shared state for main thread
let mut shared = shared_reader.lock().unwrap();
*shared = (total_bytes, digest.clone());
Ok((total_bytes, digest))
});
// Streamer thread: reads compressed bytes from pipe → POST to server
let client_url = client.base_url().to_string();
let client_password = client.password().cloned();
let tags_clone = tags.clone();
let streamer_handle = std::thread::spawn(move || -> Result<ItemInfo> {
let streaming_client = KeepClient::new(&client_url, client_password)?;
let params = [
("compress".to_string(), server_compress.to_string()),
("meta".to_string(), "false".to_string()),
("tags".to_string(), tags_clone.join(",")),
];
let param_refs: Vec<(&str, &str)> = params
.iter()
.map(|(k, v)| (k.as_str(), v.as_str()))
.collect();
let mut reader: Box<dyn Read> = Box::new(pipe_reader);
let item_info = streaming_client.post_stream("/api/item/", &mut reader, &param_refs)?;
Ok(item_info)
});
// Wait for streaming to complete, capture item info
let item_info = streamer_handle
.join()
.map_err(|e| anyhow::anyhow!("Streamer thread panicked: {:?}", e))??;
// Wait for reader thread (should complete quickly after pipe is drained)
reader_handle
.join()
.map_err(|e| anyhow::anyhow!("Reader thread panicked: {:?}", e))??;
// Read results from shared state
let (uncompressed_size, digest) = {
let shared = shared.lock().unwrap();
shared.clone()
};
// Build local metadata and send to server
let mut local_metadata = metadata;
local_metadata.insert("digest_sha256".to_string(), digest);
local_metadata.insert(
"uncompressed_size".to_string(),
uncompressed_size.to_string(),
);
// Add hostname
if let Ok(hostname) = gethostname::gethostname().into_string() {
local_metadata.insert("hostname".to_string(), hostname.clone());
let short = hostname.split('.').next().unwrap_or(&hostname).to_string();
local_metadata.insert("hostname_short".to_string(), short);
}
// Send metadata to server
if !local_metadata.is_empty() {
client.post_metadata(item_info.id, &local_metadata)?;
}
// Print status to stderr
if !settings.quiet {
if std::io::stderr().is_terminal() {
eprintln!("KEEP: New item (streaming) tags: {}", tags.join(" "));
} else {
eprintln!("KEEP: New item (streaming) tags: {tags:?}");
}
}
debug!("CLIENT_SAVE: Streaming complete, {uncompressed_size} bytes uncompressed");
Ok(())
}

View File

@@ -0,0 +1,33 @@
use crate::client::KeepClient;
use crate::modes::common::OutputFormat;
use crate::modes::common::settings_output_format;
use clap::Command;
use log::debug;
pub fn mode(
client: &KeepClient,
_cmd: &mut Command,
settings: &crate::config::Settings,
) -> Result<(), anyhow::Error> {
debug!("CLIENT_STATUS: Getting status from remote server");
let status = client.get_status()?;
let output_format = settings_output_format(settings);
match output_format {
OutputFormat::Json => {
println!("{}", serde_json::to_string_pretty(&status)?);
}
OutputFormat::Yaml => {
println!("{}", serde_yaml::to_string(&status)?);
}
OutputFormat::Table => {
println!("Remote Server Status");
println!("====================");
println!("{}", serde_json::to_string_pretty(&status)?);
}
}
Ok(())
}

View File

@@ -1,6 +1,9 @@
#[cfg(feature = "server")]
pub mod server;
#[cfg(feature = "client")]
pub mod client;
/// Common utilities for all modes, including column types and output formatting.
pub mod common;