feat: add client mode with streaming support
Add client mode enabling the keep CLI to connect to a remote keep server over HTTP. Local plugins (compression, meta, filters) run on the client; the server stores/retrieves binary blobs. Architecture: - Client save uses 3-thread streaming pipeline: reader thread (stdin → tee/stdout → hash → compress), OS pipe, streamer thread (pipe → chunked HTTP POST). Memory usage is O(PIPESIZE) regardless of data size. - Server accepts compress=false, meta=false, decompress=false query params for granular control of server-side processing. - Streaming body handling on server via async channel → sync reader bridge (ChannelReader). Key additions: - src/client.rs: KeepClient with post_stream() for chunked upload - src/modes/client/: save, get, list, info, delete, diff, status - --client-url / KEEP_CLIENT_URL configuration - --client-password / KEEP_CLIENT_PASSWORD for auth - os_pipe dependency for zero-copy pipe streaming Co-Authored-By: andrew/openrouter/hunter-alpha <noreply@opencode.ai>
This commit is contained in:
310
src/client.rs
Normal file
310
src/client.rs
Normal file
@@ -0,0 +1,310 @@
|
||||
use crate::services::error::CoreError;
|
||||
use serde::de::DeserializeOwned;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Read;
|
||||
|
||||
/// Item information returned from the server API.
|
||||
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
|
||||
pub struct ItemInfo {
|
||||
pub id: i64,
|
||||
pub ts: String,
|
||||
pub size: Option<i64>,
|
||||
pub compression: String,
|
||||
pub tags: Vec<String>,
|
||||
pub metadata: HashMap<String, String>,
|
||||
}
|
||||
|
||||
pub struct KeepClient {
|
||||
base_url: String,
|
||||
agent: ureq::Agent,
|
||||
password: Option<String>,
|
||||
}
|
||||
|
||||
impl KeepClient {
|
||||
pub fn new(base_url: &str, password: Option<String>) -> Result<Self, CoreError> {
|
||||
let base_url = base_url.trim_end_matches('/').to_string();
|
||||
let agent = ureq::Agent::new_with_defaults();
|
||||
Ok(Self {
|
||||
base_url,
|
||||
agent,
|
||||
password,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn base_url(&self) -> &str {
|
||||
&self.base_url
|
||||
}
|
||||
|
||||
pub fn password(&self) -> Option<&String> {
|
||||
self.password.as_ref()
|
||||
}
|
||||
|
||||
fn url(&self, path: &str) -> String {
|
||||
format!("{}{}", self.base_url, path)
|
||||
}
|
||||
|
||||
fn handle_error<T>(&self, result: Result<T, ureq::Error>) -> Result<T, CoreError> {
|
||||
match result {
|
||||
Ok(v) => Ok(v),
|
||||
Err(ureq::Error::StatusCode(code)) => Err(CoreError::Other(anyhow::anyhow!(
|
||||
"Server returned error: HTTP {}",
|
||||
code
|
||||
))),
|
||||
Err(e) => Err(CoreError::Other(anyhow::anyhow!("Request failed: {}", e))),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, CoreError> {
|
||||
let url = self.url(path);
|
||||
let mut req = self.agent.get(&url);
|
||||
if let Some(ref password) = self.password {
|
||||
req = req.header("Authorization", &format!("Bearer {password}"));
|
||||
}
|
||||
let response = self.handle_error(req.call())?;
|
||||
let body: T = self.handle_error(response.into_body().read_json())?;
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub fn get_json_with_query<T: DeserializeOwned>(
|
||||
&self,
|
||||
path: &str,
|
||||
params: &[(&str, &str)],
|
||||
) -> Result<T, CoreError> {
|
||||
let mut url = self.url(path);
|
||||
if !params.is_empty() {
|
||||
url.push('?');
|
||||
for (i, (key, value)) in params.iter().enumerate() {
|
||||
if i > 0 {
|
||||
url.push('&');
|
||||
}
|
||||
url.push_str(&format!("{key}={value}"));
|
||||
}
|
||||
}
|
||||
let mut req = self.agent.get(&url);
|
||||
if let Some(ref password) = self.password {
|
||||
req = req.header("Authorization", &format!("Bearer {password}"));
|
||||
}
|
||||
let response = self.handle_error(req.call())?;
|
||||
let body: T = self.handle_error(response.into_body().read_json())?;
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub fn get_bytes(&self, path: &str) -> Result<Vec<u8>, CoreError> {
|
||||
let url = self.url(path);
|
||||
let mut req = self.agent.get(&url);
|
||||
if let Some(ref password) = self.password {
|
||||
req = req.header("Authorization", &format!("Bearer {password}"));
|
||||
}
|
||||
let response = self.handle_error(req.call())?;
|
||||
let mut body = response.into_body();
|
||||
let bytes = body
|
||||
.read_to_vec()
|
||||
.map_err(|e| CoreError::Other(anyhow::anyhow!("{}", e)))?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub fn post_bytes(
|
||||
&self,
|
||||
path: &str,
|
||||
body_bytes: &[u8],
|
||||
params: &[(&str, &str)],
|
||||
) -> Result<ItemInfo, CoreError> {
|
||||
let mut cursor = std::io::Cursor::new(body_bytes);
|
||||
self.post_stream(path, &mut cursor, params)
|
||||
}
|
||||
|
||||
/// Stream data from a reader to the server using chunked transfer encoding.
|
||||
///
|
||||
/// The reader is consumed in chunks and sent to the server without buffering
|
||||
/// the entire body in memory. This enables true streaming for large payloads.
|
||||
pub fn post_stream(
|
||||
&self,
|
||||
path: &str,
|
||||
body_reader: &mut dyn Read,
|
||||
params: &[(&str, &str)],
|
||||
) -> Result<ItemInfo, CoreError> {
|
||||
let mut url = self.url(path);
|
||||
if !params.is_empty() {
|
||||
url.push('?');
|
||||
for (i, (key, value)) in params.iter().enumerate() {
|
||||
if i > 0 {
|
||||
url.push('&');
|
||||
}
|
||||
url.push_str(&format!("{key}={value}"));
|
||||
}
|
||||
}
|
||||
|
||||
let mut req = self.agent.post(&url);
|
||||
if let Some(ref password) = self.password {
|
||||
req = req.header("Authorization", &format!("Bearer {password}"));
|
||||
}
|
||||
req = req.header("Content-Type", "application/octet-stream");
|
||||
|
||||
let response = self.handle_error(req.send(ureq::SendBody::from_reader(body_reader)))?;
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct ApiResponse {
|
||||
data: Option<ItemInfo>,
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
let api_response: ApiResponse = self.handle_error(response.into_body().read_json())?;
|
||||
|
||||
if let Some(error) = api_response.error {
|
||||
return Err(CoreError::Other(anyhow::anyhow!("Server error: {}", error)));
|
||||
}
|
||||
|
||||
api_response
|
||||
.data
|
||||
.ok_or_else(|| CoreError::Other(anyhow::anyhow!("No data in response")))
|
||||
}
|
||||
|
||||
pub fn delete(&self, path: &str) -> Result<(), CoreError> {
|
||||
let url = self.url(path);
|
||||
let mut req = self.agent.delete(&url);
|
||||
if let Some(ref password) = self.password {
|
||||
req = req.header("Authorization", &format!("Bearer {password}"));
|
||||
}
|
||||
self.handle_error(req.call())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_status(&self) -> Result<serde_json::Value, CoreError> {
|
||||
self.get_json("/api/status")
|
||||
}
|
||||
|
||||
pub fn get_item_info(&self, id: i64) -> Result<ItemInfo, CoreError> {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct ApiResponse {
|
||||
data: Option<ItemInfo>,
|
||||
}
|
||||
let response: ApiResponse = self.get_json(&format!("/api/item/{id}/info"))?;
|
||||
response
|
||||
.data
|
||||
.ok_or_else(|| CoreError::Other(anyhow::anyhow!("Item not found")))
|
||||
}
|
||||
|
||||
pub fn list_items(
|
||||
&self,
|
||||
tags: &[String],
|
||||
order: &str,
|
||||
start: u64,
|
||||
count: u64,
|
||||
) -> Result<Vec<ItemInfo>, CoreError> {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct ApiResponse {
|
||||
data: Option<Vec<ItemInfo>>,
|
||||
}
|
||||
|
||||
let mut params: Vec<(String, String)> = Vec::new();
|
||||
params.push(("order".to_string(), order.to_string()));
|
||||
params.push(("start".to_string(), start.to_string()));
|
||||
params.push(("count".to_string(), count.to_string()));
|
||||
if !tags.is_empty() {
|
||||
params.push(("tags".to_string(), tags.join(",")));
|
||||
}
|
||||
|
||||
let param_refs: Vec<(&str, &str)> = params
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str(), v.as_str()))
|
||||
.collect();
|
||||
|
||||
let response: ApiResponse = self.get_json_with_query("/api/item/", ¶m_refs)?;
|
||||
Ok(response.data.unwrap_or_default())
|
||||
}
|
||||
|
||||
pub fn save_item(
|
||||
&self,
|
||||
content: &[u8],
|
||||
tags: &[String],
|
||||
metadata: &HashMap<String, String>,
|
||||
compress: bool,
|
||||
meta: bool,
|
||||
) -> Result<ItemInfo, CoreError> {
|
||||
let mut params: Vec<(String, String)> = Vec::new();
|
||||
if !tags.is_empty() {
|
||||
params.push(("tags".to_string(), tags.join(",")));
|
||||
}
|
||||
if !metadata.is_empty() {
|
||||
let meta_json = serde_json::to_string(metadata).map_err(|e| {
|
||||
CoreError::Other(anyhow::anyhow!("Failed to serialize metadata: {}", e))
|
||||
})?;
|
||||
params.push(("metadata".to_string(), meta_json));
|
||||
}
|
||||
params.push(("compress".to_string(), compress.to_string()));
|
||||
params.push(("meta".to_string(), meta.to_string()));
|
||||
|
||||
let param_refs: Vec<(&str, &str)> = params
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str(), v.as_str()))
|
||||
.collect();
|
||||
|
||||
self.post_bytes("/api/item/", content, ¶m_refs)
|
||||
}
|
||||
|
||||
pub fn delete_item(&self, id: i64) -> Result<(), CoreError> {
|
||||
self.delete(&format!("/api/item/{id}"))
|
||||
}
|
||||
|
||||
/// Add metadata to an existing item.
|
||||
pub fn post_metadata(
|
||||
&self,
|
||||
id: i64,
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<(), CoreError> {
|
||||
let url = self.url(&format!("/api/item/{id}/meta"));
|
||||
let mut req = self.agent.post(&url);
|
||||
if let Some(ref password) = self.password {
|
||||
req = req.header("Authorization", &format!("Bearer {password}"));
|
||||
}
|
||||
req = req.header("Content-Type", "application/json");
|
||||
|
||||
let body = serde_json::to_vec(metadata)
|
||||
.map_err(|e| CoreError::Other(anyhow::anyhow!("Failed to serialize metadata: {e}")))?;
|
||||
|
||||
let mut cursor = std::io::Cursor::new(body);
|
||||
self.handle_error(req.send(ureq::SendBody::from_reader(&mut cursor)))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_item_content_raw(&self, id: i64) -> Result<(Vec<u8>, String), CoreError> {
|
||||
let url = format!(
|
||||
"{}?decompress=false",
|
||||
self.url(&format!("/api/item/{id}/content"))
|
||||
);
|
||||
|
||||
let mut req = self.agent.get(&url);
|
||||
if let Some(ref password) = self.password {
|
||||
req = req.header("Authorization", &format!("Bearer {password}"));
|
||||
}
|
||||
|
||||
let response = self.handle_error(req.call())?;
|
||||
|
||||
let compression = response
|
||||
.headers()
|
||||
.get("X-Keep-Compression")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("none")
|
||||
.to_string();
|
||||
|
||||
let mut body = response.into_body();
|
||||
let bytes = body
|
||||
.read_to_vec()
|
||||
.map_err(|e| CoreError::Other(anyhow::anyhow!("{}", e)))?;
|
||||
|
||||
Ok((bytes, compression))
|
||||
}
|
||||
|
||||
pub fn diff_items(&self, id_a: i64, id_b: i64) -> Result<Vec<String>, CoreError> {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct ApiResponse {
|
||||
data: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
let params = [("id_a", id_a.to_string()), ("id_b", id_b.to_string())];
|
||||
let param_refs: Vec<(&str, &str)> = params.iter().map(|(k, v)| (*k, v.as_str())).collect();
|
||||
|
||||
let response: ApiResponse = self.get_json_with_query("/api/diff", ¶m_refs)?;
|
||||
Ok(response.data.unwrap_or_default())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user