feat: add streaming tar export/import and rename "none" to "raw"

- Add streaming tar-based export (--export produces .keep.tar)
- Add streaming tar import (--import reads .keep.tar archives)
- Add server endpoints GET /api/export and POST /api/import
- Rename CompressionType::None to CompressionType::Raw with "none" as alias
- Add DB migration to update existing "none" compression values to "raw"
- Fix export endpoint to propagate errors to client instead of swallowing
- Fix import endpoint to return 413 on max_body_size instead of truncating

Export streams items as tar archives without loading entire files into memory.
Import extracts items with new IDs, preserving original order. Both work
locally and via client/server mode.

Co-Authored-By: opencode <noreply@opencode.ai>
This commit is contained in:
2026-03-17 21:24:39 -03:00
parent 074ba64805
commit 49793a0f94
36 changed files with 1413 additions and 189 deletions

View File

@@ -397,7 +397,7 @@ impl KeepClient {
.headers()
.get("X-Keep-Compression")
.and_then(|v| v.to_str().ok())
.unwrap_or("none")
.unwrap_or("raw")
.to_string();
let reader = response.into_body().into_reader();
@@ -416,4 +416,109 @@ impl KeepClient {
let response: ApiResponse = self.get_json_with_query("/api/diff", &param_refs)?;
Ok(response.data.unwrap_or_default())
}
/// Export items to a tar archive, streaming the response to a file.
///
/// # Arguments
///
/// * `ids` - Item IDs to export (mutually exclusive with tags).
/// * `tags` - Tags to search for items (mutually exclusive with ids).
/// * `dest` - Destination file path.
pub fn export_items_to_file(
&self,
ids: &[i64],
tags: &[String],
dest: &std::path::Path,
) -> Result<(), CoreError> {
let mut params: Vec<(String, String)> = Vec::new();
if !ids.is_empty() {
let id_strs: Vec<String> = ids.iter().map(|id| id.to_string()).collect();
params.push(("ids".to_string(), id_strs.join(",")));
}
if !tags.is_empty() {
params.push(("tags".to_string(), tags.join(",")));
}
let param_refs: Vec<(&str, &str)> = params
.iter()
.map(|(k, v)| (k.as_str(), v.as_str()))
.collect();
let mut url = self.url("/api/export");
if !param_refs.is_empty() {
url.push('?');
for (i, (key, value)) in param_refs.iter().enumerate() {
if i > 0 {
url.push('&');
}
url.push_str(&format!("{}={}", url_encode(key), url_encode(value)));
}
}
let mut req = self.agent.get(&url);
if let Some(ref auth) = self.auth_header() {
req = req.header("Authorization", auth);
}
let response = self.handle_error(req.call())?;
let mut reader = response.into_body().into_reader();
let mut file = std::fs::File::create(dest).map_err(CoreError::Io)?;
let mut buf = [0u8; crate::common::PIPESIZE];
loop {
let n = reader.read(&mut buf).map_err(CoreError::Io)?;
if n == 0 {
break;
}
std::io::Write::write_all(&mut file, &buf[..n]).map_err(CoreError::Io)?;
}
Ok(())
}
/// Import items from a tar archive, streaming the file to the server.
///
/// # Arguments
///
/// * `tar_path` - Path to the `.keep.tar` file.
///
/// # Returns
///
/// A list of newly assigned item IDs.
pub fn import_tar_file(&self, tar_path: &std::path::Path) -> Result<Vec<i64>, CoreError> {
#[derive(serde::Deserialize)]
struct ApiResponse {
data: Option<ImportResponse>,
error: Option<String>,
}
#[derive(serde::Deserialize)]
struct ImportResponse {
ids: Vec<i64>,
}
let mut file = std::fs::File::open(tar_path).map_err(CoreError::Io)?;
let url = self.url("/api/import");
let mut req = self.agent.post(&url);
if let Some(ref auth) = self.auth_header() {
req = req.header("Authorization", auth);
}
req = req.header("Content-Type", "application/x-tar");
let response = self.handle_error(req.send(ureq::SendBody::from_reader(&mut file)))?;
let body = response
.into_body()
.read_to_string()
.map_err(|e| CoreError::InvalidInput(format!("Cannot read response: {e}")))?;
let api_response: ApiResponse = serde_json::from_str(&body)
.map_err(|e| CoreError::InvalidInput(format!("Cannot parse response: {e}")))?;
if let Some(error) = api_response.error {
return Err(CoreError::InvalidInput(error));
}
Ok(api_response.data.map(|d| d.ids).unwrap_or_default())
}
}