asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add export/import modes, unify service layer, fix binary detection

Browse Source 
Export/import:
- Add --export and --import modes for both local and client paths
- Use strfmt crate for --export-filename-format templates ({id}, {tags}, {ts}, {compression})
- Import preserves original timestamps via server ?ts= param
- --import-data-file for file-based import; stdin fallback streams with PIPESIZE buffers

Service unification:
- Merge SyncDataService unique methods into ItemService (delete_item now returns Result<Item>)
- Delete AsyncDataService, AsyncItemService, DataService trait (dead code / async-blocking anti-pattern)
- All server handlers use spawn_blocking + ItemService directly
- Extract shared types (ExportMeta, ImportMeta) and helpers (resolve_item_id(s), check_binary_tty)

Binary detection fix:
- Replace broken metadata.get("map") + is_binary(&[]) with actual content sampling
- Both as_meta and allow_binary paths read PIPESIZE sample before deciding
- Never load entire item into memory for binary check

Other fixes:
- Fix lock consistency: all handlers use blocking_lock() in spawn_blocking (no mixed lock().await)
- Use ISO 8601 format for {ts} in export filenames
- Fix resolve_item_ids returning only 1 item for tag lookups
- Fix client get.rs triple-buffering and export.rs whole-file buffering
- Add KeepClient::get_item_content_stream() for streaming reads
- Pass all clippy --features server lints (Path vs PathBuf, &mut conn, etc.)
This commit is contained in:
Andrew Phillips
2026-03-16 08:43:26 -03:00
parent 0a3d61a875
commit 35ee71c3cf
25 changed files with 1618 additions and 1700 deletions
Download Patch File Download Diff File Expand all files Collapse all files

273
src/services/item_service.rs
View File

@@ -8,12 +8,14 @@ use crate::services::error::CoreError;
use crate::services::filter_service::FilterService;
use crate::services::meta_service::MetaService;
use crate::services::types::{ItemWithContent, ItemWithMeta};
use chrono::DateTime;
use chrono::Utc;
use clap::Command;
use log::debug;
use rusqlite::Connection;
use std::collections::HashMap;
use std::fs;
use std::io::{IsTerminal, Read, Write};
use std::io::{Cursor, IsTerminal, Read, Write};
use std::path::PathBuf;
/// Service for managing items in the Keep application.
@@ -530,7 +532,7 @@ impl ItemService {
/// ```ignore
/// item_service.delete_item(&mut conn, 1)?;
/// ```
pub fn delete_item(&self, conn: &mut Connection, id: i64) -> Result<(), CoreError> {
pub fn delete_item(&self, conn: &mut Connection, id: i64) -> Result<Item, CoreError> {
debug!("ITEM_SERVICE: Deleting item with id: {id}");
if id <= 0 {
return Err(CoreError::InvalidInput(format!("Invalid item ID: {id}")));
@@ -542,6 +544,7 @@ impl ItemService {
item_path.push(id.to_string());
debug!("ITEM_SERVICE: Deleting file at path: {item_path:?}");
let deleted_item = item.clone();
db::delete_item(conn, item)?;
fs::remove_file(&item_path).or_else(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
@@ -552,7 +555,7 @@ impl ItemService {
})?;
debug!("ITEM_SERVICE: Successfully deleted item {id}");
Ok(())
Ok(deleted_item)
}
/// Saves content from a reader to a new item.
@@ -723,6 +726,270 @@ impl ItemService {
pub fn get_data_path(&self) -> &PathBuf {
&self.data_path
}
/// Returns a streaming reader and item metadata for the given item.
pub fn get_item_content_streaming(
&self,
conn: &Connection,
id: i64,
) -> Result<(Box<dyn Read + Send>, ItemWithMeta), CoreError> {
let (reader, _mime, _is_binary) = self.get_item_content_info_streaming(conn, id, None)?;
let item_with_meta = self.get_item(conn, id)?;
Ok((reader, item_with_meta))
}
/// Fetches multiple items by ID, silently skipping not-found items.
/// Falls back to `list_items` if the ID list is empty.
pub fn get_items(
&self,
conn: &Connection,
ids: &[i64],
tags: &[String],
meta: &HashMap<String, Option<String>>,
) -> Result<Vec<ItemWithMeta>, CoreError> {
if ids.is_empty() {
return self.list_items(conn, tags, meta);
}
let mut results = Vec::new();
for id in ids {
match self.get_item(conn, *id) {
Ok(item) => results.push(item),
Err(CoreError::ItemNotFound(_)) => continue,
Err(e) => return Err(e),
}
}
Ok(results)
}
/// Save an item with granular control over compression and meta plugins.
///
/// This method allows callers to control whether compression and meta plugins
/// run server-side or were already handled by the client.
///
/// # Arguments
///
/// * `conn` - Database connection.
/// * `content` - Raw content bytes.
/// * `tags` - Tags to associate with the item.
/// * `metadata` - Client-provided metadata.
/// * `compress` - Whether the server should compress the content.
/// * `run_meta` - Whether the server should run meta plugins.
/// * `settings` - Application settings.
///
/// # Returns
///
/// * `Result<ItemWithMeta, CoreError>` - The saved item with full details.
#[allow(clippy::too_many_arguments)]
pub fn save_item_raw(
&self,
conn: &mut Connection,
content: &[u8],
tags: Vec<String>,
metadata: HashMap<String, String>,
compress: bool,
run_meta: bool,
settings: &Settings,
) -> Result<ItemWithMeta, CoreError> {
let mut cursor = Cursor::new(content);
self.save_item_raw_streaming(
conn,
&mut cursor,
tags,
metadata,
compress,
run_meta,
None,
None,
settings,
)
}
/// Save an item from a streaming reader with granular control over compression.
///
/// Unlike `save_item_raw` which takes a pre-buffered `&[u8]`, this method
/// reads from the reader in chunks and writes directly to the compression
/// engine, avoiding buffering the entire content in memory.
#[allow(clippy::too_many_arguments)]
pub fn save_item_raw_streaming(
&self,
conn: &mut Connection,
reader: &mut dyn Read,
tags: Vec<String>,
metadata: HashMap<String, String>,
compress: bool,
run_meta: bool,
client_compression_type: Option<CompressionType>,
import_ts: Option<DateTime<Utc>>,
settings: &Settings,
) -> Result<ItemWithMeta, CoreError> {
let mut cmd = Command::new("keep");
let mut tags = tags;
crate::modes::common::ensure_default_tag(&mut tags);
let (compression_type_for_db, compression_engine) = if compress {
let ct = settings_compression_type(&mut cmd, settings);
let engine = get_compression_engine(ct.clone())?;
(ct, engine)
} else {
let ct = client_compression_type.unwrap_or(CompressionType::None);
let engine = get_compression_engine(CompressionType::None)?;
(ct, engine)
};
let item_id;
let mut item;
{
item = if let Some(ts) = import_ts {
db::insert_item_with_ts(conn, ts, &compression_type_for_db.to_string())?
} else {
db::create_item(conn, compression_type_for_db.clone())?
};
item_id = item
.id
.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?;
db::set_item_tags(conn, item.clone(), &tags)?;
}
let collected_meta: std::sync::Arc<std::sync::Mutex<Vec<(String, String)>>> =
std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let collector = collected_meta.clone();
let save_meta: crate::meta_plugin::SaveMetaFn =
std::sync::Arc::new(std::sync::Mutex::new(move |name: &str, value: &str| {
if let Ok(mut v) = collector.lock() {
v.push((name.to_string(), value.to_string()));
}
}));
let meta_service = MetaService::new(save_meta);
let mut plugins = if run_meta {
meta_service.get_plugins(&mut cmd, settings)
} else {
Vec::new()
};
if run_meta {
meta_service.initialize_plugins(&mut plugins);
}
let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string());
let mut item_out = compression_engine.create(item_path)?;
let mut total_bytes = 0i64;
crate::common::stream_copy(reader, |chunk| {
item_out.write_all(chunk)?;
total_bytes += chunk.len() as i64;
if run_meta {
meta_service.process_chunk(&mut plugins, chunk);
}
Ok(())
})?;
item_out.flush()?;
drop(item_out);
if run_meta {
meta_service.finalize_plugins(&mut plugins);
}
if run_meta && let Ok(entries) = collected_meta.lock() {
for (name, value) in entries.iter() {
db::add_meta(conn, item_id, name, value)?;
}
}
for (key, value) in &metadata {
if key != "uncompressed_size" {
db::add_meta(conn, item_id, key, value)?;
}
}
item.size = Some(total_bytes);
db::update_item(conn, item)?;
self.get_item(conn, item_id)
}
/// Runs specified meta plugins on an existing item's content and stores the results.
pub fn update_item_plugins(
&self,
conn: &mut Connection,
item_id: i64,
plugin_names: &[String],
metadata: HashMap<String, String>,
tags: &[String],
settings: &Settings,
) -> Result<ItemWithMeta, CoreError> {
let item = db::get_item(conn, item_id)?.ok_or_else(|| CoreError::ItemNotFound(item_id))?;
let collected_meta: std::sync::Arc<std::sync::Mutex<Vec<(String, String)>>> =
std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let collector = collected_meta.clone();
let save_meta: crate::meta_plugin::SaveMetaFn =
std::sync::Arc::new(std::sync::Mutex::new(move |name: &str, value: &str| {
if let Ok(mut v) = collector.lock() {
v.push((name.to_string(), value.to_string()));
}
}));
let meta_service = MetaService::new(save_meta);
let mut cmd = Command::new("keep");
let all_plugins = meta_service.get_plugins(&mut cmd, settings);
let mut plugins: Vec<Box<dyn crate::meta_plugin::MetaPlugin>> = all_plugins
.into_iter()
.filter(|p| {
let plugin_name = p.meta_type().to_string();
plugin_names.iter().any(|n| n == &plugin_name)
})
.collect();
if plugins.is_empty() && metadata.is_empty() {
return self.get_item(conn, item_id);
}
let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string());
if !item_path.exists() {
return Err(CoreError::ItemNotFound(item_id));
}
if !plugins.is_empty() {
let compression_service = CompressionService::new();
let mut reader =
compression_service.stream_item_content(item_path, &item.compression)?;
meta_service.initialize_plugins(&mut plugins);
crate::common::stream_copy(&mut reader, |chunk| {
meta_service.process_chunk(&mut plugins, chunk);
Ok(())
})?;
meta_service.finalize_plugins(&mut plugins);
if let Ok(entries) = collected_meta.lock() {
for (name, value) in entries.iter() {
db::add_meta(conn, item_id, name, value)?;
}
}
}
for (key, value) in &metadata {
db::add_meta(conn, item_id, key, value)?;
}
for tag in tags {
db::upsert_tag(conn, item_id, tag)?;
}
self.get_item(conn, item_id)
}
}
/// A reader that applies a filter chain to the data as it's read.