asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: correct critical bugs and improve pipe streaming performance

Browse Source 
Critical bug fixes:
- save_item now returns real Item from database, not a hardcoded fake
- AsyncDataService::save() reuses self.sync_service instead of creating redundant instance
- GenerateStatus trait signature mismatch fixed (CLI/API decoupling)

Performance improvements (pipe path untouched):
- CompressionEngine::open() returns Box<dyn Read + Send> enabling true streaming
- mode_get eliminates triple full-file read (was sampling then re-reading entire file)
- FilteringReader adds fast-path bypass when no filters, pre-allocates temp buffer
- text.rs meta plugin processes &[u8] slice directly, eliminates data.to_vec() clone

API correctness:
- Tag parse errors now return 400 instead of being silently discarded
- compute_diff uses similar crate (LCS-based) instead of naive positional comparison

Cleanup:
- Modernize string formatting (format!({x})) across codebase
- Remove redundant DB query in get mode
- Derive Debug/ToSchema on public types
- Delete placeholder test files with no real assertions
- Extract parse_comma_tags utility function
This commit is contained in:
Andrew Phillips
2026-03-11 20:45:05 -03:00
parent e8ea42506e
commit 8a8a6e1c4b
53 changed files with 813 additions and 640 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
Cargo.lock generated
View File

@@ -130,6 +130,28 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236"
[[package]]
name = "async-stream"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"
dependencies = [
"async-stream-impl",
"futures-core",
"pin-project-lite",
]
[[package]]
name = "async-stream-impl"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.105",
]
[[package]] [[package]]
name = "async-trait" name = "async-trait"
version = "0.1.89" version = "0.1.89"
@@ -1377,6 +1399,7 @@ name = "keep"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-stream",
"axum", "axum",
"base64 0.22.1", "base64 0.22.1",
"chrono", "chrono",
@@ -1392,6 +1415,7 @@ dependencies = [
"flate2", "flate2",
"futures", "futures",
"gethostname", "gethostname",
"http-body-util",
"humansize", "humansize",
"hyper", "hyper",
"inventory", "inventory",
@@ -1418,6 +1442,7 @@ dependencies = [
"serde_json", "serde_json",
"serde_yaml", "serde_yaml",
"sha2 0.10.9", "sha2 0.10.9",
"similar",
"smart-default", "smart-default",
"stderrlog", "stderrlog",
"strip-ansi-escapes", "strip-ansi-escapes",
@@ -2325,6 +2350,12 @@ version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
[[package]]
name = "similar"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.11" version = "0.4.11"

3
Cargo.toml
View File

@@ -27,7 +27,9 @@ flate2 = { version = "1.0.27", features = ["zlib-ng-compat"], optional = true }
futures = "0.3" futures = "0.3"
gethostname = "1.0.2" gethostname = "1.0.2"
humansize = "2.1.3" humansize = "2.1.3"
async-stream = "0.3"
hyper = { version = "1.0", features = ["full"] } hyper = { version = "1.0", features = ["full"] }
http-body-util = "0.1"
inventory = "0.3" inventory = "0.3"
is-terminal = "0.4.9" is-terminal = "0.4.9"
lazy_static = "1.4.0" lazy_static = "1.4.0"
@@ -67,6 +69,7 @@ strip-ansi-escapes = "0.2.1"
pest = "2.8.1" pest = "2.8.1"
pest_derive = "2.8.1" pest_derive = "2.8.1"
dirs = "6.0.0" dirs = "6.0.0"
similar = { version = "2.7.0", default-features = false, features = ["text"] }
[features] [features]
# Default features include core compression engines and swagger UI # Default features include core compression engines and swagger UI

6
src/common/is_binary.rs
View File

@@ -192,15 +192,15 @@ fn looks_like_tar(data: &[u8]) -> bool {
} }
// Check file mode field (should be octal digits) // Check file mode field (should be octal digits)
for i in 100..108 { for byte in data.iter().skip(100).take(8) {
if data[i] != 0 && (data[i] < b'0' || data[i] > b'7') && data[i] != b' ' { if *byte != 0 && !(b'0'..=b'7').contains(byte) && *byte != b' ' {
return false; return false;
} }
} }
// Check checksum field (should be octal digits or spaces) // Check checksum field (should be octal digits or spaces)
for &b in &data[148..156] { for &b in &data[148..156] {
if b != 0 && (b < b'0' || b > b'7') && b != b' ' { if b != 0 && !(b'0'..=b'7').contains(&b) && b != b' ' {
return false; return false;
} }
} }

14
src/common/status.rs
View File

@@ -8,7 +8,7 @@ use crate::meta_plugin::MetaPluginType;
use crate::filter_plugin::FilterOption; use crate::filter_plugin::FilterOption;
#[derive(serde::Serialize, serde::Deserialize, Clone)] #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
#[cfg_attr(feature = "server", derive(ToSchema))] #[cfg_attr(feature = "server", derive(ToSchema))]
pub struct FilterPluginInfo { pub struct FilterPluginInfo {
pub name: String, pub name: String,
@@ -34,7 +34,8 @@ pub struct PathInfo {
pub database: String, pub database: String,
} }
#[derive(serde::Serialize, serde::Deserialize)] #[derive(serde::Serialize, serde::Deserialize, Debug)]
#[cfg_attr(feature = "server", derive(ToSchema))]
pub struct CompressionInfo { pub struct CompressionInfo {
#[serde(rename = "type")] #[serde(rename = "type")]
pub compression_type: String, pub compression_type: String,
@@ -45,7 +46,7 @@ pub struct CompressionInfo {
pub decompress: String, pub decompress: String,
} }
#[derive(serde::Serialize, serde::Deserialize, Clone)] #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
#[cfg_attr(feature = "server", derive(ToSchema))] #[cfg_attr(feature = "server", derive(ToSchema))]
pub struct MetaPluginInfo { pub struct MetaPluginInfo {
pub meta_name: String, pub meta_name: String,
@@ -132,10 +133,7 @@ pub fn generate_status_info(
sorted_meta_plugins.sort_by_key(|meta_plugin_type| meta_plugin_type.to_string()); sorted_meta_plugins.sort_by_key(|meta_plugin_type| meta_plugin_type.to_string());
for meta_plugin_type in sorted_meta_plugins { for meta_plugin_type in sorted_meta_plugins {
log::debug!( log::debug!("STATUS: Processing meta plugin type: {meta_plugin_type:?}");
"STATUS: Processing meta plugin type: {:?}",
meta_plugin_type
);
log::debug!("STATUS: About to call get_meta_plugin"); log::debug!("STATUS: About to call get_meta_plugin");
let meta_plugin = crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone(), None, None); let meta_plugin = crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone(), None, None);
log::debug!("STATUS: Created meta plugin instance"); log::debug!("STATUS: Created meta plugin instance");
@@ -143,7 +141,7 @@ pub fn generate_status_info(
// Get meta name first to avoid borrowing issues // Get meta name first to avoid borrowing issues
log::debug!("STATUS: Getting meta name..."); log::debug!("STATUS: Getting meta name...");
let meta_name = meta_plugin.meta_type().to_string(); let meta_name = meta_plugin.meta_type().to_string();
log::debug!("STATUS: Got meta name: {}", meta_name); log::debug!("STATUS: Got meta name: {meta_name}");
// Check if this plugin is enabled // Check if this plugin is enabled
let is_enabled = enabled_meta_plugins.contains(&meta_plugin_type); let is_enabled = enabled_meta_plugins.contains(&meta_plugin_type);

8
src/compression_engine/gzip.rs
View File

@@ -11,12 +11,12 @@ use std::io::{Read, Write};
#[cfg(feature = "gzip")] #[cfg(feature = "gzip")]
use std::path::PathBuf; use std::path::PathBuf;
#[cfg(feature = "gzip")]
use flate2::Compression;
#[cfg(feature = "gzip")] #[cfg(feature = "gzip")]
use flate2::read::GzDecoder; use flate2::read::GzDecoder;
#[cfg(feature = "gzip")] #[cfg(feature = "gzip")]
use flate2::write::GzEncoder; use flate2::write::GzEncoder;
#[cfg(feature = "gzip")]
use flate2::Compression;
#[cfg(feature = "gzip")] #[cfg(feature = "gzip")]
use crate::compression_engine::CompressionEngine; use crate::compression_engine::CompressionEngine;
@@ -42,7 +42,7 @@ impl CompressionEngine for CompressionEngineGZip {
("<INTERNAL>".to_string(), "".to_string(), "".to_string()) ("<INTERNAL>".to_string(), "".to_string(), "".to_string())
} }
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>> { fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>> {
debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self); debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self);
let file = File::open(file_path)?; let file = File::open(file_path)?;
@@ -84,7 +84,7 @@ impl<W: Write> Drop for AutoFinishGzEncoder<W> {
if let Some(encoder) = self.encoder.take() { if let Some(encoder) = self.encoder.take() {
debug!("COMPRESSION: Finishing"); debug!("COMPRESSION: Finishing");
if let Err(e) = encoder.finish() { if let Err(e) = encoder.finish() {
warn!("Failed to finish GZip encoder: {}", e); warn!("Failed to finish GZip encoder: {e}");
} }
} }
} }

2
src/compression_engine/lz4.rs
View File

@@ -19,7 +19,7 @@ impl CompressionEngineLZ4 {
} }
impl CompressionEngine for CompressionEngineLZ4 { impl CompressionEngine for CompressionEngineLZ4 {
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>> { fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>> {
debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self); debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self);
let file = File::open(file_path)?; let file = File::open(file_path)?;

6
src/compression_engine/mod.rs
View File

@@ -1,4 +1,4 @@
use anyhow::{Result, anyhow}; use anyhow::{anyhow, Result};
use std::io; use std::io;
use std::io::{Read, Write}; use std::io::{Read, Write};
use std::path::PathBuf; use std::path::PathBuf;
@@ -73,14 +73,14 @@ pub trait CompressionEngine: Send + Sync {
/// ///
/// # Returns /// # Returns
/// ///
/// * `Result<Box<dyn Read>>` - A boxed reader that decompresses the file on read, /// * `Result<Box<dyn Read + Send>>` - A boxed reader that decompresses the file on read,
/// or an error if the file cannot be opened or is invalid. /// or an error if the file cannot be opened or is invalid.
/// ///
/// # Errors /// # Errors
/// ///
/// Returns an error if the file does not exist, is not a valid compressed file, /// Returns an error if the file does not exist, is not a valid compressed file,
/// or if decompression fails. /// or if decompression fails.
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>>; fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>>;
/// Creates a new compressed file for writing. /// Creates a new compressed file for writing.
/// ///

2
src/compression_engine/none.rs
View File

@@ -24,7 +24,7 @@ impl CompressionEngine for CompressionEngineNone {
("<INTERNAL>".to_string(), "".to_string(), "".to_string()) ("<INTERNAL>".to_string(), "".to_string(), "".to_string())
} }
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>> { fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>> {
debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self); debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self);
Ok(Box::new(File::open(file_path)?)) Ok(Box::new(File::open(file_path)?))
} }

18
src/compression_engine/program.rs
View File

@@ -1,4 +1,4 @@
use anyhow::{Context, Result, anyhow}; use anyhow::{anyhow, Context, Result};
use log::*; use log::*;
use std::fs::File; use std::fs::File;
use std::io::{Read, Write}; use std::io::{Read, Write};
@@ -94,16 +94,13 @@ impl CompressionEngine for CompressionEngineProgram {
) )
} }
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>> { fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>> {
debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self); debug!("COMPRESSION: Opening {file_path:?} using {self:?}");
let program = self.program.clone(); let program = self.program.clone();
let args = self.decompress.clone(); let args = self.decompress.clone();
debug!( debug!("COMPRESSION: Executing command: {program:?} {args:?} reading from {file_path:?}");
"COMPRESSION: Executing command: {:?} {:?} reading from {:?}",
program, args, file_path
);
let file = File::open(file_path).context("Unable to open file for reading")?; let file = File::open(file_path).context("Unable to open file for reading")?;
@@ -130,15 +127,12 @@ impl CompressionEngine for CompressionEngineProgram {
} }
fn create(&self, file_path: PathBuf) -> Result<Box<dyn Write>> { fn create(&self, file_path: PathBuf) -> Result<Box<dyn Write>> {
debug!("COMPRESSION: Writing to {:?} using {:?}", file_path, *self); debug!("COMPRESSION: Writing to {file_path:?} using {self:?}");
let program = self.program.clone(); let program = self.program.clone();
let args = self.compress.clone(); let args = self.compress.clone();
debug!( debug!("COMPRESSION: Executing command: {program:?} {args:?} writing to {file_path:?}");
"COMPRESSION: Executing command: {:?} {:?} writing to {:?}",
program, args, file_path
);
let file = File::create(file_path).context("Unable to open file for writing")?; let file = File::create(file_path).context("Unable to open file for writing")?;

27
src/config.rs
View File

@@ -189,10 +189,7 @@ pub struct Settings {
impl Settings { impl Settings {
/// Create unified settings from config and args with proper priority /// Create unified settings from config and args with proper priority
pub fn new(args: &Args, default_dir: PathBuf) -> Result<Self> { pub fn new(args: &Args, default_dir: PathBuf) -> Result<Self> {
debug!( debug!("CONFIG: Creating settings with default dir: {default_dir:?}");
"CONFIG: Creating settings with default dir: {:?}",
default_dir
);
let config_path = if let Some(config_path) = &args.options.config { let config_path = if let Some(config_path) = &args.options.config {
config_path.clone() config_path.clone()
@@ -208,21 +205,21 @@ impl Settings {
} else { } else {
PathBuf::from("~/.config/keep/config.yml") PathBuf::from("~/.config/keep/config.yml")
}; };
debug!("CONFIG: Using default config path: {:?}", default_path); debug!("CONFIG: Using default config path: {default_path:?}");
default_path default_path
}; };
debug!("CONFIG: Using config path: {:?}", config_path); debug!("CONFIG: Using config path: {config_path:?}");
let mut config_builder = config::Config::builder(); let mut config_builder = config::Config::builder();
// Load config file if it exists // Load config file if it exists
if config_path.exists() { if config_path.exists() {
debug!("CONFIG: Loading config file: {:?}", config_path); debug!("CONFIG: Loading config file: {config_path:?}");
config_builder = config_builder =
config_builder.add_source(config::File::from(config_path.clone()).required(false)); config_builder.add_source(config::File::from(config_path.clone()).required(false));
} else { } else {
debug!("CONFIG: Config file does not exist: {:?}", config_path); debug!("CONFIG: Config file does not exist: {config_path:?}");
} }
// Add environment variables // Add environment variables
@@ -234,7 +231,7 @@ impl Settings {
// Override with CLI args // Override with CLI args
if let Some(dir) = &args.options.dir { if let Some(dir) = &args.options.dir {
debug!("CONFIG: Overriding dir with CLI arg: {:?}", dir); debug!("CONFIG: Overriding dir with CLI arg: {dir:?}");
config_builder = config_builder.set_override("dir", dir.to_str().unwrap())?; config_builder = config_builder.set_override("dir", dir.to_str().unwrap())?;
} }
@@ -302,7 +299,7 @@ impl Settings {
match config.try_deserialize::<Settings>() { match config.try_deserialize::<Settings>() {
Ok(mut settings) => { Ok(mut settings) => {
debug!("CONFIG: Successfully deserialized settings: {:?}", settings); debug!("CONFIG: Successfully deserialized settings: {settings:?}");
// Set defaults for list_format if not provided // Set defaults for list_format if not provided
if settings.list_format.is_empty() { if settings.list_format.is_empty() {
@@ -393,15 +390,15 @@ impl Settings {
// Set dir to default if not provided or is empty // Set dir to default if not provided or is empty
if settings.dir == PathBuf::new() { if settings.dir == PathBuf::new() {
debug!("CONFIG: Setting default dir: {:?}", default_dir); debug!("CONFIG: Setting default dir: {default_dir:?}");
settings.dir = default_dir; settings.dir = default_dir;
} }
debug!("CONFIG: Final settings: {:?}", settings); debug!("CONFIG: Final settings: {settings:?}");
Ok(settings) Ok(settings)
} }
Err(e) => { Err(e) => {
error!("CONFIG: Failed to deserialize settings: {}", e); error!("CONFIG: Failed to deserialize settings: {e}");
Err(e.into()) Err(e.into())
} }
} }
@@ -422,9 +419,9 @@ impl Settings {
if let Some(server) = &self.server { if let Some(server) = &self.server {
// First check for password_file // First check for password_file
if let Some(password_file) = &server.password_file { if let Some(password_file) = &server.password_file {
debug!("CONFIG: Reading password from file: {:?}", password_file); debug!("CONFIG: Reading password from file: {password_file:?}");
let password = fs::read_to_string(password_file) let password = fs::read_to_string(password_file)
.with_context(|| format!("Failed to read password file: {:?}", password_file))? .with_context(|| format!("Failed to read password file: {password_file:?}"))?
.trim() .trim()
.to_string(); .to_string();
return Ok(Some(password)); return Ok(Some(password));

49
src/db.rs
View File

@@ -163,7 +163,7 @@ pub struct Meta {
/// let conn = db::open(db_path)?; /// let conn = db::open(db_path)?;
/// ``` /// ```
pub fn open(path: PathBuf) -> Result<Connection, Error> { pub fn open(path: PathBuf) -> Result<Connection, Error> {
debug!("DB: Opening file: {:?}", path); debug!("DB: Opening file: {path:?}");
let mut conn = Connection::open_with_flags( let mut conn = Connection::open_with_flags(
path, path,
OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE,
@@ -213,7 +213,7 @@ pub fn open(path: PathBuf) -> Result<Connection, Error> {
/// assert!(id > 0); /// assert!(id > 0);
/// ``` /// ```
pub fn insert_item(conn: &Connection, item: Item) -> Result<i64> { pub fn insert_item(conn: &Connection, item: Item) -> Result<i64> {
debug!("DB: Inserting item: {:?}", item); debug!("DB: Inserting item: {item:?}");
conn.execute( conn.execute(
"INSERT INTO items (ts, size, compression) VALUES (?1, ?2, ?3)", "INSERT INTO items (ts, size, compression) VALUES (?1, ?2, ?3)",
params![item.ts, item.size, item.compression], params![item.ts, item.size, item.compression],
@@ -353,7 +353,7 @@ pub fn add_meta(conn: &Connection, item_id: i64, name: &str, value: &str) -> Res
/// db::update_item(&conn, item)?; /// db::update_item(&conn, item)?;
/// ``` /// ```
pub fn update_item(conn: &Connection, item: Item) -> Result<()> { pub fn update_item(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Updating item: {:?}", item); debug!("DB: Updating item: {item:?}");
conn.execute( conn.execute(
"UPDATE items SET size=?2, compression=?3 WHERE id=?1", "UPDATE items SET size=?2, compression=?3 WHERE id=?1",
params![item.id, item.size, item.compression,], params![item.id, item.size, item.compression,],
@@ -386,7 +386,7 @@ pub fn update_item(conn: &Connection, item: Item) -> Result<()> {
/// db::delete_item(&conn, item)?; /// db::delete_item(&conn, item)?;
/// ``` /// ```
pub fn delete_item(conn: &Connection, item: Item) -> Result<()> { pub fn delete_item(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Deleting item: {:?}", item); debug!("DB: Deleting item: {item:?}");
conn.execute("DELETE FROM items WHERE id=?1", params![item.id])?; conn.execute("DELETE FROM items WHERE id=?1", params![item.id])?;
Ok(()) Ok(())
} }
@@ -416,7 +416,7 @@ pub fn delete_item(conn: &Connection, item: Item) -> Result<()> {
/// db::query_delete_meta(&conn, meta)?; /// db::query_delete_meta(&conn, meta)?;
/// ``` /// ```
pub fn query_delete_meta(conn: &Connection, meta: Meta) -> Result<()> { pub fn query_delete_meta(conn: &Connection, meta: Meta) -> Result<()> {
debug!("DB: Deleting meta: {:?}", meta); debug!("DB: Deleting meta: {meta:?}");
conn.execute( conn.execute(
"DELETE FROM metas WHERE id=?1 AND name=?2", "DELETE FROM metas WHERE id=?1 AND name=?2",
params![meta.id, meta.name], params![meta.id, meta.name],
@@ -449,7 +449,7 @@ pub fn query_delete_meta(conn: &Connection, meta: Meta) -> Result<()> {
/// db::query_upsert_meta(&conn, meta)?; /// db::query_upsert_meta(&conn, meta)?;
/// ``` /// ```
pub fn query_upsert_meta(conn: &Connection, meta: Meta) -> Result<()> { pub fn query_upsert_meta(conn: &Connection, meta: Meta) -> Result<()> {
debug!("DB: Inserting meta: {:?}", meta); debug!("DB: Inserting meta: {meta:?}");
conn.execute( conn.execute(
"INSERT INTO metas (id, name, value) VALUES (?1, ?2, ?3) "INSERT INTO metas (id, name, value) VALUES (?1, ?2, ?3)
ON CONFLICT(id, name) DO UPDATE SET value=?3", ON CONFLICT(id, name) DO UPDATE SET value=?3",
@@ -548,7 +548,7 @@ pub fn store_meta(conn: &Connection, meta: Meta) -> Result<()> {
/// db::insert_tag(&conn, tag)?; /// db::insert_tag(&conn, tag)?;
/// ``` /// ```
pub fn insert_tag(conn: &Connection, tag: Tag) -> Result<()> { pub fn insert_tag(conn: &Connection, tag: Tag) -> Result<()> {
debug!("DB: Inserting tag: {:?}", tag); debug!("DB: Inserting tag: {tag:?}");
conn.execute( conn.execute(
"INSERT INTO tags (id, name) VALUES (?1, ?2)", "INSERT INTO tags (id, name) VALUES (?1, ?2)",
params![tag.id, tag.name], params![tag.id, tag.name],
@@ -580,7 +580,7 @@ pub fn insert_tag(conn: &Connection, tag: Tag) -> Result<()> {
/// db::delete_item_tags(&conn, item)?; /// db::delete_item_tags(&conn, item)?;
/// ``` /// ```
pub fn delete_item_tags(conn: &Connection, item: Item) -> Result<()> { pub fn delete_item_tags(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Deleting all item tags: {:?}", item); debug!("DB: Deleting all item tags: {item:?}");
conn.execute("DELETE FROM tags WHERE id=?1", params![item.id])?; conn.execute("DELETE FROM tags WHERE id=?1", params![item.id])?;
Ok(()) Ok(())
} }
@@ -612,7 +612,7 @@ pub fn delete_item_tags(conn: &Connection, item: Item) -> Result<()> {
/// db::set_item_tags(&conn, item, &tags)?; /// db::set_item_tags(&conn, item, &tags)?;
/// ``` /// ```
pub fn set_item_tags(conn: &Connection, item: Item, tags: &Vec<String>) -> Result<()> { pub fn set_item_tags(conn: &Connection, item: Item, tags: &Vec<String>) -> Result<()> {
debug!("DB: Setting tags for item: {:?} ?{:?}", item, tags); debug!("DB: Setting tags for item: {item:?} ?{tags:?}");
delete_item_tags(conn, item.clone())?; delete_item_tags(conn, item.clone())?;
let item_id = item.id.unwrap(); let item_id = item.id.unwrap();
for tag_name in tags { for tag_name in tags {
@@ -695,7 +695,7 @@ pub fn query_all_items(conn: &Connection) -> Result<Vec<Item>> {
/// let tagged_items = db::query_tagged_items(&conn, &tags)?; /// let tagged_items = db::query_tagged_items(&conn, &tags)?;
/// ``` /// ```
pub fn query_tagged_items<'a>(conn: &'a Connection, tags: &'a Vec<String>) -> Result<Vec<Item>> { pub fn query_tagged_items<'a>(conn: &'a Connection, tags: &'a Vec<String>) -> Result<Vec<Item>> {
debug!("DB: Querying tagged items: {:?}", tags); debug!("DB: Querying tagged items: {tags:?}");
let mut statement = conn let mut statement = conn
.prepare_cached( .prepare_cached(
" "
@@ -789,10 +789,7 @@ pub fn get_items_matching(
tags: &Vec<String>, tags: &Vec<String>,
meta: &HashMap<String, String>, meta: &HashMap<String, String>,
) -> Result<Vec<Item>> { ) -> Result<Vec<Item>> {
debug!( debug!("DB: Getting items matching: tags={tags:?} meta={meta:?}");
"DB: Getting items matching: tags={:?} meta={:?}",
tags, meta
);
let items = match tags.is_empty() { let items = match tags.is_empty() {
true => query_all_items(conn)?, true => query_all_items(conn)?,
@@ -812,7 +809,7 @@ pub fn get_items_matching(
item_meta.insert(meta.name, meta.value); item_meta.insert(meta.name, meta.value);
} }
debug!("DB: Matching: {:?}: {:?}", item, item_meta); debug!("DB: Matching: {item:?}: {item_meta:?}");
for (k, v) in meta.iter() { for (k, v) in meta.iter() {
match item_meta.get(k) { match item_meta.get(k) {
@@ -862,7 +859,7 @@ pub fn get_item_matching(
tags: &Vec<String>, tags: &Vec<String>,
_meta: &HashMap<String, String>, _meta: &HashMap<String, String>,
) -> Result<Option<Item>> { ) -> Result<Option<Item>> {
debug!("DB: Get item matching tags: {:?}", tags); debug!("DB: Get item matching tags: {tags:?}");
let mut statement = conn let mut statement = conn
.prepare_cached( .prepare_cached(
" "
@@ -925,7 +922,7 @@ pub fn get_item_matching(
/// assert!(item.is_some()); /// assert!(item.is_some());
/// ``` /// ```
pub fn get_item(conn: &Connection, item_id: i64) -> Result<Option<Item>> { pub fn get_item(conn: &Connection, item_id: i64) -> Result<Option<Item>> {
debug!("DB: Getting item {:?}", item_id); debug!("DB: Getting item {item_id:?}");
let mut statement = conn let mut statement = conn
.prepare_cached( .prepare_cached(
" "
@@ -1018,7 +1015,7 @@ pub fn get_item_last(conn: &Connection) -> Result<Option<Item>> {
/// let tags = db::get_item_tags(&conn, &item)?; /// let tags = db::get_item_tags(&conn, &item)?;
/// ``` /// ```
pub fn get_item_tags(conn: &Connection, item: &Item) -> Result<Vec<Tag>> { pub fn get_item_tags(conn: &Connection, item: &Item) -> Result<Vec<Tag>> {
debug!("DB: Getting tags for item: {:?}", item); debug!("DB: Getting tags for item: {item:?}");
let mut statement = conn let mut statement = conn
.prepare_cached("SELECT id, name FROM tags WHERE id=?1 ORDER BY name ASC") .prepare_cached("SELECT id, name FROM tags WHERE id=?1 ORDER BY name ASC")
.context("Problem preparing SQL statement")?; .context("Problem preparing SQL statement")?;
@@ -1060,7 +1057,7 @@ pub fn get_item_tags(conn: &Connection, item: &Item) -> Result<Vec<Tag>> {
/// let meta = db::get_item_meta(&conn, &item)?; /// let meta = db::get_item_meta(&conn, &item)?;
/// ``` /// ```
pub fn get_item_meta(conn: &Connection, item: &Item) -> Result<Vec<Meta>> { pub fn get_item_meta(conn: &Connection, item: &Item) -> Result<Vec<Meta>> {
debug!("DB: Getting item meta: {:?}", item); debug!("DB: Getting item meta: {item:?}");
let mut statement = conn let mut statement = conn
.prepare_cached("SELECT id, name, value FROM metas WHERE id=?1 ORDER BY name ASC") .prepare_cached("SELECT id, name, value FROM metas WHERE id=?1 ORDER BY name ASC")
.context("Problem preparing SQL statement")?; .context("Problem preparing SQL statement")?;
@@ -1104,7 +1101,7 @@ pub fn get_item_meta(conn: &Connection, item: &Item) -> Result<Vec<Meta>> {
/// let meta = db::get_item_meta_name(&conn, &item, "mime_type".to_string())?; /// let meta = db::get_item_meta_name(&conn, &item, "mime_type".to_string())?;
/// ``` /// ```
pub fn get_item_meta_name(conn: &Connection, item: &Item, name: String) -> Result<Option<Meta>> { pub fn get_item_meta_name(conn: &Connection, item: &Item, name: String) -> Result<Option<Meta>> {
debug!("DB: Getting item meta name: {:?} {:?}", item, name); debug!("DB: Getting item meta name: {item:?} {name:?}");
let mut statement = conn let mut statement = conn
.prepare_cached("SELECT id, name, value FROM metas WHERE id=?1 AND name=?2") .prepare_cached("SELECT id, name, value FROM metas WHERE id=?1 AND name=?2")
.context("Problem preparing SQL statement")?; .context("Problem preparing SQL statement")?;
@@ -1145,7 +1142,7 @@ pub fn get_item_meta_name(conn: &Connection, item: &Item, name: String) -> Resul
/// let value = db::get_item_meta_value(&conn, &item, "source".to_string())?; /// let value = db::get_item_meta_value(&conn, &item, "source".to_string())?;
/// ``` /// ```
pub fn get_item_meta_value(conn: &Connection, item: &Item, name: String) -> Result<Option<String>> { pub fn get_item_meta_value(conn: &Connection, item: &Item, name: String) -> Result<Option<String>> {
debug!("DB: Getting item meta value: {:?} {:?}", item, name); debug!("DB: Getting item meta value: {item:?} {name:?}");
let mut statement = conn let mut statement = conn
.prepare_cached("SELECT value FROM metas WHERE id=?1 AND name=?2") .prepare_cached("SELECT value FROM metas WHERE id=?1 AND name=?2")
.context("Problem preparing SQL statement")?; .context("Problem preparing SQL statement")?;
@@ -1184,7 +1181,7 @@ pub fn get_tags_for_items(
conn: &Connection, conn: &Connection,
item_ids: &[i64], item_ids: &[i64],
) -> Result<std::collections::HashMap<i64, Vec<Tag>>> { ) -> Result<std::collections::HashMap<i64, Vec<Tag>>> {
debug!("DB: Getting tags for items: {:?}", item_ids); debug!("DB: Getting tags for items: {item_ids:?}");
if item_ids.is_empty() { if item_ids.is_empty() {
return Ok(std::collections::HashMap::new()); return Ok(std::collections::HashMap::new());
@@ -1195,8 +1192,7 @@ pub fn get_tags_for_items(
let placeholders_str = placeholders.join(","); let placeholders_str = placeholders.join(",");
let sql = format!( let sql = format!(
"SELECT id, name FROM tags WHERE id IN ({}) ORDER BY id ASC, name ASC", "SELECT id, name FROM tags WHERE id IN ({placeholders_str}) ORDER BY id ASC, name ASC"
placeholders_str
); );
let mut statement = conn let mut statement = conn
@@ -1244,7 +1240,7 @@ pub fn get_meta_for_items(
conn: &Connection, conn: &Connection,
item_ids: &[i64], item_ids: &[i64],
) -> Result<std::collections::HashMap<i64, std::collections::HashMap<String, String>>> { ) -> Result<std::collections::HashMap<i64, std::collections::HashMap<String, String>>> {
debug!("DB: Getting meta for items: {:?}", item_ids); debug!("DB: Getting meta for items: {item_ids:?}");
if item_ids.is_empty() { if item_ids.is_empty() {
return Ok(std::collections::HashMap::new()); return Ok(std::collections::HashMap::new());
@@ -1255,8 +1251,7 @@ pub fn get_meta_for_items(
let placeholders_str = placeholders.join(","); let placeholders_str = placeholders.join(",");
let sql = format!( let sql = format!(
"SELECT id, name, value FROM metas WHERE id IN ({}) ORDER BY id ASC, name ASC", "SELECT id, name, value FROM metas WHERE id IN ({placeholders_str}) ORDER BY id ASC, name ASC"
placeholders_str
); );
let mut statement = conn let mut statement = conn

2
src/filter_plugin/grep.rs
View File

@@ -73,7 +73,7 @@ impl FilterPlugin for GrepFilter {
for line in buf_reader.by_ref().lines() { for line in buf_reader.by_ref().lines() {
let line = line?; let line = line?;
if self.regex.is_match(&line) { if self.regex.is_match(&line) {
writeln!(writer, "{}", line)?; writeln!(writer, "{line}")?;
} }
} }
Ok(()) Ok(())

2
src/filter_plugin/head.rs
View File

@@ -184,7 +184,7 @@ impl FilterPlugin for HeadLinesFilter {
let mut buf_reader = std::io::BufReader::new(reader); let mut buf_reader = std::io::BufReader::new(reader);
for line in buf_reader.by_ref().lines() { for line in buf_reader.by_ref().lines() {
let line = line?; let line = line?;
writeln!(writer, "{}", line)?; writeln!(writer, "{line}")?;
self.remaining -= 1; self.remaining -= 1;
if self.remaining == 0 { if self.remaining == 0 {
break; break;

6
src/filter_plugin/mod.rs
View File

@@ -381,7 +381,7 @@ pub fn parse_filter_string(filter_str: &str) -> Result<FilterChain> {
_ => { _ => {
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput, std::io::ErrorKind::InvalidInput,
format!("Filter '{}' requires parameters", part), format!("Filter '{part}' requires parameters"),
)); ));
} }
} }
@@ -391,7 +391,7 @@ pub fn parse_filter_string(filter_str: &str) -> Result<FilterChain> {
// If we get here, the filter wasn't recognized // If we get here, the filter wasn't recognized
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput, std::io::ErrorKind::InvalidInput,
format!("Unknown filter: {}", part), format!("Unknown filter: {part}"),
)); ));
} }
@@ -454,7 +454,7 @@ fn create_filter_with_options(
if !option_defs.iter().any(|opt| &opt.name == key) { if !option_defs.iter().any(|opt| &opt.name == key) {
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput, std::io::ErrorKind::InvalidInput,
format!("Unknown option '{}'", key), format!("Unknown option '{key}'"),
)); ));
} }
options.insert(key.clone(), value.clone()); options.insert(key.clone(), value.clone());

2
src/filter_plugin/skip.rs
View File

@@ -108,7 +108,7 @@ impl FilterPlugin for SkipLinesFilter {
if self.remaining > 0 { if self.remaining > 0 {
self.remaining -= 1; self.remaining -= 1;
} else { } else {
writeln!(writer, "{}", line)?; writeln!(writer, "{line}")?;
} }
} }
Ok(()) Ok(())

2
src/filter_plugin/tail.rs
View File

@@ -127,7 +127,7 @@ impl FilterPlugin for TailLinesFilter {
// Write the buffered lines // Write the buffered lines
for line in &self.lines { for line in &self.lines {
writeln!(writer, "{}", line)?; writeln!(writer, "{line}")?;
} }
Ok(()) Ok(())
} }

30
src/main.rs
View File

@@ -44,37 +44,37 @@ fn main() -> Result<(), Error> {
// Create unified settings using the new config system // Create unified settings using the new config system
let settings = Settings::new(&args, default_dir)?; let settings = Settings::new(&args, default_dir)?;
debug!("MAIN: Loaded settings: {:?}", settings); debug!("MAIN: Loaded settings: {settings:?}");
let ids = &mut Vec::new(); let ids = &mut Vec::new();
let tags = &mut Vec::new(); let tags = &mut Vec::new();
// For --info and --get modes, treat numeric strings as IDs // For --info and --get modes, treat numeric strings as IDs
for v in args.ids_or_tags.iter() { for v in args.ids_or_tags.iter() {
debug!("MAIN: Parsed value: {:?}", v); debug!("MAIN: Parsed value: {v:?}");
match v.clone() { match v.clone() {
NumberOrString::Number(num) => { NumberOrString::Number(num) => {
debug!("MAIN: Adding to ids: {}", num); debug!("MAIN: Adding to ids: {num}");
ids.push(num) ids.push(num)
} }
NumberOrString::Str(str) => { NumberOrString::Str(str) => {
// For --info and --get, try to parse strings as numbers to treat them as IDs // For --info and --get, try to parse strings as numbers to treat them as IDs
if args.mode.info || args.mode.get { if args.mode.info || args.mode.get {
if let Ok(num) = str.parse::<i64>() { if let Ok(num) = str.parse::<i64>() {
debug!("MAIN: Adding parsed string to ids: {}", num); debug!("MAIN: Adding parsed string to ids: {num}");
ids.push(num); ids.push(num);
continue; continue;
} else if args.mode.info { } else if args.mode.info {
// --info only accepts numeric IDs // --info only accepts numeric IDs
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
format!("--info requires numeric IDs, found: '{}'", str), format!("--info requires numeric IDs, found: '{str}'"),
) )
.exit(); .exit();
} }
} }
// If not a number, or not using --info/--get, treat as tag // If not a number, or not using --info/--get, treat as tag
debug!("MAIN: Adding to tags: {}", str); debug!("MAIN: Adding to tags: {str}");
tags.push(str) tags.push(str)
} }
} }
@@ -162,11 +162,11 @@ fn main() -> Result<(), Error> {
.exit(); .exit();
} }
debug!("MAIN: args: {:?}", args); debug!("MAIN: args: {args:?}");
debug!("MAIN: ids: {:?}", ids); debug!("MAIN: ids: {ids:?}");
debug!("MAIN: tags: {:?}", tags); debug!("MAIN: tags: {tags:?}");
debug!("MAIN: mode: {:?}", mode); debug!("MAIN: mode: {mode:?}");
debug!("MAIN: settings: {:?}", settings); debug!("MAIN: settings: {settings:?}");
unsafe { unsafe {
libc::umask(0o077); libc::umask(0o077);
@@ -176,12 +176,12 @@ fn main() -> Result<(), Error> {
let mut db_path = data_path.clone(); let mut db_path = data_path.clone();
db_path.push("keep-1.db"); db_path.push("keep-1.db");
debug!("MAIN: Data directory: {:?}", data_path); debug!("MAIN: Data directory: {data_path:?}");
debug!("MAIN: DB file: {:?}", db_path); debug!("MAIN: DB file: {db_path:?}");
// Ensure data directory exists // Ensure data directory exists
fs::create_dir_all(&data_path) fs::create_dir_all(&data_path)
.with_context(|| format!("Unable to create data directory {:?}", data_path))?; .with_context(|| format!("Unable to create data directory {data_path:?}"))?;
// Initialize database // Initialize database
let mut conn = db::open(db_path.clone())?; let mut conn = db::open(db_path.clone())?;
@@ -193,7 +193,7 @@ fn main() -> Result<(), Error> {
Err(e) => { Err(e) => {
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
format!("Invalid filter string: {}", e), format!("Invalid filter string: {e}"),
) )
.exit(); .exit();
} }

8
src/meta_plugin/digest.rs
View File

@@ -42,15 +42,15 @@ impl Hasher {
match self { match self {
Hasher::Sha256(hasher) => { Hasher::Sha256(hasher) => {
let result = std::mem::replace(hasher, Sha256::new()).finalize_reset(); let result = std::mem::replace(hasher, Sha256::new()).finalize_reset();
format!("{:x}", result) format!("{result:x}")
} }
Hasher::Md5(hasher) => { Hasher::Md5(hasher) => {
let result = hasher.clone().compute(); let result = hasher.clone().compute();
format!("{:x}", result) format!("{result:x}")
} }
Hasher::Sha512(hasher) => { Hasher::Sha512(hasher) => {
let result = std::mem::replace(hasher, Sha512::new()).finalize_reset(); let result = std::mem::replace(hasher, Sha512::new()).finalize_reset();
format!("{:x}", result) format!("{result:x}")
} }
} }
} }
@@ -119,7 +119,7 @@ impl DigestMetaPlugin {
// Only the selected method's output should be enabled, others should be None // Only the selected method's output should be enabled, others should be None
let all_outputs = vec!["digest_md5", "digest_sha256", "digest_sha512"]; let all_outputs = vec!["digest_md5", "digest_sha256", "digest_sha512"];
for output_name in &all_outputs { for output_name in &all_outputs {
if output_name == &format!("digest_{}", method) { if output_name == &format!("digest_{method}") {
base.outputs.insert( base.outputs.insert(
output_name.to_string(), output_name.to_string(),
serde_yaml::Value::String(output_name.to_string()), serde_yaml::Value::String(output_name.to_string()),

6
src/meta_plugin/exec.rs
View File

@@ -174,7 +174,7 @@ impl MetaPlugin for MetaPluginExec {
if let Some(writer) = self.writer.as_mut() if let Some(writer) = self.writer.as_mut()
&& let Err(e) = writer.write_all(data) && let Err(e) = writer.write_all(data)
{ {
error!("META: Exec plugin: failed to write to stdin: {}", e); error!("META: Exec plugin: failed to write to stdin: {e}");
} }
MetaPluginResponse { MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
@@ -219,11 +219,11 @@ impl MetaPlugin for MetaPluginExec {
} }
} else { } else {
let stderr = String::from_utf8_lossy(&output.stderr); let stderr = String::from_utf8_lossy(&output.stderr);
error!("META: Exec plugin: command failed: {}", stderr); error!("META: Exec plugin: command failed: {stderr}");
} }
} }
Err(e) => { Err(e) => {
error!("META: Exec plugin: failed to wait on process: {}", e); error!("META: Exec plugin: failed to wait on process: {e}");
} }
} }
} }

2
src/meta_plugin/hostname.rs
View File

@@ -178,7 +178,7 @@ impl HostnameMetaPlugin {
{ {
let domain_str = String::from_utf8_lossy(&domain.stdout).trim().to_string(); let domain_str = String::from_utf8_lossy(&domain.stdout).trim().to_string();
if !domain_str.is_empty() && domain_str != "(none)" { if !domain_str.is_empty() && domain_str != "(none)" {
return format!("{}.{}", short_hostname, domain_str); return format!("{short_hostname}.{domain_str}");
} }
} }
} }

11
src/meta_plugin/magic_file.rs
View File

@@ -54,11 +54,11 @@ impl MagicFileMetaPluginImpl {
if let Some(cookie) = &self.cookie { if let Some(cookie) = &self.cookie {
cookie cookie
.set_flags(flags) .set_flags(flags)
.map_err(|e| io::Error::other(format!("Failed to set magic flags: {}", e)))?; .map_err(|e| io::Error::other(format!("Failed to set magic flags: {e}")))?;
let result = cookie let result = cookie
.buffer(&self.buffer) .buffer(&self.buffer)
.map_err(|e| io::Error::other(format!("Failed to analyze buffer: {}", e)))?; .map_err(|e| io::Error::other(format!("Failed to analyze buffer: {e}")))?;
// Clean up the result - remove extra whitespace // Clean up the result - remove extra whitespace
let trimmed = result.trim().to_string(); let trimmed = result.trim().to_string();
@@ -109,7 +109,7 @@ impl MetaPlugin for MagicFileMetaPluginImpl {
let cookie = match Cookie::open(CookieFlags::default()) { let cookie = match Cookie::open(CookieFlags::default()) {
Ok(cookie) => cookie, Ok(cookie) => cookie,
Err(e) => { Err(e) => {
debug!("META: MagicFile plugin: failed to create cookie: {}", e); debug!("META: MagicFile plugin: failed to create cookie: {e}");
return MetaPluginResponse { return MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
@@ -118,10 +118,7 @@ impl MetaPlugin for MagicFileMetaPluginImpl {
}; };
if let Err(e) = cookie.load(&[] as &[&Path]) { if let Err(e) = cookie.load(&[] as &[&Path]) {
debug!( debug!("META: MagicFile plugin: failed to load magic database: {e}");
"META: MagicFile plugin: failed to load magic database: {}",
e
);
return MetaPluginResponse { return MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,

14
src/meta_plugin/mod.rs
View File

@@ -251,14 +251,14 @@ pub fn process_metadata_outputs(
if let Some(mapping) = outputs.get(internal_name) { if let Some(mapping) = outputs.get(internal_name) {
// Check for null to disable the output // Check for null to disable the output
if mapping.is_null() { if mapping.is_null() {
debug!("META: Skipping disabled output (null): {}", internal_name); debug!("META: Skipping disabled output (null): {internal_name}");
return None; return None;
} }
// Check for boolean false to disable the output // Check for boolean false to disable the output
if let Some(false_val) = mapping.as_bool() if let Some(false_val) = mapping.as_bool()
&& !false_val && !false_val
{ {
debug!("META: Skipping disabled output: {}", internal_name); debug!("META: Skipping disabled output: {internal_name}");
return None; return None;
} }
if let Some(custom_name) = mapping.as_str() { if let Some(custom_name) = mapping.as_str() {
@@ -279,8 +279,7 @@ pub fn process_metadata_outputs(
} }
}; };
debug!( debug!(
"META: Processing metadata: internal_name={}, custom_name={}, value={}", "META: Processing metadata: internal_name={internal_name}, custom_name={custom_name}, value={value_str}"
internal_name, custom_name, value_str
); );
return Some(MetaData { return Some(MetaData {
name: custom_name.to_string(), name: custom_name.to_string(),
@@ -307,10 +306,7 @@ pub fn process_metadata_outputs(
}; };
// Default: use internal name as output name // Default: use internal name as output name
debug!( debug!("META: Processing metadata: name={internal_name}, value={value_str}");
"META: Processing metadata: name={}, value={}",
internal_name, value_str
);
Some(MetaData { Some(MetaData {
name: internal_name.to_string(), name: internal_name.to_string(),
value: value_str, value: value_str,
@@ -507,5 +503,5 @@ pub fn get_meta_plugin(
} }
// Fallback for unknown plugins // Fallback for unknown plugins
panic!("Meta plugin {:?} not registered", meta_plugin_type); panic!("Meta plugin {meta_plugin_type:?} not registered");
} }

70
src/meta_plugin/text.rs
View File

@@ -1,5 +1,5 @@
use crate::common::PIPESIZE;
use crate::common::is_binary::is_binary; use crate::common::is_binary::is_binary;
use crate::common::PIPESIZE;
use crate::meta_plugin::{MetaPlugin, MetaPluginResponse, MetaPluginType}; use crate::meta_plugin::{MetaPlugin, MetaPluginResponse, MetaPluginType};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -532,15 +532,14 @@ impl MetaPlugin for TextMetaPlugin {
} }
let mut metadata = Vec::new(); let mut metadata = Vec::new();
let processed_data = data.to_vec();
// If we haven't determined if content is binary yet, build buffer and check // If we haven't determined if content is binary yet, build buffer and check
if self.is_binary_content.is_none() { if self.is_binary_content.is_none() {
let should_finalize = if let Some(ref mut buffer) = self.buffer { let should_finalize = if let Some(ref mut buffer) = self.buffer {
// Add processed data to our buffer up to max_buffer_size // Add data to our buffer up to max_buffer_size
let remaining_capacity = self.max_buffer_size.saturating_sub(buffer.len()); let remaining_capacity = self.max_buffer_size.saturating_sub(buffer.len());
let bytes_to_take = std::cmp::min(processed_data.len(), remaining_capacity); let bytes_to_take = std::cmp::min(data.len(), remaining_capacity);
buffer.extend_from_slice(&processed_data[..bytes_to_take]); buffer.extend_from_slice(&data[..bytes_to_take]);
// If we have enough data to make a binary determination, do it now // If we have enough data to make a binary determination, do it now
let buffer_len = buffer.len(); let buffer_len = buffer.len();
@@ -562,7 +561,7 @@ impl MetaPlugin for TextMetaPlugin {
} }
// If it's text, count words and lines for this chunk // If it's text, count words and lines for this chunk
self.count_text_stats(&processed_data[..bytes_to_take]); self.count_text_stats(&data[..bytes_to_take]);
// If we've reached our buffer limit, drop the buffer to save memory // If we've reached our buffer limit, drop the buffer to save memory
// But don't finalize yet - we need to keep counting words and lines // But don't finalize yet - we need to keep counting words and lines
@@ -572,7 +571,7 @@ impl MetaPlugin for TextMetaPlugin {
false // Never finalize here for text content false // Never finalize here for text content
} else { } else {
// Still building up buffer, count words and lines for this chunk // Still building up buffer, count words and lines for this chunk
self.count_text_stats(&processed_data[..bytes_to_take]); self.count_text_stats(&data[..bytes_to_take]);
false false
} }
} else { } else {
@@ -587,7 +586,7 @@ impl MetaPlugin for TextMetaPlugin {
} }
} else if self.is_binary_content == Some(false) { } else if self.is_binary_content == Some(false) {
// We've already determined it's text, just count words and lines // We've already determined it's text, just count words and lines
self.count_text_stats(&processed_data); self.count_text_stats(data);
} }
// If is_binary_content == Some(true), we should have already finalized, but just in case: // If is_binary_content == Some(true), we should have already finalized, but just in case:
else if self.is_binary_content == Some(true) { else if self.is_binary_content == Some(true) {
@@ -654,26 +653,43 @@ impl MetaPlugin for TextMetaPlugin {
&& let Some(buffer) = &self.buffer && let Some(buffer) = &self.buffer
&& !buffer.is_empty() && !buffer.is_empty()
{ {
// Build filter string from individual parameters let buffer = if head_bytes.is_some()
let mut filter_parts = Vec::new(); || head_lines.is_some()
if let Some(bytes) = head_bytes { || tail_bytes.is_some()
filter_parts.push(format!("head_bytes({})", bytes)); || tail_lines.is_some()
} {
if let Some(lines) = head_lines { // Build filter string from individual parameters
filter_parts.push(format!("head_lines({})", lines)); let mut filter_parts = Vec::new();
} if let Some(bytes) = head_bytes {
if let Some(bytes) = tail_bytes { filter_parts.push(format!("head_bytes({bytes})"));
filter_parts.push(format!("tail_bytes({})", bytes)); }
} if let Some(lines) = head_lines {
if let Some(lines) = tail_lines { filter_parts.push(format!("head_lines({lines})"));
filter_parts.push(format!("tail_lines({})", lines)); }
} if let Some(bytes) = tail_bytes {
filter_parts.push(format!("tail_bytes({bytes})"));
}
if let Some(lines) = tail_lines {
filter_parts.push(format!("tail_lines({lines})"));
}
// For now, just use the buffer as-is since filtering isn't implemented // Apply filters if any are specified
let processed_buffer = buffer.clone(); let filter_string = filter_parts.join(",");
match crate::services::FilterService::new()
.process_with_filter(buffer, Some(&filter_string))
{
Ok(filtered) => filtered,
Err(e) => {
log::warn!("Failed to apply filters: {e}");
buffer.clone()
}
}
} else {
buffer.clone()
};
// Clone the processed buffer data for binary detection // Clone the processed buffer data for binary detection
let (binary_metadata, is_binary) = self.perform_binary_detection(&processed_buffer); let (binary_metadata, is_binary) = self.perform_binary_detection(&buffer);
metadata.extend(binary_metadata); metadata.extend(binary_metadata);
self.is_binary_content = Some(is_binary); self.is_binary_content = Some(is_binary);
@@ -777,7 +793,7 @@ impl MetaPlugin for TextMetaPlugin {
/// ///
/// # Returns /// # Returns
/// ///
/// A reference to the `HashMap` of options. /// A reference to the `HashMap` of outputs.
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
@@ -786,7 +802,7 @@ impl MetaPlugin for TextMetaPlugin {
/// ///
/// # Returns /// # Returns
/// ///
/// A mutable reference to the `HashMap` of options. /// A mutable reference to the `HashMap` of outputs.
fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.options_mut() self.base.options_mut()
} }

7
src/modes/common.rs
View File

@@ -216,7 +216,7 @@ pub fn settings_meta_plugin_types(
if !found { if !found {
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
format!("Unknown meta plugin type: {}", trimmed_name), format!("Unknown meta plugin type: {trimmed_name}"),
) )
.exit(); .exit();
} }
@@ -254,10 +254,7 @@ pub fn settings_compression_type(
if compression_type_opt.is_err() { if compression_type_opt.is_err() {
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
format!( format!("Invalid compression algorithm '{compression_name}'. Supported algorithms: lz4, gzip, xz, zstd"),
"Invalid compression algorithm '{}'. Supported algorithms: lz4, gzip, xz, zstd",
compression_name
),
) )
.exit(); .exit();
} }

5
src/modes/delete.rs
View File

@@ -66,8 +66,9 @@ pub fn mode_delete(
warn!("Unable to find item {item_id} in database"); warn!("Unable to find item {item_id} in database");
} }
_ => { _ => {
return Err(anyhow::Error::from(e) return Err(
.context(format!("Failed to delete item {}", item_id))); anyhow::Error::from(e).context(format!("Failed to delete item {item_id}"))
);
} }
}, },
} }

49
src/modes/diff.rs
View File

@@ -8,11 +8,7 @@ use anyhow::{Context, Result};
use clap::Command; use clap::Command;
use log::debug; use log::debug;
fn validate_diff_args( fn validate_diff_args(_cmd: &mut Command, ids: &[i64], tags: &[String]) -> anyhow::Result<()> {
_cmd: &mut Command,
ids: &Vec<i64>,
tags: &Vec<String>,
) -> anyhow::Result<()> {
if !tags.is_empty() { if !tags.is_empty() {
return Err(anyhow::anyhow!( return Err(anyhow::anyhow!(
"Tags are not supported with --diff. Please provide exactly two IDs." "Tags are not supported with --diff. Please provide exactly two IDs."
@@ -137,9 +133,46 @@ pub fn mode_diff(
let (path_a, path_b) = setup_diff_paths_and_compression(&item_service, &item_a, &item_b)?; let (path_a, path_b) = setup_diff_paths_and_compression(&item_service, &item_a, &item_b)?;
// TODO: Implement actual diff logic here run_external_diff(&path_a, &path_b)?;
// For now, just print paths or something to make it compile
println!("Diff between {:?} and {:?}", path_a, path_b);
Ok(()) Ok(())
} }
/// Runs external diff command to compare two files.
///
/// Uses the system's `diff` command to generate a unified diff output.
/// Returns an error if the diff command is not found.
///
/// # Arguments
///
/// * `path_a` - Path to the first file.
/// * `path_b` - Path to the second file.
///
/// # Returns
///
/// * `Result<()>` - Success or error.
fn run_external_diff(path_a: &std::path::Path, path_b: &std::path::Path) -> anyhow::Result<()> {
if which::which_global("diff").is_err() {
return Err(anyhow::anyhow!(
"diff command not found. Please install diffutils."
));
}
let mut child = std::process::Command::new("diff")
.arg("-u")
.arg(path_a)
.arg(path_b)
.stdout(std::process::Stdio::inherit())
.stderr(std::process::Stdio::inherit())
.spawn()
.context("Failed to spawn diff command")?;
let status = child.wait().context("Failed to wait for diff command")?;
// diff returns 0 if files are identical, 1 if different, 2 on error
if status.code() == Some(2) {
Err(anyhow::anyhow!("diff command failed with an error"))
} else {
Ok(())
}
}

4
src/modes/generate_config.rs
View File

@@ -186,13 +186,13 @@ pub fn mode_generate_config(_cmd: &mut Command, _settings: &crate::config::Setti
if line.trim().is_empty() { if line.trim().is_empty() {
line.to_string() line.to_string()
} else { } else {
format!("# {}", line) format!("# {line}")
} }
}) })
.collect::<Vec<String>>() .collect::<Vec<String>>()
.join("\n"); .join("\n");
println!("{}", commented_yaml); println!("{commented_yaml}");
Ok(()) Ok(())
} }

33
src/modes/get.rs
View File

@@ -1,8 +1,8 @@
use anyhow::{Result, anyhow}; use anyhow::{anyhow, Result};
use std::io::Write; use std::io::Write;
use crate::common::PIPESIZE;
use crate::common::is_binary::is_binary; use crate::common::is_binary::is_binary;
use crate::common::PIPESIZE;
use crate::config; use crate::config;
use crate::filter_plugin::FilterChain; use crate::filter_plugin::FilterChain;
use crate::services::item_service::ItemService; use crate::services::item_service::ItemService;
@@ -73,32 +73,35 @@ pub fn mode_get(
} }
} }
// Get a reader that applies the filters using the pre-parsed filter chain
let (mut reader, _, _) = item_service.get_item_content_info_streaming_with_chain(
conn,
item_id,
filter_chain.as_ref(),
)?;
if detect_binary { if detect_binary {
// Read only the first 8192 bytes for binary detection // Binary detection: sample first 8KB, then create a fresh reader for the full output.
let (mut sample_reader, _, _) = item_service
.get_item_content_info_streaming_with_item(item_with_meta, filter_chain.as_ref())?;
let mut sample_buffer = vec![0; PIPESIZE]; let mut sample_buffer = vec![0; PIPESIZE];
let bytes_read = reader.read(&mut sample_buffer)?; let bytes_read = sample_reader.read(&mut sample_buffer)?;
if is_binary(&sample_buffer[..bytes_read]) { if is_binary(&sample_buffer[..bytes_read]) {
return Err(anyhow!( return Err(anyhow!(
"Refusing to output binary data to TTY, use --force to override" "Refusing to output binary data to TTY, use --force to override"
)); ));
} }
// We need to create a new reader since we consumed some bytes // Create fresh reader for actual output (sampling consumed the first reader)
let (new_reader, _, _) = item_service.get_item_content_info_streaming_with_chain( let (reader, _, _) = item_service.get_item_content_info_streaming_with_chain(
conn, conn,
item_id, item_id,
filter_chain.as_ref(), filter_chain.as_ref(),
)?; )?;
reader = new_reader; stream_to_stdout(reader)?;
} else {
// No binary detection needed, use the already-fetched item with meta
let (reader, _, _) = item_service
.get_item_content_info_streaming_with_item(item_with_meta, filter_chain.as_ref())?;
stream_to_stdout(reader)?;
} }
// Stream the content to stdout Ok(())
}
fn stream_to_stdout(mut reader: Box<dyn Read + Send>) -> Result<()> {
let mut stdout = std::io::stdout(); let mut stdout = std::io::stdout();
let mut buffer = [0; PIPESIZE]; let mut buffer = [0; PIPESIZE];
loop { loop {

2
src/modes/info.rs
View File

@@ -157,7 +157,7 @@ fn show_item(
]); ]);
let mut item_path_buf = data_path.clone(); let mut item_path_buf = data_path.clone();
item_path_buf.push(item.id.unwrap().to_string()); item_path_buf.push(item_id.to_string());
let path_str = item_path_buf let path_str = item_path_buf
.to_str() .to_str()
.expect("Unable to get item path") .expect("Unable to get item path")

4
src/modes/server/api/common.rs
View File

@@ -1,9 +1,9 @@
use axum::{ use axum::{
http::{header, StatusCode}, http::{StatusCode, header},
response::Response, response::Response,
}; };
use serde::Serialize;
use log; use log;
use serde::Serialize;
pub struct ResponseBuilder; pub struct ResponseBuilder;

356
src/modes/server/api/item.rs
View File

@@ -1,27 +1,33 @@
use crate::modes::server::api::common::ResponseBuilder;
use crate::modes::server::common::{ use crate::modes::server::common::{
ApiResponse, AppState, ItemContentQuery, ItemInfo, ItemInfoListResponse, ItemInfoResponse, ApiResponse, AppState, CreateItemQuery, ItemContentQuery, ItemInfo, ItemInfoListResponse,
ItemQuery, ListItemsQuery, MetadataResponse, TagsQuery, ItemInfoResponse, ItemQuery, ListItemsQuery, MetadataResponse, TagsQuery,
}; };
use crate::services::async_item_service::AsyncItemService; use crate::services::async_data_service::AsyncDataService;
use crate::services::data_service::DataService;
use crate::services::error::CoreError; use crate::services::error::CoreError;
use crate::services::utils::parse_comma_tags;
use axum::{ use axum::{
body::Body,
extract::{Path, Query, State}, extract::{Path, Query, State},
http::{StatusCode, header}, http::{StatusCode, header},
response::{Json, Response}, response::{Json, Response},
}; };
use http_body_util::BodyExt;
use log::{debug, warn}; use log::{debug, warn};
use std::collections::HashMap; use std::collections::HashMap;
use std::io::Read; use std::io::{Cursor, Read};
use tokio::task;
// Helper functions to replace the missing binary_detection module // Helper functions to replace the missing binary_detection module
async fn check_binary_content_allowed( async fn check_binary_content_allowed(
item_service: &AsyncItemService, data_service: &AsyncDataService,
item_id: i64, item_id: i64,
metadata: &HashMap<String, String>, metadata: &HashMap<String, String>,
allow_binary: bool, allow_binary: bool,
) -> Result<(), StatusCode> { ) -> Result<(), StatusCode> {
if !allow_binary { if !allow_binary {
let is_binary = is_content_binary(item_service, item_id, metadata).await?; let is_binary = is_content_binary(data_service, item_id, metadata).await?;
if is_binary { if is_binary {
return Err(StatusCode::BAD_REQUEST); return Err(StatusCode::BAD_REQUEST);
} }
@@ -31,7 +37,7 @@ async fn check_binary_content_allowed(
/// Helper function to determine if content is binary /// Helper function to determine if content is binary
async fn is_content_binary( async fn is_content_binary(
item_service: &AsyncItemService, data_service: &AsyncDataService,
item_id: i64, item_id: i64,
metadata: &HashMap<String, String>, metadata: &HashMap<String, String>,
) -> Result<bool, StatusCode> { ) -> Result<bool, StatusCode> {
@@ -39,7 +45,7 @@ async fn is_content_binary(
Ok(text_val == "false") Ok(text_val == "false")
} else { } else {
// If text metadata isn't set, we need to check the content using streaming approach // If text metadata isn't set, we need to check the content using streaming approach
match item_service match data_service
.get_item_content_info_streaming(item_id, None) .get_item_content_info_streaming(item_id, None)
.await .await
{ {
@@ -56,44 +62,6 @@ async fn is_content_binary(
} }
} }
// Helper function to replace missing build_filter_string
fn build_filter_string(_params: &ItemQuery) -> Option<String> {
// Implement this based on your needs
None
}
// Create a simple ResponseBuilder to replace the missing one
struct ResponseBuilder;
impl ResponseBuilder {
pub fn json<T: serde::Serialize>(data: T) -> Result<Response, StatusCode> {
let json = serde_json::to_vec(&data).map_err(|e| {
log::warn!("Failed to serialize response: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
Response::builder()
.header(header::CONTENT_TYPE, "application/json")
.header(header::CONTENT_LENGTH, json.len().to_string())
.body(axum::body::Body::from(json))
.map_err(|e| {
log::warn!("Failed to build response: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})
}
pub fn binary(content: &[u8], mime_type: &str) -> Result<Response, StatusCode> {
Response::builder()
.header(header::CONTENT_TYPE, mime_type)
.header(header::CONTENT_LENGTH, content.len().to_string())
.body(axum::body::Body::from(content.to_vec()))
.map_err(|e| {
log::warn!("Failed to build response: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})
}
}
/// Helper function to get mime type from metadata /// Helper function to get mime type from metadata
fn get_mime_type(metadata: &HashMap<String, String>) -> String { fn get_mime_type(metadata: &HashMap<String, String>) -> String {
metadata metadata
@@ -130,14 +98,12 @@ fn handle_item_error(error: CoreError) -> StatusCode {
} }
} }
/// Helper function to create AsyncItemService from AppState /// Helper function to create AsyncDataService from AppState
fn create_item_service(state: &AppState) -> AsyncItemService { fn create_data_service(state: &AppState) -> AsyncDataService {
AsyncItemService::new( AsyncDataService::new(
state.data_dir.clone(), state.data_dir.clone(),
state.db.clone(),
state.item_service.clone(),
state.cmd.clone(),
state.settings.clone(), state.settings.clone(),
state.db.clone(),
) )
} }
@@ -170,11 +136,17 @@ pub async fn handle_list_items(
let tags: Vec<String> = params let tags: Vec<String> = params
.tags .tags
.as_ref() .as_ref()
.map(|s| s.split(',').map(|t| t.trim().to_string()).collect()) .map(|s| {
parse_comma_tags(s).map_err(|e| {
warn!("Failed to parse tags: {}", e);
StatusCode::BAD_REQUEST
})
})
.transpose()?
.unwrap_or_default(); .unwrap_or_default();
let item_service = create_item_service(&state); let data_service = create_data_service(&state);
let mut items_with_meta = item_service let mut items_with_meta = data_service
.list_items(tags, HashMap::new()) .list_items(tags, HashMap::new())
.await .await
.map_err(|e| { .map_err(|e| {
@@ -226,31 +198,31 @@ pub async fn handle_list_items(
/// Handle as_meta=true response by returning JSON with metadata and content /// Handle as_meta=true response by returning JSON with metadata and content
async fn handle_as_meta_response( async fn handle_as_meta_response(
item_service: &AsyncItemService, data_service: &AsyncDataService,
item_id: i64, item_id: i64,
offset: u64, offset: u64,
length: u64, length: u64,
) -> Result<Response, StatusCode> { ) -> Result<Response, StatusCode> {
// Get the item with metadata // Get the item with metadata
let item_with_meta = item_service.get_item(item_id).await.map_err(|e| { let item_with_meta = data_service.get_item(item_id).await.map_err(|e| {
warn!("Failed to get item {} for as_meta content: {}", item_id, e); warn!("Failed to get item {} for as_meta content: {}", item_id, e);
StatusCode::INTERNAL_SERVER_ERROR StatusCode::INTERNAL_SERVER_ERROR
})?; })?;
let metadata = item_with_meta.meta_as_map(); let metadata = item_with_meta.meta_as_map();
handle_as_meta_response_with_metadata(item_service, item_id, &metadata, offset, length).await handle_as_meta_response_with_metadata(data_service, item_id, &metadata, offset, length).await
} }
/// Handle as_meta=true response with pre-fetched metadata /// Handle as_meta=true response with pre-fetched metadata
async fn handle_as_meta_response_with_metadata( async fn handle_as_meta_response_with_metadata(
item_service: &AsyncItemService, data_service: &AsyncDataService,
item_id: i64, item_id: i64,
metadata: &HashMap<String, String>, metadata: &HashMap<String, String>,
offset: u64, offset: u64,
length: u64, length: u64,
) -> Result<Response, StatusCode> { ) -> Result<Response, StatusCode> {
// Check if content is binary // Check if content is binary
let is_binary = is_content_binary(item_service, item_id, metadata).await?; let is_binary = is_content_binary(data_service, item_id, metadata).await?;
// Get the content if it's not binary // Get the content if it's not binary
if is_binary { if is_binary {
@@ -268,7 +240,7 @@ async fn handle_as_meta_response_with_metadata(
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
} else { } else {
// Get the content as text // Get the content as text
match item_service.get_item_content_info(item_id, None).await { match data_service.get_item_content_info(item_id, None).await {
Ok((content, _, _)) => { Ok((content, _, _)) => {
// Apply offset and length // Apply offset and length
let content_len = content.len() as u64; let content_len = content.len() as u64;
@@ -330,7 +302,8 @@ async fn handle_as_meta_response_with_metadata(
path = "/api/item/", path = "/api/item/",
operation_id = "keep_post_item", operation_id = "keep_post_item",
summary = "Store new item", summary = "Store new item",
description = "Upload content to store as a new item. Content is compressed, analyzed for metadata, and stored.", description = "Upload content to store as a new item. Content is compressed, analyzed for metadata, and stored. \
Query parameters: tags (comma-separated), metadata (JSON string). Body: raw binary content.",
responses( responses(
(status = 201, description = "Item created", body = ItemInfoResponse), (status = 201, description = "Item created", body = ItemInfoResponse),
(status = 400, description = "Bad request"), (status = 400, description = "Bad request"),
@@ -338,26 +311,95 @@ async fn handle_as_meta_response_with_metadata(
(status = 500, description = "Internal server error") (status = 500, description = "Internal server error")
), ),
request_body( request_body(
content = String, content = Vec<u8>,
description = "Content to store", description = "Raw binary content to store",
content_type = "application/octet-stream" content_type = "application/octet-stream"
), ),
params(
("tags" = Option<String>, Query, description = "Comma-separated tags to associate with the item"),
("metadata" = Option<String>, Query, description = "Metadata as JSON string")
),
security( security(
("bearerAuth" = []) ("bearerAuth" = [])
), ),
tag = "item" tag = "item"
)] )]
pub async fn handle_post_item( pub async fn handle_post_item(
State(_state): State<AppState>, State(state): State<AppState>,
Query(params): Query<CreateItemQuery>,
body: Body,
) -> Result<Json<ApiResponse<ItemInfo>>, StatusCode> { ) -> Result<Json<ApiResponse<ItemInfo>>, StatusCode> {
// This is a simplified implementation let db = state.db.clone();
// In a real implementation, you'd need to properly parse multipart/form-data let data_dir = state.data_dir.clone();
// or JSON payload with the item data let settings = state.settings.clone();
let response = ApiResponse::<ItemInfo> { // Parse tags from query parameter
success: false, let tags: Vec<String> = params
data: None, .tags
error: Some("POST /api/item/ not yet implemented".to_string()), .as_deref()
.map(|s| {
parse_comma_tags(s).map_err(|e| {
warn!("Failed to parse tags query parameter: {}", e);
StatusCode::BAD_REQUEST
})
})
.transpose()?
.unwrap_or_default();
// Parse metadata from query parameter
let metadata: HashMap<String, String> = if let Some(ref meta_str) = params.metadata {
serde_json::from_str(meta_str).map_err(|e| {
warn!("Failed to parse metadata JSON string: {}", e);
StatusCode::BAD_REQUEST
})?
} else {
HashMap::new()
};
// Convert body to bytes first (simpler than streaming for this use case)
let body_bytes = body
.collect()
.await
.map_err(|e| {
warn!("Failed to read request body: {}", e);
StatusCode::BAD_REQUEST
})?
.to_bytes();
let item_with_meta = task::spawn_blocking(move || {
let mut conn = db.blocking_lock();
let mut cursor = Cursor::new(body_bytes.to_vec());
let sync_service =
crate::services::SyncDataService::new(data_dir, settings.as_ref().clone());
sync_service.save_item_with_reader(&mut conn, &mut cursor, tags, metadata)
})
.await
.map_err(|e| {
warn!("Failed to save item: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?
.map_err(|e| {
warn!("Failed to save item: {}", e);
StatusCode::INTERNAL_SERVER_ERROR
})?;
let compression = item_with_meta.item.compression.clone();
let tags = item_with_meta.tags.iter().map(|t| t.name.clone()).collect();
let metadata = item_with_meta.meta_as_map();
let item_info = ItemInfo {
id: item_with_meta.item.id.unwrap(),
ts: item_with_meta.item.ts.to_rfc3339(),
size: item_with_meta.item.size,
compression,
tags,
metadata,
};
let response = ApiResponse {
success: true,
data: Some(item_info),
error: None,
}; };
Ok(Json(response)) Ok(Json(response))
@@ -397,13 +439,19 @@ pub async fn handle_get_item_latest_content(
let tags: Vec<String> = params let tags: Vec<String> = params
.tags .tags
.as_ref() .as_ref()
.map(|s| s.split(',').map(|t| t.trim().to_string()).collect()) .map(|s| {
parse_comma_tags(s).map_err(|e| {
warn!("Failed to parse tags: {}", e);
StatusCode::BAD_REQUEST
})
})
.transpose()?
.unwrap_or_default(); .unwrap_or_default();
let item_service = create_item_service(&state); let data_service = create_data_service(&state);
// First find the item to get its ID and metadata // First find the item to get its ID and metadata
let item_with_meta = item_service.find_item(vec![], tags, HashMap::new()).await; let item_with_meta = data_service.find_item(vec![], tags, HashMap::new()).await;
match item_with_meta { match item_with_meta {
Ok(item) => { Ok(item) => {
@@ -413,7 +461,7 @@ pub async fn handle_get_item_latest_content(
if params.as_meta { if params.as_meta {
// Force stream=false and allow_binary=false for as_meta=true // Force stream=false and allow_binary=false for as_meta=true
handle_as_meta_response_with_metadata( handle_as_meta_response_with_metadata(
&item_service, &data_service,
item_id, item_id,
&metadata, &metadata,
params.offset, params.offset,
@@ -422,7 +470,7 @@ pub async fn handle_get_item_latest_content(
.await .await
} else { } else {
stream_item_content_response_with_metadata( stream_item_content_response_with_metadata(
&item_service, &data_service,
item_id, item_id,
&metadata, &metadata,
params.allow_binary, params.allow_binary,
@@ -484,14 +532,12 @@ pub async fn handle_get_item_content(
item_id, params.stream, params.allow_binary, params.offset, params.length item_id, params.stream, params.allow_binary, params.offset, params.length
); );
let filter = build_filter_string(&params); let data_service = create_data_service(&state);
let item_service = create_item_service(&state);
// Handle as_meta parameter // Handle as_meta parameter
if params.as_meta { if params.as_meta {
// Force stream=false and allow_binary=false for as_meta=true // Force stream=false and allow_binary=false for as_meta=true
let result = let result =
handle_as_meta_response(&item_service, item_id, params.offset, params.length).await; handle_as_meta_response(&data_service, item_id, params.offset, params.length).await;
if let Ok(response) = &result { if let Ok(response) = &result {
debug!( debug!(
"ITEM_API: Response content-length: {:?}", "ITEM_API: Response content-length: {:?}",
@@ -501,13 +547,13 @@ pub async fn handle_get_item_content(
result result
} else { } else {
let result = stream_item_content_response( let result = stream_item_content_response(
&item_service, &data_service,
item_id, item_id,
params.allow_binary, params.allow_binary,
params.offset, params.offset,
params.length, params.length,
params.stream, params.stream,
filter, None,
) )
.await; .await;
if let Ok(response) = &result { if let Ok(response) = &result {
@@ -521,44 +567,44 @@ pub async fn handle_get_item_content(
} }
async fn stream_item_content_response( async fn stream_item_content_response(
item_service: &AsyncItemService, data_service: &AsyncDataService,
item_id: i64, item_id: i64,
allow_binary: bool, allow_binary: bool,
offset: u64, offset: u64,
length: u64, length: u64,
stream: bool, stream: bool,
filter: Option<String>, _filter: Option<String>,
) -> Result<Response, StatusCode> { ) -> Result<Response, StatusCode> {
debug!("STREAM_ITEM_CONTENT_RESPONSE: stream={}", stream); debug!("STREAM_ITEM_CONTENT_RESPONSE: stream={}", stream);
// Get the item with metadata once // Get the item with metadata once
let item_with_meta = item_service.get_item(item_id).await.map_err(|e| { let item_with_meta = data_service.get_item(item_id).await.map_err(|e| {
warn!("Failed to get item {} for content: {}", item_id, e); warn!("Failed to get item {} for content: {}", item_id, e);
StatusCode::INTERNAL_SERVER_ERROR StatusCode::INTERNAL_SERVER_ERROR
})?; })?;
let metadata = item_with_meta.meta_as_map(); let metadata = item_with_meta.meta_as_map();
stream_item_content_response_with_metadata( stream_item_content_response_with_metadata(
item_service, data_service,
item_id, item_id,
&metadata, &metadata,
allow_binary, allow_binary,
offset, offset,
length, length,
stream, stream,
filter, None,
) )
.await .await
} }
async fn stream_item_content_response_with_metadata( async fn stream_item_content_response_with_metadata(
item_service: &AsyncItemService, data_service: &AsyncDataService,
item_id: i64, item_id: i64,
metadata: &HashMap<String, String>, metadata: &HashMap<String, String>,
allow_binary: bool, allow_binary: bool,
offset: u64, offset: u64,
length: u64, length: u64,
stream: bool, stream: bool,
filter: Option<String>, _filter: Option<String>,
) -> Result<Response, StatusCode> { ) -> Result<Response, StatusCode> {
debug!( debug!(
"STREAM_ITEM_CONTENT_RESPONSE_WITH_METADATA: stream={}", "STREAM_ITEM_CONTENT_RESPONSE_WITH_METADATA: stream={}",
@@ -567,14 +613,12 @@ async fn stream_item_content_response_with_metadata(
let mime_type = get_mime_type(metadata); let mime_type = get_mime_type(metadata);
// Check if content is binary when allow_binary is false // Check if content is binary when allow_binary is false
check_binary_content_allowed(item_service, item_id, metadata, allow_binary).await?; check_binary_content_allowed(data_service, item_id, metadata, allow_binary).await?;
if stream { if stream {
debug!("STREAMING: Using streaming approach"); debug!("STREAMING: Using streaming approach");
match item_service match data_service
.stream_item_content_by_id_with_metadata( .stream_item_content_by_id_with_metadata(item_id, metadata, true, offset, length, None)
item_id, metadata, true, offset, length, filter,
)
.await .await
{ {
Ok((stream, _)) => { Ok((stream, _)) => {
@@ -592,7 +636,7 @@ async fn stream_item_content_response_with_metadata(
} }
} else { } else {
debug!("NON-STREAMING: Building full response in memory"); debug!("NON-STREAMING: Building full response in memory");
match item_service.get_item_content_info(item_id, filter).await { match data_service.get_item_content_info(item_id, None).await {
Ok((content, _, _)) => { Ok((content, _, _)) => {
let response_content = apply_offset_length(&content, offset, length); let response_content = apply_offset_length(&content, offset, length);
@@ -639,12 +683,18 @@ pub async fn handle_get_item_latest_meta(
let tags: Vec<String> = params let tags: Vec<String> = params
.tags .tags
.as_ref() .as_ref()
.map(|s| s.split(',').map(|t| t.trim().to_string()).collect()) .map(|s| {
parse_comma_tags(s).map_err(|e| {
warn!("Failed to parse tags: {}", e);
StatusCode::BAD_REQUEST
})
})
.transpose()?
.unwrap_or_default(); .unwrap_or_default();
let item_service = create_item_service(&state); let data_service = create_data_service(&state);
match item_service.find_item(vec![], tags, HashMap::new()).await { match data_service.find_item(vec![], tags, HashMap::new()).await {
Ok(item_with_meta) => { Ok(item_with_meta) => {
let item_meta = item_with_meta.meta_as_map(); let item_meta = item_with_meta.meta_as_map();
@@ -685,9 +735,9 @@ pub async fn handle_get_item_meta(
State(state): State<AppState>, State(state): State<AppState>,
Path(item_id): Path<i64>, Path(item_id): Path<i64>,
) -> Result<Json<ApiResponse<HashMap<String, String>>>, StatusCode> { ) -> Result<Json<ApiResponse<HashMap<String, String>>>, StatusCode> {
let item_service = create_item_service(&state); let data_service = create_data_service(&state);
match item_service.get_item(item_id).await { match data_service.get_item(item_id).await {
Ok(item_with_meta) => { Ok(item_with_meta) => {
let item_meta = item_with_meta.meta_as_map(); let item_meta = item_with_meta.meta_as_map();
@@ -724,22 +774,24 @@ pub async fn handle_delete_item(
State(state): State<AppState>, State(state): State<AppState>,
Path(item_id): Path<i64>, Path(item_id): Path<i64>,
) -> Result<Json<ApiResponse<ItemInfo>>, StatusCode> { ) -> Result<Json<ApiResponse<ItemInfo>>, StatusCode> {
let conn = state.db.lock().await; let mut conn = state.db.lock().await;
let sync_service = let sync_service = crate::services::SyncDataService::new(
crate::services::SyncDataService::new(state.data_dir.clone(), state.settings.clone()); state.data_dir.clone(),
state.settings.as_ref().clone(),
);
let deleted_item = sync_service let deleted_item = sync_service
.delete_item(&mut conn.clone(), item_id) .delete_item(&mut conn, item_id)
.map_err(handle_item_error)?; .map_err(handle_item_error)?;
let item_info = ItemInfo { let item_info = ItemInfo {
id: deleted_item.id, id: deleted_item.id.unwrap(),
ts: deleted_item.ts, ts: deleted_item.ts.to_rfc3339(),
size: deleted_item.size, size: deleted_item.size,
compression: deleted_item.compression, compression: deleted_item.compression,
tags: vec![], tags: vec![],
meta: HashMap::new(), metadata: HashMap::new(),
}; };
let response = ApiResponse { let response = ApiResponse {
@@ -772,24 +824,27 @@ pub async fn handle_get_item_info(
State(state): State<AppState>, State(state): State<AppState>,
Path(item_id): Path<i64>, Path(item_id): Path<i64>,
) -> Result<Json<ApiResponse<ItemInfo>>, StatusCode> { ) -> Result<Json<ApiResponse<ItemInfo>>, StatusCode> {
let conn = state.db.lock().await; let mut conn = state.db.lock().await;
let sync_service = let sync_service = crate::services::SyncDataService::new(
crate::services::SyncDataService::new(state.data_dir.clone(), state.settings.clone()); state.data_dir.clone(),
state.settings.as_ref().clone(),
);
let item_with_meta = sync_service let item_with_meta = sync_service
.get_item(&mut conn.clone(), item_id) .get_item(&mut conn, item_id)
.map_err(handle_item_error)?; .map_err(handle_item_error)?;
let tags: Vec<String> = item_with_meta.tags.iter().map(|t| t.name.clone()).collect(); let tags: Vec<String> = item_with_meta.tags.iter().map(|t| t.name.clone()).collect();
let metadata = item_with_meta.meta_as_map();
let item_info = ItemInfo { let item_info = ItemInfo {
id: item_with_meta.item.id, id: item_with_meta.item.id.unwrap(),
ts: item_with_meta.item.ts, ts: item_with_meta.item.ts.to_rfc3339(),
size: item_with_meta.item.size, size: item_with_meta.item.size,
compression: item_with_meta.item.compression, compression: item_with_meta.item.compression.clone(),
tags, tags,
meta: item_with_meta.meta_as_map(), metadata,
}; };
let response = ApiResponse { let response = ApiResponse {
@@ -834,18 +889,20 @@ pub async fn handle_diff_items(
State(state): State<AppState>, State(state): State<AppState>,
Query(query): Query<DiffQuery>, Query(query): Query<DiffQuery>,
) -> Result<Json<ApiResponse<Vec<String>>>, StatusCode> { ) -> Result<Json<ApiResponse<Vec<String>>>, StatusCode> {
let conn = state.db.lock().await; let mut conn = state.db.lock().await;
let sync_service = let sync_service = crate::services::SyncDataService::new(
crate::services::SyncDataService::new(state.data_dir.clone(), state.settings.clone()); state.data_dir.clone(),
state.settings.as_ref().clone(),
);
let item_a = if let Some(id_a) = query.id_a { let item_a = if let Some(id_a) = query.id_a {
sync_service sync_service
.get_item(&mut conn.clone(), id_a) .get_item(&mut conn, id_a)
.map_err(handle_item_error)? .map_err(handle_item_error)?
} else if let Some(tag) = &query.tag_a { } else if let Some(tag) = &query.tag_a {
sync_service sync_service
.find_item(&mut conn.clone(), vec![], vec![tag.clone()], HashMap::new()) .find_item(&mut conn, vec![], vec![tag.clone()], HashMap::new())
.map_err(handle_item_error)? .map_err(handle_item_error)?
} else { } else {
return Err(StatusCode::BAD_REQUEST); return Err(StatusCode::BAD_REQUEST);
@@ -853,24 +910,24 @@ pub async fn handle_diff_items(
let item_b = if let Some(id_b) = query.id_b { let item_b = if let Some(id_b) = query.id_b {
sync_service sync_service
.get_item(&mut conn.clone(), id_b) .get_item(&mut conn, id_b)
.map_err(handle_item_error)? .map_err(handle_item_error)?
} else if let Some(tag) = &query.tag_b { } else if let Some(tag) = &query.tag_b {
sync_service sync_service
.find_item(&mut conn.clone(), vec![], vec![tag.clone()], HashMap::new()) .find_item(&mut conn, vec![], vec![tag.clone()], HashMap::new())
.map_err(handle_item_error)? .map_err(handle_item_error)?
} else { } else {
return Err(StatusCode::BAD_REQUEST); return Err(StatusCode::BAD_REQUEST);
}; };
let id_a = item_a.item.id.unwrap(); let id_a = item_a.item.id.ok_or_else(|| StatusCode::BAD_REQUEST)?;
let id_b = item_b.item.id.unwrap(); let id_b = item_b.item.id.ok_or_else(|| StatusCode::BAD_REQUEST)?;
let (reader_a, _) = sync_service let (mut reader_a, _) = sync_service
.get_content(&mut conn.clone(), id_a) .get_content(&mut conn, id_a)
.map_err(handle_item_error)?; .map_err(handle_item_error)?;
let (reader_b, _) = sync_service let (mut reader_b, _) = sync_service
.get_content(&mut conn.clone(), id_b) .get_content(&mut conn, id_b)
.map_err(handle_item_error)?; .map_err(handle_item_error)?;
let mut content_a = Vec::new(); let mut content_a = Vec::new();
@@ -900,31 +957,30 @@ fn compute_diff(a: &[u8], b: &[u8]) -> Vec<String> {
let text_a = String::from_utf8_lossy(a); let text_a = String::from_utf8_lossy(a);
let text_b = String::from_utf8_lossy(b); let text_b = String::from_utf8_lossy(b);
let lines_a: Vec<&str> = text_a.lines().collect(); let old_lines: Vec<&str> = text_a.lines().collect();
let lines_b: Vec<&str> = text_b.lines().collect(); let new_lines: Vec<&str> = text_b.lines().collect();
let ops = similar::TextDiff::from_lines(
text_a.as_ref(),
text_b.as_ref(),
)
.ops();
let mut diff_lines = Vec::new(); let mut diff_lines = Vec::new();
let max_lines = std::cmp::max(lines_a.len(), lines_b.len()); for op in ops {
for i in 0..max_lines { for change in op.iter_changes(&old_lines, &new_lines) {
let line_a = lines_a.get(i).copied(); match change.tag() {
let line_b = lines_b.get(i).copied(); similar::ChangeTag::Equal => {
diff_lines.push(format!(" {}", change.value()));
match (line_a, line_b) { }
(Some(la), Some(lb)) if la == lb => { similar::ChangeTag::Delete => {
diff_lines.push(format!(" {}", la)); diff_lines.push(format!("- {}", change.value()));
}
similar::ChangeTag::Insert => {
diff_lines.push(format!("+ {}", change.value()));
}
} }
(Some(la), Some(lb)) => {
diff_lines.push(format!("- {}", la));
diff_lines.push(format!("+ {}", lb));
}
(Some(la), None) => {
diff_lines.push(format!("- {}", la));
}
(None, Some(lb)) => {
diff_lines.push(format!("+ {}", lb));
}
(None, None) => {}
} }
} }

3
src/modes/server/api/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod common;
#[cfg(feature = "swagger")] #[cfg(feature = "swagger")]
pub mod item; pub mod item;
#[cfg(feature = "mcp")] #[cfg(feature = "mcp")]
@@ -59,7 +60,7 @@ use utoipa_swagger_ui::SwaggerUi;
struct ApiDoc; struct ApiDoc;
pub fn add_routes(router: Router<AppState>) -> Router<AppState> { pub fn add_routes(router: Router<AppState>) -> Router<AppState> {
let router = router let mut router = router
// Status endpoints // Status endpoints
.route("/api/status", get(status::handle_status)) .route("/api/status", get(status::handle_status))
.route("/api/plugins/status", get(status::handle_plugins_status)) .route("/api/plugins/status", get(status::handle_plugins_status))

6
src/modes/server/api/status.rs
View File

@@ -1,6 +1,6 @@
use axum::{extract::State, http::StatusCode, response::Json}; use axum::{extract::State, http::StatusCode, response::Json};
use crate::modes::server::common::{AppState, StatusInfoResponse}; use crate::modes::server::common::{ApiResponse, AppState, StatusInfoResponse};
#[utoipa::path( #[utoipa::path(
get, get,
@@ -76,7 +76,7 @@ pub async fn handle_status(
Ok(Json(response)) Ok(Json(response))
} }
#[derive(Debug, serde::Serialize)] #[derive(Debug, serde::Serialize, serde::Deserialize, utoipa::ToSchema)]
pub struct PluginsStatusResponse { pub struct PluginsStatusResponse {
pub meta_plugins: std::collections::HashMap<String, crate::common::status::MetaPluginInfo>, pub meta_plugins: std::collections::HashMap<String, crate::common::status::MetaPluginInfo>,
pub filter_plugins: Vec<crate::common::status::FilterPluginInfo>, pub filter_plugins: Vec<crate::common::status::FilterPluginInfo>,
@@ -90,7 +90,7 @@ pub struct PluginsStatusResponse {
summary = "Get plugins status", summary = "Get plugins status",
description = "Retrieve detailed status of all available plugins including meta, filter, and compression plugins.", description = "Retrieve detailed status of all available plugins including meta, filter, and compression plugins.",
responses( responses(
(status = 200, description = "Plugins status retrieved", body = ApiResponse), (status = 200, description = "Plugins status retrieved", body = ApiResponse<PluginsStatusResponse>),
(status = 401, description = "Unauthorized"), (status = 401, description = "Unauthorized"),
(status = 500, description = "Internal server error") (status = 500, description = "Internal server error")
), ),

11
src/modes/server/common.rs
View File

@@ -567,6 +567,17 @@ fn default_as_meta() -> bool {
false false
} }
/// Query parameters for creating an item via POST.
///
/// Query parameters for POST /api/item/ with streaming binary body.
#[derive(Debug, Deserialize)]
pub struct CreateItemQuery {
/// Optional comma-separated tags to associate with the item.
pub tags: Option<String>,
/// Optional metadata as JSON string.
pub metadata: Option<String>,
}
/// Request body for creating a new item. /// Request body for creating a new item.
/// ///
/// Contains the content to store and optional tags. /// Contains the content to store and optional tags.

2
src/modes/status.rs
View File

@@ -157,7 +157,7 @@ fn build_meta_plugins_configured_table(status_info: &StatusInfo) -> Option<Table
if key == &value_str { if key == &value_str {
enabled_output_pairs.push((key.clone(), key.clone())); enabled_output_pairs.push((key.clone(), key.clone()));
} else { } else {
enabled_output_pairs.push((key.clone(), format!("{}->{}", key, value_str))); enabled_output_pairs.push((key.clone(), format!("{key}->{value_str}")));
} }
} }

211
src/services/async_data_service.rs
View File

@@ -6,10 +6,12 @@ use crate::services::data_service::DataService;
use crate::services::error::CoreError; use crate::services::error::CoreError;
use crate::services::types::{ItemWithContent, ItemWithMeta}; use crate::services::types::{ItemWithContent, ItemWithMeta};
use clap::Command; use clap::Command;
use futures::Stream;
use rusqlite::Connection; use rusqlite::Connection;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::Read; use std::io::Read;
use std::path::PathBuf; use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::Mutex; use tokio::sync::Mutex;
@@ -17,14 +19,18 @@ pub struct AsyncDataService {
data_path: PathBuf, data_path: PathBuf,
settings: Arc<Settings>, settings: Arc<Settings>,
db: Arc<Mutex<Connection>>, db: Arc<Mutex<Connection>>,
sync_service: crate::services::SyncDataService,
} }
impl AsyncDataService { impl AsyncDataService {
pub fn new(data_path: PathBuf, settings: Arc<Settings>, db: Arc<Mutex<Connection>>) -> Self { pub fn new(data_path: PathBuf, settings: Arc<Settings>, db: Arc<Mutex<Connection>>) -> Self {
let sync_service =
crate::services::SyncDataService::new(data_path.clone(), settings.as_ref().clone());
Self { Self {
data_path, data_path,
settings, settings,
db, db,
sync_service,
} }
} }
@@ -39,6 +45,145 @@ impl AsyncDataService {
pub fn db(&self) -> Arc<Mutex<Connection>> { pub fn db(&self) -> Arc<Mutex<Connection>> {
self.db.clone() self.db.clone()
} }
pub async fn get_item(&self, id: i64) -> Result<ItemWithMeta, CoreError> {
let mut conn = self.db.lock().await;
self.get(&mut conn, id)
}
pub async fn list_items(
&self,
tags: Vec<String>,
meta: HashMap<String, String>,
) -> Result<Vec<ItemWithMeta>, CoreError> {
let mut conn = self.db.lock().await;
self.list(&mut conn, tags, meta)
}
pub async fn find_item(
&self,
ids: Vec<i64>,
tags: Vec<String>,
meta: HashMap<String, String>,
) -> Result<ItemWithMeta, CoreError> {
let mut conn = self.db.lock().await;
DataService::find_item(self, &mut conn, ids, tags, meta)
}
pub async fn get_item_content_info(
&self,
id: i64,
_filter: Option<String>,
) -> Result<(Vec<u8>, ItemWithMeta, bool), CoreError> {
let mut conn = self.db.lock().await;
let (mut reader, item_with_meta) = self.get_content(&mut conn, id)?;
let mut content = Vec::new();
reader.read_to_end(&mut content)?;
let is_binary = item_with_meta
.meta
.iter()
.find(|m| m.name == "text")
.map(|m| m.value == "false")
.unwrap_or(false);
Ok((content, item_with_meta, is_binary))
}
pub async fn get_item_content_info_streaming(
&self,
id: i64,
_filter: Option<String>,
) -> Result<
(
Pin<Box<dyn Stream<Item = Result<Vec<u8>, CoreError>> + Send>>,
ItemWithMeta,
bool,
),
CoreError,
> {
let mut conn = self.db.lock().await;
let (reader, item_with_meta) = self.get_content(&mut conn, id)?;
let is_binary = item_with_meta
.meta
.iter()
.find(|m| m.name == "text")
.map(|m| m.value == "false")
.unwrap_or(false);
// Convert reader to stream with optimized buffer reuse
let stream = async_stream::stream! {
let mut reader = reader;
let mut buf = [0u8; 8192];
loop {
match reader.read(&mut buf) {
Ok(0) => break,
Ok(n) => yield Ok(buf[..n].to_vec()),
Err(e) => yield Err(CoreError::from(e)),
}
}
};
Ok((Box::pin(stream), item_with_meta, is_binary))
}
pub async fn stream_item_content_by_id_with_metadata(
&self,
id: i64,
_metadata: &HashMap<String, String>,
_force_text: bool,
offset: u64,
length: u64,
_filter: Option<String>,
) -> Result<
(
Pin<Box<dyn Stream<Item = Result<Vec<u8>, std::io::Error>> + Send>>,
u64,
),
CoreError,
> {
let mut conn = self.db.lock().await;
let (mut reader, _item_with_meta) = self.get_content(&mut conn, id)?;
// Skip bytes for offset
if offset > 0 {
let mut skip_buf = [0u8; 8192];
let mut remaining = offset;
while remaining > 0 {
let to_read = std::cmp::min(8192, remaining as usize);
let n = reader.read(&mut skip_buf[..to_read])?;
if n == 0 {
break;
}
remaining -= n as u64;
}
}
let content_length = if length > 0 { length } else { u64::MAX };
// Optimized stream that reuses a single buffer for reading
let stream = async_stream::stream! {
let mut reader = reader;
let mut remaining = content_length;
let mut buf = [0u8; 8192];
while remaining > 0 {
let to_read = std::cmp::min(8192, remaining as usize);
match reader.read(&mut buf[..to_read]) {
Ok(0) => break,
Ok(n) => {
remaining -= n as u64;
yield Ok(buf[..n].to_vec());
}
Err(e) => {
yield Err(e);
break;
}
}
}
};
Ok((Box::pin(stream), content_length))
}
} }
impl DataService for AsyncDataService { impl DataService for AsyncDataService {
@@ -52,17 +197,11 @@ impl DataService for AsyncDataService {
tags: Vec<String>, tags: Vec<String>,
conn: &mut Connection, conn: &mut Connection,
) -> Result<Item, Self::Error> { ) -> Result<Item, Self::Error> {
let sync_service = self.sync_service.save(content, cmd, settings, tags, conn)
crate::services::SyncDataService::new(self.data_path.clone(), settings.clone());
sync_service.save(content, cmd, settings, tags, conn)
} }
fn get(&self, conn: &mut Connection, id: i64) -> Result<ItemWithMeta, Self::Error> { fn get(&self, conn: &mut Connection, id: i64) -> Result<ItemWithMeta, Self::Error> {
let sync_service = crate::services::SyncDataService::new( self.sync_service.get(conn, id)
self.data_path.clone(),
self.settings.as_ref().clone(),
);
sync_service.get(conn, id)
} }
fn get_content( fn get_content(
@@ -70,11 +209,7 @@ impl DataService for AsyncDataService {
conn: &mut Connection, conn: &mut Connection,
id: i64, id: i64,
) -> Result<(Box<dyn Read + Send>, ItemWithMeta), Self::Error> { ) -> Result<(Box<dyn Read + Send>, ItemWithMeta), Self::Error> {
let sync_service = crate::services::SyncDataService::new( self.sync_service.get_content(conn, id)
self.data_path.clone(),
self.settings.as_ref().clone(),
);
sync_service.get_content(conn, id)
} }
fn list( fn list(
@@ -83,19 +218,11 @@ impl DataService for AsyncDataService {
tags: Vec<String>, tags: Vec<String>,
meta: HashMap<String, String>, meta: HashMap<String, String>,
) -> Result<Vec<ItemWithMeta>, Self::Error> { ) -> Result<Vec<ItemWithMeta>, Self::Error> {
let sync_service = crate::services::SyncDataService::new( self.sync_service.list(conn, tags, meta)
self.data_path.clone(),
self.settings.as_ref().clone(),
);
sync_service.list(conn, tags, meta)
} }
fn delete(&self, conn: &mut Connection, id: i64) -> Result<Item, Self::Error> { fn delete(&self, conn: &mut Connection, id: i64) -> Result<Item, Self::Error> {
let sync_service = crate::services::SyncDataService::new( self.sync_service.delete(conn, id)
self.data_path.clone(),
self.settings.as_ref().clone(),
);
sync_service.delete(conn, id)
} }
fn find_item( fn find_item(
@@ -105,11 +232,7 @@ impl DataService for AsyncDataService {
tags: Vec<String>, tags: Vec<String>,
meta: HashMap<String, String>, meta: HashMap<String, String>,
) -> Result<ItemWithMeta, Self::Error> { ) -> Result<ItemWithMeta, Self::Error> {
let sync_service = crate::services::SyncDataService::new( self.sync_service.find_item(conn, ids, tags, meta)
self.data_path.clone(),
self.settings.as_ref().clone(),
);
sync_service.find_item(conn, ids, tags, meta)
} }
fn get_items( fn get_items(
@@ -119,30 +242,22 @@ impl DataService for AsyncDataService {
tags: &[String], tags: &[String],
meta: &HashMap<String, String>, meta: &HashMap<String, String>,
) -> Result<Vec<ItemWithMeta>, Self::Error> { ) -> Result<Vec<ItemWithMeta>, Self::Error> {
let sync_service = crate::services::SyncDataService::new( self.sync_service.get_items(conn, ids, tags, meta)
self.data_path.clone(),
self.settings.as_ref().clone(),
);
sync_service.get_items(conn, ids, tags, meta)
} }
fn generate_status( fn generate_status(
&self, &self,
_cmd: &Command,
settings: &Settings, settings: &Settings,
data_path: &PathBuf, data_path: &Path,
db_path: &PathBuf, db_path: &Path,
) -> Result<StatusInfo, Self::Error> { ) -> Result<StatusInfo, Self::Error> {
let mut cmd_mut = Command::new("keep"); let mut cmd = Command::new("keep");
let sync_service = let status_service = crate::services::StatusService::new();
crate::services::SyncDataService::new(self.data_path.clone(), settings.clone()); Ok(status_service.generate_status(
Ok( &mut cmd,
sync_service.generate_status( settings,
&mut cmd_mut, data_path.to_path_buf(),
settings, db_path.to_path_buf(),
data_path.clone(), ))
db_path.clone(),
),
)
} }
} }

1
src/services/async_item_service.rs
View File

@@ -244,7 +244,6 @@ impl AsyncItemService {
let reader = { let reader = {
let db = self.db.clone(); let db = self.db.clone();
let item_service = self.item_service.clone(); let item_service = self.item_service.clone();
let item_id = item_id;
let filter = filter.clone(); let filter = filter.clone();
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
let conn = db.blocking_lock(); let conn = db.blocking_lock();

14
src/services/compression_service.rs
View File

@@ -1,4 +1,4 @@
use crate::compression_engine::{CompressionType, get_compression_engine}; use crate::compression_engine::{get_compression_engine, CompressionType};
use crate::services::error::CoreError; use crate::services::error::CoreError;
use anyhow::anyhow; use anyhow::anyhow;
use std::io::Read; use std::io::Read;
@@ -91,8 +91,8 @@ impl CompressionService {
/// Opens a streaming reader for decompressing item content. /// Opens a streaming reader for decompressing item content.
/// ///
/// Due to Send requirements in async contexts, this loads the full content into a Cursor. /// Returns a boxed reader that implements `Read + Send` for streaming decompression.
/// Warning: For very large files, this consumes significant memory; consider alternatives for streaming without loading all data. /// This enables true streaming without loading the entire file into memory.
/// ///
/// # Arguments /// # Arguments
/// ///
@@ -129,13 +129,7 @@ impl CompressionService {
let reader = engine.open(item_path.clone()).map_err(|e| { let reader = engine.open(item_path.clone()).map_err(|e| {
CoreError::Other(anyhow!("Failed to open item file {:?}: {}", item_path, e)) CoreError::Other(anyhow!("Failed to open item file {:?}: {}", item_path, e))
})?; })?;
// Since we can't guarantee the reader implements Send, we need to wrap it Ok(reader)
// We'll read the content into a buffer and return a Cursor which is Send
// This is not ideal for large files, but it ensures Send is implemented
let mut content = Vec::new();
let mut temp_reader = reader;
temp_reader.read_to_end(&mut content)?;
Ok(Box::new(std::io::Cursor::new(content)))
} }
} }

9
src/services/data_service.rs
View File

@@ -1,13 +1,13 @@
use crate::common::status::StatusInfo; use crate::common::status::StatusInfo;
use crate::config::Settings; use crate::config::Settings;
use crate::db::{Item, Meta, Tag}; use crate::db::Item;
use crate::services::error::CoreError; use crate::services::error::CoreError;
use crate::services::types::{ItemWithContent, ItemWithMeta}; use crate::services::types::{ItemWithContent, ItemWithMeta};
use clap::Command; use clap::Command;
use rusqlite::Connection; use rusqlite::Connection;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::Read; use std::io::Read;
use std::path::PathBuf; use std::path::Path;
pub trait DataService { pub trait DataService {
type Error; type Error;
@@ -56,9 +56,8 @@ pub trait DataService {
fn generate_status( fn generate_status(
&self, &self,
cmd: &Command,
settings: &Settings, settings: &Settings,
data_path: &PathBuf, data_path: &Path,
db_path: &PathBuf, db_path: &Path,
) -> Result<StatusInfo, Self::Error>; ) -> Result<StatusInfo, Self::Error>;
} }

140
src/services/item_service.rs
View File

@@ -1,7 +1,7 @@
use crate::common::PIPESIZE; use crate::common::PIPESIZE;
use crate::compression_engine::{CompressionType, get_compression_engine}; use crate::compression_engine::{get_compression_engine, CompressionType};
use crate::config::Settings; use crate::config::Settings;
use crate::db::{self, Meta}; use crate::db::{self, Item, Meta};
use crate::filter_plugin; use crate::filter_plugin;
use crate::modes::common::settings_compression_type; use crate::modes::common::settings_compression_type;
use crate::services::compression_service::CompressionService; use crate::services::compression_service::CompressionService;
@@ -53,10 +53,7 @@ impl ItemService {
/// let service = ItemService::new(PathBuf::from("/data")); /// let service = ItemService::new(PathBuf::from("/data"));
/// ``` /// ```
pub fn new(data_path: PathBuf) -> Self { pub fn new(data_path: PathBuf) -> Self {
debug!( debug!("ITEM_SERVICE: Creating new ItemService with data_path: {data_path:?}");
"ITEM_SERVICE: Creating new ItemService with data_path: {:?}",
data_path
);
Self { Self {
data_path, data_path,
compression_service: CompressionService::new(), compression_service: CompressionService::new(),
@@ -90,9 +87,9 @@ impl ItemService {
/// assert_eq!(item_with_meta.item.id, Some(1)); /// assert_eq!(item_with_meta.item.id, Some(1));
/// ``` /// ```
pub fn get_item(&self, conn: &Connection, id: i64) -> Result<ItemWithMeta, CoreError> { pub fn get_item(&self, conn: &Connection, id: i64) -> Result<ItemWithMeta, CoreError> {
debug!("ITEM_SERVICE: Getting item with id: {}", id); debug!("ITEM_SERVICE: Getting item with id: {id}");
let item = db::get_item(conn, id)?.ok_or(CoreError::ItemNotFound(id))?; let item = db::get_item(conn, id)?.ok_or(CoreError::ItemNotFound(id))?;
debug!("ITEM_SERVICE: Found item: {:?}", item); debug!("ITEM_SERVICE: Found item: {item:?}");
let tags = db::get_item_tags(conn, &item)?; let tags = db::get_item_tags(conn, &item)?;
debug!("ITEM_SERVICE: Found {} tags for item {}", tags.len(), id); debug!("ITEM_SERVICE: Found {} tags for item {}", tags.len(), id);
let meta = db::get_item_meta(conn, &item)?; let meta = db::get_item_meta(conn, &item)?;
@@ -133,7 +130,7 @@ impl ItemService {
conn: &Connection, conn: &Connection,
id: i64, id: i64,
) -> Result<ItemWithContent, CoreError> { ) -> Result<ItemWithContent, CoreError> {
debug!("ITEM_SERVICE: Getting item content for id: {}", id); debug!("ITEM_SERVICE: Getting item content for id: {id}");
let item_with_meta = self.get_item(conn, id)?; let item_with_meta = self.get_item(conn, id)?;
let item_id = item_with_meta let item_id = item_with_meta
.item .item
@@ -142,14 +139,13 @@ impl ItemService {
if item_id <= 0 { if item_id <= 0 {
return Err(CoreError::InvalidInput(format!( return Err(CoreError::InvalidInput(format!(
"Invalid item ID: {}", "Invalid item ID: {item_id}"
item_id
))); )));
} }
let mut item_path = self.data_path.clone(); let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string()); item_path.push(item_id.to_string());
debug!("ITEM_SERVICE: Reading content from path: {:?}", item_path); debug!("ITEM_SERVICE: Reading content from path: {item_path:?}");
let content = self let content = self
.compression_service .compression_service
@@ -287,7 +283,7 @@ impl ItemService {
self.filter_service self.filter_service
.create_filter_chain(Some(&filter_str)) .create_filter_chain(Some(&filter_str))
.map_err(|e| { .map_err(|e| {
CoreError::InvalidInput(format!("Failed to create filter chain: {}", e)) CoreError::InvalidInput(format!("Failed to create filter chain: {e}"))
})? })?
} else { } else {
None None
@@ -333,8 +329,7 @@ impl ItemService {
if item_id <= 0 { if item_id <= 0 {
return Err(CoreError::InvalidInput(format!( return Err(CoreError::InvalidInput(format!(
"Invalid item ID: {}", "Invalid item ID: {item_id}"
item_id
))); )));
} }
@@ -361,6 +356,45 @@ impl ItemService {
Ok((filtered_reader, mime_type, is_binary)) Ok((filtered_reader, mime_type, is_binary))
} }
/// Like `get_item_content_info_streaming_with_chain` but accepts a pre-fetched `ItemWithMeta`
/// to avoid redundant DB queries.
pub fn get_item_content_info_streaming_with_item(
&self,
item_with_meta: ItemWithMeta,
filter_chain: Option<&filter_plugin::FilterChain>,
) -> Result<(Box<dyn Read + Send>, String, bool), CoreError> {
let item_id = item_with_meta
.item
.id
.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?;
if item_id <= 0 {
return Err(CoreError::InvalidInput(format!(
"Invalid item ID: {item_id}"
)));
}
let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string());
let reader = self
.compression_service
.stream_item_content(item_path.clone(), &item_with_meta.item.compression)?;
let filtered_reader = Box::new(FilteringReader::new(reader, filter_chain.cloned()));
let metadata = item_with_meta.meta_as_map();
let mime_type = metadata
.get("mime_type")
.map(|s| s.to_string())
.unwrap_or_else(|| "application/octet-stream".to_string());
let is_binary =
self.is_content_binary(item_path, &item_with_meta.item.compression, &metadata)?;
Ok((filtered_reader, mime_type, is_binary))
}
/// Finds an item by ID or tags/metadata criteria. /// Finds an item by ID or tags/metadata criteria.
/// ///
/// Supports lookup by ID, last item, or search by tags/metadata. /// Supports lookup by ID, last item, or search by tags/metadata.
@@ -393,10 +427,7 @@ impl ItemService {
tags: &[String], tags: &[String],
meta: &HashMap<String, String>, meta: &HashMap<String, String>,
) -> Result<ItemWithMeta, CoreError> { ) -> Result<ItemWithMeta, CoreError> {
debug!( debug!("ITEM_SERVICE: Finding item with ids: {ids:?}, tags: {tags:?}, meta: {meta:?}");
"ITEM_SERVICE: Finding item with ids: {:?}, tags: {:?}, meta: {:?}",
ids, tags, meta
);
let item_maybe = match (ids.is_empty(), tags.is_empty() && meta.is_empty()) { let item_maybe = match (ids.is_empty(), tags.is_empty() && meta.is_empty()) {
(false, _) => { (false, _) => {
debug!("ITEM_SERVICE: Finding by ID: {}", ids[0]); debug!("ITEM_SERVICE: Finding by ID: {}", ids[0]);
@@ -413,7 +444,7 @@ impl ItemService {
}; };
let item = item_maybe.ok_or(CoreError::ItemNotFoundGeneric)?; let item = item_maybe.ok_or(CoreError::ItemNotFoundGeneric)?;
debug!("ITEM_SERVICE: Found matching item: {:?}", item); debug!("ITEM_SERVICE: Found matching item: {item:?}");
// Get tags and meta directly instead of calling get_item which makes redundant queries // Get tags and meta directly instead of calling get_item which makes redundant queries
let item_id = item let item_id = item
@@ -464,10 +495,7 @@ impl ItemService {
tags: &[String], tags: &[String],
meta: &HashMap<String, String>, meta: &HashMap<String, String>,
) -> Result<Vec<ItemWithMeta>, CoreError> { ) -> Result<Vec<ItemWithMeta>, CoreError> {
debug!( debug!("ITEM_SERVICE: Listing items with tags: {tags:?}, meta: {meta:?}");
"ITEM_SERVICE: Listing items with tags: {:?}, meta: {:?}",
tags, meta
);
let items = db::get_items_matching(conn, &tags.to_vec(), meta)?; let items = db::get_items_matching(conn, &tags.to_vec(), meta)?;
debug!("ITEM_SERVICE: Found {} matching items", items.len()); debug!("ITEM_SERVICE: Found {} matching items", items.len());
@@ -532,16 +560,16 @@ impl ItemService {
/// item_service.delete_item(&mut conn, 1)?; /// item_service.delete_item(&mut conn, 1)?;
/// ``` /// ```
pub fn delete_item(&self, conn: &mut Connection, id: i64) -> Result<(), CoreError> { pub fn delete_item(&self, conn: &mut Connection, id: i64) -> Result<(), CoreError> {
debug!("ITEM_SERVICE: Deleting item with id: {}", id); debug!("ITEM_SERVICE: Deleting item with id: {id}");
if id <= 0 { if id <= 0 {
return Err(CoreError::InvalidInput(format!("Invalid item ID: {}", id))); return Err(CoreError::InvalidInput(format!("Invalid item ID: {id}")));
} }
let item = db::get_item(conn, id)?.ok_or(CoreError::ItemNotFound(id))?; let item = db::get_item(conn, id)?.ok_or(CoreError::ItemNotFound(id))?;
debug!("ITEM_SERVICE: Found item to delete: {:?}", item); debug!("ITEM_SERVICE: Found item to delete: {item:?}");
let mut item_path = self.data_path.clone(); let mut item_path = self.data_path.clone();
item_path.push(id.to_string()); item_path.push(id.to_string());
debug!("ITEM_SERVICE: Deleting file at path: {:?}", item_path); debug!("ITEM_SERVICE: Deleting file at path: {item_path:?}");
db::delete_item(conn, item)?; db::delete_item(conn, item)?;
fs::remove_file(&item_path).or_else(|e| { fs::remove_file(&item_path).or_else(|e| {
@@ -551,7 +579,7 @@ impl ItemService {
Err(e) Err(e)
} }
})?; })?;
debug!("ITEM_SERVICE: Successfully deleted item {}", id); debug!("ITEM_SERVICE: Successfully deleted item {id}");
Ok(()) Ok(())
} }
@@ -571,7 +599,7 @@ impl ItemService {
/// ///
/// # Returns /// # Returns
/// ///
/// * `Result<i64, CoreError>` - The ID of the new item. /// * `Result<Item, CoreError>` - The saved item with full details (id, size, compression, timestamp).
/// ///
/// # Errors /// # Errors
/// ///
@@ -582,7 +610,7 @@ impl ItemService {
/// ///
/// ``` /// ```
/// let reader = std::io::stdin(); /// let reader = std::io::stdin();
/// let id = item_service.save_item(reader, &mut cmd, &settings, &mut vec![], &mut conn)?; /// let item = item_service.save_item(reader, &mut cmd, &settings, &mut vec![], &mut conn)?;
/// ``` /// ```
pub fn save_item<R: Read>( pub fn save_item<R: Read>(
&self, &self,
@@ -591,18 +619,15 @@ impl ItemService {
settings: &Settings, settings: &Settings,
tags: &mut Vec<String>, tags: &mut Vec<String>,
conn: &mut Connection, conn: &mut Connection,
) -> Result<i64, CoreError> { ) -> Result<Item, CoreError> {
debug!("ITEM_SERVICE: Starting save_item with tags: {:?}", tags); debug!("ITEM_SERVICE: Starting save_item with tags: {tags:?}");
if tags.is_empty() { if tags.is_empty() {
tags.push("none".to_string()); tags.push("none".to_string());
debug!("ITEM_SERVICE: No tags provided, using default 'none' tag"); debug!("ITEM_SERVICE: No tags provided, using default 'none' tag");
} }
let compression_type = settings_compression_type(cmd, settings); let compression_type = settings_compression_type(cmd, settings);
debug!( debug!("ITEM_SERVICE: Using compression type: {compression_type:?}");
"ITEM_SERVICE: Using compression type: {:?}",
compression_type
);
let compression_engine = get_compression_engine(compression_type.clone())?; let compression_engine = get_compression_engine(compression_type.clone())?;
let item_id; let item_id;
@@ -610,9 +635,9 @@ impl ItemService {
{ {
item = db::create_item(conn, compression_type.clone())?; item = db::create_item(conn, compression_type.clone())?;
item_id = item.id.unwrap(); item_id = item.id.unwrap();
debug!("ITEM_SERVICE: Created new item with id: {}", item_id); debug!("ITEM_SERVICE: Created new item with id: {item_id}");
db::set_item_tags(conn, item.clone(), tags)?; db::set_item_tags(conn, item.clone(), tags)?;
debug!("ITEM_SERVICE: Set tags for item {}", item_id); debug!("ITEM_SERVICE: Set tags for item {item_id}");
let item_meta = self.meta_service.collect_initial_meta(); let item_meta = self.meta_service.collect_initial_meta();
debug!( debug!(
"ITEM_SERVICE: Collected {} initial meta entries", "ITEM_SERVICE: Collected {} initial meta entries",
@@ -643,7 +668,7 @@ impl ItemService {
let _ = std::io::stderr().flush(); let _ = std::io::stderr().flush();
} else { } else {
let mut t = std::io::stderr(); let mut t = std::io::stderr();
let _ = writeln!(t, "KEEP: New item: {} tags: {:?}", item_id, tags); let _ = writeln!(t, "KEEP: New item: {item_id} tags: {tags:?}");
} }
} }
@@ -654,7 +679,7 @@ impl ItemService {
let mut item_path = self.data_path.clone(); let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string()); item_path.push(item_id.to_string());
debug!("ITEM_SERVICE: Writing item to path: {:?}", item_path); debug!("ITEM_SERVICE: Writing item to path: {item_path:?}");
let mut item_out = compression_engine.create(item_path.clone())?; let mut item_out = compression_engine.create(item_path.clone())?;
@@ -673,7 +698,7 @@ impl ItemService {
self.meta_service self.meta_service
.process_chunk(&mut plugins, &buffer[..n], conn, item_id); .process_chunk(&mut plugins, &buffer[..n], conn, item_id);
} }
debug!("ITEM_SERVICE: Processed {} bytes total", total_bytes); debug!("ITEM_SERVICE: Processed {total_bytes} bytes total");
item_out.flush()?; item_out.flush()?;
drop(item_out); drop(item_out);
@@ -687,7 +712,7 @@ impl ItemService {
debug!("ITEM_SERVICE: Save completed successfully"); debug!("ITEM_SERVICE: Save completed successfully");
Ok(item_id) Ok(item)
} }
/// Saves pre-loaded content as a new item, typically from MCP (Machine-Common-Processing) sources. /// Saves pre-loaded content as a new item, typically from MCP (Machine-Common-Processing) sources.
@@ -744,7 +769,7 @@ impl ItemService {
{ {
item = db::create_item(conn, compression_type.clone())?; item = db::create_item(conn, compression_type.clone())?;
item_id = item.id.unwrap(); item_id = item.id.unwrap();
debug!("ITEM_SERVICE: Created MCP item with id: {}", item_id); debug!("ITEM_SERVICE: Created MCP item with id: {item_id}");
// Add tags // Add tags
for tag in tags { for tag in tags {
@@ -764,7 +789,7 @@ impl ItemService {
let mut item_path = self.data_path.clone(); let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string()); item_path.push(item_id.to_string());
debug!("ITEM_SERVICE: Writing MCP item to path: {:?}", item_path); debug!("ITEM_SERVICE: Writing MCP item to path: {item_path:?}");
let mut writer = compression_engine.create(item_path.clone())?; let mut writer = compression_engine.create(item_path.clone())?;
writer.write_all(content)?; writer.write_all(content)?;
@@ -827,6 +852,7 @@ struct FilteringReader<R: Read> {
filter_chain: Option<filter_plugin::FilterChain>, filter_chain: Option<filter_plugin::FilterChain>,
buffer: Vec<u8>, buffer: Vec<u8>,
buffer_pos: usize, buffer_pos: usize,
temp_buf: Vec<u8>,
} }
impl<R: Read> FilteringReader<R> { impl<R: Read> FilteringReader<R> {
@@ -854,6 +880,7 @@ impl<R: Read> FilteringReader<R> {
filter_chain, filter_chain,
buffer: Vec::new(), buffer: Vec::new(),
buffer_pos: 0, buffer_pos: 0,
temp_buf: vec![0; 8192],
} }
} }
} }
@@ -898,19 +925,22 @@ impl<R: Read> Read for FilteringReader<R> {
self.buffer.clear(); self.buffer.clear();
self.buffer_pos = 0; self.buffer_pos = 0;
// Read from the original reader into a temporary buffer // No filter chain — pass through directly, no intermediate buffer needed
let mut temp_buf = vec![0; buf.len()]; if self.filter_chain.is_none() {
let bytes_read = self.reader.read(&mut temp_buf)?; return self.reader.read(buf);
}
// Read from the original reader into the reusable temp buffer
let to_read = std::cmp::min(buf.len(), self.temp_buf.len());
let bytes_read = self.reader.read(&mut self.temp_buf[..to_read])?;
if bytes_read == 0 { if bytes_read == 0 {
return Ok(0); return Ok(0);
} }
// Process through the filter chain if it exists // Process through the filter chain
if let Some(ref mut chain) = self.filter_chain { if let Some(ref mut chain) = self.filter_chain {
// Use a cursor to read the input data let mut input_cursor = std::io::Cursor::new(&self.temp_buf[..bytes_read]);
let mut input_cursor = std::io::Cursor::new(&temp_buf[..bytes_read]);
// Write filtered output to our buffer
chain.filter(&mut input_cursor, &mut self.buffer)?; chain.filter(&mut input_cursor, &mut self.buffer)?;
if !self.buffer.is_empty() { if !self.buffer.is_empty() {
@@ -919,13 +949,11 @@ impl<R: Read> Read for FilteringReader<R> {
self.buffer_pos = bytes_to_copy; self.buffer_pos = bytes_to_copy;
Ok(bytes_to_copy) Ok(bytes_to_copy)
} else { } else {
// No data produced by filter, try reading more // No data produced by filter, signal to read more
Ok(0) Ok(0)
} }
} else { } else {
// No filter chain, just pass through unreachable!()
buf[..bytes_read].copy_from_slice(&temp_buf[..bytes_read]);
Ok(bytes_read)
} }
} }
} }

9
src/services/meta_service.rs
View File

@@ -16,16 +16,13 @@ impl MetaService {
pub fn get_plugins(&self, cmd: &mut Command, settings: &Settings) -> Vec<Box<dyn MetaPlugin>> { pub fn get_plugins(&self, cmd: &mut Command, settings: &Settings) -> Vec<Box<dyn MetaPlugin>> {
debug!("META_SERVICE: get_plugins called"); debug!("META_SERVICE: get_plugins called");
let meta_plugin_types: Vec<MetaPluginType> = settings_meta_plugin_types(cmd, settings); let meta_plugin_types: Vec<MetaPluginType> = settings_meta_plugin_types(cmd, settings);
debug!( debug!("META_SERVICE: Meta plugin types from settings: {meta_plugin_types:?}");
"META_SERVICE: Meta plugin types from settings: {:?}",
meta_plugin_types
);
// Create plugins with their configuration // Create plugins with their configuration
let meta_plugins: Vec<Box<dyn MetaPlugin>> = meta_plugin_types let meta_plugins: Vec<Box<dyn MetaPlugin>> = meta_plugin_types
.iter() .iter()
.map(|meta_plugin_type| { .map(|meta_plugin_type| {
debug!("META_SERVICE: Creating plugin: {:?}", meta_plugin_type); debug!("META_SERVICE: Creating plugin: {meta_plugin_type:?}");
// Get the plugin name using strum's Display implementation // Get the plugin name using strum's Display implementation
let plugin_name = meta_plugin_type.to_string(); let plugin_name = meta_plugin_type.to_string();
@@ -186,7 +183,7 @@ impl MetaService {
value: meta_data.value, value: meta_data.value,
}; };
if let Err(e) = crate::db::store_meta(conn, db_meta) { if let Err(e) = crate::db::store_meta(conn, db_meta) {
log::warn!("META_SERVICE: Failed to store metadata: {}", e); log::warn!("META_SERVICE: Failed to store metadata: {e}");
} }
} }
} }

2
src/services/mod.rs
View File

@@ -9,6 +9,7 @@ pub mod meta_service;
pub mod status_service; pub mod status_service;
pub mod sync_data_service; pub mod sync_data_service;
pub mod types; pub mod types;
pub mod utils;
pub use async_data_service::AsyncDataService; pub use async_data_service::AsyncDataService;
pub use async_item_service::AsyncItemService; pub use async_item_service::AsyncItemService;
@@ -21,3 +22,4 @@ pub use meta_service::MetaService;
pub use status_service::StatusService; pub use status_service::StatusService;
pub use sync_data_service::SyncDataService; pub use sync_data_service::SyncDataService;
pub use types::{ItemWithContent, ItemWithMeta}; pub use types::{ItemWithContent, ItemWithMeta};
pub use utils::{calc_byte_range, extract_tags, parse_comma_tags};

59
src/services/sync_data_service.rs
View File

@@ -11,7 +11,7 @@ use clap::Command;
use rusqlite::Connection; use rusqlite::Connection;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::{Cursor, Read}; use std::io::{Cursor, Read};
use std::path::PathBuf; use std::path::{Path, PathBuf};
pub struct SyncDataService { pub struct SyncDataService {
item_service: ItemService, item_service: ItemService,
@@ -39,7 +39,7 @@ impl SyncDataService {
} }
pub fn get_data_path(&self) -> &PathBuf { pub fn get_data_path(&self) -> &PathBuf {
&self.item_service.get_data_path() self.item_service.get_data_path()
} }
pub fn save_item<R: Read>( pub fn save_item<R: Read>(
@@ -49,11 +49,37 @@ impl SyncDataService {
settings: &Settings, settings: &Settings,
tags: &mut Vec<String>, tags: &mut Vec<String>,
conn: &mut Connection, conn: &mut Connection,
) -> Result<i64, CoreError> { ) -> Result<Item, CoreError> {
self.item_service self.item_service
.save_item(content, cmd, settings, tags, conn) .save_item(content, cmd, settings, tags, conn)
} }
pub fn save_item_with_reader<R: Read>(
&self,
conn: &mut Connection,
reader: &mut R,
tags: Vec<String>,
metadata: HashMap<String, String>,
) -> Result<ItemWithMeta, CoreError> {
let mut cmd = Command::new("keep");
let settings = &self.settings;
let mut tags = tags;
// Read content from reader
let mut content = Vec::new();
reader.read_to_end(&mut content)?;
let item = self.save_item(&*content, &mut cmd, settings, &mut tags, conn)?;
let item_id = item.id.unwrap();
// Set metadata
for (key, value) in metadata {
crate::db::add_meta(conn, item_id, &key, &value)?;
}
self.get_item(conn, item_id)
}
pub fn get_item(&self, conn: &mut Connection, id: i64) -> Result<ItemWithMeta, CoreError> { pub fn get_item(&self, conn: &mut Connection, id: i64) -> Result<ItemWithMeta, CoreError> {
self.item_service.get_item(conn, id) self.item_service.get_item(conn, id)
} }
@@ -131,16 +157,8 @@ impl DataService for SyncDataService {
tags.push("none".to_string()); tags.push("none".to_string());
} }
let item_id = self self.item_service
.item_service .save_item(content, cmd, settings, &mut tags, conn)
.save_item(content, cmd, settings, &mut tags, conn)?;
Ok(Item {
id: Some(item_id),
ts: chrono::Utc::now(),
size: Some(0),
compression: "lz4".to_string(),
})
} }
fn get(&self, conn: &mut Connection, id: i64) -> Result<ItemWithMeta, Self::Error> { fn get(&self, conn: &mut Connection, id: i64) -> Result<ItemWithMeta, Self::Error> {
@@ -202,12 +220,17 @@ impl DataService for SyncDataService {
fn generate_status( fn generate_status(
&self, &self,
_cmd: &Command,
settings: &Settings, settings: &Settings,
data_path: &PathBuf, data_path: &Path,
db_path: &PathBuf, db_path: &Path,
) -> Result<StatusInfo, Self::Error> { ) -> Result<StatusInfo, Self::Error> {
let mut cmd_mut = Command::new("keep"); let status_service = StatusService::new();
Ok(self.generate_status(&mut cmd_mut, settings, data_path.clone(), db_path.clone())) let mut cmd = Command::new("keep");
Ok(status_service.generate_status(
&mut cmd,
settings,
data_path.to_path_buf(),
db_path.to_path_buf(),
))
} }
} }

26
src/services/utils.rs Normal file
View File

@@ -0,0 +1,26 @@
use crate::services::types::ItemWithMeta;
use std::collections::HashMap;
pub fn extract_tags(items: &[ItemWithMeta]) -> Vec<String> {
items
.iter()
.flat_map(|item| item.tags.iter())
.map(|t| t.name.clone())
.collect()
}
pub fn parse_comma_tags(s: &str) -> Vec<String> {
s.split(',')
.map(|t| t.trim().to_string())
.filter(|t| !t.is_empty())
.collect()
}
pub fn calc_byte_range(content_len: u64, offset: u64, length: Option<u64>) -> (u64, u64) {
let start = std::cmp::min(offset, content_len);
let end = match length {
Some(len) if len > 0 => std::cmp::min(start + len, content_len),
_ => content_len,
};
(start, end)
}

2
src/tests/common/mod.rs
View File

@@ -3,6 +3,4 @@
#[cfg(test)] #[cfg(test)]
pub mod is_binary_tests; pub mod is_binary_tests;
#[cfg(test)] #[cfg(test)]
pub mod status_tests;
#[cfg(test)]
pub mod test_helpers; pub mod test_helpers;

11
src/tests/common/status_tests.rs
View File

@@ -1,11 +0,0 @@
#[cfg(test)]
mod tests {
// TODO: Add tests for common status functionality once implemented
// This would test functions related to status checking in the common module
#[test]
fn test_status_placeholder() {
// Placeholder test - to be implemented when status functionality is added
assert!(true);
}
}

43
src/tests/meta_plugin/digest_tests.rs
View File

@@ -1,47 +1,38 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::meta_plugin::MetaPlugin; use crate::meta_plugin::MetaPlugin;
use crate::meta_plugin::digest::*; use crate::meta_plugin::digest::DigestMetaPlugin;
use std::io::Write;
#[test] #[test]
fn test_digest_sha256_meta_plugin() { fn test_digest_meta_plugin() {
let mut plugin = DigestSha256MetaPlugin::new(); let plugin = DigestMetaPlugin::new(None, None);
assert_eq!(plugin.meta_name(), "digest_sha256"); assert_eq!(
plugin.meta_type(),
crate::meta_plugin::MetaPluginType::Digest
);
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work
let writer_result = plugin.create();
assert!(writer_result.is_ok());
// Writing some data
let mut writer = writer_result.unwrap();
let write_result = writer.write(b"test data");
assert!(write_result.is_ok());
} }
#[test] #[test]
fn test_read_time_meta_plugin() { fn test_read_time_meta_plugin() {
let mut plugin = ReadTimeMetaPlugin::new(); let plugin = crate::meta_plugin::ReadTimeMetaPlugin::new(None, None);
assert_eq!(plugin.meta_name(), "read_time"); assert_eq!(
plugin.meta_type(),
crate::meta_plugin::MetaPluginType::ReadTime
);
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work
let writer_result = plugin.create();
assert!(writer_result.is_ok());
} }
#[test] #[test]
fn test_read_rate_meta_plugin() { fn test_read_rate_meta_plugin() {
let mut plugin = ReadRateMetaPlugin::new(); let plugin = crate::meta_plugin::ReadRateMetaPlugin::new(None, None);
assert_eq!(plugin.meta_name(), "read_rate"); assert_eq!(
plugin.meta_type(),
crate::meta_plugin::MetaPluginType::ReadRate
);
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work
let writer_result = plugin.create();
assert!(writer_result.is_ok());
} }
} }

4
src/tests/meta_plugin/mod.rs
View File

@@ -2,7 +2,3 @@
#[cfg(test)] #[cfg(test)]
pub mod digest_tests; pub mod digest_tests;
#[cfg(test)]
pub mod program_tests;
#[cfg(test)]
pub mod system_tests;

40
src/tests/meta_plugin/program_tests.rs
View File

@@ -1,40 +0,0 @@
#[cfg(test)]
mod tests {
use crate::meta_plugin::MetaPlugin;
use crate::meta_plugin::program::MetaPluginProgram;
#[test]
fn test_meta_plugin_program_creation() {
let mut plugin =
MetaPluginProgram::new("echo", vec!["test"], "test_plugin".to_string(), false);
assert_eq!(plugin.meta_name(), "test_plugin");
// If echo is available, it should be supported
// We don't assert on is_supported() as it depends on system availability
}
#[test]
fn test_meta_plugin_program_create_writer() {
let plugin = MetaPluginProgram::new("cat", vec![], "cat_plugin".to_string(), false);
// Creating a writer should work for valid programs
let result = plugin.create();
// We don't assert success as it depends on system availability
// but we ensure it doesn't panic
let _ = result;
}
#[test]
fn test_meta_plugin_program_unsupported() {
let plugin = MetaPluginProgram::new(
"nonexistent_program_xyz",
vec![],
"bad_plugin".to_string(),
false,
);
// An unsupported plugin should report as such
// Note: This might still be supported if the program exists
let _ = plugin.is_supported();
}
}

57
src/tests/meta_plugin/system_tests.rs
View File

@@ -1,57 +0,0 @@
#[cfg(test)]
mod tests {
use crate::meta_plugin::MetaPlugin;
use crate::meta_plugin::system::*;
#[test]
fn test_cwd_meta_plugin() {
let mut plugin = CwdMetaPlugin::new();
assert_eq!(plugin.meta_name(), "cwd");
assert!(plugin.is_internal());
// Creating a writer should work
let writer_result = plugin.create();
assert!(writer_result.is_ok());
// Finalize should return current working directory
let result = plugin.finalize();
assert!(result.is_ok());
}
#[test]
fn test_binary_meta_plugin() {
let mut plugin = BinaryMetaPlugin::new();
assert_eq!(plugin.meta_name(), "binary");
assert!(plugin.is_internal());
// Creating a writer should work
let writer_result = plugin.create();
assert!(writer_result.is_ok());
}
#[test]
fn test_uid_meta_plugin() {
let mut plugin = UidMetaPlugin::new();
assert_eq!(plugin.meta_name(), "uid");
assert!(plugin.is_internal());
// Creating a writer should work
let writer_result = plugin.create();
assert!(writer_result.is_ok());
}
#[test]
fn test_user_meta_plugin() {
let mut plugin = UserMetaPlugin::new();
assert_eq!(plugin.meta_name(), "user");
assert!(plugin.is_internal());
// Creating a writer should work
let writer_result = plugin.create();
assert!(writer_result.is_ok());
}
}

14
src/tests/server/api_tests.rs
View File

@@ -1,14 +0,0 @@
#[cfg(test)]
mod tests {
#[test]
fn test_api_basic_setup() {
// Placeholder for API tests
assert!(true);
}
#[test]
fn test_api_endpoints() {
// Placeholder for testing server API endpoints
assert!(true);
}
}

2
src/tests/server/auth_tests.rs
View File

@@ -1,4 +1,4 @@
#[cfg(test)] #[cfg(all(test, feature = "server"))]
mod tests { mod tests {
use crate::modes::server::common::check_auth; use crate::modes::server::common::check_auth;
use axum::http::{HeaderMap, HeaderValue}; use axum::http::{HeaderMap, HeaderValue};

2
src/tests/server/mod.rs
View File

@@ -1,6 +1,4 @@
// Server tests module // Server tests module
#[cfg(test)]
pub mod api_tests;
#[cfg(test)] #[cfg(test)]
pub mod auth_tests; pub mod auth_tests;