diff --git a/src/core/compression_service.rs b/src/core/compression_service.rs index e69de29..49f7e05 100644 --- a/src/core/compression_service.rs +++ b/src/core/compression_service.rs @@ -0,0 +1,33 @@ +use crate::compression_engine::{get_compression_engine, CompressionType}; +use crate::core::error::CoreError; +use std::io::Read; +use std::path::PathBuf; +use std::str::FromStr; +use anyhow::anyhow; + +pub struct CompressionService; + +impl CompressionService { + pub fn new() -> Self { + Self + } + + pub fn get_item_content(&self, item_path: PathBuf, compression: &str) -> Result, CoreError> { + let compression_type = CompressionType::from_str(compression) + .map_err(|e| CoreError::Compression(e.to_string()))?; + let engine = get_compression_engine(compression_type) + .map_err(|e| CoreError::Other(anyhow!(e.to_string())))?; + + let mut reader = engine.open(item_path.clone()) + .map_err(|e| CoreError::Other(anyhow!("Failed to open item file {:?}: {}", item_path, e)))?; + let mut content = Vec::new(); + reader.read_to_end(&mut content)?; + Ok(content) + } +} + +impl Default for CompressionService { + fn default() -> Self { + Self::new() + } +} diff --git a/src/core/error.rs b/src/core/error.rs index e69de29..cd3950d 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -0,0 +1,34 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CoreError { + #[error("Database error: {0}")] + Database(#[from] rusqlite::Error), + + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + + #[error("Item not found with id {0}")] + ItemNotFound(i64), + + #[error("Item not found")] + ItemNotFoundGeneric, + + #[error("Invalid input: {0}")] + InvalidInput(String), + + #[error("Configuration error: {0}")] + Config(String), + + #[error("Compression error: {0}")] + Compression(String), + + #[error(transparent)] + Other(#[from] anyhow::Error), +} + +impl From for CoreError { + fn from(err: rusqlite_migration::Error) -> Self { + CoreError::Database(rusqlite::Error::from(err)) + } +} diff --git a/src/core/item_service.rs b/src/core/item_service.rs index e69de29..ceec086 100644 --- a/src/core/item_service.rs +++ b/src/core/item_service.rs @@ -0,0 +1,197 @@ +use crate::config::Settings; +use crate::core::compression_service::CompressionService; +use crate::core::error::CoreError; +use crate::core::meta_service::MetaService; +use crate::core::types::{ItemWithContent, ItemWithMeta}; +use crate::db::{self, Meta}; +use crate::compression_engine::{get_compression_engine, CompressionType}; +use crate::modes::common::settings_compression_type; +use clap::Command; +use log::debug; +use rusqlite::Connection; +use std::collections::HashMap; +use std::fs; +use std::io::{IsTerminal, Read, Write}; +use std::path::PathBuf; + +pub struct ItemService { + data_path: PathBuf, + compression_service: CompressionService, + meta_service: MetaService, +} + +impl ItemService { + pub fn new(data_path: PathBuf) -> Self { + Self { + data_path, + compression_service: CompressionService::new(), + meta_service: MetaService::new(), + } + } + + pub fn get_item(&self, conn: &Connection, id: i64) -> Result { + let item = db::get_item(conn, id)?.ok_or(CoreError::ItemNotFound(id))?; + let tags = db::get_item_tags(conn, &item)?; + let meta = db::get_item_meta(conn, &item)?; + Ok(ItemWithMeta { item, tags, meta }) + } + + pub fn get_item_content(&self, conn: &Connection, id: i64) -> Result { + let item_with_meta = self.get_item(conn, id)?; + let item_id = item_with_meta.item.id.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?; + + if item_id <= 0 { + return Err(CoreError::InvalidInput(format!("Invalid item ID: {}", item_id))); + } + + let mut item_path = self.data_path.clone(); + item_path.push(item_id.to_string()); + + let content = self + .compression_service + .get_item_content(item_path, &item_with_meta.item.compression)?; + + Ok(ItemWithContent { + item_with_meta, + content, + }) + } + + pub fn find_item(&self, conn: &Connection, ids: &[i64], tags: &[String], meta: &HashMap) -> Result { + let item_maybe = match (ids.is_empty(), tags.is_empty() && meta.is_empty()) { + (false, _) => db::get_item(conn, ids[0])?, + (true, true) => db::get_item_last(conn)?, + (true, false) => db::get_item_matching(conn, &tags.to_vec(), meta)? + }; + + let item = item_maybe.ok_or(CoreError::ItemNotFoundGeneric)?; + let item_id = item.id.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?; + self.get_item(conn, item_id) + } + + pub fn list_items(&self, conn: &Connection, tags: &[String], meta: &HashMap) -> Result, CoreError> { + let items = db::get_items_matching(conn, &tags.to_vec(), meta)?; + + let item_ids: Vec = items.iter().filter_map(|item| item.id).collect(); + if item_ids.is_empty() { + return Ok(Vec::new()); + } + + let tags_map = db::get_tags_for_items(conn, &item_ids)?; + let meta_map_db = db::get_meta_for_items(conn, &item_ids)?; + + let mut result = Vec::new(); + for item in items { + let item_id = item.id.unwrap(); + let tags = tags_map.get(&item_id).cloned().unwrap_or_default(); + let meta_hm = meta_map_db.get(&item_id).cloned().unwrap_or_default(); + let meta = meta_hm.into_iter().map(|(name, value)| Meta { id: item_id, name, value }).collect(); + + result.push(ItemWithMeta { item, tags, meta }); + } + + Ok(result) + } + + pub fn delete_item(&self, conn: &mut Connection, id: i64) -> Result<(), CoreError> { + if id <= 0 { + return Err(CoreError::InvalidInput(format!("Invalid item ID: {}", id))); + } + let item = db::get_item(conn, id)?.ok_or(CoreError::ItemNotFound(id))?; + + let mut item_path = self.data_path.clone(); + item_path.push(id.to_string()); + + let tx = conn.transaction()?; + db::delete_item(&tx, item)?; + fs::remove_file(&item_path).or_else(|e| if e.kind() == std::io::ErrorKind::NotFound { Ok(()) } else { Err(e) })?; + tx.commit()?; + + Ok(()) + } + + pub fn save_item( + &self, + mut input: R, + cmd: &mut Command, + settings: &Settings, + tags: &mut Vec, + conn: &mut Connection, + ) -> Result { + if tags.is_empty() { + tags.push("none".to_string()); + } + + let compression_type = settings_compression_type(cmd, settings); + let compression_engine = get_compression_engine(compression_type.clone())?; + + let tx = conn.transaction()?; + + let item_id; + let mut item; + { + item = db::create_item(&tx, compression_type.clone())?; + item_id = item.id.unwrap(); + db::set_item_tags(&tx, item.clone(), tags)?; + let item_meta = self.meta_service.collect_initial_meta(); + for (k, v) in item_meta.iter() { + db::add_meta(&tx, item_id, k, v)?; + } + } + + let mut plugins = self.meta_service.get_plugins(cmd, settings); + self.meta_service.initialize_plugins(&mut plugins, &tx, item_id); + + let mut item_path = self.data_path.clone(); + item_path.push(item_id.to_string()); + + let mut item_out = compression_engine.create(item_path.clone())?; + + let mut buffer = [0; 8192]; + let mut total_bytes = 0; + + loop { + let n = input.read(&mut buffer)?; + if n == 0 { break; } + + total_bytes += n as i64; + item_out.write_all(&buffer[..n])?; + self.meta_service.process_chunk(&mut plugins, &buffer[..n], &tx); + } + + item_out.flush()?; + drop(item_out); + + self.meta_service.finalize_plugins(&mut plugins, &tx); + + item.size = Some(total_bytes); + db::update_item(&tx, item.clone())?; + + tx.commit()?; + + if !settings.quiet { + if std::io::stderr().is_terminal() { + let mut t = term::stderr().unwrap(); + let _ = t.reset(); + let _ = t.attr(term::Attr::Bold); + let _ = write!(t, "KEEP:"); + let _ = t.reset(); + let _ = write!(t, " New item "); + let _ = t.attr(term::Attr::Bold); + let _ = write!(t, "{item_id}"); + let _ = t.reset(); + let _ = write!(t, " tags: "); + let _ = t.attr(term::Attr::Bold); + let _ = write!(t, "{}", tags.join(" ")); + let _ = t.reset(); + let _ = writeln!(t); + let _ = std::io::stderr().flush(); + } else { + let mut t = std::io::stderr(); + let _ = writeln!(t, "KEEP: New item: {} tags: {:?}", item_id, tags); + } + } + + self.get_item(conn, item_id) + } +} diff --git a/src/core/meta_service.rs b/src/core/meta_service.rs index e69de29..a7562b0 100644 --- a/src/core/meta_service.rs +++ b/src/core/meta_service.rs @@ -0,0 +1,125 @@ +use crate::config::Settings; +use crate::meta_plugin::{get_meta_plugin, MetaPlugin, MetaPluginType}; +use crate::modes::common::{settings_digest_type, settings_meta_plugin_types}; +use clap::Command; +use log::debug; +use rusqlite::Connection; +use std::collections::HashMap; + +pub struct MetaService; + +impl MetaService { + pub fn new() -> Self { + Self + } + + pub fn get_plugins(&self, cmd: &mut Command, settings: &Settings) -> Vec> { + let mut meta_plugin_types: Vec = settings_meta_plugin_types(cmd, settings); + + let digest_type = settings_digest_type(cmd, settings); + let digest_meta_plugin_type = match digest_type { + MetaPluginType::DigestSha256 => Some(MetaPluginType::DigestSha256), + MetaPluginType::DigestMd5 => Some(MetaPluginType::DigestMd5), + _ => None, + }; + + if let Some(digest_plugin_type) = digest_meta_plugin_type { + if !meta_plugin_types.contains(&digest_plugin_type) { + meta_plugin_types.push(digest_plugin_type); + } + } + + debug!("MetaService: Meta plugin types: {:?}", meta_plugin_types); + + let mut meta_plugins: Vec> = meta_plugin_types + .iter() + .map(|meta_plugin_type| get_meta_plugin(meta_plugin_type.clone())) + .collect(); + + if let Some(meta_plugin_configs) = &settings.meta_plugins { + for meta_plugin in meta_plugins.iter_mut() { + let plugin_name = meta_plugin.meta_name(); + if let Some(config) = meta_plugin_configs.iter().find(|c| c.name == plugin_name) { + let mut configured_outputs = meta_plugin.outputs().clone(); + for (key, value) in &config.outputs { + configured_outputs.insert(key.clone(), serde_yaml::Value::String(value.clone())); + } + + let mut configured_options = meta_plugin.default_options(); + for (key, value) in &config.options { + configured_options.insert(key.clone(), value.clone()); + } + + if let Err(e) = meta_plugin.configure_outputs(&configured_outputs) { + eprintln!("Warning: Failed to configure outputs for meta plugin '{}': {}", plugin_name, e); + } + + if let Err(e) = meta_plugin.configure_options(&configured_options) { + eprintln! ( + "Warning: Failed to configure options for meta plugin '{}': {}", + plugin_name, e + ); + } + } + } + } + + let original_len = meta_plugins.len(); + meta_plugins.retain(|meta_plugin| meta_plugin.is_supported()); + if meta_plugins.len() < original_len { + // This is not perfect as it doesn't say which one, but avoids complex logic from save.rs + eprintln!("Warning: Some meta plugins are enabled but not supported on this system"); + } + + meta_plugins + } + + pub fn initialize_plugins( + &self, + plugins: &mut [Box], + conn: &Connection, + item_id: i64, + ) { + for meta_plugin in plugins.iter_mut() { + if let Err(e) = meta_plugin.initialize(conn, item_id) { + eprintln!("Warning: Failed to initialize meta plugin: {}", e); + } + } + } + + pub fn process_chunk( + &self, + plugins: &mut [Box], + chunk: &[u8], + conn: &Connection, + ) { + for meta_plugin in plugins.iter_mut() { + meta_plugin.update(chunk, conn); + } + } + + pub fn finalize_plugins(&self, plugins: &mut [Box], conn: &Connection) { + for meta_plugin in plugins.iter_mut() { + if let Err(e) = meta_plugin.finalize(conn) { + eprintln!("Warning: Failed to finalize meta plugin: {}", e); + } + } + } + + pub fn collect_initial_meta(&self) -> HashMap { + let mut item_meta: HashMap = crate::modes::common::get_meta_from_env(); + + if let Ok(hostname) = gethostname::gethostname().into_string() { + if !item_meta.contains_key("hostname") { + item_meta.insert("hostname".to_string(), hostname); + } + } + item_meta + } +} + +impl Default for MetaService { + fn default() -> Self { + Self::new() + } +} diff --git a/src/core/mod.rs b/src/core/mod.rs index e69de29..9bedb73 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -0,0 +1,5 @@ +pub mod compression_service; +pub mod error; +pub mod item_service; +pub mod meta_service; +pub mod types; diff --git a/src/core/types.rs b/src/core/types.rs index e69de29..b6b981d 100644 --- a/src/core/types.rs +++ b/src/core/types.rs @@ -0,0 +1,22 @@ +use crate::db::{Item, Meta, Tag}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ItemWithMeta { + pub item: Item, + pub tags: Vec, + pub meta: Vec, +} + +impl ItemWithMeta { + pub fn meta_as_map(&self) -> HashMap { + self.meta.iter().cloned().map(|m| (m.name, m.value)).collect() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ItemWithContent { + pub item_with_meta: ItemWithMeta, + pub content: Vec, +} diff --git a/src/modes/delete.rs b/src/modes/delete.rs index fe4f290..55444ba 100644 --- a/src/modes/delete.rs +++ b/src/modes/delete.rs @@ -1,12 +1,11 @@ -use anyhow::{Context, Result, anyhow}; -use std::fs; +use anyhow::{anyhow, Result}; use std::path::PathBuf; -use crate::db; use crate::config; -use clap::Command; +use crate::core::item_service::ItemService; use clap::error::ErrorKind; -use log::{debug, warn}; +use clap::Command; +use log::warn; use rusqlite::Connection; pub fn mode_delete( @@ -32,23 +31,17 @@ pub fn mode_delete( .exit(); } + let item_service = ItemService::new(data_path); + for item_id in ids.iter() { - if let Some(item) = db::get_item(conn, *item_id)? { - debug!("MAIN: Found item {:?}", item); - db::delete_item(conn, item)?; - - // Validate that item ID is positive to prevent path traversal issues - if *item_id <= 0 { - return Err(anyhow!("Invalid item ID: {}", item_id)); - } - - let mut item_path = data_path.clone(); - item_path.push(item_id.to_string()); - - fs::remove_file(&item_path) - .context(anyhow!("Unable to remove item file {:?}", item_path))?; - } else { - warn!("Unable to find item {item_id} in database"); + match item_service.delete_item(conn, *item_id) { + Ok(_) => {} + Err(e) => match e { + crate::core::error::CoreError::ItemNotFound(_) => { + warn!("Unable to find item {item_id} in database"); + } + _ => return Err(anyhow!(e).context(format!("Failed to delete item {}", item_id))), + }, } } diff --git a/src/modes/diff.rs b/src/modes/diff.rs index d7c5f64..cf63c84 100644 --- a/src/modes/diff.rs +++ b/src/modes/diff.rs @@ -22,6 +22,8 @@ fn validate_diff_args(cmd: &mut Command, ids: &Vec, tags: &Vec) { } } +use crate::core::item_service::ItemService; + fn fetch_and_validate_items( conn: &mut rusqlite::Connection, ids: &Vec, @@ -37,16 +39,20 @@ fn fetch_and_validate_items( let item_a_id = item_a.id.ok_or_else(|| anyhow!("Item A missing ID"))?; let item_b_id = item_b.id.ok_or_else(|| anyhow!("Item B missing ID"))?; - + // Validate that item IDs are positive to prevent path traversal issues if item_a_id <= 0 || item_b_id <= 0 { - return Err(anyhow::anyhow!("Invalid item ID: {} or {}", item_a_id, item_b_id)); + return Err(anyhow::anyhow!( + "Invalid item ID: {} or {}", + item_a_id, + item_b_id + )); } Ok((item_a, item_b)) } -fn get_item_tags(conn: &mut rusqlite::Connection, item: &crate::db::Item) -> Result, anyhow::Error> { +fn get_item_tags(conn: &mut rusqlite::Connection, item: &crate::db::Item) -> Result> { let tags: Vec = crate::db::get_item_tags(conn, item)? .into_iter() .map(|x| x.name) diff --git a/src/modes/get.rs b/src/modes/get.rs index a86df32..8102d7a 100644 --- a/src/modes/get.rs +++ b/src/modes/get.rs @@ -1,13 +1,12 @@ -use anyhow::anyhow; -use std::io::{Read, Write}; +use anyhow::{anyhow, Result}; +use std::io::{Write}; -use crate::compression_engine::{CompressionType, get_compression_engine}; use crate::common::is_binary::is_binary; use crate::config; +use crate::core::item_service::ItemService; use clap::Command; -use std::path::PathBuf; -use std::str::FromStr; use is_terminal::IsTerminal; +use std::path::PathBuf; pub fn mode_get( cmd: &mut Command, @@ -16,7 +15,7 @@ pub fn mode_get( tags: &mut Vec, conn: &mut rusqlite::Connection, data_path: PathBuf, -) -> anyhow::Result<()> { +) -> Result<()> { if !ids.is_empty() && !tags.is_empty() { cmd.error(clap::error::ErrorKind::InvalidValue, "Both ID and tags given, you must supply exactly one ID or at least one tag when using --get").exit(); } else if ids.len() > 1 { @@ -32,70 +31,39 @@ pub fn mode_get( } } - let item_maybe = match tags.is_empty() && meta.is_empty() { - true => match ids.iter().next() { - Some(item_id) => crate::db::get_item(conn, *item_id)?, - None => crate::db::get_item_last(conn)?, - }, - false => crate::db::get_item_matching(conn, tags, &meta)?, - }; + let item_service = ItemService::new(data_path); + let item_with_content = + item_service.find_item(conn, ids, tags, &meta) + .and_then(|item_with_meta| { + let item_id = item_with_meta.item.id.unwrap(); + item_service.get_item_content(conn, item_id) + }) + .map_err(|e| anyhow!("Unable to find matching item in database: {}", e))?; - if let Some(item) = item_maybe { - let item_id = item.id.ok_or_else(|| anyhow!("Item missing ID"))?; - // Validate that item ID is positive to prevent path traversal issues - if item_id <= 0 { - return Err(anyhow!("Invalid item ID: {}", item_id)); - } + let content = &item_with_content.content; - let mut item_path = data_path.clone(); - item_path.push(item_id.to_string()); + // Determine if we should detect binary data + let mut detect_binary = !settings.force && std::io::stdout().is_terminal(); - // Determine if we should detect binary data - let mut detect_binary = !settings.force && std::io::stdout().is_terminal(); - - // If we're detecting binary and there's binary metadata, check it - if detect_binary { - let item_meta = crate::db::get_item_meta(conn, &item)?; - let binary_meta = item_meta.into_iter().find(|meta| meta.name == "binary"); - if let Some(binary_meta) = binary_meta { - if binary_meta.value == "false" { - // If metadata says it's not binary, don't detect - detect_binary = false; - } else if binary_meta.value == "true" { - // If metadata says it's binary, error immediately - return Err(anyhow!("Refusing to output binary data to TTY, use --force to override")); - } + if detect_binary { + let meta_map = item_with_content.item_with_meta.meta_as_map(); + if let Some(binary_val) = meta_map.get("binary") { + if binary_val == "false" { + detect_binary = false; + } else if binary_val == "true" { + return Err(anyhow!( + "Refusing to output binary data to TTY, use --force to override" + )); } } - - let compression_type = CompressionType::from_str(&item.compression)?; - let compression_engine = get_compression_engine(compression_type)?; - - // If we need to detect binary, read first 4KB and check - if detect_binary { - // Open the file through compression engine to read first 4KB - let mut reader = compression_engine.open(item_path.clone())?; - let mut buffer = [0u8; 4096]; - let bytes_read = reader.read(&mut buffer)?; - - // Check if this data is binary - if is_binary(&buffer[..bytes_read]) { - return Err(anyhow!("Refusing to output binary data to TTY, use --force to override")); - } - - // If not binary, output the data we've read - std::io::stdout().write_all(&buffer[..bytes_read])?; - - // Continue reading and outputting the rest of the data - let mut stdout = std::io::stdout(); - std::io::copy(&mut reader, &mut stdout)?; - } else { - // No binary detection needed, just output the data - compression_engine.cat(item_path.clone())?; - } - - Ok(()) - } else { - Err(anyhow!("Unable to find matching item in database")) } + + if detect_binary && is_binary(content) { + return Err(anyhow!( + "Refusing to output binary data to TTY, use --force to override" + )); + } + + std::io::stdout().write_all(content)?; + Ok(()) } diff --git a/src/modes/list.rs b/src/modes/list.rs index f533882..b17d3aa 100644 --- a/src/modes/list.rs +++ b/src/modes/list.rs @@ -1,16 +1,15 @@ -use crate::db::{get_items, get_items_matching}; +use crate::config; +use crate::core::item_service::ItemService; +use crate::core::types::ItemWithMeta; use crate::modes::common::ColumnType; use crate::modes::common::{size_column, string_column, OutputFormat}; -use crate::config; +use anyhow::{anyhow, Result}; +use prettytable::format::Alignment; +use prettytable::{color, row, Attr, Cell, Row, Table}; use serde::{Deserialize, Serialize}; use serde_json; use serde_yaml; -use anyhow::anyhow; -use log::debug; -use prettytable::color; -use prettytable::row; -use prettytable::format::Alignment; -use prettytable::{Attr, Cell, Row, Table}; +use std::str::FromStr; #[derive(Serialize, Deserialize)] struct ListItem { @@ -33,7 +32,7 @@ pub fn mode_list( tags: &Vec, conn: &mut rusqlite::Connection, data_path: std::path::PathBuf, -) -> anyhow::Result<()> { +) -> Result<()> { if !ids.is_empty() { cmd.error( clap::error::ErrorKind::InvalidValue, @@ -51,34 +50,13 @@ pub fn mode_list( } } - let items = match tags.is_empty() && meta.is_empty() { - true => get_items(conn)?, - false => get_items_matching(conn, tags, &meta)?, - }; - - debug!("MAIN: Items: {:?}", items); - - // Collect all item IDs for batch queries - let item_ids: Vec = items.iter().map(|item| item.id.unwrap()).collect(); - - // Fetch all tags for all items in a single query - let all_tags = crate::db::get_tags_for_items(conn, &item_ids)?; - let mut tags_by_item: std::collections::HashMap> = - std::collections::HashMap::new(); - - // Convert Tag structs to just names - for (item_id, tags) in all_tags { - let tag_names: Vec = tags.into_iter().map(|tag| tag.name).collect(); - tags_by_item.insert(item_id, tag_names); - } - - // Fetch all metadata for all items in a single query - let meta_by_item = crate::db::get_meta_for_items(conn, &item_ids)?; + let item_service = ItemService::new(data_path.clone()); + let items_with_meta = item_service.list_items(conn, tags, &meta)?; let output_format = crate::modes::common::settings_output_format(settings); if output_format != OutputFormat::Table { - return show_list_structured(items, tags_by_item, meta_by_item, data_path, settings, output_format); + return show_list_structured(items_with_meta, data_path, settings, output_format); } let mut table = Table::new(); @@ -87,18 +65,16 @@ pub fn mode_list( let mut title_row = row!(); for column in &settings.list_format { - let _column_type = ColumnType::from_str(&column.name) - .map_err(|_| anyhow!("Unknown column {:?}", column.name))?; - title_row.add_cell(Cell::new(&column.label).with_style(Attr::Bold)); } table.set_titles(title_row); - for item in items { - let item_id = item.id.unwrap(); - let tags = tags_by_item.get(&item_id).unwrap(); - let meta = meta_by_item.get(&item_id).unwrap(); + for item_with_meta in items_with_meta { + let item = item_with_meta.item; + let tags: Vec = item_with_meta.tags.into_iter().map(|t| t.name).collect(); + let meta = item_with_meta.meta_as_map(); + let mut item_path = data_path.clone(); item_path.push(item.id.unwrap().to_string()); @@ -107,11 +83,11 @@ pub fn mode_list( for column in &settings.list_format { let column_type = ColumnType::from_str(&column.name) .unwrap_or_else(|_| panic!("Unknown column {:?}", column.name)); - + let mut meta_name: Option<&str> = None; let column_width = 0; // We're not supporting width in the new format - if column_type == ColumnType::Meta { + if let ColumnType::Meta = column_type { let parts: Vec<&str> = column.name.split(':').collect(); if parts.len() > 1 { meta_name = Some(parts[1]); @@ -120,13 +96,18 @@ pub fn mode_list( let cell = match column_type { ColumnType::Id => { - let mut cell = Cell::new(&string_column(item.id.unwrap_or(0).to_string(), column_width)); + let mut cell = + Cell::new(&string_column(item.id.unwrap_or(0).to_string(), column_width)); match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); + } } cell - }, + } ColumnType::Time => { let mut cell = Cell::new(&string_column( item.ts @@ -136,69 +117,96 @@ pub fn mode_list( column_width, )); match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); + } } cell - }, - ColumnType::Size => { - let cell = match item.size { - Some(size) => { - let mut cell = Cell::new(&size_column(size as u64, settings.human_readable, column_width)); - match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + } + ColumnType::Size => match item.size { + Some(size) => { + let mut cell = Cell::new(&size_column( + size as u64, + settings.human_readable, + column_width, + )); + match column.align { + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); } - cell - }, - None => { - let mut cell = match item_path.metadata() { - Ok(_) => Cell::new("Unknown") - .with_style(Attr::ForegroundColor(color::YELLOW)) - .with_style(Attr::Bold), - Err(_) => Cell::new("Missing") - .with_style(Attr::ForegroundColor(color::RED)) - .with_style(Attr::Bold), - }; - match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); } - cell - }, - }; - cell + } + cell + } + None => { + let mut cell = match item_path.metadata() { + Ok(_) => Cell::new("Unknown") + .with_style(Attr::ForegroundColor(color::YELLOW)) + .with_style(Attr::Bold), + Err(_) => Cell::new("Missing") + .with_style(Attr::ForegroundColor(color::RED)) + .with_style(Attr::Bold), + }; + match column.align { + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); + } + } + cell + } }, ColumnType::Compression => { - let mut cell = Cell::new(&string_column(item.compression.to_string(), column_width)); + let mut cell = + Cell::new(&string_column(item.compression.to_string(), column_width)); match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); + } } cell - }, - ColumnType::FileSize => { - let cell = match item_path.metadata() { - Ok(metadata) => { - let mut cell = Cell::new(&size_column(metadata.len(), settings.human_readable, column_width)); - match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + } + ColumnType::FileSize => match item_path.metadata() { + Ok(metadata) => { + let mut cell = Cell::new(&size_column( + metadata.len(), + settings.human_readable, + column_width, + )); + match column.align { + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); } - cell - }, - Err(_) => { - let mut cell = Cell::new("Missing") - .with_style(Attr::ForegroundColor(color::RED)) - .with_style(Attr::Bold); - match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); } - cell - }, - }; - cell + } + cell + } + Err(_) => { + let mut cell = Cell::new("Missing") + .with_style(Attr::ForegroundColor(color::RED)) + .with_style(Attr::Bold); + match column.align { + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); + } + } + cell + } }, ColumnType::FilePath => { let mut cell = Cell::new(&string_column( @@ -206,49 +214,67 @@ pub fn mode_list( column_width, )); match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); + } } cell - }, + } ColumnType::Tags => { let mut cell = Cell::new(&string_column(tags.join(" "), column_width)); match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); + } } cell - }, - ColumnType::Meta => { - let cell = match meta_name { - Some(meta_name) => match meta.get(meta_name) { - Some(meta_value) => { - let mut cell = Cell::new(&string_column(meta_value.to_string(), column_width)); - match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + } + ColumnType::Meta => match meta_name { + Some(meta_name) => match meta.get(meta_name) { + Some(meta_value) => { + let mut cell = + Cell::new(&string_column(meta_value.to_string(), column_width)); + match column.align { + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); } - cell } - None => { - let mut cell = Cell::new(""); - match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, - } - cell - }, - }, + cell + } None => { let mut cell = Cell::new(""); match column.align { - crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); }, - crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); }, + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); + } } cell - }, - }; - cell + } + }, + None => { + let mut cell = Cell::new(""); + match column.align { + crate::config::ColumnAlignment::Right => { + cell.align(Alignment::RIGHT); + } + crate::config::ColumnAlignment::Left => { + cell.align(Alignment::LEFT); + } + } + cell + } }, }; table_row.add_cell(cell); @@ -262,20 +288,19 @@ pub fn mode_list( } fn show_list_structured( - items: Vec, - tags_by_item: std::collections::HashMap>, - meta_by_item: std::collections::HashMap>, + items_with_meta: Vec, data_path: std::path::PathBuf, settings: &config::Settings, output_format: OutputFormat, -) -> anyhow::Result<()> { +) -> Result<()> { let mut list_items = Vec::new(); - for item in items { + for item_with_meta in items_with_meta { + let item = item_with_meta.item; let item_id = item.id.unwrap(); - let tags = tags_by_item.get(&item_id).cloned().unwrap_or_default(); - let meta = meta_by_item.get(&item_id).cloned().unwrap_or_default(); - + let tags = item_with_meta.tags.into_iter().map(|t| t.name).collect(); + let meta = item_with_meta.meta_as_map(); + let mut item_path = data_path.clone(); item_path.push(item_id.to_string()); @@ -292,7 +317,11 @@ fn show_list_structured( let list_item = ListItem { id: item.id, - time: item.ts.with_timezone(&chrono::Local).format("%F %T").to_string(), + time: item + .ts + .with_timezone(&chrono::Local) + .format("%F %T") + .to_string(), size: item.size.map(|s| s as u64), size_formatted, compression: item.compression, diff --git a/src/modes/save.rs b/src/modes/save.rs index 65ca71b..361b979 100644 --- a/src/modes/save.rs +++ b/src/modes/save.rs @@ -1,11 +1,9 @@ -use anyhow::{anyhow, Result}; +use anyhow::Result; use clap::Command; -use log::debug; -use std::io::{Read, Write, IsTerminal}; +use std::io::{Read, Write}; -// Import the missing functions from common module -use crate::modes::common::{settings_compression_type, settings_meta_plugin_types}; use crate::config; +use crate::core::item_service::ItemService; fn validate_save_args(cmd: &mut Command, ids: &Vec) { if !ids.is_empty() { @@ -17,209 +15,20 @@ fn validate_save_args(cmd: &mut Command, ids: &Vec) { } } -fn initialize_tags(tags: &mut Vec) { - if tags.is_empty() { - tags.push("none".to_string()); - } +// Tee reader that writes to a writer as it is read +struct TeeReader { + reader: R, + writer: W, } -fn setup_compression_and_plugins( - cmd: &mut Command, - settings: &config::Settings, -) -> (crate::compression_engine::CompressionType, Box, Vec>) { - let compression_type = settings_compression_type(cmd, settings); - debug!("MAIN: Compression type: {:?}", compression_type); - let compression_engine = - crate::compression_engine::get_compression_engine(compression_type.clone()).expect("Unable to get compression engine"); - - // Get meta plugin types from settings - let meta_plugin_types: Vec = settings_meta_plugin_types(cmd, settings); - debug!("MAIN: Meta plugin types: {:?}", meta_plugin_types); - - // Initialize meta_plugins with MetaPlugin instances for each MetaPluginType - let mut meta_plugins: Vec> = meta_plugin_types - .iter() - .map(|meta_plugin_type| crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone())) - .collect(); - - // Configure meta plugins with their options and outputs from settings - if let Some(meta_plugin_configs) = &settings.meta_plugins { - for meta_plugin in meta_plugins.iter_mut() { - let plugin_name = meta_plugin.meta_name(); - if let Some(config) = meta_plugin_configs.iter().find(|c| c.name == plugin_name) { - // Start with default outputs and options, then apply configuration on top - let mut configured_outputs = meta_plugin.outputs().clone(); - for (key, value) in &config.outputs { - configured_outputs.insert(key.clone(), serde_yaml::Value::String(value.clone())); - } - - let mut configured_options = meta_plugin.default_options(); - for (key, value) in &config.options { - configured_options.insert(key.clone(), value.clone()); - } - - // Apply the combined configuration - if let Err(e) = meta_plugin.configure_outputs(&configured_outputs) { - eprintln!("Warning: Failed to configure outputs for meta plugin '{}': {}", plugin_name, e); - } - - if let Err(e) = meta_plugin.configure_options(&configured_options) { - eprintln!("Warning: Failed to configure options for meta plugin '{}': {}", plugin_name, e); - } - } +impl Read for TeeReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.reader.read(buf)?; + if n > 0 { + self.writer.write_all(&buf[..n])?; } + Ok(n) } - - // Check for unsupported meta plugins, warn the user, and remove them from the list - let mut i = 0; - meta_plugins.retain(|meta_plugin| { - let is_supported = meta_plugin.is_supported(); - if !is_supported { - // We need to get the meta name for the warning message - // Since we can't mutably borrow meta_plugin here, we create a temporary one - let meta_plugin_type = meta_plugin_types[i].clone(); - let mut temp_plugin = crate::meta_plugin::get_meta_plugin(meta_plugin_type); - eprintln!("Warning: Meta plugin '{}' is enabled but not supported on this system", temp_plugin.meta_name()); - } - i += 1; - is_supported - }); - - (compression_type, compression_engine, meta_plugins) -} - -fn create_and_log_item( - conn: &mut rusqlite::Connection, - settings: &config::Settings, - tags: &Vec, - compression_type: &crate::compression_engine::CompressionType, -) -> Result { - let mut item = crate::db::Item { - id: None, - ts: chrono::Utc::now(), - size: None, - compression: compression_type.to_string(), - }; - - let id = crate::db::insert_item(conn, item.clone())?; - item.id = Some(id); - debug!("MAIN: Added item {:?}", item.clone()); - - if !settings.quiet { - if std::io::stderr().is_terminal() { - let mut t = term::stderr().unwrap(); - t.reset().unwrap_or(()); - t.attr(term::Attr::Bold).unwrap_or(()); - write!(t, "KEEP:").unwrap_or(()); - t.reset().unwrap_or(()); - write!(t, " New item ").unwrap_or(()); - t.attr(term::Attr::Bold).unwrap_or(()); - write!(t, "{id}")?; - t.reset().unwrap_or(()); - write!(t, " tags: ")?; - t.attr(term::Attr::Bold).unwrap_or(()); - write!(t, "{}", tags.join(" "))?; - t.reset().unwrap_or(()); - writeln!(t)?; - std::io::stderr().flush()?; - } else { - let mut t = std::io::stderr(); - writeln!(t, "KEEP: New item: {} tags: {:?}", id, tags)?; - } - } - - Ok(item) -} - -fn setup_item_metadata( - conn: &mut rusqlite::Connection, - _settings: &config::Settings, - item: &crate::db::Item, - tags: &Vec, -) -> Result<(), anyhow::Error> { - crate::db::set_item_tags(conn, item.clone(), tags)?; - Ok(()) -} - -fn collect_item_meta(_settings: &config::Settings) -> std::collections::HashMap { - let mut item_meta: std::collections::HashMap = crate::modes::common::get_meta_from_env(); - - if let Ok(hostname) = gethostname::gethostname().into_string() { - if !item_meta.contains_key("hostname") { - item_meta.insert("hostname".to_string(), hostname); - } - } - - // Add any additional metadata from settings if needed - // (currently there's no direct metadata in settings, but this could be extended) - - item_meta -} - -fn process_input_stream( - compression_engine: &Box, - data_path: &std::path::PathBuf, - item_id: i64, - meta_plugins: &mut Vec>, - conn: &rusqlite::Connection, -) -> Result<(Box, crate::db::Item), anyhow::Error> { - let mut item = crate::db::Item { - id: Some(item_id), - ts: chrono::Utc::now(), - size: None, - compression: String::new(), // Will be set later - }; - - let mut item_path = data_path.clone(); - item_path.push(item_id.to_string()); - - let mut stdin = std::io::stdin().lock(); - let mut stdout = std::io::stdout().lock(); - let mut buffer = [0; libc::BUFSIZ as usize]; - - let mut item_out: Box = - compression_engine - .create(item_path.clone()) - .map_err(|e| anyhow!("Unable to write file {:?}: {}", item_path, e))?; - - let mut total_bytes = 0; - debug!("MAIN: Starting IO loop"); - loop { - let n = stdin.read(&mut buffer[..libc::BUFSIZ as usize])?; - item.size = match item.size { - None => Some(n as i64), - Some(prev_n) => Some(prev_n + n as i64), - }; - - if n == 0 { - debug!("MAIN: EOF on STDIN"); - break; - } - - total_bytes += n; - debug!("MAIN: Loop - {:?} bytes (total: {})", item.size, total_bytes); - - stdout.write_all(&buffer[..n])?; - item_out.write_all(&buffer[..n])?; - - // Process data with meta plugins - for meta_plugin in meta_plugins.iter_mut() { - meta_plugin.update(&buffer[..n], conn); - } - } - debug!("MAIN: Ending IO loop after {:?} bytes", item.size); - - // Finalize meta plugins - for meta_plugin in meta_plugins.iter_mut() { - if let Err(e) = meta_plugin.finalize(conn) { - eprintln!("Warning: Failed to finalize meta plugin: {}", e); - } - } - - stdout.flush()?; - item_out.flush()?; - - Ok((item_out, item)) } pub fn mode_save( @@ -231,46 +40,18 @@ pub fn mode_save( data_path: std::path::PathBuf, ) -> Result<(), anyhow::Error> { validate_save_args(cmd, ids); - initialize_tags(tags); - let (compression_type, compression_engine, mut meta_plugins) = setup_compression_and_plugins(cmd, settings); + let item_service = ItemService::new(data_path); - let mut item = create_and_log_item(conn, settings, tags, &compression_type)?; - setup_item_metadata(conn, settings, &item, tags)?; // Pass mutable reference + let stdin = std::io::stdin(); + let stdout = std::io::stdout(); - // Save as much as possible in case something breaks - don't use transactions - // This allows partial saves to succeed even if some metadata operations fail - let item_meta = collect_item_meta(settings); - let item_id = item.id.ok_or_else(|| anyhow!("Item missing ID"))?; + let tee_reader = TeeReader { + reader: stdin.lock(), + writer: stdout.lock(), + }; - for kv in item_meta.iter() { - let meta = crate::db::Meta { - id: item_id, - name: kv.0.to_string(), - value: kv.1.to_string(), - }; - crate::db::store_meta(conn, meta)?; - } - - // Initialize meta plugins with database connection - for meta_plugin in meta_plugins.iter_mut() { - if let Err(e) = meta_plugin.initialize(conn, item_id) { - eprintln!("Warning: Failed to initialize meta plugin: {}", e); - } - } - - let (_item_out, processed_item) = process_input_stream( - &compression_engine, - &data_path, - item_id, - &mut meta_plugins, - conn, - )?; - - item.size = processed_item.size; - item.compression = compression_type.to_string(); - - crate::db::update_item(conn, item.clone())?; + item_service.save_item(tee_reader, cmd, settings, tags, conn)?; Ok(()) }