asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: implement core services and refactor modes

Browse Source 
Co-authored-by: aider (openai/andrew/openrouter/google/gemini-2.5-pro) <aider@aider.chat>
This commit is contained in:
Andrew Phillips
2025-08-24 23:56:06 -03:00
parent 437a05e5d6
commit 7ec0603e00
11 changed files with 659 additions and 466 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/core/compression_service.rs
View File

@@ -0,0 +1,33 @@
use crate::compression_engine::{get_compression_engine, CompressionType};
use crate::core::error::CoreError;
use std::io::Read;
use std::path::PathBuf;
use std::str::FromStr;
use anyhow::anyhow;
pub struct CompressionService;
impl CompressionService {
pub fn new() -> Self {
Self
}
pub fn get_item_content(&self, item_path: PathBuf, compression: &str) -> Result<Vec<u8>, CoreError> {
let compression_type = CompressionType::from_str(compression)
.map_err(|e| CoreError::Compression(e.to_string()))?;
let engine = get_compression_engine(compression_type)
.map_err(|e| CoreError::Other(anyhow!(e.to_string())))?;
let mut reader = engine.open(item_path.clone())
.map_err(|e| CoreError::Other(anyhow!("Failed to open item file {:?}: {}", item_path, e)))?;
let mut content = Vec::new();
reader.read_to_end(&mut content)?;
Ok(content)
}
}
impl Default for CompressionService {
fn default() -> Self {
Self::new()
}
}

34
src/core/error.rs
View File

@@ -0,0 +1,34 @@
use thiserror::Error;
#[derive(Error, Debug)]
pub enum CoreError {
#[error("Database error: {0}")]
Database(#[from] rusqlite::Error),
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("Item not found with id {0}")]
ItemNotFound(i64),
#[error("Item not found")]
ItemNotFoundGeneric,
#[error("Invalid input: {0}")]
InvalidInput(String),
#[error("Configuration error: {0}")]
Config(String),
#[error("Compression error: {0}")]
Compression(String),
#[error(transparent)]
Other(#[from] anyhow::Error),
}
impl From<rusqlite_migration::Error> for CoreError {
fn from(err: rusqlite_migration::Error) -> Self {
CoreError::Database(rusqlite::Error::from(err))
}
}

197
src/core/item_service.rs
View File

@@ -0,0 +1,197 @@
use crate::config::Settings;
use crate::core::compression_service::CompressionService;
use crate::core::error::CoreError;
use crate::core::meta_service::MetaService;
use crate::core::types::{ItemWithContent, ItemWithMeta};
use crate::db::{self, Meta};
use crate::compression_engine::{get_compression_engine, CompressionType};
use crate::modes::common::settings_compression_type;
use clap::Command;
use log::debug;
use rusqlite::Connection;
use std::collections::HashMap;
use std::fs;
use std::io::{IsTerminal, Read, Write};
use std::path::PathBuf;
pub struct ItemService {
data_path: PathBuf,
compression_service: CompressionService,
meta_service: MetaService,
}
impl ItemService {
pub fn new(data_path: PathBuf) -> Self {
Self {
data_path,
compression_service: CompressionService::new(),
meta_service: MetaService::new(),
}
}
pub fn get_item(&self, conn: &Connection, id: i64) -> Result<ItemWithMeta, CoreError> {
let item = db::get_item(conn, id)?.ok_or(CoreError::ItemNotFound(id))?;
let tags = db::get_item_tags(conn, &item)?;
let meta = db::get_item_meta(conn, &item)?;
Ok(ItemWithMeta { item, tags, meta })
}
pub fn get_item_content(&self, conn: &Connection, id: i64) -> Result<ItemWithContent, CoreError> {
let item_with_meta = self.get_item(conn, id)?;
let item_id = item_with_meta.item.id.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?;
if item_id <= 0 {
return Err(CoreError::InvalidInput(format!("Invalid item ID: {}", item_id)));
}
let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string());
let content = self
.compression_service
.get_item_content(item_path, &item_with_meta.item.compression)?;
Ok(ItemWithContent {
item_with_meta,
content,
})
}
pub fn find_item(&self, conn: &Connection, ids: &[i64], tags: &[String], meta: &HashMap<String, String>) -> Result<ItemWithMeta, CoreError> {
let item_maybe = match (ids.is_empty(), tags.is_empty() && meta.is_empty()) {
(false, _) => db::get_item(conn, ids[0])?,
(true, true) => db::get_item_last(conn)?,
(true, false) => db::get_item_matching(conn, &tags.to_vec(), meta)?
};
let item = item_maybe.ok_or(CoreError::ItemNotFoundGeneric)?;
let item_id = item.id.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?;
self.get_item(conn, item_id)
}
pub fn list_items(&self, conn: &Connection, tags: &[String], meta: &HashMap<String, String>) -> Result<Vec<ItemWithMeta>, CoreError> {
let items = db::get_items_matching(conn, &tags.to_vec(), meta)?;
let item_ids: Vec<i64> = items.iter().filter_map(|item| item.id).collect();
if item_ids.is_empty() {
return Ok(Vec::new());
}
let tags_map = db::get_tags_for_items(conn, &item_ids)?;
let meta_map_db = db::get_meta_for_items(conn, &item_ids)?;
let mut result = Vec::new();
for item in items {
let item_id = item.id.unwrap();
let tags = tags_map.get(&item_id).cloned().unwrap_or_default();
let meta_hm = meta_map_db.get(&item_id).cloned().unwrap_or_default();
let meta = meta_hm.into_iter().map(|(name, value)| Meta { id: item_id, name, value }).collect();
result.push(ItemWithMeta { item, tags, meta });
}
Ok(result)
}
pub fn delete_item(&self, conn: &mut Connection, id: i64) -> Result<(), CoreError> {
if id <= 0 {
return Err(CoreError::InvalidInput(format!("Invalid item ID: {}", id)));
}
let item = db::get_item(conn, id)?.ok_or(CoreError::ItemNotFound(id))?;
let mut item_path = self.data_path.clone();
item_path.push(id.to_string());
let tx = conn.transaction()?;
db::delete_item(&tx, item)?;
fs::remove_file(&item_path).or_else(|e| if e.kind() == std::io::ErrorKind::NotFound { Ok(()) } else { Err(e) })?;
tx.commit()?;
Ok(())
}
pub fn save_item<R: Read>(
&self,
mut input: R,
cmd: &mut Command,
settings: &Settings,
tags: &mut Vec<String>,
conn: &mut Connection,
) -> Result<ItemWithMeta, CoreError> {
if tags.is_empty() {
tags.push("none".to_string());
}
let compression_type = settings_compression_type(cmd, settings);
let compression_engine = get_compression_engine(compression_type.clone())?;
let tx = conn.transaction()?;
let item_id;
let mut item;
{
item = db::create_item(&tx, compression_type.clone())?;
item_id = item.id.unwrap();
db::set_item_tags(&tx, item.clone(), tags)?;
let item_meta = self.meta_service.collect_initial_meta();
for (k, v) in item_meta.iter() {
db::add_meta(&tx, item_id, k, v)?;
}
}
let mut plugins = self.meta_service.get_plugins(cmd, settings);
self.meta_service.initialize_plugins(&mut plugins, &tx, item_id);
let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string());
let mut item_out = compression_engine.create(item_path.clone())?;
let mut buffer = [0; 8192];
let mut total_bytes = 0;
loop {
let n = input.read(&mut buffer)?;
if n == 0 { break; }
total_bytes += n as i64;
item_out.write_all(&buffer[..n])?;
self.meta_service.process_chunk(&mut plugins, &buffer[..n], &tx);
}
item_out.flush()?;
drop(item_out);
self.meta_service.finalize_plugins(&mut plugins, &tx);
item.size = Some(total_bytes);
db::update_item(&tx, item.clone())?;
tx.commit()?;
if !settings.quiet {
if std::io::stderr().is_terminal() {
let mut t = term::stderr().unwrap();
let _ = t.reset();
let _ = t.attr(term::Attr::Bold);
let _ = write!(t, "KEEP:");
let _ = t.reset();
let _ = write!(t, " New item ");
let _ = t.attr(term::Attr::Bold);
let _ = write!(t, "{item_id}");
let _ = t.reset();
let _ = write!(t, " tags: ");
let _ = t.attr(term::Attr::Bold);
let _ = write!(t, "{}", tags.join(" "));
let _ = t.reset();
let _ = writeln!(t);
let _ = std::io::stderr().flush();
} else {
let mut t = std::io::stderr();
let _ = writeln!(t, "KEEP: New item: {} tags: {:?}", item_id, tags);
}
}
self.get_item(conn, item_id)
}
}

125
src/core/meta_service.rs
View File

@@ -0,0 +1,125 @@
use crate::config::Settings;
use crate::meta_plugin::{get_meta_plugin, MetaPlugin, MetaPluginType};
use crate::modes::common::{settings_digest_type, settings_meta_plugin_types};
use clap::Command;
use log::debug;
use rusqlite::Connection;
use std::collections::HashMap;
pub struct MetaService;
impl MetaService {
pub fn new() -> Self {
Self
}
pub fn get_plugins(&self, cmd: &mut Command, settings: &Settings) -> Vec<Box<dyn MetaPlugin>> {
let mut meta_plugin_types: Vec<MetaPluginType> = settings_meta_plugin_types(cmd, settings);
let digest_type = settings_digest_type(cmd, settings);
let digest_meta_plugin_type = match digest_type {
MetaPluginType::DigestSha256 => Some(MetaPluginType::DigestSha256),
MetaPluginType::DigestMd5 => Some(MetaPluginType::DigestMd5),
_ => None,
};
if let Some(digest_plugin_type) = digest_meta_plugin_type {
if !meta_plugin_types.contains(&digest_plugin_type) {
meta_plugin_types.push(digest_plugin_type);
}
}
debug!("MetaService: Meta plugin types: {:?}", meta_plugin_types);
let mut meta_plugins: Vec<Box<dyn MetaPlugin>> = meta_plugin_types
.iter()
.map(|meta_plugin_type| get_meta_plugin(meta_plugin_type.clone()))
.collect();
if let Some(meta_plugin_configs) = &settings.meta_plugins {
for meta_plugin in meta_plugins.iter_mut() {
let plugin_name = meta_plugin.meta_name();
if let Some(config) = meta_plugin_configs.iter().find(|c| c.name == plugin_name) {
let mut configured_outputs = meta_plugin.outputs().clone();
for (key, value) in &config.outputs {
configured_outputs.insert(key.clone(), serde_yaml::Value::String(value.clone()));
}
let mut configured_options = meta_plugin.default_options();
for (key, value) in &config.options {
configured_options.insert(key.clone(), value.clone());
}
if let Err(e) = meta_plugin.configure_outputs(&configured_outputs) {
eprintln!("Warning: Failed to configure outputs for meta plugin '{}': {}", plugin_name, e);
}
if let Err(e) = meta_plugin.configure_options(&configured_options) {
eprintln! (
"Warning: Failed to configure options for meta plugin '{}': {}",
plugin_name, e
);
}
}
}
}
let original_len = meta_plugins.len();
meta_plugins.retain(|meta_plugin| meta_plugin.is_supported());
if meta_plugins.len() < original_len {
// This is not perfect as it doesn't say which one, but avoids complex logic from save.rs
eprintln!("Warning: Some meta plugins are enabled but not supported on this system");
}
meta_plugins
}
pub fn initialize_plugins(
&self,
plugins: &mut [Box<dyn MetaPlugin>],
conn: &Connection,
item_id: i64,
) {
for meta_plugin in plugins.iter_mut() {
if let Err(e) = meta_plugin.initialize(conn, item_id) {
eprintln!("Warning: Failed to initialize meta plugin: {}", e);
}
}
}
pub fn process_chunk(
&self,
plugins: &mut [Box<dyn MetaPlugin>],
chunk: &[u8],
conn: &Connection,
) {
for meta_plugin in plugins.iter_mut() {
meta_plugin.update(chunk, conn);
}
}
pub fn finalize_plugins(&self, plugins: &mut [Box<dyn MetaPlugin>], conn: &Connection) {
for meta_plugin in plugins.iter_mut() {
if let Err(e) = meta_plugin.finalize(conn) {
eprintln!("Warning: Failed to finalize meta plugin: {}", e);
}
}
}
pub fn collect_initial_meta(&self) -> HashMap<String, String> {
let mut item_meta: HashMap<String, String> = crate::modes::common::get_meta_from_env();
if let Ok(hostname) = gethostname::gethostname().into_string() {
if !item_meta.contains_key("hostname") {
item_meta.insert("hostname".to_string(), hostname);
}
}
item_meta
}
}
impl Default for MetaService {
fn default() -> Self {
Self::new()
}
}

5
src/core/mod.rs
View File

@@ -0,0 +1,5 @@
pub mod compression_service;
pub mod error;
pub mod item_service;
pub mod meta_service;
pub mod types;

22
src/core/types.rs
View File

@@ -0,0 +1,22 @@
use crate::db::{Item, Meta, Tag};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ItemWithMeta {
pub item: Item,
pub tags: Vec<Tag>,
pub meta: Vec<Meta>,
}
impl ItemWithMeta {
pub fn meta_as_map(&self) -> HashMap<String, String> {
self.meta.iter().cloned().map(|m| (m.name, m.value)).collect()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ItemWithContent {
pub item_with_meta: ItemWithMeta,
pub content: Vec<u8>,
}

33
src/modes/delete.rs
View File

@@ -1,12 +1,11 @@
use anyhow::{Context, Result, anyhow};
use std::fs;
use anyhow::{anyhow, Result};
use std::path::PathBuf;
use crate::db;
use crate::config;
use clap::Command;
use crate::core::item_service::ItemService;
use clap::error::ErrorKind;
use log::{debug, warn};
use clap::Command;
use log::warn;
use rusqlite::Connection;
pub fn mode_delete(
@@ -32,24 +31,18 @@ pub fn mode_delete(
.exit();
}
let item_service = ItemService::new(data_path);
for item_id in ids.iter() {
if let Some(item) = db::get_item(conn, *item_id)? {
debug!("MAIN: Found item {:?}", item);
db::delete_item(conn, item)?;
// Validate that item ID is positive to prevent path traversal issues
if *item_id <= 0 {
return Err(anyhow!("Invalid item ID: {}", item_id));
}
let mut item_path = data_path.clone();
item_path.push(item_id.to_string());
fs::remove_file(&item_path)
.context(anyhow!("Unable to remove item file {:?}", item_path))?;
} else {
match item_service.delete_item(conn, *item_id) {
Ok(_) => {}
Err(e) => match e {
crate::core::error::CoreError::ItemNotFound(_) => {
warn!("Unable to find item {item_id} in database");
}
_ => return Err(anyhow!(e).context(format!("Failed to delete item {}", item_id))),
},
}
}
Ok(())

10
src/modes/diff.rs
View File

@@ -22,6 +22,8 @@ fn validate_diff_args(cmd: &mut Command, ids: &Vec<i64>, tags: &Vec<String>) {
}
}
use crate::core::item_service::ItemService;
fn fetch_and_validate_items(
conn: &mut rusqlite::Connection,
ids: &Vec<i64>,
@@ -40,13 +42,17 @@ fn fetch_and_validate_items(
// Validate that item IDs are positive to prevent path traversal issues
if item_a_id <= 0 || item_b_id <= 0 {
return Err(anyhow::anyhow!("Invalid item ID: {} or {}", item_a_id, item_b_id));
return Err(anyhow::anyhow!(
"Invalid item ID: {} or {}",
item_a_id,
item_b_id
));
}
Ok((item_a, item_b))
}
fn get_item_tags(conn: &mut rusqlite::Connection, item: &crate::db::Item) -> Result<Vec<String>, anyhow::Error> {
fn get_item_tags(conn: &mut rusqlite::Connection, item: &crate::db::Item) -> Result<Vec<String>> {
let tags: Vec<String> = crate::db::get_item_tags(conn, item)?
.into_iter()
.map(|x| x.name)

84
src/modes/get.rs
View File

@@ -1,13 +1,12 @@
use anyhow::anyhow;
use std::io::{Read, Write};
use anyhow::{anyhow, Result};
use std::io::{Write};
use crate::compression_engine::{CompressionType, get_compression_engine};
use crate::common::is_binary::is_binary;
use crate::config;
use crate::core::item_service::ItemService;
use clap::Command;
use std::path::PathBuf;
use std::str::FromStr;
use is_terminal::IsTerminal;
use std::path::PathBuf;
pub fn mode_get(
cmd: &mut Command,
@@ -16,7 +15,7 @@ pub fn mode_get(
tags: &mut Vec<String>,
conn: &mut rusqlite::Connection,
data_path: PathBuf,
) -> anyhow::Result<()> {
) -> Result<()> {
if !ids.is_empty() && !tags.is_empty() {
cmd.error(clap::error::ErrorKind::InvalidValue, "Both ID and tags given, you must supply exactly one ID or at least one tag when using --get").exit();
} else if ids.len() > 1 {
@@ -32,70 +31,39 @@ pub fn mode_get(
}
}
let item_maybe = match tags.is_empty() && meta.is_empty() {
true => match ids.iter().next() {
Some(item_id) => crate::db::get_item(conn, *item_id)?,
None => crate::db::get_item_last(conn)?,
},
false => crate::db::get_item_matching(conn, tags, &meta)?,
};
let item_service = ItemService::new(data_path);
let item_with_content =
item_service.find_item(conn, ids, tags, &meta)
.and_then(|item_with_meta| {
let item_id = item_with_meta.item.id.unwrap();
item_service.get_item_content(conn, item_id)
})
.map_err(|e| anyhow!("Unable to find matching item in database: {}", e))?;
if let Some(item) = item_maybe {
let item_id = item.id.ok_or_else(|| anyhow!("Item missing ID"))?;
// Validate that item ID is positive to prevent path traversal issues
if item_id <= 0 {
return Err(anyhow!("Invalid item ID: {}", item_id));
}
let mut item_path = data_path.clone();
item_path.push(item_id.to_string());
let content = &item_with_content.content;
// Determine if we should detect binary data
let mut detect_binary = !settings.force && std::io::stdout().is_terminal();
// If we're detecting binary and there's binary metadata, check it
if detect_binary {
let item_meta = crate::db::get_item_meta(conn, &item)?;
let binary_meta = item_meta.into_iter().find(|meta| meta.name == "binary");
if let Some(binary_meta) = binary_meta {
if binary_meta.value == "false" {
// If metadata says it's not binary, don't detect
let meta_map = item_with_content.item_with_meta.meta_as_map();
if let Some(binary_val) = meta_map.get("binary") {
if binary_val == "false" {
detect_binary = false;
} else if binary_meta.value == "true" {
// If metadata says it's binary, error immediately
return Err(anyhow!("Refusing to output binary data to TTY, use --force to override"));
} else if binary_val == "true" {
return Err(anyhow!(
"Refusing to output binary data to TTY, use --force to override"
));
}
}
}
let compression_type = CompressionType::from_str(&item.compression)?;
let compression_engine = get_compression_engine(compression_type)?;
// If we need to detect binary, read first 4KB and check
if detect_binary {
// Open the file through compression engine to read first 4KB
let mut reader = compression_engine.open(item_path.clone())?;
let mut buffer = [0u8; 4096];
let bytes_read = reader.read(&mut buffer)?;
// Check if this data is binary
if is_binary(&buffer[..bytes_read]) {
return Err(anyhow!("Refusing to output binary data to TTY, use --force to override"));
}
// If not binary, output the data we've read
std::io::stdout().write_all(&buffer[..bytes_read])?;
// Continue reading and outputting the rest of the data
let mut stdout = std::io::stdout();
std::io::copy(&mut reader, &mut stdout)?;
} else {
// No binary detection needed, just output the data
compression_engine.cat(item_path.clone())?;
if detect_binary && is_binary(content) {
return Err(anyhow!(
"Refusing to output binary data to TTY, use --force to override"
));
}
std::io::stdout().write_all(content)?;
Ok(())
} else {
Err(anyhow!("Unable to find matching item in database"))
}
}

231
src/modes/list.rs
View File

@@ -1,16 +1,15 @@
use crate::db::{get_items, get_items_matching};
use crate::config;
use crate::core::item_service::ItemService;
use crate::core::types::ItemWithMeta;
use crate::modes::common::ColumnType;
use crate::modes::common::{size_column, string_column, OutputFormat};
use crate::config;
use anyhow::{anyhow, Result};
use prettytable::format::Alignment;
use prettytable::{color, row, Attr, Cell, Row, Table};
use serde::{Deserialize, Serialize};
use serde_json;
use serde_yaml;
use anyhow::anyhow;
use log::debug;
use prettytable::color;
use prettytable::row;
use prettytable::format::Alignment;
use prettytable::{Attr, Cell, Row, Table};
use std::str::FromStr;
#[derive(Serialize, Deserialize)]
struct ListItem {
@@ -33,7 +32,7 @@ pub fn mode_list(
tags: &Vec<String>,
conn: &mut rusqlite::Connection,
data_path: std::path::PathBuf,
) -> anyhow::Result<()> {
) -> Result<()> {
if !ids.is_empty() {
cmd.error(
clap::error::ErrorKind::InvalidValue,
@@ -51,34 +50,13 @@ pub fn mode_list(
}
}
let items = match tags.is_empty() && meta.is_empty() {
true => get_items(conn)?,
false => get_items_matching(conn, tags, &meta)?,
};
debug!("MAIN: Items: {:?}", items);
// Collect all item IDs for batch queries
let item_ids: Vec<i64> = items.iter().map(|item| item.id.unwrap()).collect();
// Fetch all tags for all items in a single query
let all_tags = crate::db::get_tags_for_items(conn, &item_ids)?;
let mut tags_by_item: std::collections::HashMap<i64, Vec<String>> =
std::collections::HashMap::new();
// Convert Tag structs to just names
for (item_id, tags) in all_tags {
let tag_names: Vec<String> = tags.into_iter().map(|tag| tag.name).collect();
tags_by_item.insert(item_id, tag_names);
}
// Fetch all metadata for all items in a single query
let meta_by_item = crate::db::get_meta_for_items(conn, &item_ids)?;
let item_service = ItemService::new(data_path.clone());
let items_with_meta = item_service.list_items(conn, tags, &meta)?;
let output_format = crate::modes::common::settings_output_format(settings);
if output_format != OutputFormat::Table {
return show_list_structured(items, tags_by_item, meta_by_item, data_path, settings, output_format);
return show_list_structured(items_with_meta, data_path, settings, output_format);
}
let mut table = Table::new();
@@ -87,18 +65,16 @@ pub fn mode_list(
let mut title_row = row!();
for column in &settings.list_format {
let _column_type = ColumnType::from_str(&column.name)
.map_err(|_| anyhow!("Unknown column {:?}", column.name))?;
title_row.add_cell(Cell::new(&column.label).with_style(Attr::Bold));
}
table.set_titles(title_row);
for item in items {
let item_id = item.id.unwrap();
let tags = tags_by_item.get(&item_id).unwrap();
let meta = meta_by_item.get(&item_id).unwrap();
for item_with_meta in items_with_meta {
let item = item_with_meta.item;
let tags: Vec<String> = item_with_meta.tags.into_iter().map(|t| t.name).collect();
let meta = item_with_meta.meta_as_map();
let mut item_path = data_path.clone();
item_path.push(item.id.unwrap().to_string());
@@ -111,7 +87,7 @@ pub fn mode_list(
let mut meta_name: Option<&str> = None;
let column_width = 0; // We're not supporting width in the new format
if column_type == ColumnType::Meta {
if let ColumnType::Meta = column_type {
let parts: Vec<&str> = column.name.split(':').collect();
if parts.len() > 1 {
meta_name = Some(parts[1]);
@@ -120,13 +96,18 @@ pub fn mode_list(
let cell = match column_type {
ColumnType::Id => {
let mut cell = Cell::new(&string_column(item.id.unwrap_or(0).to_string(), column_width));
let mut cell =
Cell::new(&string_column(item.id.unwrap_or(0).to_string(), column_width));
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
}
ColumnType::Time => {
let mut cell = Cell::new(&string_column(
item.ts
@@ -136,21 +117,32 @@ pub fn mode_list(
column_width,
));
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
ColumnType::Size => {
let cell = match item.size {
}
ColumnType::Size => match item.size {
Some(size) => {
let mut cell = Cell::new(&size_column(size as u64, settings.human_readable, column_width));
let mut cell = Cell::new(&size_column(
size as u64,
settings.human_readable,
column_width,
));
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
}
None => {
let mut cell = match item_path.metadata() {
Ok(_) => Cell::new("Unknown")
@@ -161,44 +153,60 @@ pub fn mode_list(
.with_style(Attr::Bold),
};
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
};
cell
}
},
ColumnType::Compression => {
let mut cell = Cell::new(&string_column(item.compression.to_string(), column_width));
let mut cell =
Cell::new(&string_column(item.compression.to_string(), column_width));
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
ColumnType::FileSize => {
let cell = match item_path.metadata() {
}
ColumnType::FileSize => match item_path.metadata() {
Ok(metadata) => {
let mut cell = Cell::new(&size_column(metadata.len(), settings.human_readable, column_width));
let mut cell = Cell::new(&size_column(
metadata.len(),
settings.human_readable,
column_width,
));
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
}
Err(_) => {
let mut cell = Cell::new("Missing")
.with_style(Attr::ForegroundColor(color::RED))
.with_style(Attr::Bold);
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
};
cell
}
},
ColumnType::FilePath => {
let mut cell = Cell::new(&string_column(
@@ -206,49 +214,67 @@ pub fn mode_list(
column_width,
));
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
}
ColumnType::Tags => {
let mut cell = Cell::new(&string_column(tags.join(" "), column_width));
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
ColumnType::Meta => {
let cell = match meta_name {
}
ColumnType::Meta => match meta_name {
Some(meta_name) => match meta.get(meta_name) {
Some(meta_value) => {
let mut cell = Cell::new(&string_column(meta_value.to_string(), column_width));
let mut cell =
Cell::new(&string_column(meta_value.to_string(), column_width));
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
}
None => {
let mut cell = Cell::new("");
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
}
},
None => {
let mut cell = Cell::new("");
match column.align {
crate::config::ColumnAlignment::Right => { cell.align(Alignment::RIGHT); },
crate::config::ColumnAlignment::Left => { cell.align(Alignment::LEFT); },
crate::config::ColumnAlignment::Right => {
cell.align(Alignment::RIGHT);
}
crate::config::ColumnAlignment::Left => {
cell.align(Alignment::LEFT);
}
}
cell
},
};
cell
}
},
};
table_row.add_cell(cell);
@@ -262,19 +288,18 @@ pub fn mode_list(
}
fn show_list_structured(
items: Vec<crate::db::Item>,
tags_by_item: std::collections::HashMap<i64, Vec<String>>,
meta_by_item: std::collections::HashMap<i64, std::collections::HashMap<String, String>>,
items_with_meta: Vec<ItemWithMeta>,
data_path: std::path::PathBuf,
settings: &config::Settings,
output_format: OutputFormat,
) -> anyhow::Result<()> {
) -> Result<()> {
let mut list_items = Vec::new();
for item in items {
for item_with_meta in items_with_meta {
let item = item_with_meta.item;
let item_id = item.id.unwrap();
let tags = tags_by_item.get(&item_id).cloned().unwrap_or_default();
let meta = meta_by_item.get(&item_id).cloned().unwrap_or_default();
let tags = item_with_meta.tags.into_iter().map(|t| t.name).collect();
let meta = item_with_meta.meta_as_map();
let mut item_path = data_path.clone();
item_path.push(item_id.to_string());
@@ -292,7 +317,11 @@ fn show_list_structured(
let list_item = ListItem {
id: item.id,
time: item.ts.with_timezone(&chrono::Local).format("%F %T").to_string(),
time: item
.ts
.with_timezone(&chrono::Local)
.format("%F %T")
.to_string(),
size: item.size.map(|s| s as u64),
size_formatted,
compression: item.compression,

259
src/modes/save.rs
View File

@@ -1,11 +1,9 @@
use anyhow::{anyhow, Result};
use anyhow::Result;
use clap::Command;
use log::debug;
use std::io::{Read, Write, IsTerminal};
use std::io::{Read, Write};
// Import the missing functions from common module
use crate::modes::common::{settings_compression_type, settings_meta_plugin_types};
use crate::config;
use crate::core::item_service::ItemService;
fn validate_save_args(cmd: &mut Command, ids: &Vec<i64>) {
if !ids.is_empty() {
@@ -17,209 +15,20 @@ fn validate_save_args(cmd: &mut Command, ids: &Vec<i64>) {
}
}
fn initialize_tags(tags: &mut Vec<String>) {
if tags.is_empty() {
tags.push("none".to_string());
}
// Tee reader that writes to a writer as it is read
struct TeeReader<R: Read, W: Write> {
reader: R,
writer: W,
}
fn setup_compression_and_plugins(
cmd: &mut Command,
settings: &config::Settings,
) -> (crate::compression_engine::CompressionType, Box<dyn crate::compression_engine::CompressionEngine>, Vec<Box<dyn crate::meta_plugin::MetaPlugin>>) {
let compression_type = settings_compression_type(cmd, settings);
debug!("MAIN: Compression type: {:?}", compression_type);
let compression_engine =
crate::compression_engine::get_compression_engine(compression_type.clone()).expect("Unable to get compression engine");
// Get meta plugin types from settings
let meta_plugin_types: Vec<crate::meta_plugin::MetaPluginType> = settings_meta_plugin_types(cmd, settings);
debug!("MAIN: Meta plugin types: {:?}", meta_plugin_types);
// Initialize meta_plugins with MetaPlugin instances for each MetaPluginType
let mut meta_plugins: Vec<Box<dyn crate::meta_plugin::MetaPlugin>> = meta_plugin_types
.iter()
.map(|meta_plugin_type| crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone()))
.collect();
// Configure meta plugins with their options and outputs from settings
if let Some(meta_plugin_configs) = &settings.meta_plugins {
for meta_plugin in meta_plugins.iter_mut() {
let plugin_name = meta_plugin.meta_name();
if let Some(config) = meta_plugin_configs.iter().find(|c| c.name == plugin_name) {
// Start with default outputs and options, then apply configuration on top
let mut configured_outputs = meta_plugin.outputs().clone();
for (key, value) in &config.outputs {
configured_outputs.insert(key.clone(), serde_yaml::Value::String(value.clone()));
impl<R: Read, W: Write> Read for TeeReader<R, W> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.reader.read(buf)?;
if n > 0 {
self.writer.write_all(&buf[..n])?;
}
let mut configured_options = meta_plugin.default_options();
for (key, value) in &config.options {
configured_options.insert(key.clone(), value.clone());
Ok(n)
}
// Apply the combined configuration
if let Err(e) = meta_plugin.configure_outputs(&configured_outputs) {
eprintln!("Warning: Failed to configure outputs for meta plugin '{}': {}", plugin_name, e);
}
if let Err(e) = meta_plugin.configure_options(&configured_options) {
eprintln!("Warning: Failed to configure options for meta plugin '{}': {}", plugin_name, e);
}
}
}
}
// Check for unsupported meta plugins, warn the user, and remove them from the list
let mut i = 0;
meta_plugins.retain(|meta_plugin| {
let is_supported = meta_plugin.is_supported();
if !is_supported {
// We need to get the meta name for the warning message
// Since we can't mutably borrow meta_plugin here, we create a temporary one
let meta_plugin_type = meta_plugin_types[i].clone();
let mut temp_plugin = crate::meta_plugin::get_meta_plugin(meta_plugin_type);
eprintln!("Warning: Meta plugin '{}' is enabled but not supported on this system", temp_plugin.meta_name());
}
i += 1;
is_supported
});
(compression_type, compression_engine, meta_plugins)
}
fn create_and_log_item(
conn: &mut rusqlite::Connection,
settings: &config::Settings,
tags: &Vec<String>,
compression_type: &crate::compression_engine::CompressionType,
) -> Result<crate::db::Item, anyhow::Error> {
let mut item = crate::db::Item {
id: None,
ts: chrono::Utc::now(),
size: None,
compression: compression_type.to_string(),
};
let id = crate::db::insert_item(conn, item.clone())?;
item.id = Some(id);
debug!("MAIN: Added item {:?}", item.clone());
if !settings.quiet {
if std::io::stderr().is_terminal() {
let mut t = term::stderr().unwrap();
t.reset().unwrap_or(());
t.attr(term::Attr::Bold).unwrap_or(());
write!(t, "KEEP:").unwrap_or(());
t.reset().unwrap_or(());
write!(t, " New item ").unwrap_or(());
t.attr(term::Attr::Bold).unwrap_or(());
write!(t, "{id}")?;
t.reset().unwrap_or(());
write!(t, " tags: ")?;
t.attr(term::Attr::Bold).unwrap_or(());
write!(t, "{}", tags.join(" "))?;
t.reset().unwrap_or(());
writeln!(t)?;
std::io::stderr().flush()?;
} else {
let mut t = std::io::stderr();
writeln!(t, "KEEP: New item: {} tags: {:?}", id, tags)?;
}
}
Ok(item)
}
fn setup_item_metadata(
conn: &mut rusqlite::Connection,
_settings: &config::Settings,
item: &crate::db::Item,
tags: &Vec<String>,
) -> Result<(), anyhow::Error> {
crate::db::set_item_tags(conn, item.clone(), tags)?;
Ok(())
}
fn collect_item_meta(_settings: &config::Settings) -> std::collections::HashMap<String, String> {
let mut item_meta: std::collections::HashMap<String, String> = crate::modes::common::get_meta_from_env();
if let Ok(hostname) = gethostname::gethostname().into_string() {
if !item_meta.contains_key("hostname") {
item_meta.insert("hostname".to_string(), hostname);
}
}
// Add any additional metadata from settings if needed
// (currently there's no direct metadata in settings, but this could be extended)
item_meta
}
fn process_input_stream(
compression_engine: &Box<dyn crate::compression_engine::CompressionEngine>,
data_path: &std::path::PathBuf,
item_id: i64,
meta_plugins: &mut Vec<Box<dyn crate::meta_plugin::MetaPlugin>>,
conn: &rusqlite::Connection,
) -> Result<(Box<dyn std::io::Write>, crate::db::Item), anyhow::Error> {
let mut item = crate::db::Item {
id: Some(item_id),
ts: chrono::Utc::now(),
size: None,
compression: String::new(), // Will be set later
};
let mut item_path = data_path.clone();
item_path.push(item_id.to_string());
let mut stdin = std::io::stdin().lock();
let mut stdout = std::io::stdout().lock();
let mut buffer = [0; libc::BUFSIZ as usize];
let mut item_out: Box<dyn std::io::Write> =
compression_engine
.create(item_path.clone())
.map_err(|e| anyhow!("Unable to write file {:?}: {}", item_path, e))?;
let mut total_bytes = 0;
debug!("MAIN: Starting IO loop");
loop {
let n = stdin.read(&mut buffer[..libc::BUFSIZ as usize])?;
item.size = match item.size {
None => Some(n as i64),
Some(prev_n) => Some(prev_n + n as i64),
};
if n == 0 {
debug!("MAIN: EOF on STDIN");
break;
}
total_bytes += n;
debug!("MAIN: Loop - {:?} bytes (total: {})", item.size, total_bytes);
stdout.write_all(&buffer[..n])?;
item_out.write_all(&buffer[..n])?;
// Process data with meta plugins
for meta_plugin in meta_plugins.iter_mut() {
meta_plugin.update(&buffer[..n], conn);
}
}
debug!("MAIN: Ending IO loop after {:?} bytes", item.size);
// Finalize meta plugins
for meta_plugin in meta_plugins.iter_mut() {
if let Err(e) = meta_plugin.finalize(conn) {
eprintln!("Warning: Failed to finalize meta plugin: {}", e);
}
}
stdout.flush()?;
item_out.flush()?;
Ok((item_out, item))
}
pub fn mode_save(
@@ -231,46 +40,18 @@ pub fn mode_save(
data_path: std::path::PathBuf,
) -> Result<(), anyhow::Error> {
validate_save_args(cmd, ids);
initialize_tags(tags);
let (compression_type, compression_engine, mut meta_plugins) = setup_compression_and_plugins(cmd, settings);
let item_service = ItemService::new(data_path);
let mut item = create_and_log_item(conn, settings, tags, &compression_type)?;
setup_item_metadata(conn, settings, &item, tags)?; // Pass mutable reference
let stdin = std::io::stdin();
let stdout = std::io::stdout();
// Save as much as possible in case something breaks - don't use transactions
// This allows partial saves to succeed even if some metadata operations fail
let item_meta = collect_item_meta(settings);
let item_id = item.id.ok_or_else(|| anyhow!("Item missing ID"))?;
for kv in item_meta.iter() {
let meta = crate::db::Meta {
id: item_id,
name: kv.0.to_string(),
value: kv.1.to_string(),
let tee_reader = TeeReader {
reader: stdin.lock(),
writer: stdout.lock(),
};
crate::db::store_meta(conn, meta)?;
}
// Initialize meta plugins with database connection
for meta_plugin in meta_plugins.iter_mut() {
if let Err(e) = meta_plugin.initialize(conn, item_id) {
eprintln!("Warning: Failed to initialize meta plugin: {}", e);
}
}
let (_item_out, processed_item) = process_input_stream(
&compression_engine,
&data_path,
item_id,
&mut meta_plugins,
conn,
)?;
item.size = processed_item.size;
item.compression = compression_type.to_string();
crate::db::update_item(conn, item.clone())?;
item_service.save_item(tee_reader, cmd, settings, tags, conn)?;
Ok(())
}