refactor: decouple meta plugins from DB via SaveMetaFn callback, extract shared utilities

- Add SaveMetaFn callback pattern: meta plugins receive a closure instead of
  &Connection, enabling the same plugin code to work in local, client, and
  server contexts (collect-to-Vec, collect-to-HashMap, or direct DB write)
- Client save now runs meta plugins locally during streaming (smart client
  sets meta=false, server skips its own plugins)
- Add POST /api/item/{id}/update endpoint for re-running plugins on stored
  content without downloading compressed data
- Add client update mode (--update with --meta-plugin flags)
- Extract shared utilities: stream_copy, print_serialized, build_path_table,
  ensure_default_tag to reduce duplication across modes
- Add upsert_tag for idempotent tag addition (INSERT OR IGNORE)
- Add warn logging on save_meta lock failure in BaseMetaPlugin and MetaService
This commit is contained in:
2026-03-14 22:36:59 -03:00
parent fdc5f1d744
commit 5bad7ac7a6
39 changed files with 843 additions and 290 deletions

View File

@@ -1,4 +1,3 @@
use crate::common::PIPESIZE;
use crate::compression_engine::{CompressionType, get_compression_engine};
use crate::config::Settings;
use crate::db::{self, Item, Meta};
@@ -28,8 +27,6 @@ pub struct ItemService {
data_path: PathBuf,
/// Service for handling compression and decompression.
compression_service: CompressionService,
/// Service for managing metadata plugins.
meta_service: MetaService,
/// Service for applying content filters.
filter_service: FilterService,
}
@@ -59,7 +56,6 @@ impl ItemService {
Self {
data_path,
compression_service: CompressionService::new(),
meta_service: MetaService::new(),
filter_service: FilterService::new(),
}
}
@@ -596,10 +592,8 @@ impl ItemService {
conn: &mut Connection,
) -> Result<Item, CoreError> {
debug!("ITEM_SERVICE: Starting save_item with tags: {tags:?}");
if tags.is_empty() {
tags.push("none".to_string());
debug!("ITEM_SERVICE: No tags provided, using default 'none' tag");
}
crate::modes::common::ensure_default_tag(tags);
debug!("ITEM_SERVICE: Tags after ensure_default: {tags:?}");
let compression_type = settings_compression_type(cmd, settings);
debug!("ITEM_SERVICE: Using compression type: {compression_type:?}");
@@ -615,7 +609,7 @@ impl ItemService {
debug!("ITEM_SERVICE: Created new item with id: {item_id}");
db::set_item_tags(conn, item.clone(), tags)?;
debug!("ITEM_SERVICE: Set tags for item {item_id}");
let item_meta = self.meta_service.collect_initial_meta();
let item_meta = MetaService::collect_initial_meta_static();
debug!(
"ITEM_SERVICE: Collected {} initial meta entries",
item_meta.len()
@@ -656,10 +650,23 @@ impl ItemService {
}
}
let mut plugins = self.meta_service.get_plugins(cmd, settings);
// Collect metadata from plugins into a Vec, then write to DB after plugins finish.
// This avoids capturing &Connection in the save_meta closure (which would need unsafe
// and wouldn't be Send for parallel plugins).
let collected_meta: std::sync::Arc<std::sync::Mutex<Vec<(String, String)>>> =
std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let collector = collected_meta.clone();
let save_meta: crate::meta_plugin::SaveMetaFn =
std::sync::Arc::new(std::sync::Mutex::new(move |name: &str, value: &str| {
if let Ok(mut v) = collector.lock() {
v.push((name.to_string(), value.to_string()));
}
}));
let meta_service = MetaService::new(save_meta);
let mut plugins = meta_service.get_plugins(cmd, settings);
debug!("ITEM_SERVICE: Got {} meta plugins", plugins.len());
self.meta_service
.initialize_plugins(&mut plugins, conn, item_id);
meta_service.initialize_plugins(&mut plugins);
let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string());
@@ -667,29 +674,29 @@ impl ItemService {
let mut item_out = compression_engine.create(item_path.clone())?;
let mut buffer = [0; PIPESIZE];
let mut total_bytes = 0;
let mut total_bytes: i64 = 0;
debug!("ITEM_SERVICE: Starting to read and process input data");
loop {
let n = input.read(&mut buffer)?;
if n == 0 {
break;
}
total_bytes += n as i64;
item_out.write_all(&buffer[..n])?;
self.meta_service
.process_chunk(&mut plugins, &buffer[..n], conn, item_id);
}
crate::common::stream_copy(&mut input, |chunk| {
total_bytes += chunk.len() as i64;
item_out.write_all(chunk)?;
meta_service.process_chunk(&mut plugins, chunk);
Ok(())
})?;
debug!("ITEM_SERVICE: Processed {total_bytes} bytes total");
item_out.flush()?;
drop(item_out);
debug!("ITEM_SERVICE: Finalizing meta plugins");
self.meta_service
.finalize_plugins(&mut plugins, conn, item_id);
meta_service.finalize_plugins(&mut plugins);
// Write collected plugin metadata to DB
if let Ok(entries) = collected_meta.lock() {
for (name, value) in entries.iter() {
db::add_meta(conn, item_id, name, value)?;
}
}
item.size = Some(total_bytes);
db::update_item(conn, item.clone())?;