refactor: decouple meta plugins from DB via SaveMetaFn callback, extract shared utilities
- Add SaveMetaFn callback pattern: meta plugins receive a closure instead of
&Connection, enabling the same plugin code to work in local, client, and
server contexts (collect-to-Vec, collect-to-HashMap, or direct DB write)
- Client save now runs meta plugins locally during streaming (smart client
sets meta=false, server skips its own plugins)
- Add POST /api/item/{id}/update endpoint for re-running plugins on stored
content without downloading compressed data
- Add client update mode (--update with --meta-plugin flags)
- Extract shared utilities: stream_copy, print_serialized, build_path_table,
ensure_default_tag to reduce duplication across modes
- Add upsert_tag for idempotent tag addition (INSERT OR IGNORE)
- Add warn logging on save_meta lock failure in BaseMetaPlugin and MetaService
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
use crate::common::PIPESIZE;
|
||||
use crate::compression_engine::{CompressionType, get_compression_engine};
|
||||
use crate::config::Settings;
|
||||
use crate::db::{self, Item, Meta};
|
||||
@@ -28,8 +27,6 @@ pub struct ItemService {
|
||||
data_path: PathBuf,
|
||||
/// Service for handling compression and decompression.
|
||||
compression_service: CompressionService,
|
||||
/// Service for managing metadata plugins.
|
||||
meta_service: MetaService,
|
||||
/// Service for applying content filters.
|
||||
filter_service: FilterService,
|
||||
}
|
||||
@@ -59,7 +56,6 @@ impl ItemService {
|
||||
Self {
|
||||
data_path,
|
||||
compression_service: CompressionService::new(),
|
||||
meta_service: MetaService::new(),
|
||||
filter_service: FilterService::new(),
|
||||
}
|
||||
}
|
||||
@@ -596,10 +592,8 @@ impl ItemService {
|
||||
conn: &mut Connection,
|
||||
) -> Result<Item, CoreError> {
|
||||
debug!("ITEM_SERVICE: Starting save_item with tags: {tags:?}");
|
||||
if tags.is_empty() {
|
||||
tags.push("none".to_string());
|
||||
debug!("ITEM_SERVICE: No tags provided, using default 'none' tag");
|
||||
}
|
||||
crate::modes::common::ensure_default_tag(tags);
|
||||
debug!("ITEM_SERVICE: Tags after ensure_default: {tags:?}");
|
||||
|
||||
let compression_type = settings_compression_type(cmd, settings);
|
||||
debug!("ITEM_SERVICE: Using compression type: {compression_type:?}");
|
||||
@@ -615,7 +609,7 @@ impl ItemService {
|
||||
debug!("ITEM_SERVICE: Created new item with id: {item_id}");
|
||||
db::set_item_tags(conn, item.clone(), tags)?;
|
||||
debug!("ITEM_SERVICE: Set tags for item {item_id}");
|
||||
let item_meta = self.meta_service.collect_initial_meta();
|
||||
let item_meta = MetaService::collect_initial_meta_static();
|
||||
debug!(
|
||||
"ITEM_SERVICE: Collected {} initial meta entries",
|
||||
item_meta.len()
|
||||
@@ -656,10 +650,23 @@ impl ItemService {
|
||||
}
|
||||
}
|
||||
|
||||
let mut plugins = self.meta_service.get_plugins(cmd, settings);
|
||||
// Collect metadata from plugins into a Vec, then write to DB after plugins finish.
|
||||
// This avoids capturing &Connection in the save_meta closure (which would need unsafe
|
||||
// and wouldn't be Send for parallel plugins).
|
||||
let collected_meta: std::sync::Arc<std::sync::Mutex<Vec<(String, String)>>> =
|
||||
std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let collector = collected_meta.clone();
|
||||
let save_meta: crate::meta_plugin::SaveMetaFn =
|
||||
std::sync::Arc::new(std::sync::Mutex::new(move |name: &str, value: &str| {
|
||||
if let Ok(mut v) = collector.lock() {
|
||||
v.push((name.to_string(), value.to_string()));
|
||||
}
|
||||
}));
|
||||
|
||||
let meta_service = MetaService::new(save_meta);
|
||||
let mut plugins = meta_service.get_plugins(cmd, settings);
|
||||
debug!("ITEM_SERVICE: Got {} meta plugins", plugins.len());
|
||||
self.meta_service
|
||||
.initialize_plugins(&mut plugins, conn, item_id);
|
||||
meta_service.initialize_plugins(&mut plugins);
|
||||
|
||||
let mut item_path = self.data_path.clone();
|
||||
item_path.push(item_id.to_string());
|
||||
@@ -667,29 +674,29 @@ impl ItemService {
|
||||
|
||||
let mut item_out = compression_engine.create(item_path.clone())?;
|
||||
|
||||
let mut buffer = [0; PIPESIZE];
|
||||
let mut total_bytes = 0;
|
||||
let mut total_bytes: i64 = 0;
|
||||
|
||||
debug!("ITEM_SERVICE: Starting to read and process input data");
|
||||
loop {
|
||||
let n = input.read(&mut buffer)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
total_bytes += n as i64;
|
||||
item_out.write_all(&buffer[..n])?;
|
||||
self.meta_service
|
||||
.process_chunk(&mut plugins, &buffer[..n], conn, item_id);
|
||||
}
|
||||
crate::common::stream_copy(&mut input, |chunk| {
|
||||
total_bytes += chunk.len() as i64;
|
||||
item_out.write_all(chunk)?;
|
||||
meta_service.process_chunk(&mut plugins, chunk);
|
||||
Ok(())
|
||||
})?;
|
||||
debug!("ITEM_SERVICE: Processed {total_bytes} bytes total");
|
||||
|
||||
item_out.flush()?;
|
||||
drop(item_out);
|
||||
|
||||
debug!("ITEM_SERVICE: Finalizing meta plugins");
|
||||
self.meta_service
|
||||
.finalize_plugins(&mut plugins, conn, item_id);
|
||||
meta_service.finalize_plugins(&mut plugins);
|
||||
|
||||
// Write collected plugin metadata to DB
|
||||
if let Ok(entries) = collected_meta.lock() {
|
||||
for (name, value) in entries.iter() {
|
||||
db::add_meta(conn, item_id, name, value)?;
|
||||
}
|
||||
}
|
||||
|
||||
item.size = Some(total_bytes);
|
||||
db::update_item(conn, item.clone())?;
|
||||
|
||||
Reference in New Issue
Block a user