asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: decouple meta plugins from DB via SaveMetaFn callback, extract shared utilities

Browse Source 
- Add SaveMetaFn callback pattern: meta plugins receive a closure instead of
  &Connection, enabling the same plugin code to work in local, client, and
  server contexts (collect-to-Vec, collect-to-HashMap, or direct DB write)
- Client save now runs meta plugins locally during streaming (smart client
  sets meta=false, server skips its own plugins)
- Add POST /api/item/{id}/update endpoint for re-running plugins on stored
  content without downloading compressed data
- Add client update mode (--update with --meta-plugin flags)
- Extract shared utilities: stream_copy, print_serialized, build_path_table,
  ensure_default_tag to reduce duplication across modes
- Add upsert_tag for idempotent tag addition (INSERT OR IGNORE)
- Add warn logging on save_meta lock failure in BaseMetaPlugin and MetaService
This commit is contained in:
Andrew Phillips
2026-03-14 22:36:59 -03:00
parent fdc5f1d744
commit 5bad7ac7a6
39 changed files with 843 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
src/services/meta_service.rs
View File

@@ -1,12 +1,13 @@
use crate::config::Settings;
use crate::meta_plugin::{MetaPlugin, MetaPluginResponse, MetaPluginType};
use crate::meta_plugin::{MetaPlugin, MetaPluginResponse, MetaPluginType, SaveMetaFn};
use crate::modes::common::settings_meta_plugin_types;
use clap::Command;
use log::{debug, error};
use rusqlite::Connection;
use log::{debug, error, warn};
use std::collections::HashMap;
pub struct MetaService;
pub struct MetaService {
save_meta: SaveMetaFn,
}
/// Sentinel plugin used as a placeholder when extracting plugins for parallel
/// execution. The original plugin is written back immediately after the threads
@@ -22,9 +23,28 @@ fn replace_plugin(plugins: &mut [Box<dyn MetaPlugin>], i: usize) -> Box<dyn Meta
std::mem::replace(&mut plugins[i], Box::new(NullMetaPlugin))
}
/// Stores metadata entries from a plugin response via the save_meta callback.
fn store_plugin_response(response: &MetaPluginResponse, save_meta: &SaveMetaFn) {
if let Ok(mut f) = save_meta.lock() {
for meta_data in &response.metadata {
f(&meta_data.name, &meta_data.value);
}
} else {
warn!(
"META_SERVICE: save_meta lock poisoned, dropping {} metadata entries",
response.metadata.len()
);
}
}
impl MetaService {
pub fn new() -> Self {
Self
/// Creates a new MetaService with the given save_meta callback.
///
/// All plugins created by this service will share this callback for
/// persisting metadata. The callback is wrapped in Arc<Mutex<>> so it
/// can be cloned into parallel-safe plugin threads.
pub fn new(save_meta: SaveMetaFn) -> Self {
Self { save_meta }
}
pub fn get_plugins(&self, cmd: &mut Command, settings: &Settings) -> Vec<Box<dyn MetaPlugin>> {
@@ -32,7 +52,7 @@ impl MetaService {
let meta_plugin_types: Vec<MetaPluginType> = settings_meta_plugin_types(cmd, settings);
debug!("META_SERVICE: Meta plugin types from settings: {meta_plugin_types:?}");
// Create plugins with their configuration
// Create plugins with their configuration and wire save_meta
let meta_plugins: Vec<Box<dyn MetaPlugin>> = meta_plugin_types
.iter()
.filter_map(|meta_plugin_type| {
@@ -66,7 +86,12 @@ impl MetaService {
(None, None)
};
match crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone(), options, outputs) {
match crate::meta_plugin::get_meta_plugin_with_save(
meta_plugin_type.clone(),
options,
outputs,
Some(self.save_meta.clone()),
) {
Ok(plugin) => Some(plugin),
Err(e) => {
log::warn!("META_SERVICE: Failed to create plugin {meta_plugin_type:?}: {e}, skipping");
@@ -79,12 +104,7 @@ impl MetaService {
meta_plugins
}
pub fn initialize_plugins(
&self,
plugins: &mut [Box<dyn MetaPlugin>],
conn: &Connection,
item_id: i64,
) {
pub fn initialize_plugins(&self, plugins: &mut [Box<dyn MetaPlugin>]) {
// Check for duplicate output names before initializing plugins
let mut output_names: std::collections::HashMap<String, Vec<String>> =
std::collections::HashMap::new();
@@ -135,7 +155,6 @@ impl MetaService {
parallel_plugins.push(replace_plugin(plugins, i));
}
// Write results back to original slots sequentially (DB writes are serial)
let (results, panicked): (Vec<(usize, MetaPluginResponse)>, Vec<usize>) =
std::thread::scope(|s| {
let handles: Vec<_> = parallel_plugins
@@ -157,15 +176,13 @@ impl MetaService {
});
for (j, response) in results {
store_plugin_metadata(conn, item_id, &response);
store_plugin_response(&response, &self.save_meta);
let mut plugin = replace_plugin(&mut parallel_plugins, j);
if response.is_finalized {
plugin.set_finalized(true);
}
plugins[parallel_idx[j]] = plugin;
}
// Panicked plugins: restore the NullMetaPlugin sentinel and
// mark it finalized so future phases skip it cleanly.
for j in panicked {
let mut plugin = replace_plugin(&mut parallel_plugins, j);
plugin.set_finalized(true);
@@ -176,20 +193,14 @@ impl MetaService {
// Run sequential plugins
for &i in &sequential_idx {
let response = plugins[i].initialize();
store_plugin_metadata(conn, item_id, &response);
store_plugin_response(&response, &self.save_meta);
if response.is_finalized {
plugins[i].set_finalized(true);
}
}
}
pub fn process_chunk(
&self,
plugins: &mut [Box<dyn MetaPlugin>],
chunk: &[u8],
conn: &Connection,
item_id: i64,
) {
pub fn process_chunk(&self, plugins: &mut [Box<dyn MetaPlugin>], chunk: &[u8]) {
// Partition non-finalized plugins by parallel_safe
let (parallel_idx, sequential_idx): (Vec<usize>, Vec<usize>) = plugins
.iter()
@@ -200,7 +211,6 @@ impl MetaService {
// Run parallel-safe plugins concurrently on this chunk
if !parallel_idx.is_empty() {
// Extract plugins by unique index into a flat Vec indexed by position
let mut parallel_plugins: Vec<Box<dyn MetaPlugin>> =
Vec::with_capacity(parallel_idx.len());
for &i in &parallel_idx {
@@ -228,7 +238,7 @@ impl MetaService {
});
for (j, response) in results {
store_plugin_metadata(conn, item_id, &response);
store_plugin_response(&response, &self.save_meta);
let mut plugin = replace_plugin(&mut parallel_plugins, j);
if response.is_finalized {
plugin.set_finalized(true);
@@ -245,26 +255,21 @@ impl MetaService {
// Run sequential plugins
for &i in &sequential_idx {
let response = plugins[i].update(chunk);
store_plugin_metadata(conn, item_id, &response);
store_plugin_response(&response, &self.save_meta);
if response.is_finalized {
plugins[i].set_finalized(true);
}
}
}
pub fn finalize_plugins(
&self,
plugins: &mut [Box<dyn MetaPlugin>],
conn: &Connection,
item_id: i64,
) {
pub fn finalize_plugins(&self, plugins: &mut [Box<dyn MetaPlugin>]) {
for meta_plugin in plugins.iter_mut() {
if meta_plugin.is_finalized() {
continue;
}
let response = meta_plugin.finalize();
store_plugin_metadata(conn, item_id, &response);
store_plugin_response(&response, &self.save_meta);
if response.is_finalized {
meta_plugin.set_finalized(true);
@@ -273,22 +278,12 @@ impl MetaService {
}
/// Collects initial metadata from environment variables and hostname.
///
/// Gathers metadata from `KEEP_META_*` environment variables and adds hostname
/// if not already present.
///
/// # Returns
///
/// A `HashMap` of initial metadata key-value pairs.
///
/// # Examples
///
/// ```
/// # use keep::services::MetaService;
/// let service = MetaService::new();
/// let initial_meta = service.collect_initial_meta();
/// ```
pub fn collect_initial_meta(&self) -> HashMap<String, String> {
Self::collect_initial_meta_static()
}
/// Static version of collect_initial_meta for use without a MetaService instance.
pub fn collect_initial_meta_static() -> HashMap<String, String> {
let mut item_meta: HashMap<String, String> = crate::modes::common::get_meta_from_env();
if let Ok(hostname) = gethostname::gethostname().into_string()
@@ -299,34 +294,3 @@ impl MetaService {
item_meta
}
}
/// Stores metadata entries from a plugin response into the database.
///
/// # Arguments
///
/// * `conn` - Database connection.
/// * `item_id` - Item ID to associate with the metadata.
/// * `response` - The plugin response containing metadata.
fn store_plugin_metadata(conn: &Connection, item_id: i64, response: &MetaPluginResponse) {
for meta_data in &response.metadata {
let db_meta = crate::db::Meta {
id: item_id,
name: meta_data.name.clone(),
value: meta_data.value.clone(),
};
if let Err(e) = crate::db::store_meta(conn, db_meta) {
log::warn!("META_SERVICE: Failed to store metadata: {e}");
}
}
}
impl Default for MetaService {
/// Provides a default `MetaService` instance.
///
/// # Returns
///
/// A new `MetaService` via `new()`.
fn default() -> Self {
Self::new()
}
}