feat: plugin-declared parallel execution, switch to env_logger, update deps
Parallel execution (opt-in via MetaPlugin::parallel_safe): - Add Send bound to MetaPlugin, parallel_safe() method (default false) - Override to true in digest, tokens, exec, magic_file plugins - MetaService: std::thread::scope for initialize_plugins and process_chunk - Extract plugins via NullMetaPlugin sentinel + std::mem::replace (no unsafe) - Panic tracking: join errors logged, NullMetaPlugin restored and finalized - MetaPluginExec: Box<dyn Write> -> Box<dyn Write + Send> - SendCookie wrapper for libmagic Cookie with unsafe impl Send Logging (stderrlog -> env_logger): - Custom format: [SSSSSS.mmm] LEVEL [module:] message (time-since-start ms) - Default level: Warn (matches previous behavior) - -v: Debug, -vv+: Trace, -q: off - -vv+ shows module path Maintenance: - Bump deps: thiserror 2.0, config 0.15, dns-lookup 3.0, lz4_flex 0.12, ringbuf 0.4, rand 0.9, lazy_static 1.5, env_logger 0.11 - Update Cargo.lock (186 transitive packages) - Clippy fixes: is_multiple_of, to_string_in_format_args, collapsible_if - Fix double-counting bug in TokensMetaPlugin::update - Fix schema description using plugin.description() Co-Authored-By: opencode <noreply@opencode.ai>
This commit is contained in:
@@ -12,13 +12,36 @@ use crate::meta_plugin::{
|
||||
process_metadata_outputs,
|
||||
};
|
||||
|
||||
/// Wrapper around `magic::Cookie` that is Send.
|
||||
///
|
||||
/// Libmagic cookies are thread-safe per-instance (separate cookies have
|
||||
/// independent state). The raw pointer `*mut magic_sys::magic_set` does not
|
||||
/// auto-derive Send, but concurrent access to distinct cookies is safe per
|
||||
/// the libmagic documentation.
|
||||
#[cfg(feature = "magic")]
|
||||
struct SendCookie(Cookie);
|
||||
|
||||
#[cfg(feature = "magic")]
|
||||
// SAFETY: Each SendCookie owns a distinct libmagic instance. Libmagic
|
||||
// documents that separate cookies can be used from different threads
|
||||
// concurrently without synchronization.
|
||||
#[allow(unsafe_code)]
|
||||
unsafe impl Send for SendCookie {}
|
||||
|
||||
#[cfg(feature = "magic")]
|
||||
impl std::fmt::Debug for SendCookie {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("SendCookie").finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "magic")]
|
||||
#[derive(Debug)]
|
||||
pub struct MagicFileMetaPluginImpl {
|
||||
buffer: Vec<u8>,
|
||||
max_buffer_size: usize,
|
||||
is_finalized: bool,
|
||||
cookie: Option<Cookie>,
|
||||
cookie: Option<SendCookie>,
|
||||
base: BaseMetaPlugin,
|
||||
}
|
||||
|
||||
@@ -51,7 +74,8 @@ impl MagicFileMetaPluginImpl {
|
||||
}
|
||||
|
||||
fn get_magic_result(&self, flags: CookieFlags) -> io::Result<String> {
|
||||
if let Some(cookie) = &self.cookie {
|
||||
if let Some(send_cookie) = &self.cookie {
|
||||
let cookie = &send_cookie.0;
|
||||
cookie
|
||||
.set_flags(flags)
|
||||
.map_err(|e| io::Error::other(format!("Failed to set magic flags: {e}")))?;
|
||||
@@ -125,7 +149,7 @@ impl MetaPlugin for MagicFileMetaPluginImpl {
|
||||
};
|
||||
}
|
||||
|
||||
self.cookie = Some(cookie);
|
||||
self.cookie = Some(SendCookie(cookie));
|
||||
|
||||
MetaPluginResponse {
|
||||
metadata: Vec::new(),
|
||||
@@ -210,6 +234,10 @@ impl MetaPlugin for MagicFileMetaPluginImpl {
|
||||
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
|
||||
Ok(self.base.options_mut())
|
||||
}
|
||||
|
||||
fn parallel_safe(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "magic")]
|
||||
@@ -308,16 +336,15 @@ impl FallbackMagicFileMetaPlugin {
|
||||
}
|
||||
|
||||
// Get human-readable file type via --brief
|
||||
if let Some(file_type) = self.run_file_command(&["--brief"]) {
|
||||
if !file_type.is_empty() {
|
||||
if let Some(meta_data) = process_metadata_outputs(
|
||||
"file_type",
|
||||
serde_yaml::Value::String(file_type),
|
||||
self.base.outputs(),
|
||||
) {
|
||||
metadata.push(meta_data);
|
||||
}
|
||||
}
|
||||
if let Some(file_type) = self.run_file_command(&["--brief"])
|
||||
&& !file_type.is_empty()
|
||||
&& let Some(meta_data) = process_metadata_outputs(
|
||||
"file_type",
|
||||
serde_yaml::Value::String(file_type),
|
||||
self.base.outputs(),
|
||||
)
|
||||
{
|
||||
metadata.push(meta_data);
|
||||
}
|
||||
|
||||
metadata
|
||||
@@ -415,6 +442,10 @@ impl MetaPlugin for FallbackMagicFileMetaPlugin {
|
||||
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
|
||||
Ok(self.base.options_mut())
|
||||
}
|
||||
|
||||
fn parallel_safe(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "magic"))]
|
||||
|
||||
Reference in New Issue
Block a user