asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add lz4 command fallback, remove unused magic.rs

Browse Source 
- Add program-based lz4 command fallback when lz4 feature is disabled
- Feature-gate lz4.rs and lz4 tests to compile without lz4_flex
- Delete legacy magic.rs (unused, no feature gating, superseded by magic_file.rs)
This commit is contained in:
Andrew Phillips
2026-03-13 08:51:10 -03:00
parent b166477202
commit d5d58bc52c
6 changed files with 28 additions and 371 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/compression_engine/gzip.rs
View File

@@ -11,12 +11,12 @@ use std::io::{Read, Write};
#[cfg(feature = "gzip")]
use std::path::PathBuf;
#[cfg(feature = "gzip")]
use flate2::Compression;
#[cfg(feature = "gzip")]
use flate2::read::GzDecoder;
#[cfg(feature = "gzip")]
use flate2::write::GzEncoder;
#[cfg(feature = "gzip")]
use flate2::Compression;
#[cfg(feature = "gzip")]
use crate::compression_engine::CompressionEngine;

11
src/compression_engine/lz4.rs
View File

@@ -1,23 +1,34 @@
#[cfg(feature = "lz4")]
use anyhow::Result;
#[cfg(feature = "lz4")]
use log::*;
#[cfg(feature = "lz4")]
use std::io::Write;
#[cfg(feature = "lz4")]
use lz4_flex::frame::{FrameDecoder, FrameEncoder};
#[cfg(feature = "lz4")]
use std::fs::File;
#[cfg(feature = "lz4")]
use std::io::Read;
#[cfg(feature = "lz4")]
use std::path::PathBuf;
#[cfg(feature = "lz4")]
use crate::compression_engine::CompressionEngine;
#[cfg(feature = "lz4")]
#[derive(Debug, Eq, PartialEq, Clone, Default)]
pub struct CompressionEngineLZ4 {}
#[cfg(feature = "lz4")]
impl CompressionEngineLZ4 {
pub fn new() -> CompressionEngineLZ4 {
CompressionEngineLZ4 {}
}
}
#[cfg(feature = "lz4")]
impl CompressionEngine for CompressionEngineLZ4 {
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>> {
debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self);

15
src/compression_engine/mod.rs
View File

@@ -1,4 +1,4 @@
use anyhow::{Result, anyhow};
use anyhow::{anyhow, Result};
use std::io;
use std::io::{Read, Write};
use std::path::PathBuf;
@@ -176,7 +176,11 @@ impl Clone for Box<dyn CompressionEngine> {
lazy_static! {
static ref COMPRESSION_ENGINES: EnumMap<CompressionType, Box<dyn CompressionEngine>> = {
let mut em = enum_map! {
CompressionType::LZ4 => Box::new(crate::compression_engine::lz4::CompressionEngineLZ4::new()) as Box<dyn CompressionEngine>,
CompressionType::LZ4 => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"lz4",
vec!["-c"],
vec!["-d", "-c"]
)) as Box<dyn CompressionEngine>,
CompressionType::GZip => Box::new(crate::compression_engine::program::CompressionEngineProgram::new(
"gzip",
vec!["-c"],
@@ -207,6 +211,13 @@ lazy_static! {
as Box<dyn CompressionEngine>;
}
#[cfg(feature = "lz4")]
{
em[CompressionType::LZ4] =
Box::new(crate::compression_engine::lz4::CompressionEngineLZ4::new())
as Box<dyn CompressionEngine>;
}
em
};
}

2
src/compression_engine/program.rs
View File

@@ -1,4 +1,4 @@
use anyhow::{Context, Result, anyhow};
use anyhow::{anyhow, Context, Result};
use log::*;
use std::fs::File;
use std::io::{Read, Write};

366
src/meta_plugin/magic.rs
View File

@@ -1,366 +0,0 @@
use magic::{Cookie, CookieFlags};
use std::io;
use crate::common::PIPESIZE;
use crate::meta_plugin::{MetaPlugin, MetaPluginType};
#[derive(Debug)]
pub struct MagicFileMetaPlugin {
buffer: Vec<u8>,
max_buffer_size: usize,
is_finalized: bool,
cookie: Option<Cookie>,
base: crate::meta_plugin::BaseMetaPlugin,
}
impl MagicFileMetaPlugin {
pub fn new(
options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> MagicFileMetaPlugin {
// Start with default options
let mut final_options = std::collections::HashMap::new();
final_options.insert(
"max_buffer_size".to_string(),
serde_yaml::Value::Number(PIPESIZE.into()),
);
if let Some(opts) = options {
for (key, value) in opts {
final_options.insert(key, value);
}
}
// Start with default outputs
let mut final_outputs = std::collections::HashMap::new();
let default_outputs = vec![
"mime_type".to_string(),
"mime_encoding".to_string(),
"file_type".to_string(),
];
for output_name in default_outputs {
final_outputs.insert(output_name.clone(), serde_yaml::Value::String(output_name));
}
if let Some(outs) = outputs {
for (key, value) in outs {
final_outputs.insert(key, value);
}
}
let max_buffer_size = final_options
.get("max_buffer_size")
.and_then(|v| v.as_u64())
.unwrap_or(PIPESIZE as u64) as usize;
// Ensure the default max_buffer_size is in the options
if !final_options.contains_key("max_buffer_size") {
final_options.insert(
"max_buffer_size".to_string(),
serde_yaml::Value::Number(PIPESIZE.into()),
);
}
let mut base = crate::meta_plugin::BaseMetaPlugin::new();
base.outputs = final_outputs;
base.options = final_options;
MagicFileMetaPlugin {
buffer: Vec::new(),
max_buffer_size,
is_finalized: false,
cookie: None,
base,
}
}
fn get_magic_result(&self, flags: CookieFlags) -> io::Result<String> {
// Use the existing cookie and just change flags
if let Some(cookie) = &self.cookie {
cookie.set_flags(flags).map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("Failed to set magic flags: {}", e),
)
})?;
let result = cookie.buffer(&self.buffer).map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("Failed to analyze buffer: {}", e),
)
})?;
// Clean up the result - remove extra whitespace and take first part if needed
let trimmed = result.trim();
// For some magic results, we might want just the first part before semicolon or comma
let cleaned = if trimmed.contains(';') {
trimmed.split(';').next().unwrap_or(trimmed).trim()
} else if trimmed.contains(',')
&& flags.contains(CookieFlags::MIME_TYPE | CookieFlags::MIME_ENCODING)
{
trimmed.split(',').next().unwrap_or(trimmed).trim()
} else {
trimmed
};
Ok(cleaned.to_string())
} else {
Err(io::Error::new(
io::ErrorKind::Other,
"Magic cookie not initialized",
))
}
}
/// Helper function to process all magic types and collect metadata
fn process_magic_types(&self) -> Vec<crate::meta_plugin::MetaData> {
let mut metadata = Vec::new();
// Define the types to process with their corresponding flags
let types_to_process = [
("mime_type", CookieFlags::MIME_TYPE),
("mime_encoding", CookieFlags::MIME_ENCODING),
("file_type", CookieFlags::default()),
];
for (name, flags) in types_to_process.iter() {
if let Ok(result) = self.get_magic_result(*flags) {
if !result.is_empty() {
// Use process_metadata_outputs to handle output mapping
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
name,
serde_yaml::Value::String(result),
self.base.outputs(),
) {
metadata.push(meta_data);
}
}
}
}
metadata
}
}
impl MetaPlugin for MagicFileMetaPlugin {
/// Checks if the plugin has been finalized.
///
/// # Returns
///
/// `true` if finalized, `false` otherwise.
fn is_finalized(&self) -> bool {
self.is_finalized
}
/// Sets the finalized state of the plugin.
///
/// # Arguments
///
/// * `finalized` - The new finalized state.
fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized;
}
/// Initializes the magic cookie for file type detection.
///
/// Loads the magic database; finalizes if initialization fails.
///
/// # Returns
///
/// A `MetaPluginResponse` with empty metadata; `is_finalized` is `true` on failure.
///
/// # Errors
///
/// Logs errors; returns finalized response on cookie or load failure.
///
/// # Examples
///
/// ```
/// let mut plugin = MagicFileMetaPlugin::new(None, None);
/// let response = plugin.initialize();
/// ```
fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
// Initialize the magic cookie once
let cookie = match Cookie::open(Default::default()) {
Ok(cookie) => cookie,
Err(_e) => {
return crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
};
if let Err(_e) = cookie.load(&[] as &[&str]) {
return crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
self.cookie = Some(cookie);
crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(),
is_finalized: false,
}
}
/// Finalizes the plugin and performs file type detection.
///
/// Analyzes the accumulated buffer and outputs detected types.
///
/// # Returns
///
/// A `MetaPluginResponse` with detection metadata and finalized state set to `true`.
///
/// # Examples
///
/// ```
/// let mut plugin = MagicFileMetaPlugin::new(None, None);
/// // ... after updates
/// let response = plugin.finalize();
/// assert!(response.is_finalized);
/// ```
fn finalize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
// If already finalized, don't process again
if self.is_finalized {
return crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
let metadata = self.process_magic_types();
// Mark as finalized
self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse {
metadata,
is_finalized: true,
}
}
/// Updates the plugin with new data, accumulating for analysis.
///
/// Buffers data up to `max_buffer_size`; triggers detection when full.
///
/// # Arguments
///
/// * `data` - Content chunk to buffer.
///
/// # Returns
///
/// A `MetaPluginResponse` with metadata on buffer full; finalizes then.
///
/// # Examples
///
/// ```
/// let mut plugin = MagicFileMetaPlugin::new(None, None);
/// let response = plugin.update(b"content");
/// ```
fn update(&mut self, data: &[u8]) -> crate::meta_plugin::MetaPluginResponse {
// If already finalized, don't process more data
if self.is_finalized {
return crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
let mut metadata = Vec::new();
// Only collect up to max_buffer_size
let remaining_capacity = self.max_buffer_size.saturating_sub(self.buffer.len());
if remaining_capacity > 0 {
let bytes_to_copy = std::cmp::min(data.len(), remaining_capacity);
self.buffer.extend_from_slice(&data[..bytes_to_copy]);
// Check if we've reached our buffer limit and return metadata
if self.buffer.len() >= self.max_buffer_size {
metadata = self.process_magic_types();
// Mark as finalized when we've processed enough data
self.is_finalized = true;
}
}
let is_finalized = !metadata.is_empty();
crate::meta_plugin::MetaPluginResponse {
metadata,
is_finalized,
}
}
/// Returns the type of this meta plugin.
///
/// # Returns
///
/// `MetaPluginType::MagicFile`.
fn meta_type(&self) -> MetaPluginType {
MetaPluginType::MagicFile
}
/// Returns a reference to the outputs mapping.
///
/// # Returns
///
/// A reference to the `HashMap` of outputs.
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs()
}
/// Returns a mutable reference to the outputs mapping.
///
/// # Returns
///
/// A mutable reference to the `HashMap` of outputs.
fn outputs_mut(
&mut self,
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
Ok(self.base.outputs_mut())
}
/// Returns the default output names for this plugin.
///
/// # Returns
///
/// Vector of default output field names.
fn default_outputs(&self) -> Vec<String> {
vec![
"mime_type".to_string(),
"mime_encoding".to_string(),
"file_type".to_string(),
]
}
/// Returns a reference to the options mapping.
///
/// # Returns
///
/// A reference to the `HashMap` of options.
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options()
}
/// Returns a mutable reference to the options mapping.
///
/// # Returns
///
/// A mutable reference to the `HashMap` of options.
fn options_mut(
&mut self,
) -> anyhow::Result<&mut std::collections::HashMap<String, serde_yaml::Value>> {
Ok(self.base.options_mut())
}
}
use crate::meta_plugin::register_meta_plugin;
// Register the plugin at module initialization time
#[ctor::ctor]
fn register_magic_file_plugin() {
register_meta_plugin(MetaPluginType::MagicFile, |options, outputs| {
Box::new(MagicFileMetaPlugin::new(options, outputs))
});
}

1
src/tests/compression/lz4_tests.rs
View File

@@ -1,4 +1,5 @@
#[cfg(test)]
#[cfg(feature = "lz4")]
mod tests {
use crate::compression_engine::lz4::CompressionEngineLZ4;
use crate::tests::common::test_helpers::test_compression_engine;