refactor: reduce code duplication in filter and item services

Co-authored-by: aider (openai/andrew/openrouter/mistralai/mistral-medium-3.1) <aider@aider.chat>
This commit is contained in:
Andrew Phillips
2025-08-28 20:51:39 -03:00
parent 5542f5592a
commit 4c8466bb21
5 changed files with 268 additions and 151 deletions

View File

@@ -367,42 +367,42 @@ impl TextMetaPlugin {
/// Helper method to output word and line counts
fn output_word_line_counts(&mut self) -> Vec<crate::meta_plugin::MetaData> {
let mut metadata = Vec::new();
// Process any remaining data in utf8_buffer
self.process_remaining_utf8_buffer();
// Handle the last line if tracking line lengths
self.handle_last_line_for_length_tracking();
// Output word count if tracked
if let Some(meta_data) = self.output_word_count_metadata() {
metadata.push(meta_data);
// Collect all metadata outputs
let mut metadata = Vec::new();
// Add metadata outputs using a more concise approach
let outputs_to_check = vec![
(self.output_word_count_metadata(), "word count"),
(self.output_line_count_metadata(), "line count"),
];
for (output, _) in outputs_to_check {
if let Some(meta_data) = output {
metadata.push(meta_data);
}
}
// Output line count if tracked
if let Some(meta_data) = self.output_line_count_metadata() {
metadata.push(meta_data);
}
// Output line length statistics if tracked
if self.track_line_lengths && self.line_count_for_stats > 0 {
// Calculate and output max line length if enabled
if let Some(meta_data) = self.output_max_line_length_metadata() {
metadata.push(meta_data);
}
// Calculate and output mean line length if enabled
if let Some(meta_data) = self.output_mean_line_length_metadata() {
metadata.push(meta_data);
}
// Calculate and output median line length if enabled
if let Some(meta_data) = self.output_median_line_length_metadata() {
metadata.push(meta_data);
let line_stats_outputs = vec![
(self.output_max_line_length_metadata(), "max line length"),
(self.output_mean_line_length_metadata(), "mean line length"),
(self.output_median_line_length_metadata(), "median line length"),
];
for (output, _) in line_stats_outputs {
if let Some(meta_data) = output {
metadata.push(meta_data);
}
}
}
metadata
}
}
@@ -417,19 +417,9 @@ impl MetaPlugin for TextMetaPlugin {
}
fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
// If already finalized, don't process more data
if self.is_finalized {
return MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
let mut metadata = Vec::new();
/// Helper method to create a filter chain and process data
fn create_filter_and_process_data(&self, data: &[u8]) -> Vec<u8> {
// Check if we have head/tail options that would affect processing
// These options come from the base plugin's options
let head_bytes = self.base.options.get("head_bytes")
.and_then(|v| v.as_u64())
.map(|v| v as usize);
@@ -442,7 +432,7 @@ impl MetaPlugin for TextMetaPlugin {
let tail_lines = self.base.options.get("tail_lines")
.and_then(|v| v.as_u64())
.map(|v| v as usize);
// Build filter string from individual parameters
let mut filter_parts = Vec::new();
if let Some(bytes) = head_bytes {
@@ -457,27 +447,43 @@ impl MetaPlugin for TextMetaPlugin {
if let Some(lines) = tail_lines {
filter_parts.push(format!("tail_lines({})", lines));
}
// Use the filter service to process data
let processed_data = if !filter_parts.is_empty() {
if !filter_parts.is_empty() {
let filter_str = filter_parts.join(" | ");
let filter_service = crate::services::filter_service::FilterService::new();
let mut filter_chain = filter_service.create_filter_chain(Some(&filter_str))
.map_err(|e| {
let mut filter_chain = match filter_service.create_filter_chain(Some(&filter_str)) {
Ok(chain) => chain,
Err(e) => {
log::error!("Failed to create filter chain: {}", e);
data.to_vec()
})
.unwrap_or_else(|_| data.to_vec());
return data.to_vec();
}
};
// Process the data through the filter chain
filter_service.process_data(&mut filter_chain, data)
.unwrap_or_else(|e| {
match filter_service.process_data(&mut filter_chain, data) {
Ok(processed) => processed,
Err(e) => {
log::error!("Failed to process data through filter: {}", e);
data.to_vec()
})
}
}
} else {
data.to_vec()
};
}
}
fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
// If already finalized, don't process more data
if self.is_finalized {
return MetaPluginResponse {
metadata: Vec::new(),
is_finalized: true,
};
}
let mut metadata = Vec::new();
let processed_data = self.create_filter_and_process_data(data);
// If we haven't determined if content is binary yet, build buffer and check
if self.is_binary_content.is_none() {