refactor: reduce code duplication in filter and item services
Co-authored-by: aider (openai/andrew/openrouter/mistralai/mistral-medium-3.1) <aider@aider.chat>
This commit is contained in:
@@ -367,42 +367,42 @@ impl TextMetaPlugin {
|
||||
|
||||
/// Helper method to output word and line counts
|
||||
fn output_word_line_counts(&mut self) -> Vec<crate::meta_plugin::MetaData> {
|
||||
let mut metadata = Vec::new();
|
||||
|
||||
// Process any remaining data in utf8_buffer
|
||||
self.process_remaining_utf8_buffer();
|
||||
|
||||
|
||||
// Handle the last line if tracking line lengths
|
||||
self.handle_last_line_for_length_tracking();
|
||||
|
||||
// Output word count if tracked
|
||||
if let Some(meta_data) = self.output_word_count_metadata() {
|
||||
metadata.push(meta_data);
|
||||
|
||||
// Collect all metadata outputs
|
||||
let mut metadata = Vec::new();
|
||||
|
||||
// Add metadata outputs using a more concise approach
|
||||
let outputs_to_check = vec![
|
||||
(self.output_word_count_metadata(), "word count"),
|
||||
(self.output_line_count_metadata(), "line count"),
|
||||
];
|
||||
|
||||
for (output, _) in outputs_to_check {
|
||||
if let Some(meta_data) = output {
|
||||
metadata.push(meta_data);
|
||||
}
|
||||
}
|
||||
|
||||
// Output line count if tracked
|
||||
if let Some(meta_data) = self.output_line_count_metadata() {
|
||||
metadata.push(meta_data);
|
||||
}
|
||||
|
||||
|
||||
// Output line length statistics if tracked
|
||||
if self.track_line_lengths && self.line_count_for_stats > 0 {
|
||||
// Calculate and output max line length if enabled
|
||||
if let Some(meta_data) = self.output_max_line_length_metadata() {
|
||||
metadata.push(meta_data);
|
||||
}
|
||||
|
||||
// Calculate and output mean line length if enabled
|
||||
if let Some(meta_data) = self.output_mean_line_length_metadata() {
|
||||
metadata.push(meta_data);
|
||||
}
|
||||
|
||||
// Calculate and output median line length if enabled
|
||||
if let Some(meta_data) = self.output_median_line_length_metadata() {
|
||||
metadata.push(meta_data);
|
||||
let line_stats_outputs = vec![
|
||||
(self.output_max_line_length_metadata(), "max line length"),
|
||||
(self.output_mean_line_length_metadata(), "mean line length"),
|
||||
(self.output_median_line_length_metadata(), "median line length"),
|
||||
];
|
||||
|
||||
for (output, _) in line_stats_outputs {
|
||||
if let Some(meta_data) = output {
|
||||
metadata.push(meta_data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
metadata
|
||||
}
|
||||
}
|
||||
@@ -417,19 +417,9 @@ impl MetaPlugin for TextMetaPlugin {
|
||||
}
|
||||
|
||||
|
||||
fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
|
||||
// If already finalized, don't process more data
|
||||
if self.is_finalized {
|
||||
return MetaPluginResponse {
|
||||
metadata: Vec::new(),
|
||||
is_finalized: true,
|
||||
};
|
||||
}
|
||||
|
||||
let mut metadata = Vec::new();
|
||||
|
||||
/// Helper method to create a filter chain and process data
|
||||
fn create_filter_and_process_data(&self, data: &[u8]) -> Vec<u8> {
|
||||
// Check if we have head/tail options that would affect processing
|
||||
// These options come from the base plugin's options
|
||||
let head_bytes = self.base.options.get("head_bytes")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|v| v as usize);
|
||||
@@ -442,7 +432,7 @@ impl MetaPlugin for TextMetaPlugin {
|
||||
let tail_lines = self.base.options.get("tail_lines")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|v| v as usize);
|
||||
|
||||
|
||||
// Build filter string from individual parameters
|
||||
let mut filter_parts = Vec::new();
|
||||
if let Some(bytes) = head_bytes {
|
||||
@@ -457,27 +447,43 @@ impl MetaPlugin for TextMetaPlugin {
|
||||
if let Some(lines) = tail_lines {
|
||||
filter_parts.push(format!("tail_lines({})", lines));
|
||||
}
|
||||
|
||||
|
||||
// Use the filter service to process data
|
||||
let processed_data = if !filter_parts.is_empty() {
|
||||
if !filter_parts.is_empty() {
|
||||
let filter_str = filter_parts.join(" | ");
|
||||
let filter_service = crate::services::filter_service::FilterService::new();
|
||||
let mut filter_chain = filter_service.create_filter_chain(Some(&filter_str))
|
||||
.map_err(|e| {
|
||||
let mut filter_chain = match filter_service.create_filter_chain(Some(&filter_str)) {
|
||||
Ok(chain) => chain,
|
||||
Err(e) => {
|
||||
log::error!("Failed to create filter chain: {}", e);
|
||||
data.to_vec()
|
||||
})
|
||||
.unwrap_or_else(|_| data.to_vec());
|
||||
|
||||
return data.to_vec();
|
||||
}
|
||||
};
|
||||
|
||||
// Process the data through the filter chain
|
||||
filter_service.process_data(&mut filter_chain, data)
|
||||
.unwrap_or_else(|e| {
|
||||
match filter_service.process_data(&mut filter_chain, data) {
|
||||
Ok(processed) => processed,
|
||||
Err(e) => {
|
||||
log::error!("Failed to process data through filter: {}", e);
|
||||
data.to_vec()
|
||||
})
|
||||
}
|
||||
}
|
||||
} else {
|
||||
data.to_vec()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
|
||||
// If already finalized, don't process more data
|
||||
if self.is_finalized {
|
||||
return MetaPluginResponse {
|
||||
metadata: Vec::new(),
|
||||
is_finalized: true,
|
||||
};
|
||||
}
|
||||
|
||||
let mut metadata = Vec::new();
|
||||
let processed_data = self.create_filter_and_process_data(data);
|
||||
|
||||
// If we haven't determined if content is binary yet, build buffer and check
|
||||
if self.is_binary_content.is_none() {
|
||||
|
||||
Reference in New Issue
Block a user