feat: implement buffer size limit and early metadata storage for binary plugin

Co-authored-by: aider (openai/andrew/openrouter/qwen/qwen3-coder) <aider@aider.chat>
This commit is contained in:
Andrew Phillips
2025-08-18 20:39:52 -03:00
parent 4ba8ce74cb
commit d19ef19a5b

View File

@@ -26,19 +26,11 @@ impl BinaryMetaPlugin {
} }
} }
} fn save_metadata(&mut self, conn: &Connection) -> Result<()> {
impl MetaPlugin for BinaryMetaPlugin {
fn is_internal(&self) -> bool {
true
}
fn finalize(&mut self, conn: &Connection) -> Result<()> {
// Save the binary detection result when finalizing, after all data has been collected
if !self.is_saved { if !self.is_saved {
if let Some(item_id) = self.item_id { if let Some(item_id) = self.item_id {
let is_binary = is_binary(&self.buffer); let is_binary_result = is_binary(&self.buffer);
let value = if is_binary { "true".to_string() } else { "false".to_string() }; let value = if is_binary_result { "true".to_string() } else { "false".to_string() };
// Save to database immediately using central output handler // Save to database immediately using central output handler
let _ = output_metadata(conn, item_id, &self.meta_name, value, &self.output_names); let _ = output_metadata(conn, item_id, &self.meta_name, value, &self.output_names);
@@ -48,13 +40,34 @@ impl MetaPlugin for BinaryMetaPlugin {
} }
Ok(()) Ok(())
} }
}
fn update(&mut self, data: &[u8], _conn: &Connection) { impl MetaPlugin for BinaryMetaPlugin {
// Only collect up to max_buffer_size fn is_internal(&self) -> bool {
true
}
fn finalize(&mut self, conn: &Connection) -> Result<()> {
// Save the binary detection result when finalizing, if not already saved
self.save_metadata(conn)
}
fn update(&mut self, data: &[u8], conn: &Connection) {
// If we've already saved the metadata, no need to collect more data
if self.is_saved {
return;
}
// Add data to buffer
let remaining_capacity = self.max_buffer_size.saturating_sub(self.buffer.len()); let remaining_capacity = self.max_buffer_size.saturating_sub(self.buffer.len());
if remaining_capacity > 0 { if remaining_capacity > 0 {
let bytes_to_copy = std::cmp::min(data.len(), remaining_capacity); let bytes_to_copy = std::cmp::min(data.len(), remaining_capacity);
self.buffer.extend_from_slice(&data[..bytes_to_copy]); self.buffer.extend_from_slice(&data[..bytes_to_copy]);
// If we've reached the buffer limit, save the metadata immediately
if self.buffer.len() >= self.max_buffer_size {
let _ = self.save_metadata(conn);
}
} }
} }
@@ -72,6 +85,11 @@ impl MetaPlugin for BinaryMetaPlugin {
if let Some(size) = max_buffer_size.as_u64() { if let Some(size) = max_buffer_size.as_u64() {
self.max_buffer_size = size as usize; self.max_buffer_size = size as usize;
} }
} else if let Some(max_buffer_size) = options.get("max_buffer") {
// Also support "max_buffer" for backward compatibility
if let Some(size) = max_buffer_size.as_u64() {
self.max_buffer_size = size as usize;
}
} }
if let Some(outputs) = options.get("outputs") { if let Some(outputs) = options.get("outputs") {