refactor: Optimize filter plugins to read in chunks of PIPESIZE

Co-authored-by: aider (openai/andrew/openrouter/deepseek/deepseek-chat-v3.1) <aider@aider.chat>
This commit is contained in:
Andrew Phillips
2025-09-02 11:27:50 -03:00
parent 11559c7b26
commit 9a25bdea37
4 changed files with 36 additions and 23 deletions

View File

@@ -1,5 +1,6 @@
use super::FilterPlugin; use super::FilterPlugin;
use std::io::{Result, Read, Write, BufRead}; use std::io::{Result, Read, Write, BufRead};
use crate::common::PIPESIZE;
pub struct HeadBytesFilter { pub struct HeadBytesFilter {
remaining: usize, remaining: usize,
@@ -19,10 +20,13 @@ impl FilterPlugin for HeadBytesFilter {
return Ok(()); return Ok(());
} }
// Read only up to remaining bytes let mut buffer = vec![0; PIPESIZE];
let mut buffer = vec![0; self.remaining]; while self.remaining > 0 {
let bytes_read = reader.read(&mut buffer)?; let to_read = std::cmp::min(self.remaining, PIPESIZE);
if bytes_read > 0 { let bytes_read = reader.read(&mut buffer[..to_read])?;
if bytes_read == 0 {
break;
}
writer.write_all(&buffer[..bytes_read])?; writer.write_all(&buffer[..bytes_read])?;
self.remaining -= bytes_read; self.remaining -= bytes_read;
} }

View File

@@ -1,5 +1,6 @@
use super::FilterPlugin; use super::FilterPlugin;
use std::io::{Result, Read, Write, BufRead, Seek}; use std::io::{Result, Read, Write, BufRead, Seek};
use crate::common::PIPESIZE;
pub struct SkipBytesFilter { pub struct SkipBytesFilter {
remaining: usize, remaining: usize,
@@ -15,22 +16,12 @@ impl SkipBytesFilter {
impl FilterPlugin for SkipBytesFilter { impl FilterPlugin for SkipBytesFilter {
fn filter<R: Read, W: Write>(&mut self, reader: &mut R, writer: &mut W) -> Result<()> { fn filter<R: Read, W: Write>(&mut self, reader: &mut R, writer: &mut W) -> Result<()> {
// Try to use seek if the reader supports it // Skip bytes in chunks
if self.remaining > 0 { if self.remaining > 0 {
// Create a mutable reference that we can try to cast to &mut dyn Seek let mut buffer = vec![0; PIPESIZE];
let any_reader: &mut dyn std::io::Read = reader;
// We need to use unsafe code to perform the cast, but let's find a safer way
// Instead, we can use a helper function that uses a trait object
// For now, we'll use a different approach: check if the concrete type implements Seek
// Since we can't easily do that at runtime, we'll use a different strategy
// Let's read a small buffer at a time to be efficient
const BUFFER_SIZE: usize = 8192;
while self.remaining > 0 { while self.remaining > 0 {
let to_read = std::cmp::min(self.remaining, BUFFER_SIZE); let to_read = std::cmp::min(self.remaining, PIPESIZE);
let mut buffer = vec![0; to_read]; let bytes_read = reader.read(&mut buffer[..to_read])?;
let bytes_read = reader.read(&mut buffer)?;
if bytes_read == 0 { if bytes_read == 0 {
break; break;
} }
@@ -39,7 +30,14 @@ impl FilterPlugin for SkipBytesFilter {
} }
// Copy the remaining data // Copy the remaining data
std::io::copy(reader, writer)?; let mut buffer = vec![0; PIPESIZE];
loop {
let bytes_read = reader.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
writer.write_all(&buffer[..bytes_read])?;
}
Ok(()) Ok(())
} }
} }

View File

@@ -1,5 +1,6 @@
use std::io::{Result, Read, Write}; use std::io::{Result, Read, Write};
use strip_ansi_escapes::strip as strip_ansi_escapes; use strip_ansi_escapes::strip as strip_ansi_escapes;
use crate::common::PIPESIZE;
use super::FilterPlugin; use super::FilterPlugin;
@@ -13,9 +14,18 @@ impl StripAnsiFilter {
impl FilterPlugin for StripAnsiFilter { impl FilterPlugin for StripAnsiFilter {
fn filter<R: Read, W: Write>(&mut self, reader: &mut R, writer: &mut W) -> Result<()> { fn filter<R: Read, W: Write>(&mut self, reader: &mut R, writer: &mut W) -> Result<()> {
let mut data = Vec::new(); let mut buffer = vec![0; PIPESIZE];
reader.read_to_end(&mut data)?; let mut processed_data = Vec::new();
let stripped = strip_ansi_escapes(&data);
loop {
let bytes_read = reader.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
processed_data.extend_from_slice(&buffer[..bytes_read]);
}
let stripped = strip_ansi_escapes(&processed_data);
writer.write_all(&stripped)?; writer.write_all(&stripped)?;
Ok(()) Ok(())
} }

View File

@@ -1,6 +1,7 @@
use super::FilterPlugin; use super::FilterPlugin;
use std::io::{Result, Read, Write, BufRead}; use std::io::{Result, Read, Write, BufRead};
use std::collections::VecDeque; use std::collections::VecDeque;
use crate::common::PIPESIZE;
pub struct TailBytesFilter { pub struct TailBytesFilter {
buffer: VecDeque<u8>, buffer: VecDeque<u8>,
@@ -18,7 +19,7 @@ impl TailBytesFilter {
impl FilterPlugin for TailBytesFilter { impl FilterPlugin for TailBytesFilter {
fn filter<R: Read, W: Write>(&mut self, reader: &mut R, writer: &mut W) -> Result<()> { fn filter<R: Read, W: Write>(&mut self, reader: &mut R, writer: &mut W) -> Result<()> {
let mut temp_buffer = vec![0; 8192]; let mut temp_buffer = vec![0; PIPESIZE];
loop { loop {
let bytes_read = reader.read(&mut temp_buffer)?; let bytes_read = reader.read(&mut temp_buffer)?;
if bytes_read == 0 { if bytes_read == 0 {