asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: optimize tail filter to use ring buffer directly

Browse Source 
Co-authored-by: aider (openai/andrew/openrouter/deepseek/deepseek-chat-v3.1) <aider@aider.chat>
This commit is contained in:
Andrew Phillips
2025-08-28 20:09:45 -03:00
parent 6c29c9c4c5
commit 9ef4ba2abe
2 changed files with 73 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
src/services/async_item_service.rs
View File

@@ -180,6 +180,14 @@ impl AsyncItemService {
allow_binary: bool,
offset: u64,
length: u64,
head_bytes: Option<usize>,
head_words: Option<usize>,
head_lines: Option<usize>,
tail_bytes: Option<usize>,
tail_words: Option<usize>,
tail_lines: Option<usize>,
line_start: Option<usize>,
line_end: Option<usize>,
) -> Result<(std::pin::Pin<Box<dyn tokio_stream::Stream<Item = Result<tokio_util::bytes::Bytes, std::io::Error>> + Send>>, String), CoreError> {
let _db = self.db.clone();
let _item_service = self.item_service.clone();
@@ -196,7 +204,17 @@ impl AsyncItemService {
text_val == "false"
} else {
// Get binary status using streaming approach
let (_, _, is_binary) = self.get_item_content_info_streaming(item_id).await?;
let (_, _, is_binary) = self.get_item_content_info_streaming(
item_id,
head_bytes,
head_words,
head_lines,
tail_bytes,
tail_words,
tail_lines,
line_start,
line_end
).await?;
is_binary
};
@@ -205,31 +223,30 @@ impl AsyncItemService {
}
}
// Get a streaming reader for the content
// Get a streaming reader for the content with filtering applied
let reader = {
let db = self.db.clone();
let item_service = self.item_service.clone();
tokio::task::spawn_blocking(move || {
let conn = db.blocking_lock();
let item_with_meta = item_service.get_item(&conn, item_id)?;
let item_id_val = item_with_meta.item.id.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?;
let mut item_path = item_service.get_data_path().clone();
item_path.push(item_id_val.to_string());
let reader = item_service.get_compression_service().stream_item_content(
item_path,
&item_with_meta.item.compression
)?;
Ok::<_, CoreError>(reader)
item_service.get_item_content_info_streaming(
&conn,
item_id,
head_bytes,
head_words,
head_lines,
tail_bytes,
tail_words,
tail_lines,
line_start,
line_end
).map(|(reader, _, _)| reader)
})
.await
.unwrap()?
};
// Convert the reader into an async stream manually
// Since ReaderStream requires AsyncRead, we'll create our own implementation
use tokio_util::bytes::Bytes;
// Create a channel to stream data between the blocking thread and async runtime