asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: implement streaming for large file handling

Browse Source 
Co-authored-by: aider (openai/andrew/openrouter/deepseek/deepseek-chat-v3.1) <aider@aider.chat>
This commit is contained in:
Andrew Phillips
2025-08-25 21:07:04 -03:00
parent 3478ffee2c
commit 1640932148
5 changed files with 165 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
src/modes/get.rs
View File

@@ -32,21 +32,16 @@ pub fn mode_get(
}
let item_service = ItemService::new(data_path);
let item_with_content =
item_service.find_item(conn, ids, tags, &meta)
.and_then(|item_with_meta| {
let item_id = item_with_meta.item.id.unwrap();
item_service.get_item_content(conn, item_id)
})
let item_with_meta = item_service.find_item(conn, ids, tags, &meta)
.map_err(|e| anyhow!("Unable to find matching item in database: {}", e))?;
let content = &item_with_content.content;
let item_id = item_with_meta.item.id.unwrap();
// Determine if we should detect binary data
let mut detect_binary = !settings.force && std::io::stdout().is_terminal();
if detect_binary {
let meta_map = item_with_content.item_with_meta.meta_as_map();
let meta_map = item_with_meta.meta_as_map();
if let Some(binary_val) = meta_map.get("binary") {
if binary_val == "false" {
detect_binary = false;
@@ -58,12 +53,37 @@ pub fn mode_get(
}
}
if detect_binary && is_binary(content) {
return Err(anyhow!(
"Refusing to output binary data to TTY, use --force to override"
));
// Use streaming approach to handle large files
let mut reader = item_service.compression_service.stream_item_content(
data_path.join(item_id.to_string()),
&item_with_meta.item.compression
)?;
if detect_binary {
// Read only the first 8192 bytes for binary detection
let mut sample_buffer = vec![0; 8192];
let bytes_read = reader.read(&mut sample_buffer)?;
if is_binary(&sample_buffer[..bytes_read]) {
return Err(anyhow!(
"Refusing to output binary data to TTY, use --force to override"
));
}
// We need to create a new reader since we consumed some bytes
reader = item_service.compression_service.stream_item_content(
data_path.join(item_id.to_string()),
&item_with_meta.item.compression
)?;
}
std::io::stdout().write_all(content)?;
// Stream the content to stdout
let mut stdout = std::io::stdout();
let mut buffer = [0; 8192];
loop {
let bytes_read = reader.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
stdout.write_all(&buffer[..bytes_read])?;
}
Ok(())
}