asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add streaming content support with offset and length parameters

Browse Source 
Co-authored-by: aider (openai/andrew/openrouter/qwen/qwen3-coder) <aider@aider.chat>
This commit is contained in:
Andrew Phillips
2025-08-25 18:33:22 -03:00
parent fd61b66a97
commit 06c6971d07
2 changed files with 100 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
src/modes/server/api/item.rs
View File

@@ -9,8 +9,10 @@ use std::collections::HashMap;
use crate::services::async_item_service::AsyncItemService;
use crate::services::error::CoreError;
use crate::modes::server::common::{AppState, ApiResponse, ItemInfo, TagsQuery, ListItemsQuery, ItemInfoListResponse, ItemInfoResponse, MetadataResponse, ItemQuery};
use crate::modes::server::common::{AppState, ApiResponse, ItemInfo, TagsQuery, ListItemsQuery, ItemInfoListResponse, ItemInfoResponse, MetadataResponse, ItemQuery, ItemContentQuery};
use crate::common::is_binary::is_binary;
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tokio_util::io::ReaderStream;
#[utoipa::path(
get,
@@ -138,7 +140,7 @@ pub async fn handle_post_item(
path = "/api/item/latest/content",
operation_id = "get_item_latest_content",
summary = "Download latest item content",
description = "Download the raw content of the most recently stored item. The content is automatically decompressed and returned with the appropriate MIME type header for proper browser handling. If tags are specified, returns the latest item matching ALL the given tags. If allow_binary is false and the content is detected as binary, a 400 error is returned.",
description = "Download the raw content of the most recently stored item. The content is automatically decompressed and returned with the appropriate MIME type header for proper browser handling. If tags are specified, returns the latest item matching ALL the given tags. If allow_binary is false and the content is detected as binary, a 400 error is returned. Supports offset and length parameters for partial content retrieval.",
responses(
(status = 200, description = "Successfully retrieved latest item raw content with appropriate Content-Type header set based on detected MIME type"),
(status = 400, description = "Bad request - Content is binary but allow_binary is false"),
@@ -148,7 +150,9 @@ pub async fn handle_post_item(
),
params(
("tags" = Option<String>, Query, description = "Comma-separated list of tags to filter by (e.g., 'important,work'). If specified, returns the latest item that has ALL the specified tags."),
("allow_binary" = Option<bool>, Query, description = "Whether to allow binary content to be returned (default: true). When false, returns 400 for binary files.")
("allow_binary" = Option<bool>, Query, description = "Whether to allow binary content to be returned (default: true). When false, returns 400 for binary files."),
("offset" = Option<u64>, Query, description = "Byte offset from the start of the file to begin reading (default: 0)"),
("length" = Option<u64>, Query, description = "Maximum number of bytes to return, starting at offset (default: 0 for unlimited)")
),
security(
("bearerAuth" = [])
@@ -157,7 +161,7 @@ pub async fn handle_post_item(
)]
pub async fn handle_get_item_latest_content(
State(state): State<AppState>,
Query(params): Query<TagsQuery>,
Query(params): Query<ItemContentQuery>,
) -> Result<Response, StatusCode> {
let tags: Vec<String> = params
.tags
@@ -174,38 +178,17 @@ pub async fn handle_get_item_latest_content(
match item_with_meta {
Ok(item) => {
let item_id = item.item.id.unwrap();
match item_service.get_item_content(item_id).await {
Ok(item_with_content) => {
let content = item_with_content.content;
let metadata = item_with_content.item_with_meta.meta_as_map();
// Check if content is binary when allow_binary is false
if !params.allow_binary {
let is_content_binary = if let Some(binary_meta) = metadata.get("binary") {
binary_meta == "true"
} else {
// If binary metadata not available, check the content
is_binary(&content)
};
if is_content_binary {
return Err(StatusCode::BAD_REQUEST);
}
}
let mime_type = metadata
.get("mime_type")
.map(|s| s.to_string())
.unwrap_or_else(|| "application/octet-stream".to_string());
match stream_item_content(&item_service, item_id, params.allow_binary, params.offset, params.length).await {
Ok((stream, mime_type)) => {
let body = axum::body::Body::from_stream(stream);
let response = Response::builder()
.header(header::CONTENT_TYPE, mime_type)
.body(axum::body::Body::from(content))
.body(body)
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
Ok(response)
}
Err(e) => {
warn!("Failed to get raw content for item {}: {}", item_id, e);
warn!("Failed to stream content for item {}: {}", item_id, e);
Err(StatusCode::INTERNAL_SERVER_ERROR)
}
}
@@ -218,12 +201,75 @@ pub async fn handle_get_item_latest_content(
}
}
async fn stream_item_content(
service: &AsyncItemService,
item_id: i64,
allow_binary: bool,
offset: u64,
length: u64,
) -> anyhow::Result<(impl tokio_stream::Stream<Item = Result<bytes::Bytes, std::io::Error>>, String)> {
let item_with_meta = service.get_item(item_id).await?;
let metadata = item_with_meta.meta_as_map();
// Check if content is binary when allow_binary is false
if !allow_binary {
let is_content_binary = if let Some(binary_meta) = metadata.get("binary") {
binary_meta == "true"
} else {
// If binary metadata not available, we need to check the file content
// Read the first 8192 bytes to determine if it's binary
let file_path = service.data_dir.join(format!("{}.dat", item_id));
if let Ok(mut file) = tokio::fs::File::open(&file_path).await {
let mut buffer = vec![0; 8192];
if let Ok(bytes_read) = file.read(&mut buffer).await {
buffer.truncate(bytes_read);
is_binary(&buffer)
} else {
false // Default to non-binary if we can't read
}
} else {
false // Default to non-binary if we can't open the file
}
};
if is_content_binary {
return Err(anyhow::anyhow!("Binary content not allowed"));
}
}
let mime_type = metadata
.get("mime_type")
.map(|s| s.to_string())
.unwrap_or_else(|| "application/octet-stream".to_string());
// Open the file for streaming
let file_path = service.data_dir.join(format!("{}.dat", item_id));
let file = tokio::fs::File::open(&file_path).await?;
let mut buffered_file = tokio::io::BufReader::new(file);
// Seek to the requested offset if needed
if offset > 0 {
buffered_file.seek(std::io::SeekFrom::Start(offset)).await?;
}
// Create a reader stream with optional length limit
let stream = if length > 0 {
// Limit the stream to the specified length
ReaderStream::new(buffered_file.take(length))
} else {
// Stream the entire file from the offset
ReaderStream::new(buffered_file)
};
Ok((stream, mime_type))
}
#[utoipa::path(
get,
path = "/api/item/{item_id}/content",
operation_id = "get_item_content",
summary = "Download item content",
description = "Download the raw content of a specific item by its ID. The content is automatically decompressed and returned with the appropriate MIME type header for proper browser handling. This endpoint is ideal for downloading files or viewing content directly in the browser. If allow_binary is false and the content is detected as binary, a 400 error is returned.",
description = "Download the raw content of a specific item by its ID. The content is automatically decompressed and returned with the appropriate MIME type header for proper browser handling. This endpoint is ideal for downloading files or viewing content directly in the browser. If allow_binary is false and the content is detected as binary, a 400 error is returned. Supports offset and length parameters for partial content retrieval.",
responses(
(status = 200, description = "Successfully retrieved item raw content with appropriate Content-Type header set based on detected MIME type"),
(status = 400, description = "Bad request - Invalid item ID (must be a positive integer) or content is binary but allow_binary is false"),
@@ -233,7 +279,9 @@ pub async fn handle_get_item_latest_content(
),
params(
("item_id" = i64, Path, description = "Unique identifier of the item to retrieve content for (must be a positive integer)"),
("allow_binary" = Option<bool>, Query, description = "Whether to allow binary content to be returned (default: true). When false, returns 400 for binary files.")
("allow_binary" = Option<bool>, Query, description = "Whether to allow binary content to be returned (default: true). When false, returns 400 for binary files."),
("offset" = Option<u64>, Query, description = "Byte offset from the start of the file to begin reading (default: 0)"),
("length" = Option<u64>, Query, description = "Maximum number of bytes to return, starting at offset (default: 0 for unlimited)")
),
security(
("bearerAuth" = [])
@@ -252,38 +300,17 @@ pub async fn handle_get_item_content(
let item_service = AsyncItemService::new(state.data_dir.clone(), state.db.clone());
match item_service.get_item_content(item_id).await {
Ok(item_with_content) => {
let content = item_with_content.content;
let metadata = item_with_content.item_with_meta.meta_as_map();
// Check if content is binary when allow_binary is false
if !params.allow_binary {
let is_content_binary = if let Some(binary_meta) = metadata.get("binary") {
binary_meta == "true"
} else {
// If binary metadata not available, check the content
is_binary(&content)
};
if is_content_binary {
return Err(StatusCode::BAD_REQUEST);
}
}
let mime_type = metadata
.get("mime_type")
.map(|s| s.to_string())
.unwrap_or_else(|| "application/octet-stream".to_string());
match stream_item_content(&item_service, item_id, params.allow_binary, params.offset, params.length).await {
Ok((stream, mime_type)) => {
let body = axum::body::Body::from_stream(stream);
let response = Response::builder()
.header(header::CONTENT_TYPE, mime_type)
.body(axum::body::Body::from(content))
.body(body)
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
Ok(response)
}
Err(e) => {
warn!("Failed to get raw content for item {}: {}", item_id, e);
warn!("Failed to stream content for item {}: {}", item_id, e);
Err(StatusCode::INTERNAL_SERVER_ERROR)
}
}