fix: correct critical bugs and improve pipe streaming performance
Critical bug fixes:
- save_item now returns real Item from database, not a hardcoded fake
- AsyncDataService::save() reuses self.sync_service instead of creating redundant instance
- GenerateStatus trait signature mismatch fixed (CLI/API decoupling)
Performance improvements (pipe path untouched):
- CompressionEngine::open() returns Box<dyn Read + Send> enabling true streaming
- mode_get eliminates triple full-file read (was sampling then re-reading entire file)
- FilteringReader adds fast-path bypass when no filters, pre-allocates temp buffer
- text.rs meta plugin processes &[u8] slice directly, eliminates data.to_vec() clone
API correctness:
- Tag parse errors now return 400 instead of being silently discarded
- compute_diff uses similar crate (LCS-based) instead of naive positional comparison
Cleanup:
- Modernize string formatting (format!({x})) across codebase
- Remove redundant DB query in get mode
- Derive Debug/ToSchema on public types
- Delete placeholder test files with no real assertions
- Extract parse_comma_tags utility function
This commit is contained in:
@@ -11,12 +11,12 @@ use std::io::{Read, Write};
|
||||
#[cfg(feature = "gzip")]
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[cfg(feature = "gzip")]
|
||||
use flate2::Compression;
|
||||
#[cfg(feature = "gzip")]
|
||||
use flate2::read::GzDecoder;
|
||||
#[cfg(feature = "gzip")]
|
||||
use flate2::write::GzEncoder;
|
||||
#[cfg(feature = "gzip")]
|
||||
use flate2::Compression;
|
||||
|
||||
#[cfg(feature = "gzip")]
|
||||
use crate::compression_engine::CompressionEngine;
|
||||
@@ -42,7 +42,7 @@ impl CompressionEngine for CompressionEngineGZip {
|
||||
("<INTERNAL>".to_string(), "".to_string(), "".to_string())
|
||||
}
|
||||
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>> {
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>> {
|
||||
debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self);
|
||||
|
||||
let file = File::open(file_path)?;
|
||||
@@ -84,7 +84,7 @@ impl<W: Write> Drop for AutoFinishGzEncoder<W> {
|
||||
if let Some(encoder) = self.encoder.take() {
|
||||
debug!("COMPRESSION: Finishing");
|
||||
if let Err(e) = encoder.finish() {
|
||||
warn!("Failed to finish GZip encoder: {}", e);
|
||||
warn!("Failed to finish GZip encoder: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ impl CompressionEngineLZ4 {
|
||||
}
|
||||
|
||||
impl CompressionEngine for CompressionEngineLZ4 {
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>> {
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>> {
|
||||
debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self);
|
||||
|
||||
let file = File::open(file_path)?;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use anyhow::{Result, anyhow};
|
||||
use anyhow::{anyhow, Result};
|
||||
use std::io;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::PathBuf;
|
||||
@@ -73,14 +73,14 @@ pub trait CompressionEngine: Send + Sync {
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// * `Result<Box<dyn Read>>` - A boxed reader that decompresses the file on read,
|
||||
/// * `Result<Box<dyn Read + Send>>` - A boxed reader that decompresses the file on read,
|
||||
/// or an error if the file cannot be opened or is invalid.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if the file does not exist, is not a valid compressed file,
|
||||
/// or if decompression fails.
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>>;
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>>;
|
||||
|
||||
/// Creates a new compressed file for writing.
|
||||
///
|
||||
|
||||
@@ -24,7 +24,7 @@ impl CompressionEngine for CompressionEngineNone {
|
||||
("<INTERNAL>".to_string(), "".to_string(), "".to_string())
|
||||
}
|
||||
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>> {
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>> {
|
||||
debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self);
|
||||
Ok(Box::new(File::open(file_path)?))
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use log::*;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Write};
|
||||
@@ -94,16 +94,13 @@ impl CompressionEngine for CompressionEngineProgram {
|
||||
)
|
||||
}
|
||||
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read>> {
|
||||
debug!("COMPRESSION: Opening {:?} using {:?}", file_path, *self);
|
||||
fn open(&self, file_path: PathBuf) -> Result<Box<dyn Read + Send>> {
|
||||
debug!("COMPRESSION: Opening {file_path:?} using {self:?}");
|
||||
|
||||
let program = self.program.clone();
|
||||
let args = self.decompress.clone();
|
||||
|
||||
debug!(
|
||||
"COMPRESSION: Executing command: {:?} {:?} reading from {:?}",
|
||||
program, args, file_path
|
||||
);
|
||||
debug!("COMPRESSION: Executing command: {program:?} {args:?} reading from {file_path:?}");
|
||||
|
||||
let file = File::open(file_path).context("Unable to open file for reading")?;
|
||||
|
||||
@@ -130,15 +127,12 @@ impl CompressionEngine for CompressionEngineProgram {
|
||||
}
|
||||
|
||||
fn create(&self, file_path: PathBuf) -> Result<Box<dyn Write>> {
|
||||
debug!("COMPRESSION: Writing to {:?} using {:?}", file_path, *self);
|
||||
debug!("COMPRESSION: Writing to {file_path:?} using {self:?}");
|
||||
|
||||
let program = self.program.clone();
|
||||
let args = self.compress.clone();
|
||||
|
||||
debug!(
|
||||
"COMPRESSION: Executing command: {:?} {:?} writing to {:?}",
|
||||
program, args, file_path
|
||||
);
|
||||
debug!("COMPRESSION: Executing command: {program:?} {args:?} writing to {file_path:?}");
|
||||
|
||||
let file = File::create(file_path).context("Unable to open file for writing")?;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user