asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Ugh

Browse Source
This commit is contained in:
Andrew Phillips
2026-02-19 13:57:39 -04:00
parent a72395fe83
commit fdeb5f7951
82 changed files with 2756 additions and 2018 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
AGENT.md
View File

@@ -2,25 +2,53 @@
**IMPORTANT:** Prefer to use the `write_file` tool if the edit is for the majority of a file, or if you are correcting previous problems made edits from other tools. **IMPORTANT:** Prefer to use the `write_file` tool if the edit is for the majority of a file, or if you are correcting previous problems made edits from other tools.
## Commands ## Tools
**IMPORTANT:** Always export `TERM=dumb`. **IMPORTANT**: Be very careful when quoting text in tool calls to add the right amount of escaping.
### `write_file`
When editing files use the `write_file` tool to output the complete version of the corrected file.
**IMPORTANT**: You must provide the whole file to `write_file`, even the unchanged parts.
## Build/Test Commands
**IMPORTANT**: Do not run application, start the web server, or the trunk server.
**IMPORTANT:** The cargo command cannot be ran in parallel. **IMPORTANT:** The cargo command cannot be ran in parallel.
### Build ```bash
- Build: `TERM=dumb cargo build` # Check project
- Build release: `TERM=dumb cargo build --release` TERM=dumb cargo check
### Test # Build project
- Run all tests: `TERM=dumb cargo test` TERM=dumb cargo build
- Run specific test: `TERM=dumb cargo test TEST_NAME`
- Run tests with output: `TERM=dumb cargo test -- --nocapture`
### Lint/Format Commands # DO NOT RUN RUN APPLICATION (native)
- Check formatting: `TERM=dumb cargo fmt --check` # TERM=dumb cargo run
- Format code: `TERM=dumb cargo fmt`
- Lint: `TERM=dumb cargo clippy` # Run all tests
- Lint with errors: `TERM=dumb cargo clippy -- -D warnings` TERM=dumb cargo test
# Run specific test (by name substring)
TERM=dumb cargo test test_function_name
# Run specific test with verbose output
TERM=dumb cargo test test_function_name -- --nocapture
# Check formatting
TERM=dumb cargo fmt --check
# Apply formatting
TERM=dumb cargo fmt
# Lint with clippy
TERM=dumb cargo clippy -- -D warnings
# Build for release
TERM=dumb cargo build --release
```
Prefix commands with `TERM=dumb` for consistent output.
## Code Style Guidelines ## Code Style Guidelines
@@ -28,29 +56,9 @@
- Group imports in order: standard library, external crates, local modules - Group imports in order: standard library, external crates, local modules
- Use explicit imports over glob imports (`use std::fs::File;` not `use std::fs::*;`) - Use explicit imports over glob imports (`use std::fs::File;` not `use std::fs::*;`)
### Formatting
- Use rustfmt (configured via rustfmt.toml if exists)
- Max line length: 100 characters
- Indent with 4 spaces
### Types
- Prefer explicit types in public API
- Use `&str` for string literals, `String` for owned strings
- Use `Option<T>` for optional values, `Result<T, E>` for error handling
### Naming Conventions
- Use snake_case for variables and functions
- Use PascalCase for types and traits
- Use UPPER_SNAKE_CASE for constants and statics
### Error Handling
- Use `anyhow::Result` for most error handling
- Use `anyhow::Context` to add context to errors
- Avoid `unwrap()` in production code
### Documentation ### Documentation
- Document all public APIs with rustdoc - Document all public APIs with rustdoc
- Use examples in documentation when helpful - Use examples in documentation only when helpful
## Procedures ## Procedures

1
modulefile
View File

@@ -14,3 +14,4 @@ set mydir [ file normalize $mydir_base ]
module-whatis Keep module-whatis Keep
prepend-path PATH $mydir/bin prepend-path PATH $mydir/bin
setenv KEEP_BASH_PROFILE ${mydir}/profile.bash

19
src/args.rs
View File

@@ -41,7 +41,6 @@ pub struct ModeArgs {
#[arg(help("List items, filtering on tags or metadata if given"))] #[arg(help("List items, filtering on tags or metadata if given"))]
pub list: bool, pub list: bool,
#[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "diff", "list", "info", "status"]))] #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "diff", "list", "info", "status"]))]
#[arg(help("Delete items either by ID or by matching tags"))] #[arg(help("Delete items either by ID or by matching tags"))]
#[arg(requires = "ids_or_tags")] #[arg(requires = "ids_or_tags")]
@@ -85,7 +84,12 @@ pub struct ItemArgs {
#[arg(help("Compression algorithm to use when saving items"))] #[arg(help("Compression algorithm to use when saving items"))]
pub compression: Option<String>, pub compression: Option<String>,
#[arg(help_heading("Item Options"), short('M'), long, env("KEEP_META_PLUGINS"))] #[arg(
help_heading("Item Options"),
short('M'),
long,
env("KEEP_META_PLUGINS")
)]
#[arg(help("Meta plugins to use when saving items"))] #[arg(help("Meta plugins to use when saving items"))]
pub meta_plugins: Vec<String>, pub meta_plugins: Vec<String>,
@@ -94,7 +98,6 @@ pub struct ItemArgs {
pub filters: Option<String>, pub filters: Option<String>,
} }
/// Struct for general options, including verbosity, paths, and output settings. /// Struct for general options, including verbosity, paths, and output settings.
#[derive(Parser, Debug, Default, Clone)] #[derive(Parser, Debug, Default, Clone)]
pub struct OptionsArgs { pub struct OptionsArgs {
@@ -138,7 +141,10 @@ pub struct OptionsArgs {
#[arg(help("Password hash for server authentication (requires --server)"))] #[arg(help("Password hash for server authentication (requires --server)"))]
pub server_password_hash: Option<String>, pub server_password_hash: Option<String>,
#[arg(long, help("Force output even when binary data would be sent to a TTY"))] #[arg(
long,
help("Force output even when binary data would be sent to a TTY")
)]
pub force: bool, pub force: bool,
} }
@@ -170,7 +176,7 @@ impl Args {
if self.mode.delete && self.ids_or_tags.is_empty() { if self.mode.delete && self.ids_or_tags.is_empty() {
return Err("At least one ID is required when using --delete".to_string()); return Err("At least one ID is required when using --delete".to_string());
} }
// Check if --delete is used and any of the ids_or_tags are tags (strings) // Check if --delete is used and any of the ids_or_tags are tags (strings)
if self.mode.delete { if self.mode.delete {
for item in &self.ids_or_tags { for item in &self.ids_or_tags {
@@ -179,8 +185,7 @@ impl Args {
} }
} }
} }
Ok(()) Ok(())
} }
} }

171
src/common/is_binary.rs
View File

@@ -1,44 +1,43 @@
/// Detect if data is binary or text /// Detect if data is binary or text
/// Returns true if data is likely binary, false if likely text /// Returns true if data is likely binary, false if likely text
pub fn is_binary(data: &[u8]) -> bool { pub fn is_binary(data: &[u8]) -> bool {
if data.is_empty() { if data.is_empty() {
return false; return false;
} }
// First check for known binary file signatures // First check for known binary file signatures
if has_binary_signature(data) { if has_binary_signature(data) {
return true; return true;
} }
// Check for UTF-16 BOM (text) // Check for UTF-16 BOM (text)
if data.len() >= 2 { if data.len() >= 2
if (data[0] == 0xFF && data[1] == 0xFE) || (data[0] == 0xFE && data[1] == 0xFF) { && ((data[0] == 0xFF && data[1] == 0xFE) || (data[0] == 0xFE && data[1] == 0xFF))
return false; // UTF-16 with BOM is text {
} return false; // UTF-16 with BOM is text
} }
// Check for UTF-8 BOM (text) // Check for UTF-8 BOM (text)
if data.len() >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF { if data.len() >= 3 && data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF {
return false; // UTF-8 with BOM is text return false; // UTF-8 with BOM is text
} }
// Check if it's valid UTF-8 // Check if it's valid UTF-8
if std::str::from_utf8(data).is_ok() { if std::str::from_utf8(data).is_ok() {
// Valid UTF-8, check printable character ratio // Valid UTF-8, check printable character ratio
return calculate_printable_ratio(data) < 0.7; return calculate_printable_ratio(data) < 0.7;
} }
// Not valid UTF-8, check if it might be UTF-16 without BOM // Not valid UTF-8, check if it might be UTF-16 without BOM
if looks_like_utf16(data) { if looks_like_utf16(data) {
return false; // Likely UTF-16 text return false; // Likely UTF-16 text
} }
// Check for TAR format (special case with no magic number) // Check for TAR format (special case with no magic number)
if looks_like_tar(data) { if looks_like_tar(data) {
return true; return true;
} }
// Final fallback: check printable character ratio // Final fallback: check printable character ratio
// For 1KB of random data, we expect very few printable characters // For 1KB of random data, we expect very few printable characters
calculate_printable_ratio(data) < 0.7 calculate_printable_ratio(data) < 0.7
@@ -50,51 +49,47 @@ fn has_binary_signature(data: &[u8]) -> bool {
let signatures: &[(&[u8], usize)] = &[ let signatures: &[(&[u8], usize)] = &[
// Image formats // Image formats
(&[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], 8), // PNG (&[0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], 8), // PNG
(&[0xFF, 0xD8, 0xFF], 3), // JPEG (various subtypes) (&[0xFF, 0xD8, 0xFF], 3), // JPEG (various subtypes)
(&[0x47, 0x49, 0x46, 0x38, 0x37, 0x61], 6), // GIF87a (&[0x47, 0x49, 0x46, 0x38, 0x37, 0x61], 6), // GIF87a
(&[0x47, 0x49, 0x46, 0x38, 0x39, 0x61], 6), // GIF89a (&[0x47, 0x49, 0x46, 0x38, 0x39, 0x61], 6), // GIF89a
(&[0x42, 0x4D], 2), // BMP (&[0x42, 0x4D], 2), // BMP
(&[0x00, 0x00, 0x01, 0x00], 4), // ICO (&[0x00, 0x00, 0x01, 0x00], 4), // ICO
(&[0x49, 0x49, 0x2A, 0x00], 4), // TIFF (little endian) (&[0x49, 0x49, 0x2A, 0x00], 4), // TIFF (little endian)
(&[0x4D, 0x4D, 0x00, 0x2A], 4), // TIFF (big endian) (&[0x4D, 0x4D, 0x00, 0x2A], 4), // TIFF (big endian)
(&[0x52, 0x49, 0x46, 0x46], 4), // WebP (RIFF container) (&[0x52, 0x49, 0x46, 0x46], 4), // WebP (RIFF container)
(&[0x00, 0x00, 0x00, 0x0C, 0x6A, 0x50, 0x20, 0x20], 8), // JPEG 2000 (&[0x00, 0x00, 0x00, 0x0C, 0x6A, 0x50, 0x20, 0x20], 8), // JPEG 2000
// Audio/Video formats // Audio/Video formats
(&[0x49, 0x44, 0x33], 3), // MP3 with ID3v2 (&[0x49, 0x44, 0x33], 3), // MP3 with ID3v2
(&[0xFF, 0xFB], 2), // MP3 (&[0xFF, 0xFB], 2), // MP3
(&[0xFF, 0xF3], 2), // MP3 (&[0xFF, 0xF3], 2), // MP3
(&[0xFF, 0xF2], 2), // MP3 (&[0xFF, 0xF2], 2), // MP3
(&[0x4F, 0x67, 0x67, 0x53], 4), // OGG (&[0x4F, 0x67, 0x67, 0x53], 4), // OGG
(&[0x66, 0x74, 0x79, 0x70], 4), // MP4/M4A/MOV (at offset 4) (&[0x66, 0x74, 0x79, 0x70], 4), // MP4/M4A/MOV (at offset 4)
(&[0x52, 0x49, 0x46, 0x46], 4), // WAV/AVI (RIFF) (&[0x52, 0x49, 0x46, 0x46], 4), // WAV/AVI (RIFF)
(&[0x46, 0x4C, 0x56], 3), // FLV (&[0x46, 0x4C, 0x56], 3), // FLV
(&[0x1A, 0x45, 0xDF, 0xA3], 4), // MKV/WebM (&[0x1A, 0x45, 0xDF, 0xA3], 4), // MKV/WebM
// Archive formats // Archive formats
(&[0x50, 0x4B, 0x03, 0x04], 4), // ZIP (&[0x50, 0x4B, 0x03, 0x04], 4), // ZIP
(&[0x50, 0x4B, 0x05, 0x06], 4), // ZIP (empty) (&[0x50, 0x4B, 0x05, 0x06], 4), // ZIP (empty)
(&[0x50, 0x4B, 0x07, 0x08], 4), // ZIP (spanned) (&[0x50, 0x4B, 0x07, 0x08], 4), // ZIP (spanned)
(&[0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00], 7), // RAR v1.5+ (&[0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00], 7), // RAR v1.5+
(&[0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00], 8), // RAR v5.0+ (&[0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00], 8), // RAR v5.0+
(&[0x1F, 0x8B], 2), // GZIP (&[0x1F, 0x8B], 2), // GZIP
(&[0x42, 0x5A, 0x68], 3), // BZIP2 (&[0x42, 0x5A, 0x68], 3), // BZIP2
(&[0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00], 6), // XZ (&[0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00], 6), // XZ
(&[0x28, 0xB5, 0x2F, 0xFD], 4), // Zstandard (&[0x28, 0xB5, 0x2F, 0xFD], 4), // Zstandard
(&[0x04, 0x22, 0x4D, 0x18], 4), // LZ4 (&[0x04, 0x22, 0x4D, 0x18], 4), // LZ4
(&[0x1F, 0x9D], 2), // LZW compressed (&[0x1F, 0x9D], 2), // LZW compressed
(&[0x1F, 0xA0], 2), // LZH compressed (&[0x1F, 0xA0], 2), // LZH compressed
(&[0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C], 6), // 7-Zip (&[0x37, 0x7A, 0xBC, 0xAF, 0x27, 0x1C], 6), // 7-Zip
// Document formats // Document formats
(&[0x25, 0x50, 0x44, 0x46], 4), // PDF (&[0x25, 0x50, 0x44, 0x46], 4), // PDF
(&[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1], 8), // MS Office (OLE) (&[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1], 8), // MS Office (OLE)
(&[0x50, 0x4B, 0x03, 0x04], 4), // Office Open XML (also ZIP) (&[0x50, 0x4B, 0x03, 0x04], 4), // Office Open XML (also ZIP)
(&[0x7B, 0x5C, 0x72, 0x74, 0x66], 5), // RTF (&[0x7B, 0x5C, 0x72, 0x74, 0x66], 5), // RTF
// Executables and object files // Executables and object files
(&[0x7F, 0x45, 0x4C, 0x46], 4), // ELF (&[0x7F, 0x45, 0x4C, 0x46], 4), // ELF
(&[0x4D, 0x5A], 2), // Windows PE/DOS (&[0x4D, 0x5A], 2), // Windows PE/DOS
(&[0xCA, 0xFE, 0xBA, 0xBE], 4), // Mach-O (big endian) (&[0xCA, 0xFE, 0xBA, 0xBE], 4), // Mach-O (big endian)
(&[0xFE, 0xED, 0xFA, 0xCE], 4), // Mach-O 32-bit (little endian) (&[0xFE, 0xED, 0xFA, 0xCE], 4), // Mach-O 32-bit (little endian)
(&[0xFE, 0xED, 0xFA, 0xCF], 4), // Mach-O 64-bit (little endian) (&[0xFE, 0xED, 0xFA, 0xCF], 4), // Mach-O 64-bit (little endian)
@@ -102,45 +97,53 @@ fn has_binary_signature(data: &[u8]) -> bool {
(&[0xCF, 0xFA, 0xED, 0xFE], 4), // Mach-O 64-bit (big endian) (&[0xCF, 0xFA, 0xED, 0xFE], 4), // Mach-O 64-bit (big endian)
(&[0xCA, 0xFE, 0xBA, 0xBE], 4), // Java class file (&[0xCA, 0xFE, 0xBA, 0xBE], 4), // Java class file
(&[0xDE, 0xC0, 0x17, 0x0B], 4), // Dalvik executable (&[0xDE, 0xC0, 0x17, 0x0B], 4), // Dalvik executable
// Database formats // Database formats
(&[0x53, 0x51, 0x4C, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6F, 0x72, 0x6D, 0x61, 0x74, 0x20, 0x33, 0x00], 16), // SQLite (
&[
0x53, 0x51, 0x4C, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6F, 0x72, 0x6D, 0x61, 0x74, 0x20,
0x33, 0x00,
],
16,
), // SQLite
(&[0x00, 0x01, 0x00, 0x00], 4), // Palm Database (&[0x00, 0x01, 0x00, 0x00], 4), // Palm Database
// Font formats // Font formats
(&[0x00, 0x01, 0x00, 0x00, 0x00], 5), // TrueType (&[0x00, 0x01, 0x00, 0x00, 0x00], 5), // TrueType
(&[0x4F, 0x54, 0x54, 0x4F], 4), // OpenType (&[0x4F, 0x54, 0x54, 0x4F], 4), // OpenType
(&[0x77, 0x4F, 0x46, 0x46], 4), // WOFF (&[0x77, 0x4F, 0x46, 0x46], 4), // WOFF
(&[0x77, 0x4F, 0x46, 0x32], 4), // WOFF2 (&[0x77, 0x4F, 0x46, 0x32], 4), // WOFF2
// Virtual machine formats // Virtual machine formats
(&[0x76, 0x6D, 0x64, 0x6B], 4), // VMDK (&[0x76, 0x6D, 0x64, 0x6B], 4), // VMDK
(&[0x3C, 0x3C, 0x3C, 0x20, 0x4F, 0x72, 0x61, 0x63, 0x6C, 0x65, 0x20, 0x56, 0x4D, 0x20, 0x56, 0x69, 0x72, 0x74, 0x75, 0x61, 0x6C, 0x42, 0x6F, 0x78, 0x20, 0x44, 0x69, 0x73, 0x6B, 0x20, 0x49, 0x6D, 0x61, 0x67, 0x65, 0x20, 0x3E, 0x3E, 0x3E], 39), // VirtualBox VDI (
&[
0x3C, 0x3C, 0x3C, 0x20, 0x4F, 0x72, 0x61, 0x63, 0x6C, 0x65, 0x20, 0x56, 0x4D, 0x20,
0x56, 0x69, 0x72, 0x74, 0x75, 0x61, 0x6C, 0x42, 0x6F, 0x78, 0x20, 0x44, 0x69, 0x73,
0x6B, 0x20, 0x49, 0x6D, 0x61, 0x67, 0x65, 0x20, 0x3E, 0x3E, 0x3E,
],
39,
), // VirtualBox VDI
// Disk image formats // Disk image formats
(&[0xEB, 0x3C, 0x90], 3), // FAT12/16/32 (&[0xEB, 0x3C, 0x90], 3), // FAT12/16/32
(&[0xEB, 0x58, 0x90], 3), // FAT32 (&[0xEB, 0x58, 0x90], 3), // FAT32
(&[0x55, 0xAA], 2), // Boot sector (at offset 510) (&[0x55, 0xAA], 2), // Boot sector (at offset 510)
// Other binary formats // Other binary formats
(&[0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E, 0x0A], 8), // AR archive (&[0x21, 0x3C, 0x61, 0x72, 0x63, 0x68, 0x3E, 0x0A], 8), // AR archive
(&[0x78, 0x01], 2), // zlib (default compression) (&[0x78, 0x01], 2), // zlib (default compression)
(&[0x78, 0x9C], 2), // zlib (best compression) (&[0x78, 0x9C], 2), // zlib (best compression)
(&[0x78, 0xDA], 2), // zlib (fast compression) (&[0x78, 0xDA], 2), // zlib (fast compression)
(&[0x62, 0x76, 0x78, 0x32], 4), // LZFSE (&[0x62, 0x76, 0x78, 0x32], 4), // LZFSE
]; ];
for (signature, min_len) in signatures { for (signature, min_len) in signatures {
if data.len() >= *min_len && data.starts_with(signature) { if data.len() >= *min_len && data.starts_with(signature) {
return true; return true;
} }
} }
// Special case: check for ftyp box in MP4/MOV files (at offset 4) // Special case: check for ftyp box in MP4/MOV files (at offset 4)
if data.len() >= 8 && &data[4..8] == b"ftyp" { if data.len() >= 8 && &data[4..8] == b"ftyp" {
return true; return true;
} }
false false
} }
@@ -149,22 +152,22 @@ fn looks_like_utf16(data: &[u8]) -> bool {
if data.len() < 4 || data.len() % 2 != 0 { if data.len() < 4 || data.len() % 2 != 0 {
return false; return false;
} }
// Check if it could be UTF-16 by looking at null patterns // Check if it could be UTF-16 by looking at null patterns
let mut null_pairs = 0; let mut null_pairs = 0;
let max_checks = std::cmp::min(data.len() / 2, 50); // Check up to 50 character pairs let max_checks = std::cmp::min(data.len() / 2, 50); // Check up to 50 character pairs
for i in 0..max_checks { for i in 0..max_checks {
if data[i * 2 + 1] == 0 { if data[i * 2 + 1] == 0 {
null_pairs += 1; null_pairs += 1;
} }
} }
// If most high bytes are zero, it's likely UTF-16 // If most high bytes are zero, it's likely UTF-16
if max_checks > 0 && null_pairs as f64 / max_checks as f64 > 0.7 { if max_checks > 0 && null_pairs as f64 / max_checks as f64 > 0.7 {
return true; return true;
} }
// Also check the reverse pattern (little-endian UTF-16) // Also check the reverse pattern (little-endian UTF-16)
let mut null_pairs_reverse = 0; let mut null_pairs_reverse = 0;
for i in 0..max_checks { for i in 0..max_checks {
@@ -172,7 +175,7 @@ fn looks_like_utf16(data: &[u8]) -> bool {
null_pairs_reverse += 1; null_pairs_reverse += 1;
} }
} }
null_pairs_reverse as f64 / max_checks as f64 > 0.7 null_pairs_reverse as f64 / max_checks as f64 > 0.7
} }
@@ -181,27 +184,27 @@ fn looks_like_tar(data: &[u8]) -> bool {
if data.len() < 512 { if data.len() < 512 {
return false; return false;
} }
// TAR header structure validation // TAR header structure validation
// Filename should not start with null // Filename should not start with null
if data[0] == 0 { if data[0] == 0 {
return false; return false;
} }
// Check file mode field (should be octal digits) // Check file mode field (should be octal digits)
for i in 100..108 { for i in 100..108 {
if data[i] != 0 && (data[i] < b'0' || data[i] > b'7') && data[i] != b' ' { if data[i] != 0 && (data[i] < b'0' || data[i] > b'7') && data[i] != b' ' {
return false; return false;
} }
} }
// Check checksum field (should be octal digits or spaces) // Check checksum field (should be octal digits or spaces)
for i in 148..156 { for &b in &data[148..156] {
if data[i] != 0 && (data[i] < b'0' || data[i] > b'7') && data[i] != b' ' { if b != 0 && (b < b'0' || b > b'7') && b != b' ' {
return false; return false;
} }
} }
// Check magic field for POSIX TAR // Check magic field for POSIX TAR
if data.len() >= 265 { if data.len() >= 265 {
let magic = &data[257..262]; let magic = &data[257..262];
@@ -209,20 +212,20 @@ fn looks_like_tar(data: &[u8]) -> bool {
return true; return true;
} }
} }
// Additional heuristic: check if the structure looks reasonable // Additional heuristic: check if the structure looks reasonable
let has_reasonable_structure = // Mode field
data[0] != 0 && // Filename starts
data[100..108].iter().all(|&b| b == 0 || (b >= b'0' && b <= b'7') || b == b' '); // Mode field data[0] != 0 && // Filename starts
data[100..108].iter().all(|&b| b == 0 || (b'0'..=b'7').contains(&b) || b == b' ')
has_reasonable_structure
} }
/// Calculate the ratio of printable characters in the data /// Calculate the ratio of printable characters in the data
fn calculate_printable_ratio(data: &[u8]) -> f64 { fn calculate_printable_ratio(data: &[u8]) -> f64 {
let printable_count = data.iter().filter(|&&b| { let printable_count = data
b.is_ascii_graphic() || b.is_ascii_whitespace() .iter()
}).count(); .filter(|&&b| b.is_ascii_graphic() || b.is_ascii_whitespace())
.count();
printable_count as f64 / data.len() as f64 printable_count as f64 / data.len() as f64
} }

88
src/common/status.rs
View File

@@ -3,7 +3,7 @@ use strum::IntoEnumIterator;
#[cfg(feature = "server")] #[cfg(feature = "server")]
use utoipa::ToSchema; use utoipa::ToSchema;
use crate::compression_engine::{get_compression_engine, CompressionType}; use crate::compression_engine::{CompressionType, get_compression_engine};
use crate::meta_plugin::MetaPluginType; use crate::meta_plugin::MetaPluginType;
use crate::filter_plugin::FilterOption; use crate::filter_plugin::FilterOption;
@@ -56,13 +56,19 @@ pub struct MetaPluginInfo {
pub fn generate_status_info( pub fn generate_status_info(
data_path: PathBuf, data_path: PathBuf,
db_path: PathBuf, db_path: PathBuf,
enabled_meta_plugins: &Vec<MetaPluginType>, enabled_meta_plugins: &[MetaPluginType],
enabled_compression_type: Option<CompressionType>, enabled_compression_type: Option<CompressionType>,
) -> StatusInfo { ) -> StatusInfo {
log::debug!("STATUS: Starting status info generation"); log::debug!("STATUS: Starting status info generation");
let path_info = PathInfo { let path_info = PathInfo {
data: data_path.into_os_string().into_string().expect("Unable to convert data path to string"), data: data_path
database: db_path.into_os_string().into_string().expect("Unable to convert DB path to string"), .into_os_string()
.into_string()
.expect("Unable to convert data path to string"),
database: db_path
.into_os_string()
.into_string()
.expect("Unable to convert DB path to string"),
}; };
let _default_type = crate::compression_engine::default_compression_type(); let _default_type = crate::compression_engine::default_compression_type();
@@ -73,22 +79,40 @@ pub fn generate_status_info(
sorted_compression_types.sort_by_key(|ct| ct.to_string()); sorted_compression_types.sort_by_key(|ct| ct.to_string());
for compression_type in sorted_compression_types { for compression_type in sorted_compression_types {
let (binary, compress, decompress, supported) = match get_compression_engine(compression_type.clone()) { let (binary, compress, decompress, supported) =
Ok(engine) => { match get_compression_engine(compression_type.clone()) {
let supp = engine.is_supported(); Ok(engine) => {
if supp && engine.is_internal() { let supp = engine.is_supported();
("<INTERNAL>".to_string(), "".to_string(), "".to_string(), supp) if supp && engine.is_internal() {
} else if supp { (
let (b, c, d) = engine.get_status_info(); "<INTERNAL>".to_string(),
(b, c, d, supp) "".to_string(),
} else { "".to_string(),
("<UNSUPPORTED>".to_string(), "".to_string(), "".to_string(), supp) supp,
)
} else if supp {
let (b, c, d) = engine.get_status_info();
(b, c, d, supp)
} else {
(
"<UNSUPPORTED>".to_string(),
"".to_string(),
"".to_string(),
supp,
)
}
} }
} Err(_) => (
Err(_) => ("<UNSUPPORTED>".to_string(), "".to_string(), "".to_string(), false), "<UNSUPPORTED>".to_string(),
}; "".to_string(),
"".to_string(),
false,
),
};
let is_enabled = enabled_compression_type.as_ref().map_or(false, |ct| *ct == compression_type); let is_enabled = enabled_compression_type
.as_ref()
.is_some_and(|ct| *ct == compression_type);
compression_info.push(CompressionInfo { compression_info.push(CompressionInfo {
compression_type: compression_type.to_string(), compression_type: compression_type.to_string(),
@@ -108,22 +132,25 @@ pub fn generate_status_info(
sorted_meta_plugins.sort_by_key(|meta_plugin_type| meta_plugin_type.to_string()); sorted_meta_plugins.sort_by_key(|meta_plugin_type| meta_plugin_type.to_string());
for meta_plugin_type in sorted_meta_plugins { for meta_plugin_type in sorted_meta_plugins {
log::debug!("STATUS: Processing meta plugin type: {:?}", meta_plugin_type); log::debug!(
"STATUS: Processing meta plugin type: {:?}",
meta_plugin_type
);
log::debug!("STATUS: About to call get_meta_plugin"); log::debug!("STATUS: About to call get_meta_plugin");
let meta_plugin = crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone(), None, None); let meta_plugin = crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone(), None, None);
log::debug!("STATUS: Created meta plugin instance"); log::debug!("STATUS: Created meta plugin instance");
// Get meta name first to avoid borrowing issues // Get meta name first to avoid borrowing issues
log::debug!("STATUS: Getting meta name..."); log::debug!("STATUS: Getting meta name...");
let meta_name = meta_plugin.meta_type().to_string(); let meta_name = meta_plugin.meta_type().to_string();
log::debug!("STATUS: Got meta name: {}", meta_name); log::debug!("STATUS: Got meta name: {}", meta_name);
// Check if this plugin is enabled // Check if this plugin is enabled
let is_enabled = enabled_meta_plugins.contains(&meta_plugin_type); let is_enabled = enabled_meta_plugins.contains(&meta_plugin_type);
if is_enabled { if is_enabled {
enabled_meta_plugins_vec.push(meta_name.clone()); enabled_meta_plugins_vec.push(meta_name.clone());
} }
// Create a display of outputs for status - use configured outputs if available, otherwise defaults // Create a display of outputs for status - use configured outputs if available, otherwise defaults
let outputs_display = if meta_plugin.outputs().is_empty() { let outputs_display = if meta_plugin.outputs().is_empty() {
// No configured outputs, use defaults // No configured outputs, use defaults
@@ -136,15 +163,18 @@ pub fn generate_status_info(
// Use configured outputs // Use configured outputs
meta_plugin.outputs().clone() meta_plugin.outputs().clone()
}; };
// Get options // Get options
let options = meta_plugin.options().clone(); let options = meta_plugin.options().clone();
meta_plugins_map.insert(meta_name.clone(), MetaPluginInfo { meta_plugins_map.insert(
meta_name, meta_name.clone(),
outputs: outputs_display, MetaPluginInfo {
options, meta_name,
}); outputs: outputs_display,
options,
},
);
} }
StatusInfo { StatusInfo {

11
src/compression_engine/mod.rs
View File

@@ -3,7 +3,7 @@ use std::io;
use std::io::{Read, Write}; use std::io::{Read, Write};
use std::path::PathBuf; use std::path::PathBuf;
use strum::IntoEnumIterator; use strum::IntoEnumIterator;
use strum::{Display, EnumString, EnumIter}; use strum::{Display, EnumIter, EnumString};
use log::*; use log::*;
@@ -203,7 +203,9 @@ lazy_static! {
#[cfg(feature = "gzip")] #[cfg(feature = "gzip")]
{ {
em[CompressionType::GZip] = Box::new(crate::compression_engine::gzip::CompressionEngineGZip::new()) as Box<dyn CompressionEngine>; em[CompressionType::GZip] =
Box::new(crate::compression_engine::gzip::CompressionEngineGZip::new())
as Box<dyn CompressionEngine>;
} }
em em
@@ -219,6 +221,9 @@ pub fn get_compression_engine(ct: CompressionType) -> Result<Box<dyn Compression
if engine.is_supported() { if engine.is_supported() {
Ok(engine.clone()) Ok(engine.clone())
} else { } else {
Err(anyhow!("Compression engine for {} is not supported", ct.to_string())) Err(anyhow!(
"Compression engine for {} is not supported",
ct.to_string()
))
} }
} }

17
src/compression_engine/program.rs
View File

@@ -68,7 +68,8 @@ impl CompressionEngineProgram {
let supported = program_path.is_ok(); let supported = program_path.is_ok();
CompressionEngineProgram { CompressionEngineProgram {
program: program_path.map_or_else(|_| program.to_string(), |p| p.to_string_lossy().to_string()), program: program_path
.map_or_else(|_| program.to_string(), |p| p.to_string_lossy().to_string()),
compress: compress.iter().map(|s| s.to_string()).collect(), compress: compress.iter().map(|s| s.to_string()).collect(),
decompress: decompress.iter().map(|s| s.to_string()).collect(), decompress: decompress.iter().map(|s| s.to_string()).collect(),
supported, supported,
@@ -117,9 +118,10 @@ impl CompressionEngine for CompressionEngineProgram {
args args
))?; ))?;
let stdout = process.stdout.take().ok_or_else(|| { let stdout = process
anyhow!("Failed to capture stdout from child process") .stdout
})?; .take()
.ok_or_else(|| anyhow!("Failed to capture stdout from child process"))?;
Ok(Box::new(ProgramReader { Ok(Box::new(ProgramReader {
process, process,
@@ -151,9 +153,10 @@ impl CompressionEngine for CompressionEngineProgram {
args args
))?; ))?;
let stdin = process.stdin.take().ok_or_else(|| { let stdin = process
anyhow!("Failed to capture stdin from child process") .stdin
})?; .take()
.ok_or_else(|| anyhow!("Failed to capture stdin from child process"))?;
Ok(Box::new(ProgramWriter { Ok(Box::new(ProgramWriter {
process, process,

168
src/config.rs
View File

@@ -1,10 +1,10 @@
use std::path::PathBuf; use crate::args::Args;
use std::fs; use anyhow::{Context, Result};
use anyhow::{Result, Context};
use serde::{Deserialize, Serialize};
use log::{debug, error};
use crate::args::{Args};
use dirs; use dirs;
use log::{debug, error};
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::PathBuf;
#[derive(Debug, Clone, Serialize, Deserialize, Default)] #[derive(Debug, Clone, Serialize, Deserialize, Default)]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]
@@ -122,10 +122,10 @@ impl<'de> serde::Deserialize<'de> for ColumnConfig {
#[serde(default)] #[serde(default)]
padding: Option<(u16, u16)>, padding: Option<(u16, u16)>,
} }
let helper = Helper::deserialize(deserializer)?; let helper = Helper::deserialize(deserializer)?;
let label = helper.label.unwrap_or_else(|| helper.name.clone()); let label = helper.label.unwrap_or_else(|| helper.name.clone());
Ok(ColumnConfig { Ok(ColumnConfig {
name: helper.name, name: helper.name,
label, label,
@@ -188,14 +188,17 @@ pub struct Settings {
impl Settings { impl Settings {
/// Create unified settings from config and args with proper priority /// Create unified settings from config and args with proper priority
pub fn new(args: &Args, default_dir: PathBuf) -> Result<Self> { pub fn new(args: &Args, default_dir: PathBuf) -> Result<Self> {
debug!("CONFIG: Creating settings with default dir: {:?}", default_dir); debug!(
"CONFIG: Creating settings with default dir: {:?}",
default_dir
);
let config_path = if let Some(config_path) = &args.options.config { let config_path = if let Some(config_path) = &args.options.config {
config_path.clone() config_path.clone()
} else if let Ok(env_config) = std::env::var("KEEP_CONFIG") { } else if let Ok(env_config) = std::env::var("KEEP_CONFIG") {
PathBuf::from(env_config) PathBuf::from(env_config)
} else { } else {
let default_path = if let Some(home_dir) = std::env::var("HOME").ok() { let default_path = if let Ok(home_dir) = std::env::var("HOME") {
let mut path = PathBuf::from(home_dir); let mut path = PathBuf::from(home_dir);
path.push(".config"); path.push(".config");
path.push("keep"); path.push("keep");
@@ -207,74 +210,82 @@ impl Settings {
debug!("CONFIG: Using default config path: {:?}", default_path); debug!("CONFIG: Using default config path: {:?}", default_path);
default_path default_path
}; };
debug!("CONFIG: Using config path: {:?}", config_path); debug!("CONFIG: Using config path: {:?}", config_path);
let mut config_builder = config::Config::builder(); let mut config_builder = config::Config::builder();
// Load config file if it exists // Load config file if it exists
if config_path.exists() { if config_path.exists() {
debug!("CONFIG: Loading config file: {:?}", config_path); debug!("CONFIG: Loading config file: {:?}", config_path);
config_builder = config_builder.add_source(config::File::from(config_path.clone()).required(false)); config_builder =
config_builder.add_source(config::File::from(config_path.clone()).required(false));
} else { } else {
debug!("CONFIG: Config file does not exist: {:?}", config_path); debug!("CONFIG: Config file does not exist: {:?}", config_path);
} }
// Add environment variables // Add environment variables
debug!("CONFIG: Adding environment variables"); debug!("CONFIG: Adding environment variables");
let env_source = config::Environment::with_prefix("KEEP").separator("__").ignore_empty(true); let env_source = config::Environment::with_prefix("KEEP")
.separator("__")
.ignore_empty(true);
config_builder = config_builder.add_source(env_source); config_builder = config_builder.add_source(env_source);
// Override with CLI args // Override with CLI args
if let Some(dir) = &args.options.dir { if let Some(dir) = &args.options.dir {
debug!("CONFIG: Overriding dir with CLI arg: {:?}", dir); debug!("CONFIG: Overriding dir with CLI arg: {:?}", dir);
config_builder = config_builder.set_override("dir", dir.to_str().unwrap())?; config_builder = config_builder.set_override("dir", dir.to_str().unwrap())?;
} }
if args.options.human_readable { if args.options.human_readable {
config_builder = config_builder.set_override("human_readable", true)?; config_builder = config_builder.set_override("human_readable", true)?;
} }
if let Some(output_format) = &args.options.output_format { if let Some(output_format) = &args.options.output_format {
config_builder = config_builder.set_override("output_format", output_format.as_str())?; config_builder =
config_builder.set_override("output_format", output_format.as_str())?;
} }
if args.options.verbose > 0 { if args.options.verbose > 0 {
config_builder = config_builder.set_override("verbose", args.options.verbose)?; config_builder = config_builder.set_override("verbose", args.options.verbose)?;
} }
if args.options.quiet { if args.options.quiet {
config_builder = config_builder.set_override("quiet", true)?; config_builder = config_builder.set_override("quiet", true)?;
} }
if args.options.force { if args.options.force {
config_builder = config_builder.set_override("force", true)?; config_builder = config_builder.set_override("force", true)?;
} }
if let Some(server_password) = &args.options.server_password { if let Some(server_password) = &args.options.server_password {
config_builder = config_builder.set_override("server.password", server_password.as_str())?; config_builder =
config_builder.set_override("server.password", server_password.as_str())?;
} }
if let Some(server_password_hash) = &args.options.server_password_hash { if let Some(server_password_hash) = &args.options.server_password_hash {
config_builder = config_builder.set_override("server.password_hash", server_password_hash.as_str())?; config_builder = config_builder
.set_override("server.password_hash", server_password_hash.as_str())?;
} }
if let Some(server_address) = &args.mode.server_address { if let Some(server_address) = &args.mode.server_address {
config_builder = config_builder.set_override("server.address", server_address.as_str())?; config_builder =
config_builder.set_override("server.address", server_address.as_str())?;
} }
if let Some(server_port) = args.mode.server_port { if let Some(server_port) = args.mode.server_port {
config_builder = config_builder.set_override("server.port", server_port)?; config_builder = config_builder.set_override("server.port", server_port)?;
} }
if let Some(compression) = &args.item.compression { if let Some(compression) = &args.item.compression {
config_builder = config_builder.set_override("compression_plugin.name", compression.as_str())?; config_builder =
config_builder.set_override("compression_plugin.name", compression.as_str())?;
} }
if !args.item.meta_plugins.is_empty() { if !args.item.meta_plugins.is_empty() {
let meta_plugins: Vec<std::collections::HashMap<String, String>> = args.item.meta_plugins let meta_plugins: Vec<std::collections::HashMap<String, String>> = args
.item
.meta_plugins
.iter() .iter()
.map(|name| { .map(|name| {
let mut map = std::collections::HashMap::new(); let mut map = std::collections::HashMap::new();
@@ -284,21 +295,21 @@ impl Settings {
.collect(); .collect();
config_builder = config_builder.set_override("meta_plugins", meta_plugins)?; config_builder = config_builder.set_override("meta_plugins", meta_plugins)?;
} }
let config = config_builder.build()?; let config = config_builder.build()?;
debug!("CONFIG: Built config, attempting to deserialize"); debug!("CONFIG: Built config, attempting to deserialize");
match config.try_deserialize::<Settings>() { match config.try_deserialize::<Settings>() {
Ok(mut settings) => { Ok(mut settings) => {
debug!("CONFIG: Successfully deserialized settings: {:?}", settings); debug!("CONFIG: Successfully deserialized settings: {:?}", settings);
// Set defaults for list_format if not provided // Set defaults for list_format if not provided
if settings.list_format.is_empty() { if settings.list_format.is_empty() {
debug!("CONFIG: Setting default list_format"); debug!("CONFIG: Setting default list_format");
settings.list_format = vec![ settings.list_format = vec![
ColumnConfig { ColumnConfig {
name: "id".to_string(), name: "id".to_string(),
label: "Item".to_string(), label: "Item".to_string(),
align: ColumnAlignment::Right, align: ColumnAlignment::Right,
max_len: None, max_len: None,
fg_color: None, fg_color: None,
@@ -306,9 +317,9 @@ impl Settings {
attributes: Vec::new(), attributes: Vec::new(),
padding: None, padding: None,
}, },
ColumnConfig { ColumnConfig {
name: "time".to_string(), name: "time".to_string(),
label: "Time".to_string(), label: "Time".to_string(),
align: ColumnAlignment::Right, align: ColumnAlignment::Right,
max_len: None, max_len: None,
fg_color: None, fg_color: None,
@@ -316,9 +327,9 @@ impl Settings {
attributes: Vec::new(), attributes: Vec::new(),
padding: None, padding: None,
}, },
ColumnConfig { ColumnConfig {
name: "size".to_string(), name: "size".to_string(),
label: "Size".to_string(), label: "Size".to_string(),
align: ColumnAlignment::Right, align: ColumnAlignment::Right,
max_len: None, max_len: None,
fg_color: None, fg_color: None,
@@ -326,9 +337,9 @@ impl Settings {
attributes: Vec::new(), attributes: Vec::new(),
padding: None, padding: None,
}, },
ColumnConfig { ColumnConfig {
name: "meta:text_line_count".to_string(), name: "meta:text_line_count".to_string(),
label: "Lines".to_string(), label: "Lines".to_string(),
align: ColumnAlignment::Right, align: ColumnAlignment::Right,
max_len: None, max_len: None,
fg_color: None, fg_color: None,
@@ -336,9 +347,9 @@ impl Settings {
attributes: Vec::new(), attributes: Vec::new(),
padding: None, padding: None,
}, },
ColumnConfig { ColumnConfig {
name: "tags".to_string(), name: "tags".to_string(),
label: "Tags".to_string(), label: "Tags".to_string(),
align: ColumnAlignment::Left, align: ColumnAlignment::Left,
max_len: None, max_len: None,
fg_color: None, fg_color: None,
@@ -346,9 +357,9 @@ impl Settings {
attributes: Vec::new(), attributes: Vec::new(),
padding: None, padding: None,
}, },
ColumnConfig { ColumnConfig {
name: "meta:hostname_short".to_string(), name: "meta:hostname_short".to_string(),
label: "Host".to_string(), label: "Host".to_string(),
align: ColumnAlignment::Left, align: ColumnAlignment::Left,
max_len: None, max_len: None,
fg_color: None, fg_color: None,
@@ -356,9 +367,9 @@ impl Settings {
attributes: Vec::new(), attributes: Vec::new(),
padding: None, padding: None,
}, },
ColumnConfig { ColumnConfig {
name: "meta:command".to_string(), name: "meta:command".to_string(),
label: "Command".to_string(), label: "Command".to_string(),
align: ColumnAlignment::Left, align: ColumnAlignment::Left,
max_len: None, max_len: None,
fg_color: None, fg_color: None,
@@ -372,21 +383,19 @@ impl Settings {
// Set default meta_plugins to include 'env' if not provided // Set default meta_plugins to include 'env' if not provided
if settings.meta_plugins.is_none() { if settings.meta_plugins.is_none() {
debug!("CONFIG: Setting default meta_plugins to include 'env'"); debug!("CONFIG: Setting default meta_plugins to include 'env'");
settings.meta_plugins = Some(vec![ settings.meta_plugins = Some(vec![MetaPluginConfig {
MetaPluginConfig { name: "env".to_string(),
name: "env".to_string(), options: std::collections::HashMap::new(),
options: std::collections::HashMap::new(), outputs: std::collections::HashMap::new(),
outputs: std::collections::HashMap::new(), }]);
}
]);
} }
// Set dir to default if not provided or is empty // Set dir to default if not provided or is empty
if settings.dir == PathBuf::new() { if settings.dir == PathBuf::new() {
debug!("CONFIG: Setting default dir: {:?}", default_dir); debug!("CONFIG: Setting default dir: {:?}", default_dir);
settings.dir = default_dir; settings.dir = default_dir;
} }
debug!("CONFIG: Final settings: {:?}", settings); debug!("CONFIG: Final settings: {:?}", settings);
Ok(settings) Ok(settings)
} }
@@ -398,8 +407,8 @@ impl Settings {
} }
pub fn default_dir() -> anyhow::Result<PathBuf> { pub fn default_dir() -> anyhow::Result<PathBuf> {
let mut path = dirs::home_dir() let mut path =
.ok_or_else(|| anyhow::anyhow!("No home directory found"))?; dirs::home_dir().ok_or_else(|| anyhow::anyhow!("No home directory found"))?;
path.push(".keep"); path.push(".keep");
if !path.exists() { if !path.exists() {
std::fs::create_dir_all(&path)?; std::fs::create_dir_all(&path)?;
@@ -419,7 +428,7 @@ impl Settings {
.to_string(); .to_string();
return Ok(Some(password)); return Ok(Some(password));
} }
// Fall back to direct password field // Fall back to direct password field
if let Some(password) = &server.password { if let Some(password) = &server.password {
debug!("CONFIG: Using password from config"); debug!("CONFIG: Using password from config");
@@ -428,30 +437,31 @@ impl Settings {
} }
Ok(None) Ok(None)
} }
// Helper methods to access configuration values // Helper methods to access configuration values
pub fn server_password(&self) -> Option<String> { pub fn server_password(&self) -> Option<String> {
self.get_server_password().ok().flatten() self.get_server_password().ok().flatten()
} }
pub fn server_password_hash(&self) -> Option<String> { pub fn server_password_hash(&self) -> Option<String> {
self.server.as_ref().and_then(|s| s.password_hash.clone()) self.server.as_ref().and_then(|s| s.password_hash.clone())
} }
pub fn server_address(&self) -> Option<String> { pub fn server_address(&self) -> Option<String> {
self.server.as_ref().and_then(|s| s.address.clone()) self.server.as_ref().and_then(|s| s.address.clone())
} }
pub fn server_port(&self) -> Option<u16> { pub fn server_port(&self) -> Option<u16> {
self.server.as_ref().and_then(|s| s.port) self.server.as_ref().and_then(|s| s.port)
} }
pub fn compression(&self) -> Option<String> { pub fn compression(&self) -> Option<String> {
self.compression_plugin.as_ref().map(|c| c.name.clone()) self.compression_plugin.as_ref().map(|c| c.name.clone())
} }
pub fn meta_plugins_names(&self) -> Vec<String> { pub fn meta_plugins_names(&self) -> Vec<String> {
self.meta_plugins.as_ref() self.meta_plugins
.as_ref()
.map(|plugins| plugins.iter().map(|p| p.name.clone()).collect()) .map(|plugins| plugins.iter().map(|p| p.name.clone()).collect())
.unwrap_or_default() .unwrap_or_default()
} }

76
src/db.rs
View File

@@ -1,10 +1,10 @@
use anyhow::{Context, Error, Result}; use anyhow::{Context, Error, Result};
use chrono::prelude::*; use chrono::prelude::*;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use log::*; use log::*;
use rusqlite::{Connection, OpenFlags, params}; use rusqlite::{Connection, OpenFlags, params};
use rusqlite_migration::{M, Migrations}; use rusqlite_migration::{M, Migrations};
use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
use std::path::PathBuf; use std::path::PathBuf;
use std::rc::Rc; use std::rc::Rc;
@@ -245,7 +245,10 @@ pub fn insert_item(conn: &Connection, item: Item) -> Result<i64> {
/// let item = db::create_item(&conn, compression)?; /// let item = db::create_item(&conn, compression)?;
/// assert!(item.id.is_some()); /// assert!(item.id.is_some());
/// ``` /// ```
pub fn create_item(conn: &Connection, compression_type: crate::compression_engine::CompressionType) -> Result<Item> { pub fn create_item(
conn: &Connection,
compression_type: crate::compression_engine::CompressionType,
) -> Result<Item> {
let item = Item { let item = Item {
id: None, id: None,
ts: chrono::Utc::now(), ts: chrono::Utc::now(),
@@ -353,11 +356,7 @@ pub fn update_item(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Updating item: {:?}", item); debug!("DB: Updating item: {:?}", item);
conn.execute( conn.execute(
"UPDATE items SET size=?2, compression=?3 WHERE id=?1", "UPDATE items SET size=?2, compression=?3 WHERE id=?1",
params![ params![item.id, item.size, item.compression,],
item.id,
item.size,
item.compression,
],
)?; )?;
Ok(()) Ok(())
} }
@@ -1181,34 +1180,40 @@ pub fn get_item_meta_value(conn: &Connection, item: &Item, name: String) -> Resu
/// let ids = vec![1, 2, 3]; /// let ids = vec![1, 2, 3];
/// let tags_map = db::get_tags_for_items(&conn, &ids)?; /// let tags_map = db::get_tags_for_items(&conn, &ids)?;
/// ``` /// ```
pub fn get_tags_for_items(conn: &Connection, item_ids: &[i64]) -> Result<std::collections::HashMap<i64, Vec<Tag>>> { pub fn get_tags_for_items(
conn: &Connection,
item_ids: &[i64],
) -> Result<std::collections::HashMap<i64, Vec<Tag>>> {
debug!("DB: Getting tags for items: {:?}", item_ids); debug!("DB: Getting tags for items: {:?}", item_ids);
if item_ids.is_empty() { if item_ids.is_empty() {
return Ok(std::collections::HashMap::new()); return Ok(std::collections::HashMap::new());
} }
// Create placeholders for the IN clause // Create placeholders for the IN clause
let placeholders: Vec<String> = item_ids.iter().map(|_| "?".to_string()).collect(); let placeholders: Vec<String> = item_ids.iter().map(|_| "?".to_string()).collect();
let placeholders_str = placeholders.join(","); let placeholders_str = placeholders.join(",");
let sql = format!("SELECT id, name FROM tags WHERE id IN ({}) ORDER BY id ASC, name ASC", placeholders_str); let sql = format!(
"SELECT id, name FROM tags WHERE id IN ({}) ORDER BY id ASC, name ASC",
placeholders_str
);
let mut statement = conn let mut statement = conn
.prepare_cached(&sql) .prepare_cached(&sql)
.context("Problem preparing SQL statement")?; .context("Problem preparing SQL statement")?;
let mut rows = statement.query(rusqlite::params_from_iter(item_ids))?; let mut rows = statement.query(rusqlite::params_from_iter(item_ids))?;
let mut tags_map: std::collections::HashMap<i64, Vec<Tag>> = std::collections::HashMap::new(); let mut tags_map: std::collections::HashMap<i64, Vec<Tag>> = std::collections::HashMap::new();
while let Some(row) = rows.next()? { while let Some(row) = rows.next()? {
let id: i64 = row.get(0)?; let id: i64 = row.get(0)?;
let name: String = row.get(1)?; let name: String = row.get(1)?;
tags_map.entry(id).or_insert_with(Vec::new).push(Tag { id, name }); tags_map.entry(id).or_default().push(Tag { id, name });
} }
Ok(tags_map) Ok(tags_map)
} }
@@ -1235,34 +1240,41 @@ pub fn get_tags_for_items(conn: &Connection, item_ids: &[i64]) -> Result<std::co
/// let ids = vec![1, 2, 3]; /// let ids = vec![1, 2, 3];
/// let meta_map = db::get_meta_for_items(&conn, &ids)?; /// let meta_map = db::get_meta_for_items(&conn, &ids)?;
/// ``` /// ```
pub fn get_meta_for_items(conn: &Connection, item_ids: &[i64]) -> Result<std::collections::HashMap<i64, std::collections::HashMap<String, String>>> { pub fn get_meta_for_items(
conn: &Connection,
item_ids: &[i64],
) -> Result<std::collections::HashMap<i64, std::collections::HashMap<String, String>>> {
debug!("DB: Getting meta for items: {:?}", item_ids); debug!("DB: Getting meta for items: {:?}", item_ids);
if item_ids.is_empty() { if item_ids.is_empty() {
return Ok(std::collections::HashMap::new()); return Ok(std::collections::HashMap::new());
} }
// Create placeholders for the IN clause // Create placeholders for the IN clause
let placeholders: Vec<String> = item_ids.iter().map(|_| "?".to_string()).collect(); let placeholders: Vec<String> = item_ids.iter().map(|_| "?".to_string()).collect();
let placeholders_str = placeholders.join(","); let placeholders_str = placeholders.join(",");
let sql = format!("SELECT id, name, value FROM metas WHERE id IN ({}) ORDER BY id ASC, name ASC", placeholders_str); let sql = format!(
"SELECT id, name, value FROM metas WHERE id IN ({}) ORDER BY id ASC, name ASC",
placeholders_str
);
let mut statement = conn let mut statement = conn
.prepare_cached(&sql) .prepare_cached(&sql)
.context("Problem preparing SQL statement")?; .context("Problem preparing SQL statement")?;
let mut rows = statement.query(rusqlite::params_from_iter(item_ids))?; let mut rows = statement.query(rusqlite::params_from_iter(item_ids))?;
let mut meta_map: std::collections::HashMap<i64, std::collections::HashMap<String, String>> = std::collections::HashMap::new(); let mut meta_map: std::collections::HashMap<i64, std::collections::HashMap<String, String>> =
std::collections::HashMap::new();
while let Some(row) = rows.next()? { while let Some(row) = rows.next()? {
let id: i64 = row.get(0)?; let id: i64 = row.get(0)?;
let name: String = row.get(1)?; let name: String = row.get(1)?;
let value: String = row.get(2)?; let value: String = row.get(2)?;
meta_map.entry(id).or_insert_with(std::collections::HashMap::new).insert(name, value); meta_map.entry(id).or_default().insert(name, value);
} }
Ok(meta_map) Ok(meta_map)
} }

22
src/filter_plugin/grep.rs
View File

@@ -1,6 +1,6 @@
use super::{FilterPlugin, FilterOption}; use super::{FilterOption, FilterPlugin};
use std::io::{Result, Read, Write, BufRead};
use regex::Regex; use regex::Regex;
use std::io::{BufRead, Read, Result, Write};
/// A filter that matches lines against a regular expression pattern. /// A filter that matches lines against a regular expression pattern.
/// ///
@@ -40,9 +40,7 @@ impl GrepFilter {
pub fn new(pattern: String) -> Result<Self> { pub fn new(pattern: String) -> Result<Self> {
let regex = Regex::new(&pattern) let regex = Regex::new(&pattern)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?;
Ok(Self { Ok(Self { regex })
regex,
})
} }
} }
@@ -99,7 +97,7 @@ impl FilterPlugin for GrepFilter {
regex: self.regex.clone(), regex: self.regex.clone(),
}) })
} }
/// Returns the configuration options for this filter. /// Returns the configuration options for this filter.
/// ///
/// The only option is the required "pattern" for the regex. /// The only option is the required "pattern" for the regex.
@@ -116,12 +114,10 @@ impl FilterPlugin for GrepFilter {
/// assert!(opts[0].required); /// assert!(opts[0].required);
/// ``` /// ```
fn options(&self) -> Vec<FilterOption> { fn options(&self) -> Vec<FilterOption> {
vec![ vec![FilterOption {
FilterOption { name: "pattern".to_string(),
name: "pattern".to_string(), default: None,
default: None, required: true,
required: true, }]
}
]
} }
} }

75
src/filter_plugin/head.rs
View File

@@ -1,7 +1,7 @@
use super::{FilterPlugin, FilterOption}; use super::{FilterOption, FilterPlugin};
use std::io::{Result, Read, Write, BufRead};
use crate::common::PIPESIZE; use crate::common::PIPESIZE;
use crate::services::filter_service::register_filter_plugin; use crate::services::filter_service::register_filter_plugin;
use std::io::{BufRead, Read, Result, Write};
/// A filter that reads the first N bytes from the input stream. /// A filter that reads the first N bytes from the input stream.
/// ///
@@ -41,9 +41,7 @@ impl HeadBytesFilter {
/// assert_eq!(filter.remaining, 1024); /// assert_eq!(filter.remaining, 1024);
/// ``` /// ```
pub fn new(count: usize) -> Self { pub fn new(count: usize) -> Self {
Self { Self { remaining: count }
remaining: count,
}
} }
} }
@@ -76,7 +74,7 @@ impl FilterPlugin for HeadBytesFilter {
if self.remaining == 0 { if self.remaining == 0 {
return Ok(()); return Ok(());
} }
let mut buffer = vec![0; PIPESIZE]; let mut buffer = vec![0; PIPESIZE];
while self.remaining > 0 { while self.remaining > 0 {
let to_read = std::cmp::min(self.remaining, PIPESIZE); let to_read = std::cmp::min(self.remaining, PIPESIZE);
@@ -102,7 +100,7 @@ impl FilterPlugin for HeadBytesFilter {
remaining: self.remaining, remaining: self.remaining,
}) })
} }
/// Returns the configuration options for this filter. /// Returns the configuration options for this filter.
/// ///
/// Defines the "count" parameter as required with no default. /// Defines the "count" parameter as required with no default.
@@ -111,13 +109,11 @@ impl FilterPlugin for HeadBytesFilter {
/// ///
/// Vector of `FilterOption` describing parameters. /// Vector of `FilterOption` describing parameters.
fn options(&self) -> Vec<FilterOption> { fn options(&self) -> Vec<FilterOption> {
vec![ vec![FilterOption {
FilterOption { name: "count".to_string(),
name: "count".to_string(), default: None,
default: None, required: true,
required: true, }]
}
]
} }
} }
@@ -152,42 +148,39 @@ impl HeadLinesFilter {
/// assert_eq!(filter.remaining, 3); /// assert_eq!(filter.remaining, 3);
/// ``` /// ```
pub fn new(count: usize) -> Self { pub fn new(count: usize) -> Self {
Self { Self { remaining: count }
remaining: count,
}
} }
} }
/// Filters input by reading only the first N lines and writing them to the output. /// Filters input by reading only the first N lines and writing them to the output.
/// ///
/// Uses buffered line reading to process input line-by-line until the limit or EOF. /// Uses buffered line reading to process input line-by-line until the limit or EOF.
/// ///
/// # Arguments /// # Arguments
/// ///
/// * `reader` - Mutable reference to the input data stream. /// * `reader` - Mutable reference to the input data stream.
/// * `writer` - Mutable reference to the output stream. /// * `writer` - Mutable reference to the output stream.
/// ///
/// # Returns /// # Returns
/// ///
/// * `Result<()>` - Success if filtering completes, or I/O error. /// * `Result<()>` - Success if filtering completes, or I/O error.
/// ///
/// # Errors /// # Errors
/// ///
/// * `io::Error` from line reading or writing operations. /// * `io::Error` from line reading or writing operations.
/// ///
/// # Examples /// # Examples
/// ///
/// ``` /// ```
/// // Assuming a filter chain with head_lines(2) /// // Assuming a filter chain with head_lines(2)
/// // Input: "Line1\nLine2\nLine3" becomes "Line1\nLine2\n" /// // Input: "Line1\nLine2\nLine3" becomes "Line1\nLine2\n"
/// ``` /// ```
impl FilterPlugin for HeadLinesFilter { impl FilterPlugin for HeadLinesFilter {
fn filter(&mut self, reader: &mut dyn Read, writer: &mut dyn Write) -> Result<()> { fn filter(&mut self, reader: &mut dyn Read, writer: &mut dyn Write) -> Result<()> {
if self.remaining == 0 { if self.remaining == 0 {
return Ok(()); return Ok(());
} }
let mut buf_reader = std::io::BufReader::new(reader); let mut buf_reader = std::io::BufReader::new(reader);
for line in buf_reader.by_ref().lines() { for line in buf_reader.by_ref().lines() {
let line = line?; let line = line?;
@@ -201,33 +194,31 @@ impl FilterPlugin for HeadLinesFilter {
} }
/// Clones this filter into a new boxed instance. /// Clones this filter into a new boxed instance.
/// ///
/// Creates an independent copy with the same configuration. /// Creates an independent copy with the same configuration.
/// ///
/// # Returns /// # Returns
/// ///
/// A new `Box<dyn FilterPlugin>` clone. /// A new `Box<dyn FilterPlugin>` clone.
fn clone_box(&self) -> Box<dyn FilterPlugin> { fn clone_box(&self) -> Box<dyn FilterPlugin> {
Box::new(Self { Box::new(Self {
remaining: self.remaining, remaining: self.remaining,
}) })
} }
/// Returns the configuration options for this filter. /// Returns the configuration options for this filter.
/// ///
/// Defines the "count" parameter as required with no default. /// Defines the "count" parameter as required with no default.
/// ///
/// # Returns /// # Returns
/// ///
/// Vector of `FilterOption` describing parameters. /// Vector of `FilterOption` describing parameters.
fn options(&self) -> Vec<FilterOption> { fn options(&self) -> Vec<FilterOption> {
vec![ vec![FilterOption {
FilterOption { name: "count".to_string(),
name: "count".to_string(), default: None,
default: None, required: true,
required: true, }]
}
]
} }
} }

168
src/filter_plugin/mod.rs
View File

@@ -1,7 +1,8 @@
use std::io::{Result, Read, Write}; use std::io::{Read, Result, Write};
use std::str::FromStr; use std::str::FromStr;
use strum::EnumString; use strum::EnumString;
pub mod grep;
/// Filter plugin module for processing input streams. /// Filter plugin module for processing input streams.
/// ///
/// This module defines the `FilterPlugin` trait and `FilterChain` for chaining filters, /// This module defines the `FilterPlugin` trait and `FilterChain` for chaining filters,
@@ -17,19 +18,18 @@ use strum::EnumString;
/// chain.filter(&mut reader, &mut writer)?; /// chain.filter(&mut reader, &mut writer)?;
/// ``` /// ```
pub mod head; pub mod head;
pub mod tail;
pub mod skip; pub mod skip;
pub mod grep;
pub mod strip_ansi; pub mod strip_ansi;
pub mod tail;
pub mod utils; pub mod utils;
use std::collections::HashMap; use std::collections::HashMap;
pub use head::{HeadBytesFilter, HeadLinesFilter};
pub use tail::{TailBytesFilter, TailLinesFilter};
pub use skip::{SkipBytesFilter, SkipLinesFilter};
pub use grep::GrepFilter; pub use grep::GrepFilter;
pub use head::{HeadBytesFilter, HeadLinesFilter};
pub use skip::{SkipBytesFilter, SkipLinesFilter};
pub use strip_ansi::StripAnsiFilter; pub use strip_ansi::StripAnsiFilter;
pub use tail::{TailBytesFilter, TailLinesFilter};
/// Represents an option for a filter plugin. /// Represents an option for a filter plugin.
/// ///
@@ -195,7 +195,6 @@ pub struct FilterChain {
/// chain.add_plugin(Box::new(HeadLinesFilter::new(10))); /// chain.add_plugin(Box::new(HeadLinesFilter::new(10)));
/// chain.filter(&mut reader, &mut writer)?; /// chain.filter(&mut reader, &mut writer)?;
/// ``` /// ```
impl Clone for FilterChain { impl Clone for FilterChain {
/// Clones this filter chain. /// Clones this filter chain.
/// ///
@@ -222,6 +221,12 @@ impl Clone for Box<dyn FilterPlugin> {
} }
} }
impl Default for FilterChain {
fn default() -> Self {
Self::new()
}
}
impl FilterChain { impl FilterChain {
/// Creates a new empty filter chain. /// Creates a new empty filter chain.
/// ///
@@ -286,19 +291,19 @@ impl FilterChain {
std::io::copy(reader, writer)?; std::io::copy(reader, writer)?;
return Ok(()); return Ok(());
} }
// For multiple plugins, we need to chain them together // For multiple plugins, we need to chain them together
// We'll use a temporary buffer to hold intermediate results // We'll use a temporary buffer to hold intermediate results
let mut current_data = Vec::new(); let mut current_data = Vec::new();
std::io::copy(reader, &mut current_data)?; std::io::copy(reader, &mut current_data)?;
// Store the plugins length to avoid borrowing issues // Store the plugins length to avoid borrowing issues
let plugins_len = self.plugins.len(); let plugins_len = self.plugins.len();
for i in 0..plugins_len { for i in 0..plugins_len {
// Create a cursor for the current data // Create a cursor for the current data
let mut input = std::io::Cursor::new(std::mem::take(&mut current_data)); let mut input = std::io::Cursor::new(std::mem::take(&mut current_data));
// For the last plugin, write directly to the output writer // For the last plugin, write directly to the output writer
if i == plugins_len - 1 { if i == plugins_len - 1 {
self.plugins[i].filter(&mut input, writer)?; self.plugins[i].filter(&mut input, writer)?;
@@ -337,14 +342,14 @@ pub fn parse_filter_string(filter_str: &str) -> Result<FilterChain> {
// Parse parameters // Parse parameters
let mut options = HashMap::new(); let mut options = HashMap::new();
let mut unnamed_params = Vec::new(); let mut unnamed_params = Vec::new();
// Split parameters by commas // Split parameters by commas
for param in params.split(',') { for param in params.split(',') {
let param = param.trim(); let param = param.trim();
if param.is_empty() { if param.is_empty() {
continue; continue;
} }
// Check if it's a named parameter (key=value) // Check if it's a named parameter (key=value)
if let Some((key, value)) = param.split_once('=') { if let Some((key, value)) = param.split_once('=') {
let key = key.trim(); let key = key.trim();
@@ -356,10 +361,11 @@ pub fn parse_filter_string(filter_str: &str) -> Result<FilterChain> {
unnamed_params.push(value); unnamed_params.push(value);
} }
} }
// Create the appropriate filter plugin // Create the appropriate filter plugin
if let Ok(filter_type) = FilterType::from_str(filter_name) { if let Ok(filter_type) = FilterType::from_str(filter_name) {
let plugin = create_filter_with_options(filter_type, &unnamed_params, &options)?; let plugin =
create_filter_with_options(filter_type, &unnamed_params, &options)?;
chain.add_plugin(plugin); chain.add_plugin(plugin);
continue; continue;
} }
@@ -375,7 +381,7 @@ pub fn parse_filter_string(filter_str: &str) -> Result<FilterChain> {
_ => { _ => {
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput, std::io::ErrorKind::InvalidInput,
format!("Filter '{}' requires parameters", part) format!("Filter '{}' requires parameters", part),
)); ));
} }
} }
@@ -385,7 +391,7 @@ pub fn parse_filter_string(filter_str: &str) -> Result<FilterChain> {
// If we get here, the filter wasn't recognized // If we get here, the filter wasn't recognized
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput, std::io::ErrorKind::InvalidInput,
format!("Unknown filter: {}", part) format!("Unknown filter: {}", part),
)); ));
} }
@@ -420,17 +426,20 @@ fn create_filter_with_options(
FilterType::SkipLines => skip::SkipLinesFilter::new(0).options(), FilterType::SkipLines => skip::SkipLinesFilter::new(0).options(),
FilterType::StripAnsi => strip_ansi::StripAnsiFilter::new().options(), FilterType::StripAnsi => strip_ansi::StripAnsiFilter::new().options(),
}; };
let mut options = HashMap::new(); let mut options = HashMap::new();
// Process unnamed parameters // Process unnamed parameters
if unnamed_params.len() > option_defs.len() { if unnamed_params.len() > option_defs.len() {
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput, std::io::ErrorKind::InvalidInput,
format!("Too many unnamed parameters (expected at most {})", option_defs.len()) format!(
"Too many unnamed parameters (expected at most {})",
option_defs.len()
),
)); ));
} }
for (i, param) in unnamed_params.iter().enumerate() { for (i, param) in unnamed_params.iter().enumerate() {
if i >= option_defs.len() { if i >= option_defs.len() {
break; break;
@@ -438,19 +447,19 @@ fn create_filter_with_options(
let option_name = &option_defs[i].name; let option_name = &option_defs[i].name;
options.insert(option_name.clone(), param.clone()); options.insert(option_name.clone(), param.clone());
} }
// Process named options // Process named options
for (key, value) in named_options { for (key, value) in named_options {
// Check if the option exists // Check if the option exists
if !option_defs.iter().any(|opt| &opt.name == key) { if !option_defs.iter().any(|opt| &opt.name == key) {
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput, std::io::ErrorKind::InvalidInput,
format!("Unknown option '{}'", key) format!("Unknown option '{}'", key),
)); ));
} }
options.insert(key.clone(), value.clone()); options.insert(key.clone(), value.clone());
} }
// Fill in defaults and check required options // Fill in defaults and check required options
for opt_def in option_defs { for opt_def in option_defs {
if !options.contains_key(&opt_def.name) { if !options.contains_key(&opt_def.name) {
@@ -459,12 +468,12 @@ fn create_filter_with_options(
} else if opt_def.required { } else if opt_def.required {
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput, std::io::ErrorKind::InvalidInput,
format!("Missing required option '{}'", opt_def.name) format!("Missing required option '{}'", opt_def.name),
)); ));
} }
} }
} }
// Create the specific filter with the processed options // Create the specific filter with the processed options
create_specific_filter(filter_type, &options) create_specific_filter(filter_type, &options)
} }
@@ -485,72 +494,93 @@ fn create_specific_filter(
) -> Result<Box<dyn FilterPlugin>> { ) -> Result<Box<dyn FilterPlugin>> {
match filter_type { match filter_type {
FilterType::Grep => { FilterType::Grep => {
let pattern = options.get("pattern") let pattern = options
.get("pattern")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.ok_or_else(|| std::io::Error::new( .ok_or_else(|| {
std::io::ErrorKind::InvalidInput, std::io::Error::new(
"grep filter requires 'pattern' parameter" std::io::ErrorKind::InvalidInput,
))?; "grep filter requires 'pattern' parameter",
)
})?;
grep::GrepFilter::new(pattern.to_string()).map(|f| Box::new(f) as Box<dyn FilterPlugin>) grep::GrepFilter::new(pattern.to_string()).map(|f| Box::new(f) as Box<dyn FilterPlugin>)
} }
FilterType::HeadBytes => { FilterType::HeadBytes => {
let count = options.get("count") let count = options
.get("count")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|n| n as usize) .map(|n| n as usize)
.ok_or_else(|| std::io::Error::new( .ok_or_else(|| {
std::io::ErrorKind::InvalidInput, std::io::Error::new(
"head_bytes filter requires 'count' parameter" std::io::ErrorKind::InvalidInput,
))?; "head_bytes filter requires 'count' parameter",
)
})?;
Ok(Box::new(head::HeadBytesFilter::new(count))) Ok(Box::new(head::HeadBytesFilter::new(count)))
} }
FilterType::HeadLines => { FilterType::HeadLines => {
let count = options.get("count") let count = options
.get("count")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|n| n as usize) .map(|n| n as usize)
.ok_or_else(|| std::io::Error::new( .ok_or_else(|| {
std::io::ErrorKind::InvalidInput, std::io::Error::new(
"head_lines filter requires 'count' parameter" std::io::ErrorKind::InvalidInput,
))?; "head_lines filter requires 'count' parameter",
)
})?;
Ok(Box::new(head::HeadLinesFilter::new(count))) Ok(Box::new(head::HeadLinesFilter::new(count)))
} }
FilterType::TailBytes => { FilterType::TailBytes => {
let count = options.get("count") let count = options
.get("count")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|n| n as usize) .map(|n| n as usize)
.ok_or_else(|| std::io::Error::new( .ok_or_else(|| {
std::io::ErrorKind::InvalidInput, std::io::Error::new(
"tail_bytes filter requires 'count' parameter" std::io::ErrorKind::InvalidInput,
))?; "tail_bytes filter requires 'count' parameter",
)
})?;
Ok(Box::new(tail::TailBytesFilter::new(count))) Ok(Box::new(tail::TailBytesFilter::new(count)))
} }
FilterType::TailLines => { FilterType::TailLines => {
let count = options.get("count") let count = options
.get("count")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|n| n as usize) .map(|n| n as usize)
.ok_or_else(|| std::io::Error::new( .ok_or_else(|| {
std::io::ErrorKind::InvalidInput, std::io::Error::new(
"tail_lines filter requires 'count' parameter" std::io::ErrorKind::InvalidInput,
))?; "tail_lines filter requires 'count' parameter",
)
})?;
Ok(Box::new(tail::TailLinesFilter::new(count))) Ok(Box::new(tail::TailLinesFilter::new(count)))
} }
FilterType::SkipBytes => { FilterType::SkipBytes => {
let count = options.get("count") let count = options
.get("count")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|n| n as usize) .map(|n| n as usize)
.ok_or_else(|| std::io::Error::new( .ok_or_else(|| {
std::io::ErrorKind::InvalidInput, std::io::Error::new(
"skip_bytes filter requires 'count' parameter" std::io::ErrorKind::InvalidInput,
))?; "skip_bytes filter requires 'count' parameter",
)
})?;
Ok(Box::new(skip::SkipBytesFilter::new(count))) Ok(Box::new(skip::SkipBytesFilter::new(count)))
} }
FilterType::SkipLines => { FilterType::SkipLines => {
let count = options.get("count") let count = options
.get("count")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|n| n as usize) .map(|n| n as usize)
.ok_or_else(|| std::io::Error::new( .ok_or_else(|| {
std::io::ErrorKind::InvalidInput, std::io::Error::new(
"skip_lines filter requires 'count' parameter" std::io::ErrorKind::InvalidInput,
))?; "skip_lines filter requires 'count' parameter",
)
})?;
Ok(Box::new(skip::SkipLinesFilter::new(count))) Ok(Box::new(skip::SkipLinesFilter::new(count)))
} }
FilterType::StripAnsi => { FilterType::StripAnsi => {
@@ -558,7 +588,7 @@ fn create_specific_filter(
if !options.is_empty() { if !options.is_empty() {
return Err(std::io::Error::new( return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput, std::io::ErrorKind::InvalidInput,
"strip_ansi filter doesn't take parameters" "strip_ansi filter doesn't take parameters",
)); ));
} }
Ok(Box::new(strip_ansi::StripAnsiFilter::new())) Ok(Box::new(strip_ansi::StripAnsiFilter::new()))
@@ -578,17 +608,17 @@ fn create_specific_filter(
fn parse_option_value(input: &str) -> Result<serde_json::Value> { fn parse_option_value(input: &str) -> Result<serde_json::Value> {
// Remove quotes if present // Remove quotes if present
let input = input.trim_matches(|c| c == '\'' || c == '"'); let input = input.trim_matches(|c| c == '\'' || c == '"');
// Try to parse as number // Try to parse as number
if let Ok(num) = input.parse::<i64>() { if let Ok(num) = input.parse::<i64>() {
return Ok(serde_json::Value::Number(num.into())); return Ok(serde_json::Value::Number(num.into()));
} }
if let Ok(num) = input.parse::<f64>() { if let Ok(num) = input.parse::<f64>()
if let Some(number) = serde_json::Number::from_f64(num) { && let Some(number) = serde_json::Number::from_f64(num)
return Ok(serde_json::Value::Number(number)); {
} return Ok(serde_json::Value::Number(number));
} }
// Try to parse as boolean // Try to parse as boolean
if input.eq_ignore_ascii_case("true") { if input.eq_ignore_ascii_case("true") {
return Ok(serde_json::Value::Bool(true)); return Ok(serde_json::Value::Bool(true));
@@ -596,7 +626,7 @@ fn parse_option_value(input: &str) -> Result<serde_json::Value> {
if input.eq_ignore_ascii_case("false") { if input.eq_ignore_ascii_case("false") {
return Ok(serde_json::Value::Bool(false)); return Ok(serde_json::Value::Bool(false));
} }
// Treat as string // Treat as string
Ok(serde_json::Value::String(input.to_string())) Ok(serde_json::Value::String(input.to_string()))
} }

42
src/filter_plugin/skip.rs
View File

@@ -1,7 +1,7 @@
use super::{FilterPlugin, FilterOption}; use super::{FilterOption, FilterPlugin};
use std::io::{Result, Read, Write, BufRead};
use crate::common::PIPESIZE; use crate::common::PIPESIZE;
use crate::services::filter_service::register_filter_plugin; use crate::services::filter_service::register_filter_plugin;
use std::io::{BufRead, Read, Result, Write};
/// A filter that skips the first N bytes from the input stream. /// A filter that skips the first N bytes from the input stream.
pub struct SkipBytesFilter { pub struct SkipBytesFilter {
@@ -15,9 +15,7 @@ impl SkipBytesFilter {
/// ///
/// * `count` - The number of bytes to skip from the beginning of the input. /// * `count` - The number of bytes to skip from the beginning of the input.
pub fn new(count: usize) -> Self { pub fn new(count: usize) -> Self {
Self { Self { remaining: count }
remaining: count,
}
} }
} }
@@ -45,7 +43,7 @@ impl FilterPlugin for SkipBytesFilter {
self.remaining -= bytes_read; self.remaining -= bytes_read;
} }
} }
// Copy the remaining data using io::copy for efficiency // Copy the remaining data using io::copy for efficiency
std::io::copy(reader, writer)?; std::io::copy(reader, writer)?;
Ok(()) Ok(())
@@ -61,20 +59,18 @@ impl FilterPlugin for SkipBytesFilter {
remaining: self.remaining, remaining: self.remaining,
}) })
} }
/// Returns the configuration options for this filter. /// Returns the configuration options for this filter.
/// ///
/// # Returns /// # Returns
/// ///
/// A vector of `FilterOption` describing the filter's configurable parameters. /// A vector of `FilterOption` describing the filter's configurable parameters.
fn options(&self) -> Vec<FilterOption> { fn options(&self) -> Vec<FilterOption> {
vec![ vec![FilterOption {
FilterOption { name: "count".to_string(),
name: "count".to_string(), default: None,
default: None, required: true,
required: true, }]
}
]
} }
} }
@@ -90,9 +86,7 @@ impl SkipLinesFilter {
/// ///
/// * `count` - The number of lines to skip from the beginning of the input. /// * `count` - The number of lines to skip from the beginning of the input.
pub fn new(count: usize) -> Self { pub fn new(count: usize) -> Self {
Self { Self { remaining: count }
remaining: count,
}
} }
} }
@@ -130,20 +124,18 @@ impl FilterPlugin for SkipLinesFilter {
remaining: self.remaining, remaining: self.remaining,
}) })
} }
/// Returns the configuration options for this filter. /// Returns the configuration options for this filter.
/// ///
/// # Returns /// # Returns
/// ///
/// A vector of `FilterOption` describing the filter's configurable parameters. /// A vector of `FilterOption` describing the filter's configurable parameters.
fn options(&self) -> Vec<FilterOption> { fn options(&self) -> Vec<FilterOption> {
vec![ vec![FilterOption {
FilterOption { name: "count".to_string(),
name: "count".to_string(), default: None,
default: None, required: true,
required: true, }]
}
]
} }
} }

6
src/filter_plugin/strip_ansi.rs
View File

@@ -1,6 +1,6 @@
use std::io::{Result, Read, Write}; use super::{FilterOption, FilterPlugin};
use std::io::{Read, Result, Write};
use strip_ansi_escapes::Writer; use strip_ansi_escapes::Writer;
use super::{FilterPlugin, FilterOption};
/// A filter that removes ANSI escape sequences from the input. /// A filter that removes ANSI escape sequences from the input.
/// ///
@@ -47,7 +47,7 @@ impl FilterPlugin for StripAnsiFilter {
fn clone_box(&self) -> Box<dyn FilterPlugin> { fn clone_box(&self) -> Box<dyn FilterPlugin> {
Box::new(Self) Box::new(Self)
} }
/// Returns the configuration options for this filter (none required). /// Returns the configuration options for this filter (none required).
/// ///
/// # Returns /// # Returns

40
src/filter_plugin/tail.rs
View File

@@ -1,8 +1,8 @@
use super::{FilterPlugin, FilterOption}; use super::{FilterOption, FilterPlugin};
use std::io::{Result, Read, Write, BufRead};
use std::collections::VecDeque;
use crate::common::PIPESIZE; use crate::common::PIPESIZE;
use crate::services::filter_service::register_filter_plugin; use crate::services::filter_service::register_filter_plugin;
use std::collections::VecDeque;
use std::io::{BufRead, Read, Result, Write};
/// A filter that reads the last N bytes from the input stream. /// A filter that reads the last N bytes from the input stream.
pub struct TailBytesFilter { pub struct TailBytesFilter {
@@ -42,7 +42,7 @@ impl FilterPlugin for TailBytesFilter {
if bytes_read == 0 { if bytes_read == 0 {
break; break;
} }
// Add new data to the buffer // Add new data to the buffer
for &byte in &temp_buffer[..bytes_read] { for &byte in &temp_buffer[..bytes_read] {
if self.buffer.len() == self.count { if self.buffer.len() == self.count {
@@ -51,7 +51,7 @@ impl FilterPlugin for TailBytesFilter {
self.buffer.push_back(byte); self.buffer.push_back(byte);
} }
} }
// Write the buffered data at the end // Write the buffered data at the end
let result: Vec<u8> = self.buffer.iter().cloned().collect(); let result: Vec<u8> = self.buffer.iter().cloned().collect();
writer.write_all(&result)?; writer.write_all(&result)?;
@@ -69,20 +69,18 @@ impl FilterPlugin for TailBytesFilter {
count: self.count, count: self.count,
}) })
} }
/// Returns the configuration options for this filter. /// Returns the configuration options for this filter.
/// ///
/// # Returns /// # Returns
/// ///
/// A vector of `FilterOption` describing the filter's configurable parameters. /// A vector of `FilterOption` describing the filter's configurable parameters.
fn options(&self) -> Vec<FilterOption> { fn options(&self) -> Vec<FilterOption> {
vec![ vec![FilterOption {
FilterOption { name: "count".to_string(),
name: "count".to_string(), default: None,
default: None, required: true,
required: true, }]
}
]
} }
} }
@@ -126,7 +124,7 @@ impl FilterPlugin for TailLinesFilter {
} }
self.lines.push_back(line); self.lines.push_back(line);
} }
// Write the buffered lines // Write the buffered lines
for line in &self.lines { for line in &self.lines {
writeln!(writer, "{}", line)?; writeln!(writer, "{}", line)?;
@@ -145,20 +143,18 @@ impl FilterPlugin for TailLinesFilter {
count: self.count, count: self.count,
}) })
} }
/// Returns the configuration options for this filter. /// Returns the configuration options for this filter.
/// ///
/// # Returns /// # Returns
/// ///
/// A vector of `FilterOption` describing the filter's configurable parameters. /// A vector of `FilterOption` describing the filter's configurable parameters.
fn options(&self) -> Vec<FilterOption> { fn options(&self) -> Vec<FilterOption> {
vec![ vec![FilterOption {
FilterOption { name: "count".to_string(),
name: "count".to_string(), default: None,
default: None, required: true,
required: true, }]
}
]
} }
} }

13
src/lib.rs
View File

@@ -29,15 +29,15 @@
//! - `magic`: File type detection via libmagic. //! - `magic`: File type detection via libmagic.
// Re-export modules for testing // Re-export modules for testing
pub mod args;
pub mod common; pub mod common;
pub mod compression_engine; pub mod compression_engine;
pub mod config; pub mod config;
pub mod services;
pub mod db; pub mod db;
pub mod filter_plugin;
pub mod meta_plugin; pub mod meta_plugin;
pub mod modes; pub mod modes;
pub mod filter_plugin; pub mod services;
pub mod args;
// Re-export Args struct for library usage // Re-export Args struct for library usage
pub use args::Args; pub use args::Args;
@@ -46,13 +46,10 @@ pub use common::PIPESIZE;
// Import all filter plugins to ensure they register themselves // Import all filter plugins to ensure they register themselves
#[allow(unused_imports)] #[allow(unused_imports)]
use filter_plugin::{ use filter_plugin::{grep, head, skip, strip_ansi, tail};
head, tail, skip, grep, strip_ansi
};
use crate::meta_plugin::{ use crate::meta_plugin::{
cwd, user, shell, shell_pid, keep_pid, digest, cwd, digest, env, exec, hostname, keep_pid, read_rate, read_time, shell, shell_pid, user,
read_time, read_rate, hostname, exec, env
}; };
#[cfg(feature = "magic")] #[cfg(feature = "magic")]

80
src/main.rs
View File

@@ -1,6 +1,6 @@
use anyhow::{Context, Error, Result, anyhow}; use anyhow::{Context, Error, Result, anyhow};
use clap::*;
use clap::error::ErrorKind; use clap::error::ErrorKind;
use clap::*;
use log::*; use log::*;
use directories::ProjectDirs; use directories::ProjectDirs;
@@ -19,7 +19,7 @@ fn main() -> Result<(), Error> {
let mut cmd = Args::command(); let mut cmd = Args::command();
let args = Args::parse(); let args = Args::parse();
// Validate arguments based on mode // Validate arguments based on mode
if let Err(e) = args.validate() { if let Err(e) = args.validate() {
cmd.error(ErrorKind::ValueValidation, e).exit(); cmd.error(ErrorKind::ValueValidation, e).exit();
@@ -43,7 +43,7 @@ fn main() -> Result<(), Error> {
// Create unified settings using the new config system // Create unified settings using the new config system
let settings = Settings::new(&args, default_dir)?; let settings = Settings::new(&args, default_dir)?;
debug!("MAIN: Loaded settings: {:?}", settings); debug!("MAIN: Loaded settings: {:?}", settings);
let ids = &mut Vec::new(); let ids = &mut Vec::new();
@@ -56,7 +56,7 @@ fn main() -> Result<(), Error> {
NumberOrString::Number(num) => { NumberOrString::Number(num) => {
debug!("MAIN: Adding to ids: {}", num); debug!("MAIN: Adding to ids: {}", num);
ids.push(num) ids.push(num)
}, }
NumberOrString::Str(str) => { NumberOrString::Str(str) => {
// For --info and --get, try to parse strings as numbers to treat them as IDs // For --info and --get, try to parse strings as numbers to treat them as IDs
if args.mode.info || args.mode.get { if args.mode.info || args.mode.get {
@@ -68,14 +68,15 @@ fn main() -> Result<(), Error> {
// --info only accepts numeric IDs // --info only accepts numeric IDs
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
format!("--info requires numeric IDs, found: '{}'", str) format!("--info requires numeric IDs, found: '{}'", str),
).exit(); )
.exit();
} }
} }
// If not a number, or not using --info/--get, treat as tag // If not a number, or not using --info/--get, treat as tag
debug!("MAIN: Adding to tags: {}", str); debug!("MAIN: Adding to tags: {}", str);
tags.push(str) tags.push(str)
}, }
} }
} }
tags.sort(); tags.sort();
@@ -130,29 +131,35 @@ fn main() -> Result<(), Error> {
} }
// Validate output format usage // Validate output format usage
if let Some(output_format_str) = &settings.output_format { if let Some(output_format_str) = &settings.output_format
if output_format_str != "table" && mode != KeepModes::Info && mode != KeepModes::Status && mode != KeepModes::StatusPlugins && mode != KeepModes::List { && output_format_str != "table"
cmd.error( && mode != KeepModes::Info
&& mode != KeepModes::Status
&& mode != KeepModes::StatusPlugins
&& mode != KeepModes::List
{
cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
"--output-format can only be used with --info, --status, --status-plugins, or --list modes" "--output-format can only be used with --info, --status, --status-plugins, or --list modes"
).exit(); ).exit();
}
} }
// Validate human-readable usage // Validate human-readable usage
if settings.human_readable && mode != KeepModes::List && mode != KeepModes::Info { if settings.human_readable && mode != KeepModes::List && mode != KeepModes::Info {
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
"--human-readable can only be used with --list and --info modes" "--human-readable can only be used with --list and --info modes",
).exit(); )
.exit();
} }
// Validate server password usage // Validate server password usage
if settings.server_password().is_some() && mode != KeepModes::Server { if settings.server_password().is_some() && mode != KeepModes::Server {
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
"--server-password can only be used with --server mode" "--server-password can only be used with --server mode",
).exit(); )
.exit();
} }
debug!("MAIN: args: {:?}", args); debug!("MAIN: args: {:?}", args);
@@ -186,23 +193,42 @@ fn main() -> Result<(), Error> {
Err(e) => { Err(e) => {
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
format!("Invalid filter string: {}", e) format!("Invalid filter string: {}", e),
).exit(); )
.exit();
} }
} }
} else { } else {
None None
}; };
match mode { match mode {
KeepModes::Save => modes::save::mode_save(&mut cmd, &settings, ids, tags, &mut conn, data_path), KeepModes::Save => {
KeepModes::Get => modes::get::mode_get(&mut cmd, &settings, ids, tags, &mut conn, data_path, filter_chain), modes::save::mode_save(&mut cmd, &settings, ids, tags, &mut conn, data_path)
}
KeepModes::Get => modes::get::mode_get(
&mut cmd,
&settings,
ids,
tags,
&mut conn,
data_path,
filter_chain,
),
KeepModes::Diff => modes::diff::mode_diff(&mut cmd, &args, &mut conn), KeepModes::Diff => modes::diff::mode_diff(&mut cmd, &args, &mut conn),
KeepModes::List => modes::list::mode_list(&mut cmd, &settings, ids, tags, &mut conn, data_path), KeepModes::List => {
KeepModes::Delete => modes::delete::mode_delete(&mut cmd, &settings, &settings, ids, tags, &mut conn, data_path), modes::list::mode_list(&mut cmd, &settings, ids, tags, &mut conn, data_path)
KeepModes::Info => modes::info::mode_info(&mut cmd, &settings, ids, tags, &mut conn, data_path), }
KeepModes::Delete => modes::delete::mode_delete(
&mut cmd, &settings, &settings, ids, tags, &mut conn, data_path,
),
KeepModes::Info => {
modes::info::mode_info(&mut cmd, &settings, ids, tags, &mut conn, data_path)
}
KeepModes::Status => modes::status::mode_status(&mut cmd, &settings, data_path, db_path), KeepModes::Status => modes::status::mode_status(&mut cmd, &settings, data_path, db_path),
KeepModes::StatusPlugins => modes::status_plugins::mode_status_plugins(&mut cmd, &settings, data_path, db_path), KeepModes::StatusPlugins => {
modes::status_plugins::mode_status_plugins(&mut cmd, &settings, data_path, db_path)
}
KeepModes::Server => { KeepModes::Server => {
#[cfg(feature = "server")] #[cfg(feature = "server")]
{ {
@@ -215,8 +241,10 @@ fn main() -> Result<(), Error> {
"This binary was not compiled with server support. Recompile with --features server" "This binary was not compiled with server support. Recompile with --features server"
).exit(); ).exit();
} }
}, }
KeepModes::GenerateConfig => modes::generate_config::mode_generate_config(&mut cmd, &settings), KeepModes::GenerateConfig => {
modes::generate_config::mode_generate_config(&mut cmd, &settings)
}
KeepModes::Unknown => unreachable!(), KeepModes::Unknown => unreachable!(),
} }
} }

44
src/meta_plugin/cwd.rs
View File

@@ -1,5 +1,5 @@
use std::env;
use crate::meta_plugin::{MetaPlugin, MetaPluginType}; use crate::meta_plugin::{MetaPlugin, MetaPluginType};
use std::env;
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct CwdMetaPlugin { pub struct CwdMetaPlugin {
@@ -13,13 +13,14 @@ impl CwdMetaPlugin {
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>, outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> CwdMetaPlugin { ) -> CwdMetaPlugin {
let mut base = crate::meta_plugin::BaseMetaPlugin::new(); let mut base = crate::meta_plugin::BaseMetaPlugin::new();
// Set default outputs // Set default outputs
let default_outputs = vec!["cwd".to_string()]; let default_outputs = vec!["cwd".to_string()];
for output_name in default_outputs { for output_name in default_outputs {
base.outputs.insert(output_name.clone(), serde_yaml::Value::String(output_name)); base.outputs
.insert(output_name.clone(), serde_yaml::Value::String(output_name));
} }
// Apply provided options and outputs // Apply provided options and outputs
if let Some(opts) = options { if let Some(opts) = options {
for (key, value) in opts { for (key, value) in opts {
@@ -31,24 +32,23 @@ impl CwdMetaPlugin {
base.outputs.insert(key, value); base.outputs.insert(key, value);
} }
} }
CwdMetaPlugin { CwdMetaPlugin {
is_finalized: false, is_finalized: false,
base, base,
} }
} }
} }
impl MetaPlugin for CwdMetaPlugin { impl MetaPlugin for CwdMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
fn set_finalized(&mut self, finalized: bool) { fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
fn finalize(&mut self) -> crate::meta_plugin::MetaPluginResponse { fn finalize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
// If already finalized, don't process again // If already finalized, don't process again
if self.is_finalized { if self.is_finalized {
@@ -57,10 +57,10 @@ impl MetaPlugin for CwdMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
// Mark as finalized // Mark as finalized
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
@@ -70,7 +70,7 @@ impl MetaPlugin for CwdMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::Cwd MetaPluginType::Cwd
} }
fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse { fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
// If already finalized, don't process again // If already finalized, don't process again
if self.is_finalized { if self.is_finalized {
@@ -79,42 +79,40 @@ impl MetaPlugin for CwdMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
let mut metadata = Vec::new(); let mut metadata = Vec::new();
let cwd = match env::current_dir() { let cwd = match env::current_dir() {
Ok(path) => path.to_string_lossy().to_string(), Ok(path) => path.to_string_lossy().to_string(),
Err(_) => "unknown".to_string(), Err(_) => "unknown".to_string(),
}; };
// Use process_metadata_outputs to handle output mapping // Use process_metadata_outputs to handle output mapping
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"cwd", "cwd",
serde_yaml::Value::String(cwd), serde_yaml::Value::String(cwd),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata, metadata,
is_finalized: false, is_finalized: false,
} }
} }
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.options_mut() self.base.options_mut()
} }

77
src/meta_plugin/digest.rs
View File

@@ -1,6 +1,6 @@
use sha2::{Digest, Sha256, Sha512}; use crate::meta_plugin::{BaseMetaPlugin, MetaPlugin, MetaPluginType};
use md5; use md5;
use crate::meta_plugin::{MetaPlugin, MetaPluginType, BaseMetaPlugin}; use sha2::{Digest, Sha256, Sha512};
use std::io::Write; use std::io::Write;
#[derive(Clone)] #[derive(Clone)]
@@ -33,11 +33,11 @@ impl Hasher {
Hasher::Sha256(hasher) => hasher.update(data), Hasher::Sha256(hasher) => hasher.update(data),
Hasher::Md5(hasher) => { Hasher::Md5(hasher) => {
let _ = hasher.write(data); let _ = hasher.write(data);
}, }
Hasher::Sha512(hasher) => hasher.update(data), Hasher::Sha512(hasher) => hasher.update(data),
} }
} }
fn finalize(&mut self) -> String { fn finalize(&mut self) -> String {
match self { match self {
Hasher::Sha256(hasher) => { Hasher::Sha256(hasher) => {
@@ -54,7 +54,7 @@ impl Hasher {
} }
} }
} }
fn output_name(&self) -> &'static str { fn output_name(&self) -> &'static str {
match self { match self {
Hasher::Sha256(_) => "digest_sha256", Hasher::Sha256(_) => "digest_sha256",
@@ -71,21 +71,20 @@ pub struct DigestMetaPlugin {
base: BaseMetaPlugin, base: BaseMetaPlugin,
} }
impl DigestMetaPlugin { impl DigestMetaPlugin {
pub fn new( pub fn new(
options: Option<std::collections::HashMap<String, serde_yaml::Value>>, options: Option<std::collections::HashMap<String, serde_yaml::Value>>,
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>, outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> DigestMetaPlugin { ) -> DigestMetaPlugin {
let mut base = BaseMetaPlugin::new(); let mut base = BaseMetaPlugin::new();
// Apply provided options // Apply provided options
if let Some(opts) = options { if let Some(opts) = options {
for (key, value) in opts { for (key, value) in opts {
base.options.insert(key, value); base.options.insert(key, value);
} }
} }
// Determine the selected method // Determine the selected method
let method = if let Some(method_value) = base.options.get("method") { let method = if let Some(method_value) = base.options.get("method") {
if let Some(method_str) = method_value.as_str() { if let Some(method_str) = method_value.as_str() {
@@ -101,7 +100,7 @@ impl DigestMetaPlugin {
} else { } else {
"sha256" "sha256"
}; };
// Initialize the hasher based on the method // Initialize the hasher based on the method
let hasher = match method { let hasher = match method {
"md5" => Some(Hasher::Md5(md5::Context::new())), "md5" => Some(Hasher::Md5(md5::Context::new())),
@@ -109,32 +108,40 @@ impl DigestMetaPlugin {
"sha512" => Some(Hasher::Sha512(Sha512::new())), "sha512" => Some(Hasher::Sha512(Sha512::new())),
_ => Some(Hasher::Sha256(Sha256::new())), _ => Some(Hasher::Sha256(Sha256::new())),
}; };
// Add the method to options so it shows up in the status // Add the method to options so it shows up in the status
base.options.insert("method".to_string(), serde_yaml::Value::String(method.to_string())); base.options.insert(
"method".to_string(),
serde_yaml::Value::String(method.to_string()),
);
// Set outputs based on the selected hash method // Set outputs based on the selected hash method
// Only the selected method's output should be enabled, others should be None // Only the selected method's output should be enabled, others should be None
let all_outputs = vec!["digest_md5", "digest_sha256", "digest_sha512"]; let all_outputs = vec!["digest_md5", "digest_sha256", "digest_sha512"];
for output_name in &all_outputs { for output_name in &all_outputs {
if output_name == &format!("digest_{}", method) { if output_name == &format!("digest_{}", method) {
base.outputs.insert(output_name.to_string(), serde_yaml::Value::String(output_name.to_string())); base.outputs.insert(
output_name.to_string(),
serde_yaml::Value::String(output_name.to_string()),
);
} else { } else {
base.outputs.insert(output_name.to_string(), serde_yaml::Value::Null); base.outputs
.insert(output_name.to_string(), serde_yaml::Value::Null);
} }
} }
// Apply provided outputs, but only for enabled outputs // Apply provided outputs, but only for enabled outputs
if let Some(outs) = outputs { if let Some(outs) = outputs {
for (key, value) in outs { for (key, value) in outs {
// Only update if the output is not disabled (not None) // Only update if the output is not disabled (not None)
if let Some(current_value) = base.outputs.get_mut(&key) if let Some(current_value) = base.outputs.get_mut(&key)
&& !current_value.is_null() { && !current_value.is_null()
{
*current_value = value; *current_value = value;
} }
} }
} }
DigestMetaPlugin { DigestMetaPlugin {
hasher, hasher,
is_finalized: false, is_finalized: false,
@@ -147,11 +154,11 @@ impl MetaPlugin for DigestMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
fn set_finalized(&mut self, finalized: bool) { fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse { fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
@@ -166,32 +173,34 @@ impl MetaPlugin for DigestMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
let mut metadata = Vec::new(); let mut metadata = Vec::new();
// Update outputs based on the selected hash method // Update outputs based on the selected hash method
if let Some(hasher) = &mut self.hasher { if let Some(hasher) = &mut self.hasher {
let hash_value = hasher.finalize(); let hash_value = hasher.finalize();
let output_name = hasher.output_name(); let output_name = hasher.output_name();
// Use process_metadata_outputs to handle output mapping // Use process_metadata_outputs to handle output mapping
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
output_name, output_name,
serde_yaml::Value::String(hash_value), serde_yaml::Value::String(hash_value),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
// Set all other digest outputs to None // Set all other digest outputs to None
let all_outputs = vec!["digest_md5", "digest_sha256", "digest_sha512"]; let all_outputs = vec!["digest_md5", "digest_sha256", "digest_sha512"];
for output_name in all_outputs { for output_name in all_outputs {
if output_name != hasher.output_name() { if output_name != hasher.output_name() {
self.base.outputs.insert(output_name.to_string(), serde_yaml::Value::Null); self.base
.outputs
.insert(output_name.to_string(), serde_yaml::Value::Null);
} }
} }
} }
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata, metadata,
@@ -206,12 +215,12 @@ impl MetaPlugin for DigestMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
// Update the active hasher // Update the active hasher
if let Some(hasher) = &mut self.hasher { if let Some(hasher) = &mut self.hasher {
hasher.update(data); hasher.update(data);
} }
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: false, is_finalized: false,
@@ -221,15 +230,15 @@ impl MetaPlugin for DigestMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::Digest MetaPluginType::Digest
} }
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec![ vec![
"digest_md5".to_string(), "digest_md5".to_string(),
@@ -237,11 +246,11 @@ impl MetaPlugin for DigestMetaPlugin {
"digest_sha512".to_string(), "digest_sha512".to_string(),
] ]
} }
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.options_mut() self.base.options_mut()
} }

48
src/meta_plugin/env.rs
View File

@@ -1,4 +1,4 @@
use super::{MetaPlugin, MetaPluginType, process_metadata_outputs, BaseMetaPlugin}; use super::{BaseMetaPlugin, MetaPlugin, MetaPluginType, process_metadata_outputs};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
/// Meta plugin that extracts environment variables prefixed with KEEP_META_ as metadata. /// Meta plugin that extracts environment variables prefixed with KEEP_META_ as metadata.
@@ -28,29 +28,29 @@ impl EnvMetaPlugin {
// Collect environment variables starting with KEEP_META_ // Collect environment variables starting with KEEP_META_
let mut env_vars = Vec::new(); let mut env_vars = Vec::new();
let mut outputs_map = std::collections::HashMap::new(); let mut outputs_map = std::collections::HashMap::new();
for (key, value) in std::env::vars() { for (key, value) in std::env::vars() {
if let Some(stripped_key) = key.strip_prefix("KEEP_META_") { if let Some(stripped_key) = key.strip_prefix("KEEP_META_") {
// Add to env_vars to process later // Add to env_vars to process later
env_vars.push((stripped_key.to_string(), value)); env_vars.push((stripped_key.to_string(), value));
// Add to outputs with default mapping to the stripped name // Add to outputs with default mapping to the stripped name
outputs_map.insert( outputs_map.insert(
stripped_key.to_string(), stripped_key.to_string(),
serde_yaml::Value::String(stripped_key.to_string()) serde_yaml::Value::String(stripped_key.to_string()),
); );
} }
} }
// Override with provided outputs // Override with provided outputs
if let Some(provided_outputs) = outputs { if let Some(provided_outputs) = outputs {
for (key, value) in provided_outputs { for (key, value) in provided_outputs {
outputs_map.insert(key, value); outputs_map.insert(key, value);
} }
} }
let mut base = BaseMetaPlugin::new(); let mut base = BaseMetaPlugin::new();
base.outputs = outputs_map; base.outputs = outputs_map;
EnvMetaPlugin { EnvMetaPlugin {
is_finalized: false, is_finalized: false,
base, base,
@@ -68,7 +68,7 @@ impl MetaPlugin for EnvMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::Env MetaPluginType::Env
} }
/// Checks if the plugin has been finalized. /// Checks if the plugin has been finalized.
/// ///
/// # Returns /// # Returns
@@ -77,7 +77,7 @@ impl MetaPlugin for EnvMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
/// Sets the finalized state of the plugin. /// Sets the finalized state of the plugin.
/// ///
/// # Arguments /// # Arguments
@@ -86,7 +86,7 @@ impl MetaPlugin for EnvMetaPlugin {
fn set_finalized(&mut self, finalized: bool) { fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
/// Initializes the plugin, processing environment variables. /// Initializes the plugin, processing environment variables.
/// ///
/// Processes all KEEP_META_* variables and generates metadata using output mappings. /// Processes all KEEP_META_* variables and generates metadata using output mappings.
@@ -102,28 +102,28 @@ impl MetaPlugin for EnvMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
// Process all collected environment variables // Process all collected environment variables
let mut metadata = Vec::new(); let mut metadata = Vec::new();
for (name, value) in &self.env_vars { for (name, value) in &self.env_vars {
if let Some(meta_data) = process_metadata_outputs( if let Some(meta_data) = process_metadata_outputs(
name, name,
serde_yaml::Value::String(value.clone()), serde_yaml::Value::String(value.clone()),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
} }
// Mark as finalized since this plugin only needs to run once // Mark as finalized since this plugin only needs to run once
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata, metadata,
is_finalized: true, is_finalized: true,
} }
} }
/// Updates the plugin with new data (unused in this implementation). /// Updates the plugin with new data (unused in this implementation).
/// ///
/// This plugin does not process streaming data; returns empty response. /// This plugin does not process streaming data; returns empty response.
@@ -143,7 +143,7 @@ impl MetaPlugin for EnvMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: false, is_finalized: false,
@@ -162,13 +162,13 @@ impl MetaPlugin for EnvMetaPlugin {
if !self.is_finalized { if !self.is_finalized {
return self.initialize(); return self.initialize();
} }
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
} }
} }
/// Returns a reference to the outputs mapping. /// Returns a reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -177,7 +177,7 @@ impl MetaPlugin for EnvMetaPlugin {
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
/// Returns a mutable reference to the outputs mapping. /// Returns a mutable reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -186,18 +186,16 @@ impl MetaPlugin for EnvMetaPlugin {
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
/// Returns the default output names based on collected env vars. /// Returns the default output names based on collected env vars.
/// ///
/// # Returns /// # Returns
/// ///
/// A vector of environment variable names (stripped of KEEP_META_ prefix). /// A vector of environment variable names (stripped of KEEP_META_ prefix).
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
self.env_vars.iter() self.env_vars.iter().map(|(name, _)| name.clone()).collect()
.map(|(name, _)| name.clone())
.collect()
} }
/// Returns a reference to the options mapping (empty for this plugin). /// Returns a reference to the options mapping (empty for this plugin).
/// ///
/// This plugin has no configurable options. /// This plugin has no configurable options.
@@ -208,7 +206,7 @@ impl MetaPlugin for EnvMetaPlugin {
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
/// Returns a mutable reference to the options mapping. /// Returns a mutable reference to the options mapping.
/// ///
/// # Panics /// # Panics

44
src/meta_plugin/exec.rs
View File

@@ -1,9 +1,9 @@
use log::*; use log::*;
use std::io::{self, Write}; use std::io::{self, Write};
use std::process::{Command, Stdio, Child}; use std::process::{Child, Command, Stdio};
use which::which; use which::which;
use crate::meta_plugin::{MetaPlugin, MetaPluginResponse, MetaPluginType, BaseMetaPlugin}; use crate::meta_plugin::{BaseMetaPlugin, MetaPlugin, MetaPluginResponse, MetaPluginType};
/// External program execution meta plugin. /// External program execution meta plugin.
/// ///
@@ -44,7 +44,6 @@ impl std::fmt::Debug for MetaPluginExec {
} }
} }
impl MetaPluginExec { impl MetaPluginExec {
/// Creates a new MetaPluginExec instance. /// Creates a new MetaPluginExec instance.
/// ///
@@ -113,7 +112,10 @@ impl MetaPluginExec {
} }
if !self.supported { if !self.supported {
debug!("META: Exec plugin: program '{}' not supported", self.program); debug!(
"META: Exec plugin: program '{}' not supported",
self.program
);
return MetaPluginResponse { return MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
@@ -138,7 +140,10 @@ impl MetaPluginExec {
} }
} }
Err(e) => { Err(e) => {
error!("META: Exec plugin: failed to start '{}': {}", self.program, e); error!(
"META: Exec plugin: failed to start '{}': {}",
self.program, e
);
MetaPluginResponse { MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
@@ -166,10 +171,10 @@ impl MetaPlugin for MetaPluginExec {
} }
fn update(&mut self, data: &[u8]) -> MetaPluginResponse { fn update(&mut self, data: &[u8]) -> MetaPluginResponse {
if let Some(writer) = self.writer.as_mut() { if let Some(writer) = self.writer.as_mut()
if let Err(e) = writer.write_all(data) { && let Err(e) = writer.write_all(data)
error!("META: Exec plugin: failed to write to stdin: {}", e); {
} error!("META: Exec plugin: failed to write to stdin: {}", e);
} }
MetaPluginResponse { MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
@@ -190,7 +195,11 @@ impl MetaPlugin for MetaPluginExec {
if output.status.success() { if output.status.success() {
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
let result = if self.split_whitespace { let result = if self.split_whitespace {
stdout.split_whitespace().next().unwrap_or(&stdout).to_string() stdout
.split_whitespace()
.next()
.unwrap_or(&stdout)
.to_string()
} else { } else {
stdout.trim().to_string() stdout.trim().to_string()
}; };
@@ -198,7 +207,11 @@ impl MetaPlugin for MetaPluginExec {
self.result = Some(result.clone()); self.result = Some(result.clone());
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
self.base.outputs().keys().next().unwrap_or(&"exec".to_string()), self.base
.outputs()
.keys()
.next()
.unwrap_or(&"exec".to_string()),
serde_yaml::Value::String(result), serde_yaml::Value::String(result),
self.base.outputs(), self.base.outputs(),
) { ) {
@@ -261,7 +274,8 @@ fn register_exec_plugin() {
if let Some(opts) = &options { if let Some(opts) = &options {
if let Some(command_value) = opts.get("command") if let Some(command_value) = opts.get("command")
&& let Some(command_str) = command_value.as_str() { && let Some(command_str) = command_value.as_str()
{
let parts: Vec<&str> = command_str.split_whitespace().collect(); let parts: Vec<&str> = command_str.split_whitespace().collect();
if !parts.is_empty() { if !parts.is_empty() {
program_name = parts[0].to_string(); program_name = parts[0].to_string();
@@ -269,11 +283,13 @@ fn register_exec_plugin() {
} }
} }
if let Some(split_value) = opts.get("split_whitespace") if let Some(split_value) = opts.get("split_whitespace")
&& let Some(split_bool) = split_value.as_bool() { && let Some(split_bool) = split_value.as_bool()
{
split_whitespace = split_bool; split_whitespace = split_bool;
} }
if let Some(name_value) = opts.get("name") if let Some(name_value) = opts.get("name")
&& let Some(name_str) = name_value.as_str() { && let Some(name_str) = name_value.as_str()
{
meta_name = name_str.to_string(); meta_name = name_str.to_string();
} }
} }

210
src/meta_plugin/hostname.rs
View File

@@ -1,4 +1,4 @@
use crate::meta_plugin::{MetaPlugin, MetaPluginType, BaseMetaPlugin}; use crate::meta_plugin::{BaseMetaPlugin, MetaPlugin, MetaPluginType};
use smart_default::SmartDefault; use smart_default::SmartDefault;
@@ -21,21 +21,27 @@ impl HostnameMetaPlugin {
base.initialize_plugin(default_outputs, &options, &outputs); base.initialize_plugin(default_outputs, &options, &outputs);
// Start with default options - hostname is now boolean only // Start with default options - hostname is now boolean only
base.options.insert("hostname".to_string(), serde_yaml::Value::Bool(true)); base.options
base.options.insert("hostname_full".to_string(), serde_yaml::Value::Bool(true)); .insert("hostname".to_string(), serde_yaml::Value::Bool(true));
base.options.insert("hostname_short".to_string(), serde_yaml::Value::Bool(true)); base.options
.insert("hostname_full".to_string(), serde_yaml::Value::Bool(true));
base.options
.insert("hostname_short".to_string(), serde_yaml::Value::Bool(true));
// Override with provided options // Override with provided options
if let Some(opts) = &options { if let Some(opts) = &options {
for (key, value) in opts { for (key, value) in opts {
// Convert string "true"/"false" to boolean for hostname option // Convert string "true"/"false" to boolean for hostname option
if key == "hostname" if key == "hostname"
&& let serde_yaml::Value::String(s) = value { && let serde_yaml::Value::String(s) = value
{
if s == "false" { if s == "false" {
base.options.insert(key.clone(), serde_yaml::Value::Bool(false)); base.options
.insert(key.clone(), serde_yaml::Value::Bool(false));
continue; continue;
} else if s == "true" { } else if s == "true" {
base.options.insert(key.clone(), serde_yaml::Value::Bool(true)); base.options
.insert(key.clone(), serde_yaml::Value::Bool(true));
continue; continue;
} }
} }
@@ -44,15 +50,21 @@ impl HostnameMetaPlugin {
} }
// Determine which outputs are enabled based on options // Determine which outputs are enabled based on options
let hostname_enabled = base.options.get("hostname") let hostname_enabled = base
.options
.get("hostname")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
let hostname_full_enabled = base.options.get("hostname_full") let hostname_full_enabled = base
.options
.get("hostname_full")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
let hostname_short_enabled = base.options.get("hostname_short") let hostname_short_enabled = base
.options
.get("hostname_short")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
@@ -61,21 +73,30 @@ impl HostnameMetaPlugin {
// Handle hostname output // Handle hostname output
if hostname_enabled { if hostname_enabled {
final_outputs.insert("hostname".to_string(), serde_yaml::Value::String("hostname".to_string())); final_outputs.insert(
"hostname".to_string(),
serde_yaml::Value::String("hostname".to_string()),
);
} else { } else {
final_outputs.insert("hostname".to_string(), serde_yaml::Value::Null); final_outputs.insert("hostname".to_string(), serde_yaml::Value::Null);
} }
// Handle hostname_full output // Handle hostname_full output
if hostname_full_enabled { if hostname_full_enabled {
final_outputs.insert("hostname_full".to_string(), serde_yaml::Value::String("hostname_full".to_string())); final_outputs.insert(
"hostname_full".to_string(),
serde_yaml::Value::String("hostname_full".to_string()),
);
} else { } else {
final_outputs.insert("hostname_full".to_string(), serde_yaml::Value::Null); final_outputs.insert("hostname_full".to_string(), serde_yaml::Value::Null);
} }
// Handle hostname_short output // Handle hostname_short output
if hostname_short_enabled { if hostname_short_enabled {
final_outputs.insert("hostname_short".to_string(), serde_yaml::Value::String("hostname_short".to_string())); final_outputs.insert(
"hostname_short".to_string(),
serde_yaml::Value::String("hostname_short".to_string()),
);
} else { } else {
final_outputs.insert("hostname_short".to_string(), serde_yaml::Value::Null); final_outputs.insert("hostname_short".to_string(), serde_yaml::Value::Null);
} }
@@ -85,15 +106,21 @@ impl HostnameMetaPlugin {
for (key, value) in outs { for (key, value) in outs {
// Only add if the output is enabled // Only add if the output is enabled
match key.as_str() { match key.as_str() {
"hostname" => if hostname_enabled { "hostname" => {
final_outputs.insert(key.clone(), value.clone()); if hostname_enabled {
}, final_outputs.insert(key.clone(), value.clone());
"hostname_full" => if hostname_full_enabled { }
final_outputs.insert(key.clone(), value.clone()); }
}, "hostname_full" => {
"hostname_short" => if hostname_short_enabled { if hostname_full_enabled {
final_outputs.insert(key.clone(), value.clone()); final_outputs.insert(key.clone(), value.clone());
}, }
}
"hostname_short" => {
if hostname_short_enabled {
final_outputs.insert(key.clone(), value.clone());
}
}
_ => { _ => {
final_outputs.insert(key.clone(), value.clone()); final_outputs.insert(key.clone(), value.clone());
} }
@@ -108,21 +135,20 @@ impl HostnameMetaPlugin {
base, base,
} }
} }
fn get_hostname(&self) -> String { fn get_hostname(&self) -> String {
// First get the short hostname // First get the short hostname
let short_hostname = match gethostname::gethostname().into_string() { let short_hostname = match gethostname::gethostname().into_string() {
Ok(hostname) => hostname, Ok(hostname) => hostname,
Err(_) => return "unknown".to_string(), Err(_) => return "unknown".to_string(),
}; };
// First try DNS resolution for both IPv4 and IPv6 addresses // First try DNS resolution for both IPv4 and IPv6 addresses
// lookup_host should handle both A and AAAA records // lookup_host should handle both A and AAAA records
if let Ok(addrs_iter) = dns_lookup::lookup_host(&short_hostname) { if let Ok(addrs_iter) = dns_lookup::lookup_host(&short_hostname) {
// Collect addresses into a Vec to be able to use first() // Collect addresses into a Vec to be able to use first()
let addrs: Vec<std::net::IpAddr> = addrs_iter.collect(); let addrs: Vec<std::net::IpAddr> = addrs_iter.collect();
// Try each address (both IPv4 and IPv6) // Try each address (both IPv4 and IPv6)
for addr in &addrs { for addr in &addrs {
// Convert to IpAddr for lookup_addr // Convert to IpAddr for lookup_addr
@@ -141,14 +167,15 @@ impl HostnameMetaPlugin {
Err(_) => continue, Err(_) => continue,
} }
} }
// If no reverse lookup worked, but we have addresses, try to construct FQDN // If no reverse lookup worked, but we have addresses, try to construct FQDN
// from the first address's domain if the short hostname is part of a domain // from the first address's domain if the short hostname is part of a domain
if let Some(_first_addr) = addrs.first() { if let Some(_first_addr) = addrs.first() {
// For local addresses, we might not get a reverse lookup, so try to infer // For local addresses, we might not get a reverse lookup, so try to infer
// from the system's domain name // from the system's domain name
if let Ok(domain) = std::process::Command::new("domainname").output() if let Ok(domain) = std::process::Command::new("domainname").output()
&& domain.status.success() { && domain.status.success()
{
let domain_str = String::from_utf8_lossy(&domain.stdout).trim().to_string(); let domain_str = String::from_utf8_lossy(&domain.stdout).trim().to_string();
if !domain_str.is_empty() && domain_str != "(none)" { if !domain_str.is_empty() && domain_str != "(none)" {
return format!("{}.{}", short_hostname, domain_str); return format!("{}.{}", short_hostname, domain_str);
@@ -156,19 +183,20 @@ impl HostnameMetaPlugin {
} }
} }
} }
// Fallback: try to get the FQDN using the system's hostname resolution // Fallback: try to get the FQDN using the system's hostname resolution
// This should give us the full hostname if configured // This should give us the full hostname if configured
if let Ok(full_hostname) = std::process::Command::new("hostname") if let Ok(full_hostname) = std::process::Command::new("hostname").arg("-f").output()
.arg("-f") && full_hostname.status.success()
.output() {
&& full_hostname.status.success() { let full_hostname_str = String::from_utf8_lossy(&full_hostname.stdout)
let full_hostname_str = String::from_utf8_lossy(&full_hostname.stdout).trim().to_string(); .trim()
.to_string();
if !full_hostname_str.is_empty() && full_hostname_str != short_hostname { if !full_hostname_str.is_empty() && full_hostname_str != short_hostname {
return full_hostname_str; return full_hostname_str;
} }
} }
// Final fallback: return the short hostname // Final fallback: return the short hostname
short_hostname short_hostname
} }
@@ -178,11 +206,11 @@ impl MetaPlugin for HostnameMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
fn set_finalized(&mut self, finalized: bool) { fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
fn finalize(&mut self) -> crate::meta_plugin::MetaPluginResponse { fn finalize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
// If already finalized, don't process again // If already finalized, don't process again
if self.is_finalized { if self.is_finalized {
@@ -191,10 +219,10 @@ impl MetaPlugin for HostnameMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
// Mark as finalized // Mark as finalized
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
@@ -209,7 +237,7 @@ impl MetaPlugin for HostnameMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: false, is_finalized: false,
@@ -219,7 +247,7 @@ impl MetaPlugin for HostnameMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::Hostname MetaPluginType::Hostname
} }
fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse { fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
// If already finalized, don't process again // If already finalized, don't process again
if self.is_finalized { if self.is_finalized {
@@ -228,60 +256,78 @@ impl MetaPlugin for HostnameMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
// Get the full hostname // Get the full hostname
let full_hostname = self.get_hostname(); let full_hostname = self.get_hostname();
let short_hostname = full_hostname.split('.').next().unwrap_or(&full_hostname).to_string(); let short_hostname = full_hostname
.split('.')
.next()
.unwrap_or(&full_hostname)
.to_string();
// Determine which hostnames to include based on options // Determine which hostnames to include based on options
let hostname_enabled = self.base.options.get("hostname") let hostname_enabled = self
.base
.options
.get("hostname")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
let hostname_full_enabled = self.base.options.get("hostname_full") let hostname_full_enabled = self
.base
.options
.get("hostname_full")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
let hostname_short_enabled = self.base.options.get("hostname_short") let hostname_short_enabled = self
.base
.options
.get("hostname_short")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
// Always use gethostname() for the 'hostname' output when enabled // Always use gethostname() for the 'hostname' output when enabled
let hostname_value = if hostname_enabled { let hostname_value = if hostname_enabled {
gethostname::gethostname().into_string().unwrap_or_else(|_| "unknown".to_string()) gethostname::gethostname()
.into_string()
.unwrap_or_else(|_| "unknown".to_string())
} else { } else {
String::new() String::new()
}; };
// Prepare metadata to return // Prepare metadata to return
let mut metadata = Vec::new(); let mut metadata = Vec::new();
// Add enabled metadata to the response using process_metadata_outputs // Add enabled metadata to the response using process_metadata_outputs
if hostname_enabled if hostname_enabled
&& let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( && let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"hostname", "hostname",
serde_yaml::Value::String(hostname_value.clone()), serde_yaml::Value::String(hostname_value.clone()),
self.base.outputs() self.base.outputs(),
) { )
{
metadata.push(meta_data); metadata.push(meta_data);
} }
if hostname_full_enabled if hostname_full_enabled
&& let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( && let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"hostname_full", "hostname_full",
serde_yaml::Value::String(full_hostname.clone()), serde_yaml::Value::String(full_hostname.clone()),
self.base.outputs() self.base.outputs(),
) { )
{
metadata.push(meta_data); metadata.push(meta_data);
} }
if hostname_short_enabled if hostname_short_enabled
&& let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( && let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"hostname_short", "hostname_short",
serde_yaml::Value::String(short_hostname.clone()), serde_yaml::Value::String(short_hostname.clone()),
self.base.outputs() self.base.outputs(),
) { )
{
metadata.push(meta_data); metadata.push(meta_data);
} }
// Update outputs based on enabled status // Update outputs based on enabled status
// Handle hostname output // Handle hostname output
if hostname_enabled { if hostname_enabled {
@@ -289,44 +335,50 @@ impl MetaPlugin for HostnameMetaPlugin {
*output_value = serde_yaml::Value::String(hostname_value); *output_value = serde_yaml::Value::String(hostname_value);
} }
} else { } else {
self.base.outputs_mut().insert("hostname".to_string(), serde_yaml::Value::Null); self.base
.outputs_mut()
.insert("hostname".to_string(), serde_yaml::Value::Null);
} }
// Handle hostname_full output // Handle hostname_full output
if hostname_full_enabled { if hostname_full_enabled {
if let Some(output_value) = self.base.outputs_mut().get_mut("hostname_full") { if let Some(output_value) = self.base.outputs_mut().get_mut("hostname_full") {
*output_value = serde_yaml::Value::String(full_hostname); *output_value = serde_yaml::Value::String(full_hostname);
} }
} else { } else {
self.base.outputs_mut().insert("hostname_full".to_string(), serde_yaml::Value::Null); self.base
.outputs_mut()
.insert("hostname_full".to_string(), serde_yaml::Value::Null);
} }
// Handle hostname_short output // Handle hostname_short output
if hostname_short_enabled { if hostname_short_enabled {
if let Some(output_value) = self.base.outputs_mut().get_mut("hostname_short") { if let Some(output_value) = self.base.outputs_mut().get_mut("hostname_short") {
*output_value = serde_yaml::Value::String(short_hostname); *output_value = serde_yaml::Value::String(short_hostname);
} }
} else { } else {
self.base.outputs_mut().insert("hostname_short".to_string(), serde_yaml::Value::Null); self.base
.outputs_mut()
.insert("hostname_short".to_string(), serde_yaml::Value::Null);
} }
// Mark as finalized since this plugin only needs to run once // Mark as finalized since this plugin only needs to run once
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata, metadata,
is_finalized: true, is_finalized: true,
} }
} }
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec![ vec![
"hostname".to_string(), "hostname".to_string(),
@@ -334,16 +386,14 @@ impl MetaPlugin for HostnameMetaPlugin {
"hostname_short".to_string(), "hostname_short".to_string(),
] ]
} }
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.options_mut() self.base.options_mut()
} }
} }
use crate::meta_plugin::register_meta_plugin; use crate::meta_plugin::register_meta_plugin;

43
src/meta_plugin/keep_pid.rs
View File

@@ -1,5 +1,5 @@
use crate::meta_plugin::{BaseMetaPlugin, MetaPlugin, MetaPluginType};
use std::process; use std::process;
use crate::meta_plugin::{MetaPlugin, MetaPluginType, BaseMetaPlugin};
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct KeepPidMetaPlugin { pub struct KeepPidMetaPlugin {
@@ -23,17 +23,16 @@ impl KeepPidMetaPlugin {
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>, outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> KeepPidMetaPlugin { ) -> KeepPidMetaPlugin {
let mut base = BaseMetaPlugin::new(); let mut base = BaseMetaPlugin::new();
// Set default outputs // Set default outputs
let default_outputs = &["keep_pid"]; let default_outputs = &["keep_pid"];
base.initialize_plugin(default_outputs, &_options, &outputs); base.initialize_plugin(default_outputs, &_options, &outputs);
KeepPidMetaPlugin { KeepPidMetaPlugin {
is_finalized: false, is_finalized: false,
base, base,
} }
} }
} }
impl MetaPlugin for KeepPidMetaPlugin { impl MetaPlugin for KeepPidMetaPlugin {
@@ -45,7 +44,7 @@ impl MetaPlugin for KeepPidMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
/// Sets the finalized state of the plugin. /// Sets the finalized state of the plugin.
/// ///
/// # Arguments /// # Arguments
@@ -54,7 +53,7 @@ impl MetaPlugin for KeepPidMetaPlugin {
fn set_finalized(&mut self, finalized: bool) { fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
/// Finalizes the plugin, processing any remaining data if needed. /// Finalizes the plugin, processing any remaining data if needed.
/// ///
/// # Returns /// # Returns
@@ -68,10 +67,10 @@ impl MetaPlugin for KeepPidMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
// Mark as finalized // Mark as finalized
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
@@ -95,7 +94,7 @@ impl MetaPlugin for KeepPidMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: false, is_finalized: false,
@@ -110,7 +109,7 @@ impl MetaPlugin for KeepPidMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::KeepPid MetaPluginType::KeepPid
} }
/// Initializes the plugin and captures the process PID. /// Initializes the plugin and captures the process PID.
/// ///
/// Retrieves the current process ID and adds it to metadata. /// Retrieves the current process ID and adds it to metadata.
@@ -127,28 +126,28 @@ impl MetaPlugin for KeepPidMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
let mut metadata = Vec::new(); let mut metadata = Vec::new();
let pid = process::id().to_string(); let pid = process::id().to_string();
// Use process_metadata_outputs to handle output mapping // Use process_metadata_outputs to handle output mapping
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"keep_pid", "keep_pid",
serde_yaml::Value::String(pid), serde_yaml::Value::String(pid),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
// Mark as finalized since this plugin only needs to run once // Mark as finalized since this plugin only needs to run once
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata, metadata,
is_finalized: true, is_finalized: true,
} }
} }
/// Returns a reference to the outputs mapping. /// Returns a reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -157,7 +156,7 @@ impl MetaPlugin for KeepPidMetaPlugin {
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
/// Returns a mutable reference to the outputs mapping. /// Returns a mutable reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -166,7 +165,7 @@ impl MetaPlugin for KeepPidMetaPlugin {
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
/// Returns the default output names for this plugin. /// Returns the default output names for this plugin.
/// ///
/// # Returns /// # Returns
@@ -175,7 +174,7 @@ impl MetaPlugin for KeepPidMetaPlugin {
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec!["keep_pid".to_string()] vec!["keep_pid".to_string()]
} }
/// Returns a reference to the options mapping. /// Returns a reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -184,7 +183,7 @@ impl MetaPlugin for KeepPidMetaPlugin {
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
/// Returns a mutable reference to the options mapping. /// Returns a mutable reference to the options mapping.
/// ///
/// # Returns /// # Returns

72
src/meta_plugin/magic_file.rs
View File

@@ -3,11 +3,14 @@ use magic::{Cookie, CookieFlags};
#[cfg(not(feature = "magic"))] #[cfg(not(feature = "magic"))]
use std::process::{Command, Stdio}; use std::process::{Command, Stdio};
use log::debug;
use std::io::{self, Write}; use std::io::{self, Write};
use std::path::Path; use std::path::Path;
use log::debug;
use crate::meta_plugin::{MetaPlugin, MetaPluginType, BaseMetaPlugin, MetaPluginResponse, MetaData, process_metadata_outputs}; use crate::meta_plugin::{
BaseMetaPlugin, MetaData, MetaPlugin, MetaPluginResponse, MetaPluginType,
process_metadata_outputs,
};
#[cfg(feature = "magic")] #[cfg(feature = "magic")]
#[derive(Debug)] #[derive(Debug)]
@@ -32,7 +35,8 @@ impl MagicFileMetaPluginImpl {
base.initialize_plugin(default_outputs, &options, &outputs); base.initialize_plugin(default_outputs, &options, &outputs);
// Get max_buffer_size from options, default to PIPESIZE // Get max_buffer_size from options, default to PIPESIZE
let max_buffer_size = base.options let max_buffer_size = base
.options
.get("max_buffer_size") .get("max_buffer_size")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.unwrap_or(crate::common::PIPESIZE as u64) as usize; .unwrap_or(crate::common::PIPESIZE as u64) as usize;
@@ -48,18 +52,20 @@ impl MagicFileMetaPluginImpl {
fn get_magic_result(&self, flags: CookieFlags) -> io::Result<String> { fn get_magic_result(&self, flags: CookieFlags) -> io::Result<String> {
if let Some(cookie) = &self.cookie { if let Some(cookie) = &self.cookie {
cookie.set_flags(flags) cookie
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("Failed to set magic flags: {}", e)))?; .set_flags(flags)
.map_err(|e| io::Error::other(format!("Failed to set magic flags: {}", e)))?;
let result = cookie.buffer(&self.buffer) let result = cookie
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("Failed to analyze buffer: {}", e)))?; .buffer(&self.buffer)
.map_err(|e| io::Error::other(format!("Failed to analyze buffer: {}", e)))?;
// Clean up the result - remove extra whitespace // Clean up the result - remove extra whitespace
let trimmed = result.trim().to_string(); let trimmed = result.trim().to_string();
Ok(trimmed) Ok(trimmed)
} else { } else {
Err(io::Error::new(io::ErrorKind::Other, "Magic cookie not initialized")) Err(io::Error::other("Magic cookie not initialized"))
} }
} }
@@ -73,16 +79,15 @@ impl MagicFileMetaPluginImpl {
]; ];
for (name, flags) in types_to_process.iter() { for (name, flags) in types_to_process.iter() {
if let Ok(result) = self.get_magic_result(*flags) { if let Ok(result) = self.get_magic_result(*flags)
if !result.is_empty() { && !result.is_empty()
if let Some(meta_data) = process_metadata_outputs( && let Some(meta_data) = process_metadata_outputs(
name, name,
serde_yaml::Value::String(result), serde_yaml::Value::String(result),
self.base.outputs(), self.base.outputs(),
) { )
metadata.push(meta_data); {
} metadata.push(meta_data);
}
} }
} }
@@ -113,7 +118,10 @@ impl MetaPlugin for MagicFileMetaPluginImpl {
}; };
if let Err(e) = cookie.load(&[] as &[&Path]) { if let Err(e) = cookie.load(&[] as &[&Path]) {
debug!("META: MagicFile plugin: failed to load magic database: {}", e); debug!(
"META: MagicFile plugin: failed to load magic database: {}",
e
);
return MetaPluginResponse { return MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
@@ -187,7 +195,11 @@ impl MetaPlugin for MagicFileMetaPluginImpl {
} }
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec!["mime_type".to_string(), "mime_encoding".to_string(), "file_type".to_string()] vec![
"mime_type".to_string(),
"mime_encoding".to_string(),
"file_type".to_string(),
]
} }
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
@@ -221,7 +233,8 @@ impl FallbackMagicFileMetaPlugin {
base.initialize_plugin(default_outputs, &options, &outputs); base.initialize_plugin(default_outputs, &options, &outputs);
// Get max_buffer_size from options, default to PIPESIZE // Get max_buffer_size from options, default to PIPESIZE
let max_buffer_size = base.options let max_buffer_size = base
.options
.get("max_buffer_size") .get("max_buffer_size")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.unwrap_or(crate::common::PIPESIZE as u64) as usize; .unwrap_or(crate::common::PIPESIZE as u64) as usize;
@@ -244,7 +257,12 @@ impl FallbackMagicFileMetaPlugin {
.arg("all") .arg("all")
.arg(temp_file.path()) .arg(temp_file.path())
.output() .output()
.map_err(|e| io::Error::new(io::ErrorKind::Other, format!("Failed to run file command: {}", e)))?; .map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("Failed to run file command: {}", e),
)
})?;
if !output.status.success() { if !output.status.success() {
return Err(io::Error::new(io::ErrorKind::Other, "File command failed")); return Err(io::Error::new(io::ErrorKind::Other, "File command failed"));
@@ -261,7 +279,8 @@ impl FallbackMagicFileMetaPlugin {
// file -m all output format is typically: type; charset=encoding // file -m all output format is typically: type; charset=encoding
let parts: Vec<&str> = result.split(';').map(|s| s.trim()).collect(); let parts: Vec<&str> = result.split(';').map(|s| s.trim()).collect();
let file_type = parts.first().cloned().unwrap_or(result); let file_type = parts.first().cloned().unwrap_or(result);
let mime_encoding = parts.get(1) let mime_encoding = parts
.get(1)
.and_then(|s| s.strip_prefix("charset=")) .and_then(|s| s.strip_prefix("charset="))
.cloned() .cloned()
.unwrap_or(""); .unwrap_or("");
@@ -392,7 +411,11 @@ impl MetaPlugin for FallbackMagicFileMetaPlugin {
} }
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec!["mime_type".to_string(), "mime_encoding".to_string(), "file_type".to_string()] vec![
"mime_type".to_string(),
"mime_encoding".to_string(),
"file_type".to_string(),
]
} }
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
@@ -418,4 +441,3 @@ fn register_magic_file_plugin() {
Box::new(MagicFileMetaPlugin::new(options, outputs)) Box::new(MagicFileMetaPlugin::new(options, outputs))
}); });
} }

184
src/meta_plugin/mod.rs
View File

@@ -1,44 +1,47 @@
use log::debug; use log::debug;
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Mutex; use std::sync::Mutex;
use once_cell::sync::Lazy;
pub mod cwd;
pub mod digest;
pub mod env;
pub mod exec;
pub mod hostname;
pub mod keep_pid;
#[cfg(feature = "magic")] #[cfg(feature = "magic")]
pub mod magic_file; pub mod magic_file;
pub mod exec;
pub mod digest;
pub mod read_time;
pub mod read_rate; pub mod read_rate;
pub mod hostname; pub mod read_time;
pub mod cwd;
pub mod user;
pub mod shell; pub mod shell;
pub mod shell_pid; pub mod shell_pid;
pub mod keep_pid;
pub mod env;
pub mod text; pub mod text;
pub mod user;
// pub mod text; // Removed duplicate // pub mod text; // Removed duplicate
pub use digest::DigestMetaPlugin;
pub use exec::MetaPluginExec;
#[cfg(feature = "magic")] #[cfg(feature = "magic")]
pub use magic_file::MagicFileMetaPlugin; pub use magic_file::MagicFileMetaPlugin;
pub use exec::MetaPluginExec;
pub use digest::DigestMetaPlugin;
// pub use text::TextMetaPlugin; // Removed duplicate // pub use text::TextMetaPlugin; // Removed duplicate
pub use read_time::ReadTimeMetaPlugin;
pub use read_rate::ReadRateMetaPlugin;
pub use hostname::HostnameMetaPlugin;
pub use cwd::CwdMetaPlugin; pub use cwd::CwdMetaPlugin;
pub use user::UserMetaPlugin; pub use env::EnvMetaPlugin;
pub use hostname::HostnameMetaPlugin;
pub use keep_pid::KeepPidMetaPlugin;
pub use read_rate::ReadRateMetaPlugin;
pub use read_time::ReadTimeMetaPlugin;
pub use shell::ShellMetaPlugin; pub use shell::ShellMetaPlugin;
pub use shell_pid::ShellPidMetaPlugin; pub use shell_pid::ShellPidMetaPlugin;
pub use keep_pid::KeepPidMetaPlugin; pub use user::UserMetaPlugin;
pub use env::EnvMetaPlugin;
#[cfg(not(feature = "magic"))] #[cfg(not(feature = "magic"))]
pub use magic_file::FallbackMagicFileMetaPlugin as MagicFileMetaPlugin; pub use magic_file::FallbackMagicFileMetaPlugin as MagicFileMetaPlugin;
type PluginConstructor = fn(Option<HashMap<String, serde_yaml::Value>>, Option<HashMap<String, serde_yaml::Value>>) -> Box<dyn MetaPlugin>; type PluginConstructor = fn(
Option<HashMap<String, serde_yaml::Value>>,
Option<HashMap<String, serde_yaml::Value>>,
) -> Box<dyn MetaPlugin>;
/// Represents metadata to be stored. /// Represents metadata to be stored.
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -78,7 +81,7 @@ impl BaseMetaPlugin {
pub fn new() -> Self { pub fn new() -> Self {
Self::default() Self::default()
} }
/// Returns a reference to the outputs mapping. /// Returns a reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -87,7 +90,7 @@ impl BaseMetaPlugin {
pub fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { pub fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
&self.outputs &self.outputs
} }
/// Returns a mutable reference to the outputs mapping. /// Returns a mutable reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -96,7 +99,7 @@ impl BaseMetaPlugin {
pub fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { pub fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
&mut self.outputs &mut self.outputs
} }
/// Returns a reference to the options mapping. /// Returns a reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -105,7 +108,7 @@ impl BaseMetaPlugin {
pub fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { pub fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
&self.options &self.options
} }
/// Returns a mutable reference to the options mapping. /// Returns a mutable reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -114,7 +117,7 @@ impl BaseMetaPlugin {
pub fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { pub fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
&mut self.options &mut self.options
} }
/// Helper function to initialize plugin options and outputs. /// Helper function to initialize plugin options and outputs.
/// ///
/// # Arguments /// # Arguments
@@ -130,9 +133,12 @@ impl BaseMetaPlugin {
) { ) {
// Set default outputs // Set default outputs
for output_name in default_outputs { for output_name in default_outputs {
self.outputs.insert(output_name.to_string(), serde_yaml::Value::String(output_name.to_string())); self.outputs.insert(
output_name.to_string(),
serde_yaml::Value::String(output_name.to_string()),
);
} }
// Apply provided options and outputs // Apply provided options and outputs
if let Some(opts) = options { if let Some(opts) = options {
for (key, value) in opts { for (key, value) in opts {
@@ -158,7 +164,7 @@ impl MetaPlugin for BaseMetaPlugin {
// This might not be used, but we need to satisfy the trait // This might not be used, but we need to satisfy the trait
MetaPluginType::Text MetaPluginType::Text
} }
/// Returns a reference to the outputs mapping. /// Returns a reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -167,7 +173,7 @@ impl MetaPlugin for BaseMetaPlugin {
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
&self.outputs &self.outputs
} }
/// Returns a mutable reference to the outputs mapping. /// Returns a mutable reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -176,7 +182,7 @@ impl MetaPlugin for BaseMetaPlugin {
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
&mut self.outputs &mut self.outputs
} }
/// Returns a reference to the options mapping. /// Returns a reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -185,7 +191,7 @@ impl MetaPlugin for BaseMetaPlugin {
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
&self.options &self.options
} }
/// Returns a mutable reference to the options mapping. /// Returns a mutable reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -196,7 +202,18 @@ impl MetaPlugin for BaseMetaPlugin {
} }
} }
#[derive(Debug, Eq, PartialEq, Clone, Hash, strum::EnumIter, strum::Display, strum::EnumString, Serialize, Deserialize)] #[derive(
Debug,
Eq,
PartialEq,
Clone,
Hash,
strum::EnumIter,
strum::Display,
strum::EnumString,
Serialize,
Deserialize,
)]
#[strum(serialize_all = "snake_case", ascii_case_insensitive)] #[strum(serialize_all = "snake_case", ascii_case_insensitive)]
pub enum MetaPluginType { pub enum MetaPluginType {
MagicFile, MagicFile,
@@ -225,7 +242,11 @@ pub enum MetaPluginType {
/// # Returns /// # Returns
/// ///
/// An optional `MetaData` if the output is enabled, `None` if disabled. /// An optional `MetaData` if the output is enabled, `None` if disabled.
pub fn process_metadata_outputs(internal_name: &str, value: serde_yaml::Value, outputs: &std::collections::HashMap<String, serde_yaml::Value>) -> Option<MetaData> { pub fn process_metadata_outputs(
internal_name: &str,
value: serde_yaml::Value,
outputs: &std::collections::HashMap<String, serde_yaml::Value>,
) -> Option<MetaData> {
// Check if this output is disabled // Check if this output is disabled
if let Some(mapping) = outputs.get(internal_name) { if let Some(mapping) = outputs.get(internal_name) {
// Check for null to disable the output // Check for null to disable the output
@@ -235,7 +256,8 @@ pub fn process_metadata_outputs(internal_name: &str, value: serde_yaml::Value, o
} }
// Check for boolean false to disable the output // Check for boolean false to disable the output
if let Some(false_val) = mapping.as_bool() if let Some(false_val) = mapping.as_bool()
&& !false_val { && !false_val
{
debug!("META: Skipping disabled output: {}", internal_name); debug!("META: Skipping disabled output: {}", internal_name);
return None; return None;
} }
@@ -246,45 +268,66 @@ pub fn process_metadata_outputs(internal_name: &str, value: serde_yaml::Value, o
serde_yaml::Value::Bool(b) => b.to_string(), serde_yaml::Value::Bool(b) => b.to_string(),
serde_yaml::Value::Number(n) => n.to_string(), serde_yaml::Value::Number(n) => n.to_string(),
serde_yaml::Value::String(s) => s.clone(), serde_yaml::Value::String(s) => s.clone(),
serde_yaml::Value::Sequence(_) => serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string()), serde_yaml::Value::Sequence(_) => {
serde_yaml::Value::Mapping(_) => serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string()), serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string())
serde_yaml::Value::Tagged(_) => serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string()), }
serde_yaml::Value::Mapping(_) => {
serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string())
}
serde_yaml::Value::Tagged(_) => {
serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string())
}
}; };
debug!("META: Processing metadata: internal_name={}, custom_name={}, value={}", internal_name, custom_name, value_str); debug!(
"META: Processing metadata: internal_name={}, custom_name={}, value={}",
internal_name, custom_name, value_str
);
return Some(MetaData { return Some(MetaData {
name: custom_name.to_string(), name: custom_name.to_string(),
value: value_str, value: value_str,
}); });
} }
} }
// Convert the value to a string representation // Convert the value to a string representation
let value_str = match &value { let value_str = match &value {
serde_yaml::Value::Null => "null".to_string(), serde_yaml::Value::Null => "null".to_string(),
serde_yaml::Value::Bool(b) => b.to_string(), serde_yaml::Value::Bool(b) => b.to_string(),
serde_yaml::Value::Number(n) => n.to_string(), serde_yaml::Value::Number(n) => n.to_string(),
serde_yaml::Value::String(s) => s.clone(), serde_yaml::Value::String(s) => s.clone(),
serde_yaml::Value::Sequence(_) => serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string()), serde_yaml::Value::Sequence(_) => {
serde_yaml::Value::Mapping(_) => serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string()), serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string())
serde_yaml::Value::Tagged(_) => serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string()), }
serde_yaml::Value::Mapping(_) => {
serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string())
}
serde_yaml::Value::Tagged(_) => {
serde_yaml::to_string(&value).unwrap_or_else(|_| "".to_string())
}
}; };
// Default: use internal name as output name // Default: use internal name as output name
debug!("META: Processing metadata: name={}, value={}", internal_name, value_str); debug!(
"META: Processing metadata: name={}, value={}",
internal_name, value_str
);
Some(MetaData { Some(MetaData {
name: internal_name.to_string(), name: internal_name.to_string(),
value: value_str, value: value_str,
}) })
} }
pub trait MetaPlugin where Self: 'static { pub trait MetaPlugin
where
Self: 'static,
{
/// Returns the type of this meta plugin. /// Returns the type of this meta plugin.
/// ///
/// # Returns /// # Returns
/// ///
/// The `MetaPluginType` enum variant for this plugin. /// The `MetaPluginType` enum variant for this plugin.
fn meta_type(&self) -> MetaPluginType; fn meta_type(&self) -> MetaPluginType;
/// Checks if the plugin is supported on the current system. /// Checks if the plugin is supported on the current system.
/// ///
/// # Returns /// # Returns
@@ -293,7 +336,7 @@ pub trait MetaPlugin where Self: 'static {
fn is_supported(&self) -> bool { fn is_supported(&self) -> bool {
true true
} }
/// Checks if the plugin is internal (built-in). /// Checks if the plugin is internal (built-in).
/// ///
/// # Returns /// # Returns
@@ -302,7 +345,7 @@ pub trait MetaPlugin where Self: 'static {
fn is_internal(&self) -> bool { fn is_internal(&self) -> bool {
true true
} }
/// Checks if the plugin is already finalized. /// Checks if the plugin is already finalized.
/// ///
/// # Returns /// # Returns
@@ -311,14 +354,14 @@ pub trait MetaPlugin where Self: 'static {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
false false
} }
/// Sets the finalized state (only for plugins that can track this). /// Sets the finalized state (only for plugins that can track this).
/// ///
/// # Arguments /// # Arguments
/// ///
/// * `_finalized` - The new finalized state (unused in default). /// * `_finalized` - The new finalized state (unused in default).
fn set_finalized(&mut self, _finalized: bool) {} fn set_finalized(&mut self, _finalized: bool) {}
/// Updates the meta plugin with new data. /// Updates the meta plugin with new data.
/// ///
/// # Arguments /// # Arguments
@@ -348,7 +391,7 @@ pub trait MetaPlugin where Self: 'static {
is_finalized: true, is_finalized: true,
} }
} }
/// Gets program information for display in status. /// Gets program information for display in status.
/// ///
/// # Returns /// # Returns
@@ -357,8 +400,7 @@ pub trait MetaPlugin where Self: 'static {
fn program_info(&self) -> Option<(&str, Vec<&str>)> { fn program_info(&self) -> Option<(&str, Vec<&str>)> {
None None
} }
/// Initializes the plugin. /// Initializes the plugin.
/// ///
/// # Returns /// # Returns
@@ -371,7 +413,7 @@ pub trait MetaPlugin where Self: 'static {
is_finalized: false, is_finalized: false,
} }
} }
/// Returns a reference to the outputs mapping. /// Returns a reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -379,11 +421,11 @@ pub trait MetaPlugin where Self: 'static {
/// An empty `HashMap` (default implementation). /// An empty `HashMap` (default implementation).
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
static EMPTY: Lazy<std::collections::HashMap<String, serde_yaml::Value>> = static EMPTY: Lazy<std::collections::HashMap<String, serde_yaml::Value>> =
Lazy::new(|| std::collections::HashMap::new()); Lazy::new(std::collections::HashMap::new);
&EMPTY &EMPTY
} }
/// Returns a mutable reference to the outputs mapping. /// Returns a mutable reference to the outputs mapping.
/// ///
/// # Panics /// # Panics
@@ -392,7 +434,7 @@ pub trait MetaPlugin where Self: 'static {
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
panic!("outputs_mut() not implemented for this plugin") panic!("outputs_mut() not implemented for this plugin")
} }
/// Returns a reference to the options mapping. /// Returns a reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -400,11 +442,11 @@ pub trait MetaPlugin where Self: 'static {
/// An empty `HashMap` (default implementation). /// An empty `HashMap` (default implementation).
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
static EMPTY: Lazy<std::collections::HashMap<String, serde_yaml::Value>> = static EMPTY: Lazy<std::collections::HashMap<String, serde_yaml::Value>> =
Lazy::new(|| std::collections::HashMap::new()); Lazy::new(std::collections::HashMap::new);
&EMPTY &EMPTY
} }
/// Returns a mutable reference to the options mapping. /// Returns a mutable reference to the options mapping.
/// ///
/// # Panics /// # Panics
@@ -413,7 +455,7 @@ pub trait MetaPlugin where Self: 'static {
fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
panic!("options_mut() not implemented for this plugin") panic!("options_mut() not implemented for this plugin")
} }
/// Gets the default output names this plugin can produce. /// Gets the default output names this plugin can produce.
/// ///
/// # Returns /// # Returns
@@ -423,20 +465,22 @@ pub trait MetaPlugin where Self: 'static {
// Default implementation returns the meta type as a string // Default implementation returns the meta type as a string
vec![self.meta_type().to_string()] vec![self.meta_type().to_string()]
} }
/// Method to downcast to concrete type (for checking finalization state). /// Method to downcast to concrete type (for checking finalization state).
/// ///
/// # Returns /// # Returns
/// ///
/// A mutable reference to `self` as `dyn Any`. /// A mutable reference to `self` as `dyn Any`.
fn as_any_mut(&mut self) -> &mut dyn std::any::Any where Self: Sized { fn as_any_mut(&mut self) -> &mut dyn std::any::Any
where
Self: Sized,
{
self self
} }
} }
/// Global registry for meta plugins. /// Global registry for meta plugins.
static META_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<MetaPluginType, PluginConstructor>>> = static META_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<MetaPluginType, PluginConstructor>>> =
Lazy::new(|| Mutex::new(HashMap::new())); Lazy::new(|| Mutex::new(HashMap::new()));
/// Register a meta plugin with the global registry. /// Register a meta plugin with the global registry.
@@ -445,11 +489,11 @@ static META_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<MetaPluginType, PluginConstructo
/// ///
/// * `meta_plugin_type` - The type of the meta plugin to register. /// * `meta_plugin_type` - The type of the meta plugin to register.
/// * `constructor` - The constructor function for creating plugin instances. /// * `constructor` - The constructor function for creating plugin instances.
pub fn register_meta_plugin( pub fn register_meta_plugin(meta_plugin_type: MetaPluginType, constructor: PluginConstructor) {
meta_plugin_type: MetaPluginType, META_PLUGIN_REGISTRY
constructor: PluginConstructor .lock()
) { .unwrap()
META_PLUGIN_REGISTRY.lock().unwrap().insert(meta_plugin_type, constructor); .insert(meta_plugin_type, constructor);
} }
pub fn get_meta_plugin( pub fn get_meta_plugin(
@@ -461,7 +505,7 @@ pub fn get_meta_plugin(
if let Some(constructor) = registry.get(&meta_plugin_type) { if let Some(constructor) = registry.get(&meta_plugin_type) {
return constructor(options, outputs); return constructor(options, outputs);
} }
// Fallback for unknown plugins // Fallback for unknown plugins
panic!("Meta plugin {:?} not registered", meta_plugin_type); panic!("Meta plugin {:?} not registered", meta_plugin_type);
} }

40
src/meta_plugin/read_rate.rs
View File

@@ -1,11 +1,11 @@
use std::time::Instant; use std::time::Instant;
use crate::meta_plugin::{MetaPlugin, MetaPluginType, BaseMetaPlugin}; use crate::meta_plugin::{BaseMetaPlugin, MetaPlugin, MetaPluginType};
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
/// Meta plugin that calculates the read rate (KB/s) of input data. /// Meta plugin that calculates the read rate (KB/s) of input data.
/// ///
/// Tracks bytes read and elapsed time, then computes the rate in finalize(). /// Tracks bytes read and elapsed time, then computes the rate in finalize().
/// Outputs the rate via configured mappings. Supports options for customization /// Outputs the rate via configured mappings. Supports options for customization
/// (though defaults are used here). /// (though defaults are used here).
/// ///
@@ -48,11 +48,11 @@ impl ReadRateMetaPlugin {
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>, outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> ReadRateMetaPlugin { ) -> ReadRateMetaPlugin {
let mut base = BaseMetaPlugin::new(); let mut base = BaseMetaPlugin::new();
// Set default outputs // Set default outputs
let default_outputs = &["read_rate"]; let default_outputs = &["read_rate"];
base.initialize_plugin(default_outputs, &_options, &outputs); base.initialize_plugin(default_outputs, &_options, &outputs);
ReadRateMetaPlugin { ReadRateMetaPlugin {
start_time: None, start_time: None,
bytes_read: 0, bytes_read: 0,
@@ -60,7 +60,6 @@ impl ReadRateMetaPlugin {
base, base,
} }
} }
} }
impl MetaPlugin for ReadRateMetaPlugin { impl MetaPlugin for ReadRateMetaPlugin {
@@ -72,7 +71,7 @@ impl MetaPlugin for ReadRateMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
/// Sets the finalized state of the plugin. /// Sets the finalized state of the plugin.
/// ///
/// Marks the plugin as complete or resets it. /// Marks the plugin as complete or resets it.
@@ -83,7 +82,7 @@ impl MetaPlugin for ReadRateMetaPlugin {
fn set_finalized(&mut self, finalized: bool) { fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
/// Finalizes the plugin, calculating the read rate. /// Finalizes the plugin, calculating the read rate.
/// ///
/// Computes KB/s from bytes read and elapsed time. Outputs via mappings. /// Computes KB/s from bytes read and elapsed time. Outputs via mappings.
@@ -104,27 +103,30 @@ impl MetaPlugin for ReadRateMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
let mut metadata = Vec::new(); let mut metadata = Vec::new();
if let Some(start_time) = self.start_time { if let Some(start_time) = self.start_time {
let duration = start_time.elapsed(); let duration = start_time.elapsed();
let rate = if duration.as_secs_f64() > 0.0 { let rate = if duration.as_secs_f64() > 0.0 {
format!("{:.2} KB/s", (self.bytes_read as f64 / 1024.0) / duration.as_secs_f64()) format!(
"{:.2} KB/s",
(self.bytes_read as f64 / 1024.0) / duration.as_secs_f64()
)
} else { } else {
"N/A".to_string() "N/A".to_string()
}; };
// Use process_metadata_outputs to handle output mapping // Use process_metadata_outputs to handle output mapping
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"read_rate", "read_rate",
serde_yaml::Value::String(rate), serde_yaml::Value::String(rate),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
} }
// Mark as finalized // Mark as finalized
self.is_finalized = true; self.is_finalized = true;
@@ -154,7 +156,7 @@ impl MetaPlugin for ReadRateMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
if self.start_time.is_none() { if self.start_time.is_none() {
self.start_time = Some(Instant::now()); self.start_time = Some(Instant::now());
} }
@@ -173,7 +175,7 @@ impl MetaPlugin for ReadRateMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::ReadRate MetaPluginType::ReadRate
} }
/// Returns a reference to the outputs mapping. /// Returns a reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -182,7 +184,7 @@ impl MetaPlugin for ReadRateMetaPlugin {
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
/// Returns a mutable reference to the outputs mapping. /// Returns a mutable reference to the outputs mapping.
/// ///
/// Allows modification of output configurations. /// Allows modification of output configurations.
@@ -193,7 +195,7 @@ impl MetaPlugin for ReadRateMetaPlugin {
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
/// Returns the default output names for this plugin. /// Returns the default output names for this plugin.
/// ///
/// # Returns /// # Returns
@@ -202,7 +204,7 @@ impl MetaPlugin for ReadRateMetaPlugin {
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec!["read_rate".to_string()] vec!["read_rate".to_string()]
} }
/// Returns a reference to the options mapping. /// Returns a reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -211,7 +213,7 @@ impl MetaPlugin for ReadRateMetaPlugin {
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
/// Returns a mutable reference to the options mapping. /// Returns a mutable reference to the options mapping.
/// ///
/// Allows modification of plugin options. /// Allows modification of plugin options.

33
src/meta_plugin/read_time.rs
View File

@@ -1,6 +1,6 @@
use std::time::Instant; use std::time::Instant;
use crate::meta_plugin::{MetaPlugin, MetaPluginType, BaseMetaPlugin}; use crate::meta_plugin::{BaseMetaPlugin, MetaPlugin, MetaPluginType};
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct ReadTimeMetaPlugin { pub struct ReadTimeMetaPlugin {
@@ -15,29 +15,28 @@ impl ReadTimeMetaPlugin {
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>, outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> ReadTimeMetaPlugin { ) -> ReadTimeMetaPlugin {
let mut base = BaseMetaPlugin::new(); let mut base = BaseMetaPlugin::new();
// Set default outputs // Set default outputs
let default_outputs = &["read_time"]; let default_outputs = &["read_time"];
base.initialize_plugin(default_outputs, &_options, &outputs); base.initialize_plugin(default_outputs, &_options, &outputs);
ReadTimeMetaPlugin { ReadTimeMetaPlugin {
start_time: None, start_time: None,
is_finalized: false, is_finalized: false,
base, base,
} }
} }
} }
impl MetaPlugin for ReadTimeMetaPlugin { impl MetaPlugin for ReadTimeMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
fn set_finalized(&mut self, finalized: bool) { fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
fn finalize(&mut self) -> crate::meta_plugin::MetaPluginResponse { fn finalize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
// If already finalized, don't process again // If already finalized, don't process again
if self.is_finalized { if self.is_finalized {
@@ -46,7 +45,7 @@ impl MetaPlugin for ReadTimeMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
let mut metadata = Vec::new(); let mut metadata = Vec::new();
if let Some(start_time) = self.start_time { if let Some(start_time) = self.start_time {
@@ -55,14 +54,14 @@ impl MetaPlugin for ReadTimeMetaPlugin {
// Use process_metadata_outputs to handle output mapping // Use process_metadata_outputs to handle output mapping
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"read_time", "read_time",
serde_yaml::Value::String(duration_str), serde_yaml::Value::String(duration_str),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
} }
// Mark as finalized // Mark as finalized
self.is_finalized = true; self.is_finalized = true;
@@ -80,7 +79,7 @@ impl MetaPlugin for ReadTimeMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
if self.start_time.is_none() { if self.start_time.is_none() {
self.start_time = Some(Instant::now()); self.start_time = Some(Instant::now());
} }
@@ -93,23 +92,23 @@ impl MetaPlugin for ReadTimeMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::ReadTime MetaPluginType::ReadTime
} }
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec!["read_time".to_string()] vec!["read_time".to_string()]
} }
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.options_mut() self.base.options_mut()
} }

43
src/meta_plugin/shell.rs
View File

@@ -1,6 +1,6 @@
use std::env; use std::env;
use crate::meta_plugin::{MetaPlugin, MetaPluginType, BaseMetaPlugin}; use crate::meta_plugin::{BaseMetaPlugin, MetaPlugin, MetaPluginType};
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
/// Meta plugin for capturing shell environment information. /// Meta plugin for capturing shell environment information.
@@ -38,17 +38,16 @@ impl ShellMetaPlugin {
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>, outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> ShellMetaPlugin { ) -> ShellMetaPlugin {
let mut base = BaseMetaPlugin::new(); let mut base = BaseMetaPlugin::new();
// Set default outputs // Set default outputs
let default_outputs = &["shell"]; let default_outputs = &["shell"];
base.initialize_plugin(default_outputs, &_options, &outputs); base.initialize_plugin(default_outputs, &_options, &outputs);
ShellMetaPlugin { ShellMetaPlugin {
is_finalized: false, is_finalized: false,
base, base,
} }
} }
} }
impl MetaPlugin for ShellMetaPlugin { impl MetaPlugin for ShellMetaPlugin {
@@ -60,7 +59,7 @@ impl MetaPlugin for ShellMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
/// Sets the finalized state of the plugin. /// Sets the finalized state of the plugin.
/// ///
/// # Arguments /// # Arguments
@@ -69,7 +68,7 @@ impl MetaPlugin for ShellMetaPlugin {
fn set_finalized(&mut self, finalized: bool) { fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
/// Finalizes the plugin without processing data. /// Finalizes the plugin without processing data.
/// ///
/// For this plugin, finalization is handled in `initialize`, so this returns empty metadata. /// For this plugin, finalization is handled in `initialize`, so this returns empty metadata.
@@ -85,10 +84,10 @@ impl MetaPlugin for ShellMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
// Mark as finalized // Mark as finalized
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
@@ -114,7 +113,7 @@ impl MetaPlugin for ShellMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: false, is_finalized: false,
@@ -129,7 +128,7 @@ impl MetaPlugin for ShellMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::Shell MetaPluginType::Shell
} }
/// Initializes the plugin and extracts shell metadata. /// Initializes the plugin and extracts shell metadata.
/// ///
/// Retrieves the SHELL environment variable and adds it to metadata. /// Retrieves the SHELL environment variable and adds it to metadata.
@@ -154,31 +153,31 @@ impl MetaPlugin for ShellMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
let mut metadata = Vec::new(); let mut metadata = Vec::new();
let shell = match env::var("SHELL") { let shell = match env::var("SHELL") {
Ok(shell) => shell, Ok(shell) => shell,
Err(_) => "unknown".to_string(), Err(_) => "unknown".to_string(),
}; };
// Use process_metadata_outputs to handle output mapping // Use process_metadata_outputs to handle output mapping
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"shell", "shell",
serde_yaml::Value::String(shell), serde_yaml::Value::String(shell),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
// Mark as finalized since this plugin only needs to run once // Mark as finalized since this plugin only needs to run once
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata, metadata,
is_finalized: true, is_finalized: true,
} }
} }
/// Returns a reference to the plugin's outputs. /// Returns a reference to the plugin's outputs.
/// ///
/// # Returns /// # Returns
@@ -187,7 +186,7 @@ impl MetaPlugin for ShellMetaPlugin {
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
/// Returns a mutable reference to the plugin's outputs. /// Returns a mutable reference to the plugin's outputs.
/// ///
/// # Returns /// # Returns
@@ -196,7 +195,7 @@ impl MetaPlugin for ShellMetaPlugin {
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
/// Returns the default output names for this plugin. /// Returns the default output names for this plugin.
/// ///
/// # Returns /// # Returns
@@ -205,7 +204,7 @@ impl MetaPlugin for ShellMetaPlugin {
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec!["shell".to_string()] vec!["shell".to_string()]
} }
/// Returns a reference to the plugin's options. /// Returns a reference to the plugin's options.
/// ///
/// # Returns /// # Returns
@@ -214,7 +213,7 @@ impl MetaPlugin for ShellMetaPlugin {
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
/// Returns a mutable reference to the plugin's options. /// Returns a mutable reference to the plugin's options.
/// ///
/// # Returns /// # Returns

39
src/meta_plugin/shell_pid.rs
View File

@@ -1,6 +1,6 @@
use crate::meta_plugin::{BaseMetaPlugin, MetaPlugin, MetaPluginType};
use std::env; use std::env;
use std::process; use std::process;
use crate::meta_plugin::{MetaPlugin, MetaPluginType, BaseMetaPlugin};
#[derive(Debug, Clone, Default)] #[derive(Debug, Clone, Default)]
pub struct ShellPidMetaPlugin { pub struct ShellPidMetaPlugin {
@@ -24,18 +24,17 @@ impl ShellPidMetaPlugin {
base, base,
} }
} }
} }
impl MetaPlugin for ShellPidMetaPlugin { impl MetaPlugin for ShellPidMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
fn set_finalized(&mut self, finalized: bool) { fn set_finalized(&mut self, finalized: bool) {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
fn finalize(&mut self) -> crate::meta_plugin::MetaPluginResponse { fn finalize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
// If already finalized, don't process again // If already finalized, don't process again
if self.is_finalized { if self.is_finalized {
@@ -44,10 +43,10 @@ impl MetaPlugin for ShellPidMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
// Mark as finalized // Mark as finalized
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: true, is_finalized: true,
@@ -62,7 +61,7 @@ impl MetaPlugin for ShellPidMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata: Vec::new(), metadata: Vec::new(),
is_finalized: false, is_finalized: false,
@@ -72,7 +71,7 @@ impl MetaPlugin for ShellPidMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::ShellPid MetaPluginType::ShellPid
} }
fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse { fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
// If already finalized, don't process again // If already finalized, don't process again
if self.is_finalized { if self.is_finalized {
@@ -81,45 +80,43 @@ impl MetaPlugin for ShellPidMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
let mut metadata = Vec::new(); let mut metadata = Vec::new();
let pid = match env::var("PPID") { let pid = match env::var("PPID") {
Ok(ppid) => ppid, Ok(ppid) => ppid,
Err(_) => process::id().to_string(), Err(_) => process::id().to_string(),
}; };
// Use process_metadata_outputs to handle output mapping // Use process_metadata_outputs to handle output mapping
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"shell_pid", "shell_pid",
serde_yaml::Value::String(pid), serde_yaml::Value::String(pid),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
// Mark as finalized since this plugin only needs to run once // Mark as finalized since this plugin only needs to run once
self.is_finalized = true; self.is_finalized = true;
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata, metadata,
is_finalized: true, is_finalized: true,
} }
} }
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.options_mut() self.base.options_mut()
} }

372
src/meta_plugin/text.rs
View File

@@ -1,5 +1,5 @@
use crate::common::is_binary::is_binary;
use crate::common::PIPESIZE; use crate::common::PIPESIZE;
use crate::common::is_binary::is_binary;
use crate::meta_plugin::{MetaPlugin, MetaPluginResponse, MetaPluginType}; use crate::meta_plugin::{MetaPlugin, MetaPluginResponse, MetaPluginType};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@@ -38,15 +38,21 @@ impl TextMetaPlugin {
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>, outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> TextMetaPlugin { ) -> TextMetaPlugin {
let mut base = crate::meta_plugin::BaseMetaPlugin::new(); let mut base = crate::meta_plugin::BaseMetaPlugin::new();
// Initialize with helper function // Initialize with helper function
base.initialize_plugin( base.initialize_plugin(
&["text", "text_word_count", "text_line_count", &[
"text_line_max_len", "text_line_mean_len", "text_line_median_len"], "text",
"text_word_count",
"text_line_count",
"text_line_max_len",
"text_line_mean_len",
"text_line_median_len",
],
&options, &options,
&outputs, &outputs,
); );
// Set disabled outputs to null based on options // Set disabled outputs to null based on options
let outputs_to_disable = vec![ let outputs_to_disable = vec![
("text_word_count", "text_word_count"), ("text_word_count", "text_word_count"),
@@ -55,7 +61,7 @@ impl TextMetaPlugin {
("text_line_mean_len", "text_line_mean_len"), ("text_line_mean_len", "text_line_mean_len"),
("text_line_median_len", "text_line_median_len"), ("text_line_median_len", "text_line_median_len"),
]; ];
for (option_name, output_name) in outputs_to_disable { for (option_name, output_name) in outputs_to_disable {
if let Some(value) = base.options.get(option_name) { if let Some(value) = base.options.get(option_name) {
// Handle both boolean false and string "false" // Handle both boolean false and string "false"
@@ -65,53 +71,69 @@ impl TextMetaPlugin {
_ => false, _ => false,
}; };
if should_disable { if should_disable {
base.outputs.insert(output_name.to_string(), serde_yaml::Value::Null); base.outputs
.insert(output_name.to_string(), serde_yaml::Value::Null);
} }
} }
} }
// Set default options if not provided // Set default options if not provided
let default_options = vec![ let default_options = vec![
("text_detect_size", serde_yaml::Value::Number(PIPESIZE.into())), (
"text_detect_size",
serde_yaml::Value::Number(PIPESIZE.into()),
),
("text_word_count", serde_yaml::Value::Bool(true)), ("text_word_count", serde_yaml::Value::Bool(true)),
("text_line_count", serde_yaml::Value::Bool(true)), ("text_line_count", serde_yaml::Value::Bool(true)),
("text_line_max_len", serde_yaml::Value::Bool(true)), ("text_line_max_len", serde_yaml::Value::Bool(true)),
("text_line_mean_len", serde_yaml::Value::Bool(true)), ("text_line_mean_len", serde_yaml::Value::Bool(true)),
("text_line_median_len", serde_yaml::Value::Bool(false)), ("text_line_median_len", serde_yaml::Value::Bool(false)),
]; ];
for (key, value) in default_options { for (key, value) in default_options {
if !base.options.contains_key(key) { if !base.options.contains_key(key) {
base.options.insert(key.to_string(), value); base.options.insert(key.to_string(), value);
} }
} }
// Get text_detect_size (previously max_buffer_size) // Get text_detect_size (previously max_buffer_size)
let max_buffer_size = base.options.get("text_detect_size") let max_buffer_size = base
.options
.get("text_detect_size")
.or_else(|| base.options.get("max_buffer_size")) // Handle backward compatibility .or_else(|| base.options.get("max_buffer_size")) // Handle backward compatibility
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.unwrap_or(PIPESIZE as u64) as usize; .unwrap_or(PIPESIZE as u64) as usize;
// Get which statistics to track // Get which statistics to track
let track_word_count = base.options.get("text_word_count") let track_word_count = base
.options
.get("text_word_count")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
let track_line_count = base.options.get("text_line_count") let track_line_count = base
.options
.get("text_line_count")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
let track_line_max_len = base.options.get("text_line_max_len") let track_line_max_len = base
.options
.get("text_line_max_len")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
let track_line_mean_len = base.options.get("text_line_mean_len") let track_line_mean_len = base
.options
.get("text_line_mean_len")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(true); .unwrap_or(true);
let track_line_median_len = base.options.get("text_line_median_len") let track_line_median_len = base
.options
.get("text_line_median_len")
.and_then(|v| v.as_bool()) .and_then(|v| v.as_bool())
.unwrap_or(false); .unwrap_or(false);
// Track line lengths if any of the line length options are enabled // Track line lengths if any of the line length options are enabled
let track_line_lengths = track_line_max_len || track_line_mean_len || track_line_median_len; let track_line_lengths = track_line_max_len || track_line_mean_len || track_line_median_len;
TextMetaPlugin { TextMetaPlugin {
buffer: Some(Vec::new()), buffer: Some(Vec::new()),
max_buffer_size, max_buffer_size,
@@ -130,7 +152,11 @@ impl TextMetaPlugin {
output_line_max_len: track_line_max_len, output_line_max_len: track_line_max_len,
output_line_mean_len: track_line_mean_len, output_line_mean_len: track_line_mean_len,
output_line_median_len: track_line_median_len, output_line_median_len: track_line_median_len,
line_lengths: if track_line_lengths { Some(Vec::new()) } else { None }, line_lengths: if track_line_lengths {
Some(Vec::new())
} else {
None
},
current_line_length: 0, current_line_length: 0,
// Initialize incremental tracking for max and mean // Initialize incremental tracking for max and mean
max_line_length: 0, max_line_length: 0,
@@ -138,8 +164,7 @@ impl TextMetaPlugin {
line_count_for_stats: 0, line_count_for_stats: 0,
} }
} }
/// Count words and lines in a text chunk, handling block boundaries correctly. /// Count words and lines in a text chunk, handling block boundaries correctly.
/// ///
/// Processes UTF-8 data, tracks word transitions, and updates line length statistics. /// Processes UTF-8 data, tracks word transitions, and updates line length statistics.
@@ -152,7 +177,7 @@ impl TextMetaPlugin {
if self.track_line_count { if self.track_line_count {
self.line_count += data.iter().filter(|&&b| b == b'\n').count(); self.line_count += data.iter().filter(|&&b| b == b'\n').count();
} }
// Handle UTF-8 character boundaries by combining with any buffered bytes // Handle UTF-8 character boundaries by combining with any buffered bytes
let combined_data = if !self.utf8_buffer.is_empty() { let combined_data = if !self.utf8_buffer.is_empty() {
let mut combined = self.utf8_buffer.clone(); let mut combined = self.utf8_buffer.clone();
@@ -161,10 +186,10 @@ impl TextMetaPlugin {
} else { } else {
data.to_vec() data.to_vec()
}; };
// Clear the UTF-8 buffer // Clear the UTF-8 buffer
self.utf8_buffer.clear(); self.utf8_buffer.clear();
// Convert to string, handling potential UTF-8 boundaries // Convert to string, handling potential UTF-8 boundaries
let text = match std::str::from_utf8(&combined_data) { let text = match std::str::from_utf8(&combined_data) {
Ok(text) => text, Ok(text) => text,
@@ -172,7 +197,8 @@ impl TextMetaPlugin {
// If we have incomplete UTF-8 at the end, buffer those bytes for next chunk // If we have incomplete UTF-8 at the end, buffer those bytes for next chunk
let valid_up_to = e.valid_up_to(); let valid_up_to = e.valid_up_to();
if valid_up_to < combined_data.len() { if valid_up_to < combined_data.len() {
self.utf8_buffer.extend_from_slice(&combined_data[valid_up_to..]); self.utf8_buffer
.extend_from_slice(&combined_data[valid_up_to..]);
} }
match std::str::from_utf8(&combined_data[..valid_up_to]) { match std::str::from_utf8(&combined_data[..valid_up_to]) {
Ok(text) => text, Ok(text) => text,
@@ -180,12 +206,12 @@ impl TextMetaPlugin {
} }
} }
}; };
// Count words if needed // Count words if needed
if self.track_word_count { if self.track_word_count {
for ch in text.chars() { for ch in text.chars() {
let is_whitespace = ch.is_whitespace(); let is_whitespace = ch.is_whitespace();
if !self.in_word && !is_whitespace { if !self.in_word && !is_whitespace {
// Transition from whitespace to word - start of new word // Transition from whitespace to word - start of new word
self.word_count += 1; self.word_count += 1;
@@ -196,7 +222,7 @@ impl TextMetaPlugin {
} }
} }
} }
// Track line lengths if needed // Track line lengths if needed
if self.track_line_lengths { if self.track_line_lengths {
for ch in text.chars() { for ch in text.chars() {
@@ -205,16 +231,16 @@ impl TextMetaPlugin {
if self.current_line_length > self.max_line_length { if self.current_line_length > self.max_line_length {
self.max_line_length = self.current_line_length; self.max_line_length = self.current_line_length;
} }
// Update total for mean calculation // Update total for mean calculation
self.total_line_length += self.current_line_length; self.total_line_length += self.current_line_length;
self.line_count_for_stats += 1; self.line_count_for_stats += 1;
// Only store individual lengths if median is needed // Only store individual lengths if median is needed
if let Some(ref mut lengths) = self.line_lengths { if let Some(ref mut lengths) = self.line_lengths {
lengths.push(self.current_line_length); lengths.push(self.current_line_length);
} }
self.current_line_length = 0; self.current_line_length = 0;
} else { } else {
self.current_line_length += 1; self.current_line_length += 1;
@@ -222,7 +248,7 @@ impl TextMetaPlugin {
} }
} }
} }
/// Helper method to perform binary detection and return appropriate metadata. /// Helper method to perform binary detection and return appropriate metadata.
/// ///
/// Uses the is_binary function to check the buffer and sets text-related outputs accordingly. /// Uses the is_binary function to check the buffer and sets text-related outputs accordingly.
@@ -234,23 +260,30 @@ impl TextMetaPlugin {
/// # Returns /// # Returns
/// ///
/// * `(Vec<MetaData>, bool)` - Metadata updates and whether content is binary. /// * `(Vec<MetaData>, bool)` - Metadata updates and whether content is binary.
fn perform_binary_detection(&mut self, buffer: &[u8]) -> (Vec<crate::meta_plugin::MetaData>, bool) { fn perform_binary_detection(
&mut self,
buffer: &[u8],
) -> (Vec<crate::meta_plugin::MetaData>, bool) {
let mut metadata = Vec::new(); let mut metadata = Vec::new();
let is_binary_result = is_binary(buffer); let is_binary_result = is_binary(buffer);
self.is_binary_content = Some(is_binary_result); self.is_binary_content = Some(is_binary_result);
// Output text status // Output text status
let text_value = if is_binary_result { "false".to_string() } else { "true".to_string() }; let text_value = if is_binary_result {
"false".to_string()
} else {
"true".to_string()
};
// Use process_metadata_outputs to handle output mapping // Use process_metadata_outputs to handle output mapping
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
"text", "text",
serde_yaml::Value::String(text_value), serde_yaml::Value::String(text_value),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
// If content is binary, set all text-related outputs to None // If content is binary, set all text-related outputs to None
if is_binary_result { if is_binary_result {
let text_outputs = vec![ let text_outputs = vec![
@@ -262,15 +295,15 @@ impl TextMetaPlugin {
]; ];
for output_name in text_outputs { for output_name in text_outputs {
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
output_name, output_name,
serde_yaml::Value::Null, serde_yaml::Value::Null,
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
} }
} }
(metadata, is_binary_result) (metadata, is_binary_result)
} }
@@ -292,11 +325,11 @@ impl TextMetaPlugin {
if self.current_line_length > self.max_line_length { if self.current_line_length > self.max_line_length {
self.max_line_length = self.current_line_length; self.max_line_length = self.current_line_length;
} }
// Update total for mean calculation for the last line // Update total for mean calculation for the last line
self.total_line_length += self.current_line_length; self.total_line_length += self.current_line_length;
self.line_count_for_stats += 1; self.line_count_for_stats += 1;
// Only store individual lengths if median is needed // Only store individual lengths if median is needed
if let Some(ref mut lengths) = self.line_lengths { if let Some(ref mut lengths) = self.line_lengths {
lengths.push(self.current_line_length); lengths.push(self.current_line_length);
@@ -312,9 +345,9 @@ impl TextMetaPlugin {
fn output_word_count_metadata(&self) -> Option<crate::meta_plugin::MetaData> { fn output_word_count_metadata(&self) -> Option<crate::meta_plugin::MetaData> {
if self.track_word_count { if self.track_word_count {
crate::meta_plugin::process_metadata_outputs( crate::meta_plugin::process_metadata_outputs(
"text_word_count", "text_word_count",
serde_yaml::Value::String(self.word_count.to_string()), serde_yaml::Value::String(self.word_count.to_string()),
self.base.outputs() self.base.outputs(),
) )
} else { } else {
None None
@@ -329,9 +362,9 @@ impl TextMetaPlugin {
fn output_line_count_metadata(&self) -> Option<crate::meta_plugin::MetaData> { fn output_line_count_metadata(&self) -> Option<crate::meta_plugin::MetaData> {
if self.track_line_count { if self.track_line_count {
crate::meta_plugin::process_metadata_outputs( crate::meta_plugin::process_metadata_outputs(
"text_line_count", "text_line_count",
serde_yaml::Value::String(self.line_count.to_string()), serde_yaml::Value::String(self.line_count.to_string()),
self.base.outputs() self.base.outputs(),
) )
} else { } else {
None None
@@ -346,9 +379,9 @@ impl TextMetaPlugin {
fn output_max_line_length_metadata(&self) -> Option<crate::meta_plugin::MetaData> { fn output_max_line_length_metadata(&self) -> Option<crate::meta_plugin::MetaData> {
if self.output_line_max_len && self.line_count_for_stats > 0 { if self.output_line_max_len && self.line_count_for_stats > 0 {
crate::meta_plugin::process_metadata_outputs( crate::meta_plugin::process_metadata_outputs(
"text_line_max_len", "text_line_max_len",
serde_yaml::Value::String(self.max_line_length.to_string()), serde_yaml::Value::String(self.max_line_length.to_string()),
self.base.outputs() self.base.outputs(),
) )
} else { } else {
None None
@@ -368,9 +401,9 @@ impl TextMetaPlugin {
// Round to nearest integer // Round to nearest integer
let mean_len_int = mean_len.round() as usize; let mean_len_int = mean_len.round() as usize;
crate::meta_plugin::process_metadata_outputs( crate::meta_plugin::process_metadata_outputs(
"text_line_mean_len", "text_line_mean_len",
serde_yaml::Value::String(mean_len_int.to_string()), serde_yaml::Value::String(mean_len_int.to_string()),
self.base.outputs() self.base.outputs(),
) )
} else { } else {
None None
@@ -386,26 +419,27 @@ impl TextMetaPlugin {
/// * `Option<MetaData>` - Metadata entry if enabled and data exists. /// * `Option<MetaData>` - Metadata entry if enabled and data exists.
fn output_median_line_length_metadata(&self) -> Option<crate::meta_plugin::MetaData> { fn output_median_line_length_metadata(&self) -> Option<crate::meta_plugin::MetaData> {
if self.output_line_median_len if self.output_line_median_len
&& let Some(lengths) = &self.line_lengths { && let Some(lengths) = &self.line_lengths
if !lengths.is_empty() { && !lengths.is_empty()
let mut sorted_lengths = lengths.clone(); {
sorted_lengths.sort(); let mut sorted_lengths = lengths.clone();
let median_len = if lengths.len() % 2 == 0 { sorted_lengths.sort();
(sorted_lengths[lengths.len() / 2 - 1] + sorted_lengths[lengths.len() / 2]) as f64 / 2.0 let median_len = if lengths.len() % 2 == 0 {
} else { (sorted_lengths[lengths.len() / 2 - 1] + sorted_lengths[lengths.len() / 2]) as f64
sorted_lengths[lengths.len() / 2] as f64 / 2.0
}; } else {
sorted_lengths[lengths.len() / 2] as f64
return crate::meta_plugin::process_metadata_outputs( };
"text_line_median_len",
serde_yaml::Value::String(median_len.to_string()), return crate::meta_plugin::process_metadata_outputs(
self.base.outputs() "text_line_median_len",
); serde_yaml::Value::String(median_len.to_string()),
} self.base.outputs(),
);
} }
None None
} }
/// Helper method to output word and line counts. /// Helper method to output word and line counts.
/// ///
/// Finalizes pending data and collects all enabled text statistics metadata. /// Finalizes pending data and collects all enabled text statistics metadata.
@@ -440,7 +474,10 @@ impl TextMetaPlugin {
let line_stats_outputs = vec![ let line_stats_outputs = vec![
(self.output_max_line_length_metadata(), "max line length"), (self.output_max_line_length_metadata(), "max line length"),
(self.output_mean_line_length_metadata(), "mean line length"), (self.output_mean_line_length_metadata(), "mean line length"),
(self.output_median_line_length_metadata(), "median line length"), (
self.output_median_line_length_metadata(),
"median line length",
),
]; ];
for (output, _) in line_stats_outputs { for (output, _) in line_stats_outputs {
@@ -463,7 +500,7 @@ impl MetaPlugin for TextMetaPlugin {
fn is_finalized(&self) -> bool { fn is_finalized(&self) -> bool {
self.is_finalized self.is_finalized
} }
/// Sets the finalized state of the plugin. /// Sets the finalized state of the plugin.
/// ///
/// # Arguments /// # Arguments
@@ -473,7 +510,6 @@ impl MetaPlugin for TextMetaPlugin {
self.is_finalized = finalized; self.is_finalized = finalized;
} }
/// Updates the plugin with new data chunk. /// Updates the plugin with new data chunk.
/// ///
/// Accumulates data for binary detection (if pending) or text statistics. /// Accumulates data for binary detection (if pending) or text statistics.
@@ -497,7 +533,7 @@ impl MetaPlugin for TextMetaPlugin {
let mut metadata = Vec::new(); let mut metadata = Vec::new();
let processed_data = data.to_vec(); let processed_data = data.to_vec();
// If we haven't determined if content is binary yet, build buffer and check // If we haven't determined if content is binary yet, build buffer and check
if self.is_binary_content.is_none() { if self.is_binary_content.is_none() {
let should_finalize = if let Some(ref mut buffer) = self.buffer { let should_finalize = if let Some(ref mut buffer) = self.buffer {
@@ -505,7 +541,7 @@ impl MetaPlugin for TextMetaPlugin {
let remaining_capacity = self.max_buffer_size.saturating_sub(buffer.len()); let remaining_capacity = self.max_buffer_size.saturating_sub(buffer.len());
let bytes_to_take = std::cmp::min(processed_data.len(), remaining_capacity); let bytes_to_take = std::cmp::min(processed_data.len(), remaining_capacity);
buffer.extend_from_slice(&processed_data[..bytes_to_take]); buffer.extend_from_slice(&processed_data[..bytes_to_take]);
// If we have enough data to make a binary determination, do it now // If we have enough data to make a binary determination, do it now
let buffer_len = buffer.len(); let buffer_len = buffer.len();
if buffer_len >= std::cmp::min(1024, self.max_buffer_size) { if buffer_len >= std::cmp::min(1024, self.max_buffer_size) {
@@ -514,7 +550,7 @@ impl MetaPlugin for TextMetaPlugin {
let (binary_metadata, is_binary) = self.perform_binary_detection(&buffer_clone); let (binary_metadata, is_binary) = self.perform_binary_detection(&buffer_clone);
metadata.extend(binary_metadata); metadata.extend(binary_metadata);
self.is_binary_content = Some(is_binary); self.is_binary_content = Some(is_binary);
// If it's binary, we're done with this plugin // If it's binary, we're done with this plugin
if is_binary { if is_binary {
self.buffer = None; // Drop the buffer self.buffer = None; // Drop the buffer
@@ -524,16 +560,16 @@ impl MetaPlugin for TextMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
// If it's text, count words and lines for this chunk // If it's text, count words and lines for this chunk
self.count_text_stats(&processed_data[..bytes_to_take]); self.count_text_stats(&processed_data[..bytes_to_take]);
// If we've reached our buffer limit, drop the buffer to save memory // If we've reached our buffer limit, drop the buffer to save memory
// But don't finalize yet - we need to keep counting words and lines // But don't finalize yet - we need to keep counting words and lines
if buffer_len >= self.max_buffer_size { if buffer_len >= self.max_buffer_size {
self.buffer = None; // Drop the buffer self.buffer = None; // Drop the buffer
} }
false // Never finalize here for text content false // Never finalize here for text content
} else { } else {
// Still building up buffer, count words and lines for this chunk // Still building up buffer, count words and lines for this chunk
self.count_text_stats(&processed_data[..bytes_to_take]); self.count_text_stats(&processed_data[..bytes_to_take]);
@@ -542,7 +578,7 @@ impl MetaPlugin for TextMetaPlugin {
} else { } else {
false false
}; };
if should_finalize { if should_finalize {
return MetaPluginResponse { return MetaPluginResponse {
metadata, metadata,
@@ -584,97 +620,108 @@ impl MetaPlugin for TextMetaPlugin {
is_finalized: true, is_finalized: true,
}; };
} }
let mut metadata = Vec::new(); let mut metadata = Vec::new();
// Check if we have head/tail options // Check if we have head/tail options
let head_bytes = self.base.options.get("head_bytes") let head_bytes = self
.base
.options
.get("head_bytes")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|v| v as usize); .map(|v| v as usize);
let head_lines = self.base.options.get("head_lines") let head_lines = self
.base
.options
.get("head_lines")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|v| v as usize); .map(|v| v as usize);
let tail_bytes = self.base.options.get("tail_bytes") let tail_bytes = self
.base
.options
.get("tail_bytes")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|v| v as usize); .map(|v| v as usize);
let tail_lines = self.base.options.get("tail_lines") let tail_lines = self
.base
.options
.get("tail_lines")
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|v| v as usize); .map(|v| v as usize);
// If we haven't determined binary status yet, do it now with whatever we have // If we haven't determined binary status yet, do it now with whatever we have
if self.is_binary_content.is_none() { if self.is_binary_content.is_none()
if let Some(buffer) = &self.buffer { && let Some(buffer) = &self.buffer
if !buffer.is_empty() { && !buffer.is_empty()
// Build filter string from individual parameters {
let mut filter_parts = Vec::new(); // Build filter string from individual parameters
if let Some(bytes) = head_bytes { let mut filter_parts = Vec::new();
filter_parts.push(format!("head_bytes({})", bytes)); if let Some(bytes) = head_bytes {
} filter_parts.push(format!("head_bytes({})", bytes));
if let Some(lines) = head_lines { }
filter_parts.push(format!("head_lines({})", lines)); if let Some(lines) = head_lines {
} filter_parts.push(format!("head_lines({})", lines));
if let Some(bytes) = tail_bytes { }
filter_parts.push(format!("tail_bytes({})", bytes)); if let Some(bytes) = tail_bytes {
} filter_parts.push(format!("tail_bytes({})", bytes));
if let Some(lines) = tail_lines { }
filter_parts.push(format!("tail_lines({})", lines)); if let Some(lines) = tail_lines {
} filter_parts.push(format!("tail_lines({})", lines));
}
// For now, just use the buffer as-is since filtering isn't implemented
let processed_buffer = buffer.clone(); // For now, just use the buffer as-is since filtering isn't implemented
let processed_buffer = buffer.clone();
// Clone the processed buffer data for binary detection
let (binary_metadata, is_binary) = self.perform_binary_detection(&processed_buffer); // Clone the processed buffer data for binary detection
metadata.extend(binary_metadata); let (binary_metadata, is_binary) = self.perform_binary_detection(&processed_buffer);
self.is_binary_content = Some(is_binary); metadata.extend(binary_metadata);
self.is_binary_content = Some(is_binary);
// If it's binary, we're done
if is_binary { // If it's binary, we're done
self.buffer = None; // Drop the buffer if is_binary {
self.is_finalized = true; self.buffer = None; // Drop the buffer
// Set all text-related outputs to None since content is binary self.is_finalized = true;
// Only include outputs that are enabled in the configuration // Set all text-related outputs to None since content is binary
let text_outputs = vec![ // Only include outputs that are enabled in the configuration
("text_word_count", self.track_word_count), let text_outputs = vec![
("text_line_count", self.track_line_count), ("text_word_count", self.track_word_count),
("text_line_max_len", self.output_line_max_len), ("text_line_count", self.track_line_count),
("text_line_mean_len", self.output_line_mean_len), ("text_line_max_len", self.output_line_max_len),
("text_line_median_len", self.output_line_median_len), ("text_line_mean_len", self.output_line_mean_len),
]; ("text_line_median_len", self.output_line_median_len),
];
for (output_name, is_enabled) in text_outputs {
if is_enabled { for (output_name, is_enabled) in text_outputs {
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if is_enabled
output_name, && let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
serde_yaml::Value::Null, output_name,
self.base.outputs() serde_yaml::Value::Null,
) { self.base.outputs(),
metadata.push(meta_data); )
} {
} metadata.push(meta_data);
}
return MetaPluginResponse {
metadata,
is_finalized: true,
};
} }
} }
return MetaPluginResponse {
metadata,
is_finalized: true,
};
} }
} }
// If content is text, output word and line counts // If content is text, output word and line counts
if self.is_binary_content == Some(false) { if self.is_binary_content == Some(false) {
let word_line_metadata = self.output_word_line_counts(); let word_line_metadata = self.output_word_line_counts();
metadata.extend(word_line_metadata); metadata.extend(word_line_metadata);
} }
// Only include outputs that are enabled in the configuration // Only include outputs that are enabled in the configuration
// Disabled outputs should not be emitted at all (not even as null) // Disabled outputs should not be emitted at all (not even as null)
// So we don't need to add anything for disabled outputs // So we don't need to add anything for disabled outputs
// Drop the buffer since we're done with it // Drop the buffer since we're done with it
self.buffer = None; self.buffer = None;
// Mark as finalized // Mark as finalized
self.is_finalized = true; self.is_finalized = true;
MetaPluginResponse { MetaPluginResponse {
@@ -691,7 +738,7 @@ impl MetaPlugin for TextMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::Text MetaPluginType::Text
} }
/// Returns a reference to the outputs mapping. /// Returns a reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -700,7 +747,7 @@ impl MetaPlugin for TextMetaPlugin {
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
/// Returns a mutable reference to the outputs mapping. /// Returns a mutable reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -709,7 +756,7 @@ impl MetaPlugin for TextMetaPlugin {
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
/// Returns the default output names for this plugin. /// Returns the default output names for this plugin.
/// ///
/// # Returns /// # Returns
@@ -717,15 +764,15 @@ impl MetaPlugin for TextMetaPlugin {
/// Vector of default output field names. /// Vector of default output field names.
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec![ vec![
"text".to_string(), "text".to_string(),
"text_word_count".to_string(), "text_word_count".to_string(),
"text_line_count".to_string(), "text_line_count".to_string(),
"text_line_max_len".to_string(), "text_line_max_len".to_string(),
"text_line_mean_len".to_string(), "text_line_mean_len".to_string(),
"text_line_median_len".to_string() "text_line_median_len".to_string(),
] ]
} }
/// Returns a reference to the options mapping. /// Returns a reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -734,7 +781,7 @@ impl MetaPlugin for TextMetaPlugin {
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
/// Returns a mutable reference to the options mapping. /// Returns a mutable reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -743,7 +790,6 @@ impl MetaPlugin for TextMetaPlugin {
fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn options_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.options_mut() self.base.options_mut()
} }
} }
use crate::meta_plugin::register_meta_plugin; use crate::meta_plugin::register_meta_plugin;
@@ -753,4 +799,4 @@ fn register_text_plugin() {
register_meta_plugin(MetaPluginType::Text, |options, outputs| { register_meta_plugin(MetaPluginType::Text, |options, outputs| {
Box::new(TextMetaPlugin::new(options, outputs)) Box::new(TextMetaPlugin::new(options, outputs))
}); });
} }

45
src/meta_plugin/user.rs
View File

@@ -25,20 +25,17 @@ impl UserMetaPlugin {
outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>, outputs: Option<std::collections::HashMap<String, serde_yaml::Value>>,
) -> UserMetaPlugin { ) -> UserMetaPlugin {
let mut base = crate::meta_plugin::BaseMetaPlugin::new(); let mut base = crate::meta_plugin::BaseMetaPlugin::new();
// Initialize with helper function // Initialize with helper function
base.initialize_plugin( base.initialize_plugin(
&["user_uid", "user_gid", "user_name", "user_group"], &["user_uid", "user_gid", "user_name", "user_group"],
&options, &options,
&outputs, &outputs,
); );
UserMetaPlugin { UserMetaPlugin { base }
base,
}
} }
/// Gets the current username. /// Gets the current username.
/// ///
/// # Returns /// # Returns
@@ -48,7 +45,7 @@ impl UserMetaPlugin {
uzers::get_user_by_uid(uzers::get_current_uid()) uzers::get_user_by_uid(uzers::get_current_uid())
.map(|user| user.name().to_string_lossy().to_string()) .map(|user| user.name().to_string_lossy().to_string())
} }
/// Gets the current group name. /// Gets the current group name.
/// ///
/// # Returns /// # Returns
@@ -68,13 +65,13 @@ impl MetaPlugin for UserMetaPlugin {
/// A `MetaPluginResponse` with user metadata and `is_finalized` set to `true`. /// A `MetaPluginResponse` with user metadata and `is_finalized` set to `true`.
fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse { fn initialize(&mut self) -> crate::meta_plugin::MetaPluginResponse {
let mut metadata = Vec::new(); let mut metadata = Vec::new();
// Get user info // Get user info
let uid = uzers::get_current_uid().to_string(); let uid = uzers::get_current_uid().to_string();
let gid = uzers::get_current_gid().to_string(); let gid = uzers::get_current_gid().to_string();
let username = Self::get_current_username().unwrap_or_else(|| "unknown".to_string()); let username = Self::get_current_username().unwrap_or_else(|| "unknown".to_string());
let groupname = Self::get_current_groupname().unwrap_or_else(|| "unknown".to_string()); let groupname = Self::get_current_groupname().unwrap_or_else(|| "unknown".to_string());
// Process each output // Process each output
let values = [ let values = [
("user_uid", uid), ("user_uid", uid),
@@ -82,17 +79,17 @@ impl MetaPlugin for UserMetaPlugin {
("user_name", username), ("user_name", username),
("user_group", groupname), ("user_group", groupname),
]; ];
for (name, value) in values { for (name, value) in values {
if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs( if let Some(meta_data) = crate::meta_plugin::process_metadata_outputs(
name, name,
serde_yaml::Value::String(value), serde_yaml::Value::String(value),
self.base.outputs() self.base.outputs(),
) { ) {
metadata.push(meta_data); metadata.push(meta_data);
} }
} }
crate::meta_plugin::MetaPluginResponse { crate::meta_plugin::MetaPluginResponse {
metadata, metadata,
is_finalized: true, is_finalized: true,
@@ -107,7 +104,7 @@ impl MetaPlugin for UserMetaPlugin {
fn meta_type(&self) -> MetaPluginType { fn meta_type(&self) -> MetaPluginType {
MetaPluginType::User MetaPluginType::User
} }
/// Returns a reference to the outputs mapping. /// Returns a reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -116,7 +113,7 @@ impl MetaPlugin for UserMetaPlugin {
fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn outputs(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs() self.base.outputs()
} }
/// Returns a mutable reference to the outputs mapping. /// Returns a mutable reference to the outputs mapping.
/// ///
/// # Returns /// # Returns
@@ -125,17 +122,21 @@ impl MetaPlugin for UserMetaPlugin {
fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> { fn outputs_mut(&mut self) -> &mut std::collections::HashMap<String, serde_yaml::Value> {
self.base.outputs_mut() self.base.outputs_mut()
} }
/// Returns the default output names. /// Returns the default output names.
/// ///
/// # Returns /// # Returns
/// ///
/// A vector of default output names. /// A vector of default output names.
fn default_outputs(&self) -> Vec<String> { fn default_outputs(&self) -> Vec<String> {
vec!["user_uid".to_string(), "user_gid".to_string(), "user_name".to_string(), "user_group".to_string()] vec![
"user_uid".to_string(),
"user_gid".to_string(),
"user_name".to_string(),
"user_group".to_string(),
]
} }
/// Returns a reference to the options mapping. /// Returns a reference to the options mapping.
/// ///
/// # Returns /// # Returns
@@ -144,7 +145,7 @@ impl MetaPlugin for UserMetaPlugin {
fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> { fn options(&self) -> &std::collections::HashMap<String, serde_yaml::Value> {
self.base.options() self.base.options()
} }
/// Returns a mutable reference to the options mapping. /// Returns a mutable reference to the options mapping.
/// ///
/// # Returns /// # Returns

100
src/modes/common.rs
View File

@@ -1,3 +1,4 @@
use crate::compression_engine::CompressionType;
/// Common utilities shared across different modes in the Keep application. /// Common utilities shared across different modes in the Keep application.
/// ///
/// This module provides helper functions for formatting, configuration parsing, /// This module provides helper functions for formatting, configuration parsing,
@@ -13,11 +14,10 @@
/// let format = OutputFormat::from_str("json")?; /// let format = OutputFormat::from_str("json")?;
/// ``` /// ```
use crate::config; use crate::config;
use crate::compression_engine::CompressionType;
use crate::meta_plugin::MetaPluginType; use crate::meta_plugin::MetaPluginType;
use clap::Command; use clap::Command;
use clap::error::ErrorKind; use clap::error::ErrorKind;
use comfy_table::{Table, ContentArrangement}; use comfy_table::{ContentArrangement, Table};
use log::debug; use log::debug;
use regex::Regex; use regex::Regex;
use std::collections::HashMap; use std::collections::HashMap;
@@ -116,7 +116,7 @@ pub fn format_size(size: u64, human_readable: bool) -> String {
} }
} }
#[derive(Debug, Eq, PartialEq, Clone, strum::EnumIter, strum::Display, strum::EnumString)] #[derive(Debug, Eq, PartialEq, Clone, strum::EnumIter, strum::Display)]
#[strum(ascii_case_insensitive)] #[strum(ascii_case_insensitive)]
/// Enum representing column types for table display. /// Enum representing column types for table display.
/// ///
@@ -151,34 +151,20 @@ pub enum ColumnType {
Meta, Meta,
} }
impl ColumnType { impl std::str::FromStr for ColumnType {
/// Parses a string to a ColumnType, handling "meta:<name>" pattern. type Err = anyhow::Error;
///
/// Supports direct enum variants or "meta:<name>" for metadata columns. fn from_str(s: &str) -> anyhow::Result<Self> {
/// let lower_s = s.to_lowercase();
/// # Arguments
///
/// * `s` - Input string to parse, e.g., "size" or "meta:hostname".
///
/// # Returns
///
/// * `Ok(ColumnType)` - Parsed type on success.
/// * `Err(anyhow::Error)` - If the string doesn't match any variant.
///
/// # Examples
///
/// ```
/// use keep::modes::common::ColumnType;
/// let meta = ColumnType::from_str("meta:hostname").unwrap();
/// assert_eq!(meta, ColumnType::Meta);
/// ```
pub fn from_str(s: &str) -> anyhow::Result<Self> {
if s.starts_with("meta:") { if s.starts_with("meta:") {
// Handle meta:<name> pattern - this is still a Meta column type
Ok(ColumnType::Meta) Ok(ColumnType::Meta)
} else { } else {
// Handle regular column types for variant in ColumnType::iter() {
Ok(Self::try_from(s)?) if variant.to_string().to_lowercase() == lower_s {
return Ok(variant);
}
}
Err(anyhow::anyhow!("Invalid column type: {}", s))
} }
} }
} }
@@ -199,30 +185,34 @@ impl ColumnType {
/// # Panics /// # Panics
/// ///
/// Exits via Clap error if unknown plugin type specified. /// Exits via Clap error if unknown plugin type specified.
pub fn settings_meta_plugin_types(cmd: &mut Command, settings: &config::Settings) -> Vec<MetaPluginType> { pub fn settings_meta_plugin_types(
cmd: &mut Command,
settings: &config::Settings,
) -> Vec<MetaPluginType> {
let mut meta_plugin_types = Vec::new(); let mut meta_plugin_types = Vec::new();
// Handle comma-separated values in each meta_plugins argument // Handle comma-separated values in each meta_plugins argument
for meta_plugin_names_str in &settings.meta_plugins_names() { for meta_plugin_names_str in &settings.meta_plugins_names() {
let meta_plugin_names: Vec<&str> = meta_plugin_names_str.split(',').collect(); let meta_plugin_names: Vec<&str> = meta_plugin_names_str.split(',').collect();
for name in meta_plugin_names { for name in meta_plugin_names {
let trimmed_name = name.trim(); let trimmed_name = name.trim();
if trimmed_name.is_empty() { if trimmed_name.is_empty() {
continue; continue;
} }
// Try to find the MetaPluginType by meta name // Try to find the MetaPluginType by meta name
let mut found = false; let mut found = false;
for meta_plugin_type in MetaPluginType::iter() { for meta_plugin_type in MetaPluginType::iter() {
let meta_plugin = crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone(), None, None); let meta_plugin =
crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone(), None, None);
if meta_plugin.meta_type().to_string() == trimmed_name { if meta_plugin.meta_type().to_string() == trimmed_name {
meta_plugin_types.push(meta_plugin_type); meta_plugin_types.push(meta_plugin_type);
found = true; found = true;
break; break;
} }
} }
if !found { if !found {
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
@@ -252,7 +242,10 @@ pub fn settings_meta_plugin_types(cmd: &mut Command, settings: &config::Settings
/// # Panics /// # Panics
/// ///
/// Exits via Clap error if invalid compression specified. /// Exits via Clap error if invalid compression specified.
pub fn settings_compression_type(cmd: &mut Command, settings: &config::Settings) -> CompressionType { pub fn settings_compression_type(
cmd: &mut Command,
settings: &config::Settings,
) -> CompressionType {
let compression_name = settings let compression_name = settings
.compression() .compression()
.unwrap_or(CompressionType::LZ4.to_string()); .unwrap_or(CompressionType::LZ4.to_string());
@@ -261,7 +254,10 @@ pub fn settings_compression_type(cmd: &mut Command, settings: &config::Settings)
if compression_type_opt.is_err() { if compression_type_opt.is_err() {
cmd.error( cmd.error(
ErrorKind::InvalidValue, ErrorKind::InvalidValue,
format!("Invalid compression algorithm '{}'. Supported algorithms: lz4, gzip, xz, zstd", compression_name), format!(
"Invalid compression algorithm '{}'. Supported algorithms: lz4, gzip, xz, zstd",
compression_name
),
) )
.exit(); .exit();
} }
@@ -288,7 +284,8 @@ pub fn settings_compression_type(cmd: &mut Command, settings: &config::Settings)
/// assert_eq!(format, OutputFormat::Json); // If settings.output_format = Some("json") /// assert_eq!(format, OutputFormat::Json); // If settings.output_format = Some("json")
/// ``` /// ```
pub fn settings_output_format(settings: &config::Settings) -> OutputFormat { pub fn settings_output_format(settings: &config::Settings) -> OutputFormat {
settings.output_format settings
.output_format
.as_ref() .as_ref()
.and_then(|s| OutputFormat::from_str(s).ok()) .and_then(|s| OutputFormat::from_str(s).ok())
.unwrap_or(OutputFormat::Table) .unwrap_or(OutputFormat::Table)
@@ -340,7 +337,7 @@ pub fn trim_lines_end(s: &str) -> String {
pub fn create_table(use_styling: bool) -> Table { pub fn create_table(use_styling: bool) -> Table {
let mut table = Table::new(); let mut table = Table::new();
table.set_content_arrangement(ContentArrangement::Dynamic); table.set_content_arrangement(ContentArrangement::Dynamic);
if use_styling { if use_styling {
if std::io::stdout().is_terminal() { if std::io::stdout().is_terminal() {
table table
@@ -352,7 +349,7 @@ pub fn create_table(use_styling: bool) -> Table {
} else { } else {
table.load_preset(comfy_table::presets::NOTHING); table.load_preset(comfy_table::presets::NOTHING);
} }
if !std::io::stdout().is_terminal() { if !std::io::stdout().is_terminal() {
table.force_no_tty(); table.force_no_tty();
} }
@@ -379,14 +376,20 @@ pub fn create_table(use_styling: bool) -> Table {
/// ``` /// ```
pub fn create_table_with_config(table_config: &crate::config::TableConfig) -> Table { pub fn create_table_with_config(table_config: &crate::config::TableConfig) -> Table {
let mut table = Table::new(); let mut table = Table::new();
// Set content arrangement // Set content arrangement
match table_config.content_arrangement { match table_config.content_arrangement {
crate::config::ContentArrangement::Dynamic => table.set_content_arrangement(comfy_table::ContentArrangement::Dynamic), crate::config::ContentArrangement::Dynamic => {
crate::config::ContentArrangement::DynamicFullWidth => table.set_content_arrangement(comfy_table::ContentArrangement::DynamicFullWidth), table.set_content_arrangement(comfy_table::ContentArrangement::Dynamic)
crate::config::ContentArrangement::Disabled => table.set_content_arrangement(comfy_table::ContentArrangement::Disabled), }
crate::config::ContentArrangement::DynamicFullWidth => {
table.set_content_arrangement(comfy_table::ContentArrangement::DynamicFullWidth)
}
crate::config::ContentArrangement::Disabled => {
table.set_content_arrangement(comfy_table::ContentArrangement::Disabled)
}
}; };
// Set style preset // Set style preset
match &table_config.style { match &table_config.style {
crate::config::TableStyle::Ascii => { crate::config::TableStyle::Ascii => {
@@ -414,7 +417,7 @@ pub fn create_table_with_config(table_config: &crate::config::TableConfig) -> Ta
// Add more presets as needed // Add more presets as needed
} }
}; };
// Apply modifiers // Apply modifiers
for modifier in &table_config.modifiers { for modifier in &table_config.modifiers {
match modifier.as_str() { match modifier.as_str() {
@@ -427,16 +430,15 @@ pub fn create_table_with_config(table_config: &crate::config::TableConfig) -> Ta
_ => {} // Ignore unknown modifiers _ => {} // Ignore unknown modifiers
} }
} }
// Set truncation indicator if specified // Set truncation indicator if specified
if !table_config.truncation_indicator.is_empty() { if !table_config.truncation_indicator.is_empty() {
table.set_truncation_indicator(&table_config.truncation_indicator); table.set_truncation_indicator(&table_config.truncation_indicator);
} }
if !std::io::stdout().is_terminal() { if !std::io::stdout().is_terminal() {
table.force_no_tty(); table.force_no_tty();
} }
table table
} }

9
src/modes/delete.rs
View File

@@ -48,8 +48,8 @@ pub fn mode_delete(
_cmd: &mut Command, _cmd: &mut Command,
_settings: &config::Settings, _settings: &config::Settings,
_config: &config::Settings, _config: &config::Settings,
ids: &mut Vec<i64>, ids: &mut [i64],
_tags: &mut Vec<String>, _tags: &mut [String],
conn: &mut Connection, conn: &mut Connection,
data_path: PathBuf, data_path: PathBuf,
) -> Result<()> { ) -> Result<()> {
@@ -65,7 +65,10 @@ pub fn mode_delete(
CoreError::ItemNotFound(_) => { CoreError::ItemNotFound(_) => {
warn!("Unable to find item {item_id} in database"); warn!("Unable to find item {item_id} in database");
} }
_ => return Err(anyhow::Error::from(e).context(format!("Failed to delete item {}", item_id))), _ => {
return Err(anyhow::Error::from(e)
.context(format!("Failed to delete item {}", item_id)));
}
}, },
} }
} }

40
src/modes/diff.rs
View File

@@ -1,19 +1,27 @@
use crate::config;
use crate::services::item_service::ItemService;
/// Diff mode implementation. /// Diff mode implementation.
/// ///
/// This module provides functionality for comparing two items and displaying their /// This module provides functionality for comparing two items and displaying their
/// differences using external diff tools. /// differences using external diff tools.
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use clap::Command; use clap::Command;
use crate::config;
use crate::services::item_service::ItemService;
use log::debug; use log::debug;
fn validate_diff_args(_cmd: &mut Command, ids: &Vec<i64>, tags: &Vec<String>) -> anyhow::Result<()> { fn validate_diff_args(
_cmd: &mut Command,
ids: &Vec<i64>,
tags: &Vec<String>,
) -> anyhow::Result<()> {
if !tags.is_empty() { if !tags.is_empty() {
return Err(anyhow::anyhow!("Tags are not supported with --diff. Please provide exactly two IDs.")); return Err(anyhow::anyhow!(
"Tags are not supported with --diff. Please provide exactly two IDs."
));
} }
if ids.len() != 2 { if ids.len() != 2 {
return Err(anyhow::anyhow!("You must supply exactly two IDs when using --diff.")); return Err(anyhow::anyhow!(
"You must supply exactly two IDs when using --diff."
));
} }
Ok(()) Ok(())
} }
@@ -34,9 +42,12 @@ fn validate_diff_args(_cmd: &mut Command, ids: &Vec<i64>, tags: &Vec<String>) ->
/// * `Result<(ItemWithMeta, ItemWithMeta)>` - Tuple of items with metadata or error. /// * `Result<(ItemWithMeta, ItemWithMeta)>` - Tuple of items with metadata or error.
fn fetch_and_validate_items( fn fetch_and_validate_items(
conn: &mut rusqlite::Connection, conn: &mut rusqlite::Connection,
ids: &Vec<i64>, ids: &[i64],
item_service: &ItemService, item_service: &ItemService,
) -> Result<(crate::services::types::ItemWithMeta, crate::services::types::ItemWithMeta)> { ) -> Result<(
crate::services::types::ItemWithMeta,
crate::services::types::ItemWithMeta,
)> {
// Fetch items using the service, which handles validation // Fetch items using the service, which handles validation
let item_a = item_service let item_a = item_service
.get_item(conn, ids[0]) .get_item(conn, ids[0])
@@ -69,12 +80,15 @@ fn setup_diff_paths_and_compression(
item_service: &ItemService, item_service: &ItemService,
item_a: &crate::services::types::ItemWithMeta, item_a: &crate::services::types::ItemWithMeta,
item_b: &crate::services::types::ItemWithMeta, item_b: &crate::services::types::ItemWithMeta,
) -> Result<( ) -> Result<(std::path::PathBuf, std::path::PathBuf)> {
std::path::PathBuf, let item_a_id = item_a
std::path::PathBuf, .item
)> { .id
let item_a_id = item_a.item.id.ok_or_else(|| anyhow::anyhow!("Item A missing ID"))?; .ok_or_else(|| anyhow::anyhow!("Item A missing ID"))?;
let item_b_id = item_b.item.id.ok_or_else(|| anyhow::anyhow!("Item B missing ID"))?; let item_b_id = item_b
.item
.id
.ok_or_else(|| anyhow::anyhow!("Item B missing ID"))?;
// Use the service's data path to construct proper file paths // Use the service's data path to construct proper file paths
let data_path = item_service.get_data_path(); let data_path = item_service.get_data_path();

257
src/modes/generate_config.rs
View File

@@ -1,8 +1,8 @@
use crate::meta_plugin::MetaPlugin;
use anyhow::Result; use anyhow::Result;
use clap::Command; use clap::Command;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_yaml; use serde_yaml;
use crate::meta_plugin::MetaPlugin;
/// Mode for generating a default configuration file. /// Mode for generating a default configuration file.
/// ///
@@ -71,131 +71,131 @@ struct MetaPluginConfig {
outputs: std::collections::HashMap<String, String>, outputs: std::collections::HashMap<String, String>,
} }
/// Generates and prints a default commented YAML configuration template. /// Generates and prints a default commented YAML configuration template.
/// ///
/// Creates instances of available meta plugins to populate default options and outputs, /// Creates instances of available meta plugins to populate default options and outputs,
/// then serializes the config to YAML with all lines commented for easy editing. /// then serializes the config to YAML with all lines commented for easy editing.
/// ///
/// # Arguments /// # Arguments
/// ///
/// * `_cmd` - Unused Clap command reference. /// * `_cmd` - Unused Clap command reference.
/// * `_settings` - Unused settings reference. /// * `_settings` - Unused settings reference.
/// ///
/// # Returns /// # Returns
/// ///
/// `Ok(())` on success. /// `Ok(())` on success.
/// ///
/// # Examples /// # Examples
/// ///
/// ``` /// ```
/// mode_generate_config(&mut cmd, &settings)?; /// mode_generate_config(&mut cmd, &settings)?;
/// ``` /// ```
pub fn mode_generate_config(_cmd: &mut Command, _settings: &crate::config::Settings) -> Result<()> { pub fn mode_generate_config(_cmd: &mut Command, _settings: &crate::config::Settings) -> Result<()> {
// Create instances of each meta plugin to get their default options and outputs // Create instances of each meta plugin to get their default options and outputs
let cwd_plugin = crate::meta_plugin::cwd::CwdMetaPlugin::new(None, None); let cwd_plugin = crate::meta_plugin::cwd::CwdMetaPlugin::new(None, None);
let digest_plugin = crate::meta_plugin::digest::DigestMetaPlugin::new(None, None); let digest_plugin = crate::meta_plugin::digest::DigestMetaPlugin::new(None, None);
let hostname_plugin = crate::meta_plugin::hostname::HostnameMetaPlugin::new(None, None); let hostname_plugin = crate::meta_plugin::hostname::HostnameMetaPlugin::new(None, None);
#[cfg(feature = "magic")] #[cfg(feature = "magic")]
let magic_file_plugin = crate::meta_plugin::magic_file::MagicFileMetaPlugin::new(None, None); let magic_file_plugin = crate::meta_plugin::magic_file::MagicFileMetaPlugin::new(None, None);
let env_plugin = crate::meta_plugin::env::EnvMetaPlugin::new(None, None); let env_plugin = crate::meta_plugin::env::EnvMetaPlugin::new(None, None);
// Create a default configuration // Create a default configuration
let default_config = DefaultConfig { let default_config = DefaultConfig {
dir: Some("~/.local/share/keep".to_string()), dir: Some("~/.local/share/keep".to_string()),
list_format: vec![ list_format: vec![
ColumnConfig { ColumnConfig {
name: "id".to_string(), name: "id".to_string(),
label: Some("Item".to_string()), label: Some("Item".to_string()),
align: ColumnAlignment::Right, align: ColumnAlignment::Right,
max_len: None, max_len: None,
}, },
ColumnConfig { ColumnConfig {
name: "time".to_string(), name: "time".to_string(),
label: Some("Time".to_string()), label: Some("Time".to_string()),
align: ColumnAlignment::Right, align: ColumnAlignment::Right,
max_len: None, max_len: None,
}, },
ColumnConfig { ColumnConfig {
name: "size".to_string(), name: "size".to_string(),
label: Some("Size".to_string()), label: Some("Size".to_string()),
align: ColumnAlignment::Right, align: ColumnAlignment::Right,
max_len: None, max_len: None,
}, },
ColumnConfig { ColumnConfig {
name: "tags".to_string(), name: "tags".to_string(),
label: Some("Tags".to_string()), label: Some("Tags".to_string()),
align: ColumnAlignment::Left, align: ColumnAlignment::Left,
max_len: Some("40".to_string()), max_len: Some("40".to_string()),
}, },
ColumnConfig { ColumnConfig {
name: "meta:hostname_full".to_string(), name: "meta:hostname_full".to_string(),
label: Some("Hostname".to_string()), label: Some("Hostname".to_string()),
align: ColumnAlignment::Left, align: ColumnAlignment::Left,
max_len: Some("28".to_string()), max_len: Some("28".to_string()),
}, },
], ],
human_readable: false, human_readable: false,
output_format: Some("table".to_string()), output_format: Some("table".to_string()),
quiet: false, quiet: false,
force: false, force: false,
server: Some(ServerConfig { server: Some(ServerConfig {
address: Some("127.0.0.1".to_string()), address: Some("127.0.0.1".to_string()),
port: Some(8080), port: Some(8080),
password_file: None, password_file: None,
password: None, password: None,
password_hash: None, password_hash: None,
}), }),
compression_plugin: None, compression_plugin: None,
meta_plugins: Some(vec![ meta_plugins: Some(vec![
MetaPluginConfig { MetaPluginConfig {
name: "cwd".to_string(), name: "cwd".to_string(),
options: cwd_plugin.options().clone(), options: cwd_plugin.options().clone(),
outputs: convert_outputs_to_string_map(cwd_plugin.outputs()), outputs: convert_outputs_to_string_map(cwd_plugin.outputs()),
}, },
MetaPluginConfig { MetaPluginConfig {
name: "digest".to_string(), name: "digest".to_string(),
options: digest_plugin.options().clone(), options: digest_plugin.options().clone(),
outputs: convert_outputs_to_string_map(digest_plugin.outputs()), outputs: convert_outputs_to_string_map(digest_plugin.outputs()),
}, },
MetaPluginConfig { MetaPluginConfig {
name: "hostname".to_string(), name: "hostname".to_string(),
options: hostname_plugin.options().clone(), options: hostname_plugin.options().clone(),
outputs: convert_outputs_to_string_map(hostname_plugin.outputs()), outputs: convert_outputs_to_string_map(hostname_plugin.outputs()),
}, },
#[cfg(feature = "magic")] #[cfg(feature = "magic")]
MetaPluginConfig { MetaPluginConfig {
name: "magic_file".to_string(), name: "magic_file".to_string(),
options: magic_file_plugin.options().clone(), options: magic_file_plugin.options().clone(),
outputs: convert_outputs_to_string_map(magic_file_plugin.outputs()), outputs: convert_outputs_to_string_map(magic_file_plugin.outputs()),
}, },
MetaPluginConfig { MetaPluginConfig {
name: "env".to_string(), name: "env".to_string(),
options: env_plugin.options().clone(), options: env_plugin.options().clone(),
outputs: convert_outputs_to_string_map(env_plugin.outputs()), outputs: convert_outputs_to_string_map(env_plugin.outputs()),
}, },
]), ]),
}; };
// Serialize to YAML and comment out all lines // Serialize to YAML and comment out all lines
let yaml = serde_yaml::to_string(&default_config)?; let yaml = serde_yaml::to_string(&default_config)?;
// Comment out every line // Comment out every line
let commented_yaml = yaml let commented_yaml = yaml
.lines() .lines()
.map(|line| { .map(|line| {
if line.trim().is_empty() { if line.trim().is_empty() {
line.to_string() line.to_string()
} else { } else {
format!("# {}", line) format!("# {}", line)
} }
}) })
.collect::<Vec<String>>() .collect::<Vec<String>>()
.join("\n"); .join("\n");
println!("{}", commented_yaml); println!("{}", commented_yaml);
Ok(()) Ok(())
} }
/// Helper function to convert outputs from serde_yaml::Value to String. /// Helper function to convert outputs from serde_yaml::Value to String.
/// ///
@@ -223,7 +223,10 @@ fn convert_outputs_to_string_map(
} }
_ => { _ => {
// Convert other values to their YAML string representation // Convert other values to their YAML string representation
result.insert(key.clone(), serde_yaml::to_string(value).unwrap_or_default()); result.insert(
key.clone(),
serde_yaml::to_string(value).unwrap_or_default(),
);
} }
} }
} }

29
src/modes/get.rs
View File

@@ -1,15 +1,15 @@
use anyhow::{anyhow, Result}; use anyhow::{Result, anyhow};
use std::io::Write; use std::io::Write;
use crate::common::is_binary::is_binary;
use crate::common::PIPESIZE; use crate::common::PIPESIZE;
use crate::common::is_binary::is_binary;
use crate::config; use crate::config;
use crate::filter_plugin::FilterChain; use crate::filter_plugin::FilterChain;
use crate::services::item_service::ItemService; use crate::services::item_service::ItemService;
use clap::Command; use clap::Command;
use is_terminal::IsTerminal; use is_terminal::IsTerminal;
use std::path::PathBuf;
use std::io::Read; use std::io::Read;
use std::path::PathBuf;
/// Handles the get mode: retrieves and streams item content to stdout, applying filters if specified. /// Handles the get mode: retrieves and streams item content to stdout, applying filters if specified.
/// ///
@@ -29,25 +29,34 @@ use std::io::Read;
pub fn mode_get( pub fn mode_get(
cmd: &mut Command, cmd: &mut Command,
settings: &config::Settings, settings: &config::Settings,
ids: &mut Vec<i64>, ids: &mut [i64],
tags: &mut Vec<String>, tags: &mut [String],
conn: &mut rusqlite::Connection, conn: &mut rusqlite::Connection,
data_path: PathBuf, data_path: PathBuf,
filter_chain: Option<FilterChain>, filter_chain: Option<FilterChain>,
) -> Result<()> { ) -> Result<()> {
if !ids.is_empty() && !tags.is_empty() { if !ids.is_empty() && !tags.is_empty() {
cmd.error(clap::error::ErrorKind::InvalidValue, "Both ID and tags given, you must supply either IDs or tags when using --get").exit(); cmd.error(
clap::error::ErrorKind::InvalidValue,
"Both ID and tags given, you must supply either IDs or tags when using --get",
)
.exit();
} else if ids.len() > 1 { } else if ids.len() > 1 {
cmd.error(clap::error::ErrorKind::InvalidValue, "More than one ID given, you must supply exactly one ID when using --get").exit(); cmd.error(
clap::error::ErrorKind::InvalidValue,
"More than one ID given, you must supply exactly one ID when using --get",
)
.exit();
} }
// If both are empty, find_item will find the last item // If both are empty, find_item will find the last item
let item_service = ItemService::new(data_path.clone()); let item_service = ItemService::new(data_path.clone());
let item_with_meta = item_service.find_item(conn, ids, tags, &std::collections::HashMap::new()) let item_with_meta = item_service
.find_item(conn, ids, tags, &std::collections::HashMap::new())
.map_err(|e| anyhow!("Unable to find matching item in database: {}", e))?; .map_err(|e| anyhow!("Unable to find matching item in database: {}", e))?;
let item_id = item_with_meta.item.id.unwrap(); let item_id = item_with_meta.item.id.unwrap();
// Determine if we should detect binary data // Determine if we should detect binary data
let mut detect_binary = !settings.force && std::io::stdout().is_terminal(); let mut detect_binary = !settings.force && std::io::stdout().is_terminal();

36
src/modes/info.rs
View File

@@ -1,15 +1,15 @@
use crate::config; use crate::config;
use crate::modes::common::{OutputFormat, format_size};
use crate::services::types::ItemWithMeta; use crate::services::types::ItemWithMeta;
use crate::modes::common::{format_size, OutputFormat}; use anyhow::{Result, anyhow};
use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use clap::Command; use clap::Command;
use clap::error::ErrorKind; use clap::error::ErrorKind;
use serde::{Deserialize, Serialize};
use std::path::PathBuf; use std::path::PathBuf;
use crate::services::item_service::ItemService; use crate::services::item_service::ItemService;
use chrono::prelude::*; use chrono::prelude::*;
use comfy_table::{Cell, Attribute}; use comfy_table::{Attribute, Cell};
/// Displays detailed information about an item or the last item if no ID/tags specified. /// Displays detailed information about an item or the last item if no ID/tags specified.
/// ///
@@ -42,16 +42,24 @@ use comfy_table::{Cell, Attribute};
pub fn mode_info( pub fn mode_info(
cmd: &mut Command, cmd: &mut Command,
settings: &config::Settings, settings: &config::Settings,
ids: &mut Vec<i64>, ids: &mut [i64],
tags: &mut Vec<String>, tags: &mut [String],
conn: &mut rusqlite::Connection, conn: &mut rusqlite::Connection,
data_path: PathBuf, data_path: PathBuf,
) -> Result<()> { ) -> Result<()> {
// For --info, we can use either IDs or tags, but not both // For --info, we can use either IDs or tags, but not both
if !ids.is_empty() && !tags.is_empty() { if !ids.is_empty() && !tags.is_empty() {
cmd.error(ErrorKind::InvalidValue, "Both ID and tags given, you must supply either IDs or tags when using --info").exit(); cmd.error(
ErrorKind::InvalidValue,
"Both ID and tags given, you must supply either IDs or tags when using --info",
)
.exit();
} else if ids.len() > 1 { } else if ids.len() > 1 {
cmd.error(ErrorKind::InvalidValue, "More than one ID given, you must supply exactly one ID when using --info").exit(); cmd.error(
ErrorKind::InvalidValue,
"More than one ID given, you must supply exactly one ID when using --info",
)
.exit();
} }
// If both are empty, find_item will find the last item // If both are empty, find_item will find the last item
@@ -139,7 +147,7 @@ fn show_item(
// Add all the rows // Add all the rows
table.add_row(vec![ table.add_row(vec![
Cell::new("ID").add_attribute(Attribute::Bold), Cell::new("ID").add_attribute(Attribute::Bold),
Cell::new(&item_id.to_string()), Cell::new(item_id.to_string()),
]); ]);
let timestamp_str = item.ts.with_timezone(&Local).format("%F %T %Z").to_string(); let timestamp_str = item.ts.with_timezone(&Local).format("%F %T %Z").to_string();
@@ -150,7 +158,10 @@ fn show_item(
let mut item_path_buf = data_path.clone(); let mut item_path_buf = data_path.clone();
item_path_buf.push(item.id.unwrap().to_string()); item_path_buf.push(item.id.unwrap().to_string());
let path_str = item_path_buf.to_str().expect("Unable to get item path").to_string(); let path_str = item_path_buf
.to_str()
.expect("Unable to get item path")
.to_string();
table.add_row(vec![ table.add_row(vec![
Cell::new("Path").add_attribute(Attribute::Bold), Cell::new("Path").add_attribute(Attribute::Bold),
Cell::new(&path_str), Cell::new(&path_str),
@@ -194,7 +205,10 @@ fn show_item(
]); ]);
} }
println!("{}", crate::modes::common::trim_lines_end(&table.trim_fmt())); println!(
"{}",
crate::modes::common::trim_lines_end(&table.trim_fmt())
);
Ok(()) Ok(())
} }

71
src/modes/list.rs
View File

@@ -4,13 +4,13 @@
/// formatting, filtering by tags, and support for different output formats /// formatting, filtering by tags, and support for different output formats
/// including table, JSON, and YAML. /// including table, JSON, and YAML.
use crate::config; use crate::config;
use crate::modes::common::ColumnType;
use crate::modes::common::{OutputFormat, format_size};
use crate::services::item_service::ItemService; use crate::services::item_service::ItemService;
use crate::services::types::ItemWithMeta; use crate::services::types::ItemWithMeta;
use crate::modes::common::ColumnType; use anyhow::Result;
use crate::modes::common::{format_size, OutputFormat};
use anyhow::{Result};
use comfy_table::{Cell, Row, Color, Attribute};
use comfy_table::CellAlignment; use comfy_table::CellAlignment;
use comfy_table::{Attribute, Cell, Color, Row};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json; use serde_json;
use serde_yaml; use serde_yaml;
@@ -80,7 +80,7 @@ struct ListItem {
fn apply_color(mut cell: Cell, color: &crate::config::TableColor, is_foreground: bool) -> Cell { fn apply_color(mut cell: Cell, color: &crate::config::TableColor, is_foreground: bool) -> Cell {
use crate::config::TableColor::*; use crate::config::TableColor::*;
use comfy_table::Color; use comfy_table::Color;
let comfy_color = match color { let comfy_color = match color {
Black => Color::Black, Black => Color::Black,
Red => Color::Red, Red => Color::Red,
@@ -97,15 +97,19 @@ fn apply_color(mut cell: Cell, color: &crate::config::TableColor, is_foreground:
DarkBlue => Color::DarkBlue, DarkBlue => Color::DarkBlue,
DarkMagenta => Color::DarkMagenta, DarkMagenta => Color::DarkMagenta,
DarkCyan => Color::DarkCyan, DarkCyan => Color::DarkCyan,
Rgb(r, g, b) => Color::Rgb { r: *r, g: *g, b: *b }, Rgb(r, g, b) => Color::Rgb {
r: *r,
g: *g,
b: *b,
},
}; };
if is_foreground { if is_foreground {
cell = cell.fg(comfy_color); cell = cell.fg(comfy_color);
} else { } else {
cell = cell.bg(comfy_color); cell = cell.bg(comfy_color);
} }
cell cell
} }
@@ -125,7 +129,7 @@ fn apply_color(mut cell: Cell, color: &crate::config::TableColor, is_foreground:
fn apply_attribute(mut cell: Cell, attribute: &crate::config::TableAttribute) -> Cell { fn apply_attribute(mut cell: Cell, attribute: &crate::config::TableAttribute) -> Cell {
use crate::config::TableAttribute::*; use crate::config::TableAttribute::*;
use comfy_table::Attribute; use comfy_table::Attribute;
match attribute { match attribute {
Bold => cell = cell.add_attribute(Attribute::Bold), Bold => cell = cell.add_attribute(Attribute::Bold),
Dim => cell = cell.add_attribute(Attribute::Dim), Dim => cell = cell.add_attribute(Attribute::Dim),
@@ -137,7 +141,7 @@ fn apply_attribute(mut cell: Cell, attribute: &crate::config::TableAttribute) ->
Hidden => cell = cell.add_attribute(Attribute::Hidden), Hidden => cell = cell.add_attribute(Attribute::Hidden),
CrossedOut => cell = cell.add_attribute(Attribute::CrossedOut), CrossedOut => cell = cell.add_attribute(Attribute::CrossedOut),
} }
cell cell
} }
@@ -161,8 +165,8 @@ fn apply_attribute(mut cell: Cell, attribute: &crate::config::TableAttribute) ->
pub fn mode_list( pub fn mode_list(
cmd: &mut clap::Command, cmd: &mut clap::Command,
settings: &config::Settings, settings: &config::Settings,
ids: &mut Vec<i64>, ids: &mut [i64],
tags: &Vec<String>, tags: &[String],
conn: &mut rusqlite::Connection, conn: &mut rusqlite::Connection,
data_path: std::path::PathBuf, data_path: std::path::PathBuf,
) -> Result<()> { ) -> Result<()> {
@@ -203,7 +207,9 @@ pub fn mode_list(
let mut table_row = Row::new(); let mut table_row = Row::new();
for column in &settings.list_format { for column in &settings.list_format {
let column_type = ColumnType::from_str(&column.name) let column_type = column
.name
.parse::<ColumnType>()
.unwrap_or_else(|_| panic!("Unknown column {:?}", column.name)); .unwrap_or_else(|_| panic!("Unknown column {:?}", column.name));
let mut meta_name: Option<&str> = None; let mut meta_name: Option<&str> = None;
@@ -217,7 +223,8 @@ pub fn mode_list(
let cell_content = match column_type { let cell_content = match column_type {
ColumnType::Id => item.id.unwrap_or(0).to_string(), ColumnType::Id => item.id.unwrap_or(0).to_string(),
ColumnType::Time => item.ts ColumnType::Time => item
.ts
.with_timezone(&chrono::Local) .with_timezone(&chrono::Local)
.format("%F %T") .format("%F %T")
.to_string(), .to_string(),
@@ -243,9 +250,10 @@ pub fn mode_list(
None => "".to_string(), None => "".to_string(),
}, },
}; };
// Truncate content to max 3 lines // Truncate content to max 3 lines
let mut cell_lines: Vec<String> = cell_content.split('\n').map(|s| s.to_string()).collect(); let mut cell_lines: Vec<String> =
cell_content.split('\n').map(|s| s.to_string()).collect();
if cell_lines.len() > 3 { if cell_lines.len() > 3 {
cell_lines.truncate(3); cell_lines.truncate(3);
// Add ellipsis to the last line if we truncated // Add ellipsis to the last line if we truncated
@@ -257,47 +265,53 @@ pub fn mode_list(
} }
} }
let truncated_content = cell_lines.join("\n"); let truncated_content = cell_lines.join("\n");
let mut cell = Cell::new(truncated_content); let mut cell = Cell::new(truncated_content);
// Apply column-specific styling // Apply column-specific styling
if let Some(fg_color) = &column.fg_color { if let Some(fg_color) = &column.fg_color {
cell = apply_color(cell, fg_color, true); cell = apply_color(cell, fg_color, true);
} }
if let Some(bg_color) = &column.bg_color { if let Some(bg_color) = &column.bg_color {
cell = apply_color(cell, bg_color, false); cell = apply_color(cell, bg_color, false);
} }
for attribute in &column.attributes { for attribute in &column.attributes {
cell = apply_attribute(cell, attribute); cell = apply_attribute(cell, attribute);
} }
// Apply padding if specified // Apply padding if specified
if let Some((_left_padding, _right_padding)) = column.padding { if let Some((_left_padding, _right_padding)) = column.padding {
// Note: comfy-table doesn't directly support padding, so we'd need to handle this // Note: comfy-table doesn't directly support padding, so we'd need to handle this
// by adding spaces to the content, or use a different approach // by adding spaces to the content, or use a different approach
} }
// Apply styling for specific cases // Apply styling for specific cases
match column_type { match column_type {
ColumnType::Size => { ColumnType::Size => {
if item.size.is_none() { if item.size.is_none() {
if item_path.metadata().is_ok() { if item_path.metadata().is_ok() {
cell = cell.fg(comfy_table::Color::Yellow).add_attribute(Attribute::Bold); cell = cell
.fg(comfy_table::Color::Yellow)
.add_attribute(Attribute::Bold);
} else { } else {
cell = cell.fg(comfy_table::Color::Red).add_attribute(Attribute::Bold); cell = cell
.fg(comfy_table::Color::Red)
.add_attribute(Attribute::Bold);
} }
} }
} }
ColumnType::FileSize => { ColumnType::FileSize => {
if item_path.metadata().is_err() { if item_path.metadata().is_err() {
cell = cell.fg(comfy_table::Color::Red).add_attribute(Attribute::Bold); cell = cell
.fg(comfy_table::Color::Red)
.add_attribute(Attribute::Bold);
} }
} }
_ => {} _ => {}
} }
// Apply alignment // Apply alignment
cell = match column.align { cell = match column.align {
crate::config::ColumnAlignment::Right => cell.set_alignment(CellAlignment::Right), crate::config::ColumnAlignment::Right => cell.set_alignment(CellAlignment::Right),
@@ -309,7 +323,10 @@ pub fn mode_list(
table.add_row(table_row); table.add_row(table_row);
} }
println!("{}", crate::modes::common::trim_lines_end(&table.trim_fmt())); println!(
"{}",
crate::modes::common::trim_lines_end(&table.trim_fmt())
);
Ok(()) Ok(())
} }

4
src/modes/save.rs
View File

@@ -18,7 +18,7 @@ use crate::services::item_service::ItemService;
/// # Panics /// # Panics
/// ///
/// Exits the program via Clap error if IDs are provided. /// Exits the program via Clap error if IDs are provided.
fn validate_save_args(cmd: &mut Command, ids: &Vec<i64>) { fn validate_save_args(cmd: &mut Command, ids: &[i64]) {
if !ids.is_empty() { if !ids.is_empty() {
cmd.error( cmd.error(
clap::error::ErrorKind::InvalidValue, clap::error::ErrorKind::InvalidValue,
@@ -111,7 +111,7 @@ impl<R: Read, W: Write> Read for TeeReader<R, W> {
pub fn mode_save( pub fn mode_save(
cmd: &mut Command, cmd: &mut Command,
settings: &config::Settings, settings: &config::Settings,
ids: &mut Vec<i64>, ids: &mut [i64],
tags: &mut Vec<String>, tags: &mut Vec<String>,
conn: &mut rusqlite::Connection, conn: &mut rusqlite::Connection,
data_path: std::path::PathBuf, data_path: std::path::PathBuf,

181
src/modes/server/api/item.rs
View File

@@ -1,3 +1,9 @@
use crate::modes::server::common::{
ApiResponse, AppState, ItemContentQuery, ItemInfo, ItemInfoListResponse, ItemInfoResponse,
ItemQuery, ListItemsQuery, MetadataResponse, TagsQuery,
};
use crate::services::async_item_service::AsyncItemService;
use crate::services::error::CoreError;
use axum::{ use axum::{
extract::{Path, Query, State}, extract::{Path, Query, State},
http::{StatusCode, header}, http::{StatusCode, header},
@@ -5,9 +11,6 @@ use axum::{
}; };
use log::{debug, warn}; use log::{debug, warn};
use std::collections::HashMap; use std::collections::HashMap;
use crate::services::async_item_service::AsyncItemService;
use crate::services::error::CoreError;
use crate::modes::server::common::{AppState, ApiResponse, ItemInfo, TagsQuery, ListItemsQuery, ItemInfoListResponse, ItemInfoResponse, MetadataResponse, ItemQuery, ItemContentQuery};
// Helper functions to replace the missing binary_detection module // Helper functions to replace the missing binary_detection module
async fn check_binary_content_allowed( async fn check_binary_content_allowed(
@@ -35,13 +38,17 @@ async fn is_content_binary(
Ok(text_val == "false") Ok(text_val == "false")
} else { } else {
// If text metadata isn't set, we need to check the content using streaming approach // If text metadata isn't set, we need to check the content using streaming approach
match item_service.get_item_content_info_streaming( match item_service
item_id, .get_item_content_info_streaming(item_id, None)
None .await
).await { {
Ok((_, _, is_binary)) => Ok(is_binary), Ok((_, _, is_binary)) => Ok(is_binary),
Err(e) => { Err(e) => {
log::warn!("Failed to get content info for binary check for item {}: {}", item_id, e); log::warn!(
"Failed to get content info for binary check for item {}: {}",
item_id,
e
);
Err(StatusCode::INTERNAL_SERVER_ERROR) Err(StatusCode::INTERNAL_SERVER_ERROR)
} }
} }
@@ -63,7 +70,7 @@ impl ResponseBuilder {
log::warn!("Failed to serialize response: {}", e); log::warn!("Failed to serialize response: {}", e);
StatusCode::INTERNAL_SERVER_ERROR StatusCode::INTERNAL_SERVER_ERROR
})?; })?;
Response::builder() Response::builder()
.header(header::CONTENT_TYPE, "application/json") .header(header::CONTENT_TYPE, "application/json")
.header(header::CONTENT_LENGTH, json.len().to_string()) .header(header::CONTENT_LENGTH, json.len().to_string())
@@ -73,7 +80,7 @@ impl ResponseBuilder {
StatusCode::INTERNAL_SERVER_ERROR StatusCode::INTERNAL_SERVER_ERROR
}) })
} }
pub fn binary(content: &[u8], mime_type: &str) -> Result<Response, StatusCode> { pub fn binary(content: &[u8], mime_type: &str) -> Result<Response, StatusCode> {
Response::builder() Response::builder()
.header(header::CONTENT_TYPE, mime_type) .header(header::CONTENT_TYPE, mime_type)
@@ -86,7 +93,6 @@ impl ResponseBuilder {
} }
} }
/// Helper function to get mime type from metadata /// Helper function to get mime type from metadata
fn get_mime_type(metadata: &HashMap<String, String>) -> String { fn get_mime_type(metadata: &HashMap<String, String>) -> String {
metadata metadata
@@ -104,7 +110,7 @@ fn apply_offset_length(content: &[u8], offset: u64, length: u64) -> &[u8] {
} else { } else {
content_len content_len
}; };
if start < content_len { if start < content_len {
&content[start as usize..end as usize] &content[start as usize..end as usize]
} else { } else {
@@ -126,11 +132,11 @@ fn handle_item_error(error: CoreError) -> StatusCode {
/// Helper function to create AsyncItemService from AppState /// Helper function to create AsyncItemService from AppState
fn create_item_service(state: &AppState) -> AsyncItemService { fn create_item_service(state: &AppState) -> AsyncItemService {
AsyncItemService::new( AsyncItemService::new(
state.data_dir.clone(), state.data_dir.clone(),
state.db.clone(), state.db.clone(),
state.item_service.clone(), state.item_service.clone(),
state.cmd.clone(), state.cmd.clone(),
state.settings.clone() state.settings.clone(),
) )
} }
@@ -185,13 +191,18 @@ pub async fn handle_list_items(
// Apply pagination // Apply pagination
let start = params.start.unwrap_or(0) as usize; let start = params.start.unwrap_or(0) as usize;
let count = params.count.unwrap_or(100) as usize; let count = params.count.unwrap_or(100) as usize;
let items_with_meta: Vec<_> = items_with_meta.into_iter().skip(start).take(count).collect(); let items_with_meta: Vec<_> = items_with_meta
.into_iter()
.skip(start)
.take(count)
.collect();
let item_infos: Vec<ItemInfo> = items_with_meta let item_infos: Vec<ItemInfo> = items_with_meta
.into_iter() .into_iter()
.map(|item_with_meta| { .map(|item_with_meta| {
let item_id = item_with_meta.item.id.unwrap_or(0); let item_id = item_with_meta.item.id.unwrap_or(0);
let item_tags: Vec<String> = item_with_meta.tags.iter().map(|t| t.name.clone()).collect(); let item_tags: Vec<String> =
item_with_meta.tags.iter().map(|t| t.name.clone()).collect();
let item_meta = item_with_meta.meta_as_map(); let item_meta = item_with_meta.meta_as_map();
ItemInfo { ItemInfo {
@@ -239,7 +250,7 @@ async fn handle_as_meta_response_with_metadata(
) -> Result<Response, StatusCode> { ) -> Result<Response, StatusCode> {
// Check if content is binary // Check if content is binary
let is_binary = is_content_binary(item_service, item_id, metadata).await?; let is_binary = is_content_binary(item_service, item_id, metadata).await?;
// Get the content if it's not binary // Get the content if it's not binary
if is_binary { if is_binary {
// Return JSON with content as None and error message // Return JSON with content as None and error message
@@ -248,7 +259,7 @@ async fn handle_as_meta_response_with_metadata(
"content": serde_json::Value::Null, "content": serde_json::Value::Null,
"error": "Content is binary" "error": "Content is binary"
}); });
Response::builder() Response::builder()
.header(header::CONTENT_TYPE, "application/json") .header(header::CONTENT_TYPE, "application/json")
.status(StatusCode::UNPROCESSABLE_ENTITY) .status(StatusCode::UNPROCESSABLE_ENTITY)
@@ -256,10 +267,7 @@ async fn handle_as_meta_response_with_metadata(
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
} else { } else {
// Get the content as text // Get the content as text
match item_service.get_item_content_info( match item_service.get_item_content_info(item_id, None).await {
item_id,
None
).await {
Ok((content, _, _)) => { Ok((content, _, _)) => {
// Apply offset and length // Apply offset and length
let content_len = content.len() as u64; let content_len = content.len() as u64;
@@ -269,13 +277,13 @@ async fn handle_as_meta_response_with_metadata(
} else { } else {
content_len content_len
}; };
let response_content = if start < content_len { let response_content = if start < content_len {
&content[start as usize..end as usize] &content[start as usize..end as usize]
} else { } else {
&[] &[]
}; };
// Convert to UTF-8 string // Convert to UTF-8 string
let content_str = match String::from_utf8(response_content.to_vec()) { let content_str = match String::from_utf8(response_content.to_vec()) {
Ok(s) => s, Ok(s) => s,
@@ -286,7 +294,7 @@ async fn handle_as_meta_response_with_metadata(
"content": serde_json::Value::Null, "content": serde_json::Value::Null,
"error": "Content is not valid UTF-8" "error": "Content is not valid UTF-8"
}); });
let response = Response::builder() let response = Response::builder()
.header(header::CONTENT_TYPE, "application/json") .header(header::CONTENT_TYPE, "application/json")
.status(StatusCode::UNPROCESSABLE_ENTITY) .status(StatusCode::UNPROCESSABLE_ENTITY)
@@ -295,14 +303,14 @@ async fn handle_as_meta_response_with_metadata(
return Ok(response); return Ok(response);
} }
}; };
// Return JSON with metadata and content // Return JSON with metadata and content
let response_body = serde_json::json!({ let response_body = serde_json::json!({
"metadata": metadata, "metadata": metadata,
"content": content_str, "content": content_str,
"error": serde_json::Value::Null "error": serde_json::Value::Null
}); });
Response::builder() Response::builder()
.header(header::CONTENT_TYPE, "application/json") .header(header::CONTENT_TYPE, "application/json")
.body(axum::body::Body::from(response_body.to_string())) .body(axum::body::Body::from(response_body.to_string()))
@@ -316,7 +324,6 @@ async fn handle_as_meta_response_with_metadata(
} }
} }
#[utoipa::path( #[utoipa::path(
post, post,
path = "/api/item/", path = "/api/item/",
@@ -342,21 +349,19 @@ async fn handle_as_meta_response_with_metadata(
pub async fn handle_post_item( pub async fn handle_post_item(
State(_state): State<AppState>, State(_state): State<AppState>,
) -> Result<Json<ApiResponse<ItemInfo>>, StatusCode> { ) -> Result<Json<ApiResponse<ItemInfo>>, StatusCode> {
// This is a simplified implementation // This is a simplified implementation
// In a real implementation, you'd need to properly parse multipart/form-data // In a real implementation, you'd need to properly parse multipart/form-data
// or JSON payload with the item data // or JSON payload with the item data
let response = ApiResponse::<ItemInfo> { let response = ApiResponse::<ItemInfo> {
success: false, success: false,
data: None, data: None,
error: Some("POST /api/item/ not yet implemented".to_string()), error: Some("POST /api/item/ not yet implemented".to_string()),
}; };
Ok(Json(response)) Ok(Json(response))
} }
#[utoipa::path( #[utoipa::path(
get, get,
path = "/api/item/latest/content", path = "/api/item/latest/content",
@@ -397,9 +402,7 @@ pub async fn handle_get_item_latest_content(
let item_service = create_item_service(&state); let item_service = create_item_service(&state);
// First find the item to get its ID and metadata // First find the item to get its ID and metadata
let item_with_meta = item_service let item_with_meta = item_service.find_item(vec![], tags, HashMap::new()).await;
.find_item(vec![], tags, HashMap::new())
.await;
match item_with_meta { match item_with_meta {
Ok(item) => { Ok(item) => {
@@ -408,9 +411,26 @@ pub async fn handle_get_item_latest_content(
// Handle as_meta parameter // Handle as_meta parameter
if params.as_meta { if params.as_meta {
// Force stream=false and allow_binary=false for as_meta=true // Force stream=false and allow_binary=false for as_meta=true
handle_as_meta_response_with_metadata(&item_service, item_id, &metadata, params.offset, params.length).await handle_as_meta_response_with_metadata(
&item_service,
item_id,
&metadata,
params.offset,
params.length,
)
.await
} else { } else {
stream_item_content_response_with_metadata(&item_service, item_id, &metadata, params.allow_binary, params.offset, params.length, params.stream, None).await stream_item_content_response_with_metadata(
&item_service,
item_id,
&metadata,
params.allow_binary,
params.offset,
params.length,
params.stream,
None,
)
.await
} }
} }
Err(CoreError::ItemNotFoundGeneric) => Err(StatusCode::NOT_FOUND), Err(CoreError::ItemNotFoundGeneric) => Err(StatusCode::NOT_FOUND),
@@ -421,7 +441,6 @@ pub async fn handle_get_item_latest_content(
} }
} }
#[utoipa::path( #[utoipa::path(
get, get,
path = "/api/item/{item_id}/content", path = "/api/item/{item_id}/content",
@@ -459,8 +478,10 @@ pub async fn handle_get_item_content(
return Err(StatusCode::BAD_REQUEST); return Err(StatusCode::BAD_REQUEST);
} }
debug!("ITEM_API: Getting content for item {} with stream={}, allow_binary={}, offset={}, length={}", debug!(
item_id, params.stream, params.allow_binary, params.offset, params.length); "ITEM_API: Getting content for item {} with stream={}, allow_binary={}, offset={}, length={}",
item_id, params.stream, params.allow_binary, params.offset, params.length
);
let filter = build_filter_string(&params); let filter = build_filter_string(&params);
@@ -468,15 +489,31 @@ pub async fn handle_get_item_content(
// Handle as_meta parameter // Handle as_meta parameter
if params.as_meta { if params.as_meta {
// Force stream=false and allow_binary=false for as_meta=true // Force stream=false and allow_binary=false for as_meta=true
let result = handle_as_meta_response(&item_service, item_id, params.offset, params.length).await; let result =
handle_as_meta_response(&item_service, item_id, params.offset, params.length).await;
if let Ok(response) = &result { if let Ok(response) = &result {
debug!("ITEM_API: Response content-length: {:?}", response.headers().get("content-length")); debug!(
"ITEM_API: Response content-length: {:?}",
response.headers().get("content-length")
);
} }
result result
} else { } else {
let result = stream_item_content_response(&item_service, item_id, params.allow_binary, params.offset, params.length, params.stream, filter).await; let result = stream_item_content_response(
&item_service,
item_id,
params.allow_binary,
params.offset,
params.length,
params.stream,
filter,
)
.await;
if let Ok(response) = &result { if let Ok(response) = &result {
debug!("ITEM_API: Response content-length: {:?}", response.headers().get("content-length")); debug!(
"ITEM_API: Response content-length: {:?}",
response.headers().get("content-length")
);
} }
result result
} }
@@ -499,7 +536,17 @@ async fn stream_item_content_response(
})?; })?;
let metadata = item_with_meta.meta_as_map(); let metadata = item_with_meta.meta_as_map();
stream_item_content_response_with_metadata(item_service, item_id, &metadata, allow_binary, offset, length, stream, filter).await stream_item_content_response_with_metadata(
item_service,
item_id,
&metadata,
allow_binary,
offset,
length,
stream,
filter,
)
.await
} }
async fn stream_item_content_response_with_metadata( async fn stream_item_content_response_with_metadata(
@@ -512,22 +559,23 @@ async fn stream_item_content_response_with_metadata(
stream: bool, stream: bool,
filter: Option<String>, filter: Option<String>,
) -> Result<Response, StatusCode> { ) -> Result<Response, StatusCode> {
debug!("STREAM_ITEM_CONTENT_RESPONSE_WITH_METADATA: stream={}", stream); debug!(
"STREAM_ITEM_CONTENT_RESPONSE_WITH_METADATA: stream={}",
stream
);
let mime_type = get_mime_type(metadata); let mime_type = get_mime_type(metadata);
// Check if content is binary when allow_binary is false // Check if content is binary when allow_binary is false
check_binary_content_allowed(item_service, item_id, metadata, allow_binary).await?; check_binary_content_allowed(item_service, item_id, metadata, allow_binary).await?;
if stream { if stream {
debug!("STREAMING: Using streaming approach"); debug!("STREAMING: Using streaming approach");
match item_service.stream_item_content_by_id_with_metadata( match item_service
item_id, .stream_item_content_by_id_with_metadata(
metadata, item_id, metadata, true, offset, length, filter,
true, )
offset, .await
length, {
filter
).await {
Ok((stream, _)) => { Ok((stream, _)) => {
let body = axum::body::Body::from_stream(stream); let body = axum::body::Body::from_stream(stream);
let response = Response::builder() let response = Response::builder()
@@ -543,16 +591,16 @@ async fn stream_item_content_response_with_metadata(
} }
} else { } else {
debug!("NON-STREAMING: Building full response in memory"); debug!("NON-STREAMING: Building full response in memory");
match item_service.get_item_content_info( match item_service.get_item_content_info(item_id, filter).await {
item_id,
filter
).await {
Ok((content, _, _)) => { Ok((content, _, _)) => {
let response_content = apply_offset_length(&content, offset, length); let response_content = apply_offset_length(&content, offset, length);
debug!("NON-STREAMING: Content length: {}, response length: {}", debug!(
content.len(), response_content.len()); "NON-STREAMING: Content length: {}, response length: {}",
content.len(),
response_content.len()
);
ResponseBuilder::binary(response_content, &mime_type) ResponseBuilder::binary(response_content, &mime_type)
} }
Err(e) => { Err(e) => {
@@ -563,8 +611,6 @@ async fn stream_item_content_response_with_metadata(
} }
} }
#[utoipa::path( #[utoipa::path(
get, get,
path = "/api/item/latest/meta", path = "/api/item/latest/meta",
@@ -655,4 +701,3 @@ pub async fn handle_get_item_meta(
Err(e) => Err(handle_item_error(e)), Err(e) => Err(handle_item_error(e)),
} }
} }

14
src/modes/server/api/mcp.rs
View File

@@ -1,7 +1,7 @@
use axum::{ use axum::{
extract::State, extract::State,
response::sse::{Event, KeepAlive, Sse},
http::StatusCode, http::StatusCode,
response::sse::{Event, KeepAlive, Sse},
}; };
use futures::stream::{self, Stream}; use futures::stream::{self, Stream};
use log::{debug, info}; use log::{debug, info};
@@ -31,15 +31,15 @@ pub async fn handle_mcp_sse(
State(state): State<AppState>, State(state): State<AppState>,
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, StatusCode> { ) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, StatusCode> {
debug!("MCP: Starting SSE endpoint"); debug!("MCP: Starting SSE endpoint");
let _mcp_server = KeepMcpServer::new(state); let _mcp_server = KeepMcpServer::new(state);
// Create a simple message channel for SSE communication // Create a simple message channel for SSE communication
let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<String>(); let (tx, rx) = tokio::sync::mpsc::unbounded_channel::<String>();
// Send initial connection message // Send initial connection message
let _ = tx.send("data: {\"type\":\"connection\",\"status\":\"connected\"}\n\n".to_string()); let _ = tx.send("data: {\"type\":\"connection\",\"status\":\"connected\"}\n\n".to_string());
// For now, create a simple stream that sends periodic keep-alive messages // For now, create a simple stream that sends periodic keep-alive messages
// In a full implementation, this would integrate with the rmcp transport layer // In a full implementation, this would integrate with the rmcp transport layer
let stream = stream::unfold((rx, tx), |(mut rx, tx)| async move { let stream = stream::unfold((rx, tx), |(mut rx, tx)| async move {
@@ -61,9 +61,9 @@ pub async fn handle_mcp_sse(
} }
} }
}); });
info!("MCP: SSE endpoint established"); info!("MCP: SSE endpoint established");
Ok(Sse::new(stream).keep_alive( Ok(Sse::new(stream).keep_alive(
KeepAlive::new() KeepAlive::new()
.interval(Duration::from_secs(30)) .interval(Duration::from_secs(30))

35
src/modes/server/api/mod.rs
View File

@@ -1,13 +1,10 @@
#[cfg(feature = "swagger")] #[cfg(feature = "swagger")]
pub mod item; pub mod item;
pub mod status;
#[cfg(feature = "mcp")] #[cfg(feature = "mcp")]
pub mod mcp; pub mod mcp;
pub mod status;
use axum::{ use axum::{Router, routing::get};
routing::get,
Router,
};
use crate::modes::server::common::AppState; use crate::modes::server::common::AppState;
use utoipa::OpenApi; use utoipa::OpenApi;
@@ -62,26 +59,36 @@ pub fn add_routes(router: Router<AppState>) -> Router<AppState> {
let router = router let router = router
// Status endpoints // Status endpoints
.route("/api/status", get(status::handle_status)) .route("/api/status", get(status::handle_status))
// Item endpoints // Item endpoints
.route("/api/item/", get(item::handle_list_items).post(item::handle_post_item)) .route(
.route("/api/item/latest/meta", get(item::handle_get_item_latest_meta)) "/api/item/",
.route("/api/item/latest/content", get(item::handle_get_item_latest_content)) get(item::handle_list_items).post(item::handle_post_item),
)
.route(
"/api/item/latest/meta",
get(item::handle_get_item_latest_meta),
)
.route(
"/api/item/latest/content",
get(item::handle_get_item_latest_content),
)
.route("/api/item/{item_id}/meta", get(item::handle_get_item_meta)) .route("/api/item/{item_id}/meta", get(item::handle_get_item_meta))
.route("/api/item/{item_id}/content", get(item::handle_get_item_content)); .route(
"/api/item/{item_id}/content",
get(item::handle_get_item_content),
);
#[cfg(feature = "mcp")] #[cfg(feature = "mcp")]
{ {
router = router.route("/mcp/sse", get(mcp::handle_mcp_sse)); router = router.route("/mcp/sse", get(mcp::handle_mcp_sse));
} }
router router
} }
#[cfg(feature = "swagger")] #[cfg(feature = "swagger")]
pub fn add_docs_routes(router: Router<AppState>) -> Router<AppState> { pub fn add_docs_routes(router: Router<AppState>) -> Router<AppState> {
router router.merge(SwaggerUi::new("/swagger").url("/openapi.json", ApiDoc::openapi()))
.merge(SwaggerUi::new("/swagger").url("/openapi.json", ApiDoc::openapi()))
} }
#[cfg(not(feature = "swagger"))] #[cfg(not(feature = "swagger"))]

17
src/modes/server/api/status.rs
View File

@@ -1,8 +1,4 @@
use axum::{ use axum::{extract::State, http::StatusCode, response::Json};
extract::State,
http::StatusCode,
response::Json,
};
use crate::modes::server::common::{AppState, StatusInfoResponse}; use crate::modes::server::common::{AppState, StatusInfoResponse};
@@ -52,10 +48,15 @@ use crate::modes::server::common::{AppState, StatusInfoResponse};
pub async fn handle_status( pub async fn handle_status(
State(state): State<AppState>, State(state): State<AppState>,
) -> Result<Json<StatusInfoResponse>, StatusCode> { ) -> Result<Json<StatusInfoResponse>, StatusCode> {
// Get database path // Get database path
let db_path = state.db.lock().await.path().unwrap_or("unknown").to_string(); let db_path = state
.db
.lock()
.await
.path()
.unwrap_or("unknown")
.to_string();
// Use the status service to generate status info showing configured plugins // Use the status service to generate status info showing configured plugins
let status_service = crate::services::status_service::StatusService::new(); let status_service = crate::services::status_service::StatusService::new();
let mut cmd = state.cmd.lock().await; let mut cmd = state.cmd.lock().await;

89
src/modes/server/common.rs
View File

@@ -1,3 +1,4 @@
use crate::services::item_service::ItemService;
/// Common utilities and types for the server module. /// Common utilities and types for the server module.
/// ///
/// This module provides shared structures, functions, and middleware used across /// This module provides shared structures, functions, and middleware used across
@@ -13,7 +14,7 @@
/// ``` /// ```
use anyhow::Result; use anyhow::Result;
use axum::{ use axum::{
extract::{Request, ConnectInfo}, extract::{ConnectInfo, Request},
http::{HeaderMap, StatusCode}, http::{HeaderMap, StatusCode},
middleware::Next, middleware::Next,
response::Response, response::Response,
@@ -28,7 +29,6 @@ use std::sync::Arc;
use std::time::Instant; use std::time::Instant;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use utoipa::ToSchema; use utoipa::ToSchema;
use crate::services::item_service::ItemService;
/// Server configuration structure. /// Server configuration structure.
/// ///
@@ -133,7 +133,9 @@ pub struct AppState {
/// }; /// };
/// ``` /// ```
#[derive(Debug, Serialize, Deserialize, ToSchema)] #[derive(Debug, Serialize, Deserialize, ToSchema)]
#[schema(description = "Standard API response wrapper containing success status, data payload, and error information")] #[schema(
description = "Standard API response wrapper containing success status, data payload, and error information"
)]
pub struct ApiResponse<T> { pub struct ApiResponse<T> {
/// Success indicator. /// Success indicator.
/// ///
@@ -584,18 +586,22 @@ fn default_as_meta() -> bool {
/// # Errors /// # Errors
/// ///
/// None; returns false on failure. /// None; returns false on failure.
fn check_bearer_auth(auth_str: &str, expected_password: &str, expected_hash: &Option<String>) -> bool { fn check_bearer_auth(
auth_str: &str,
expected_password: &str,
expected_hash: &Option<String>,
) -> bool {
if !auth_str.starts_with("Bearer ") { if !auth_str.starts_with("Bearer ") {
return false; return false;
} }
let provided_password = &auth_str[7..]; let provided_password = &auth_str[7..];
// If we have a password hash, verify against it // If we have a password hash, verify against it
if let Some(hash) = expected_hash { if let Some(hash) = expected_hash {
return pwhash::unix::verify(provided_password, hash); return pwhash::unix::verify(provided_password, hash);
} }
// Otherwise, do direct comparison // Otherwise, do direct comparison
provided_password == expected_password provided_password == expected_password
} }
@@ -619,22 +625,26 @@ fn check_bearer_auth(auth_str: &str, expected_password: &str, expected_hash: &Op
/// # Errors /// # Errors
/// ///
/// Returns false on decode or validation failure. /// Returns false on decode or validation failure.
fn check_basic_auth(auth_str: &str, expected_password: &str, expected_hash: &Option<String>) -> bool { fn check_basic_auth(
auth_str: &str,
expected_password: &str,
expected_hash: &Option<String>,
) -> bool {
if !auth_str.starts_with("Basic ") { if !auth_str.starts_with("Basic ") {
return false; return false;
} }
let encoded = &auth_str[6..]; let encoded = &auth_str[6..];
if let Ok(decoded_bytes) = base64::engine::general_purpose::STANDARD.decode(encoded) { if let Ok(decoded_bytes) = base64::engine::general_purpose::STANDARD.decode(encoded) {
if let Ok(decoded_str) = String::from_utf8(decoded_bytes) { if let Ok(decoded_str) = String::from_utf8(decoded_bytes) {
if let Some(colon_pos) = decoded_str.find(':') { if let Some(colon_pos) = decoded_str.find(':') {
let provided_password = &decoded_str[colon_pos + 1..]; let provided_password = &decoded_str[colon_pos + 1..];
// If we have a password hash, verify against it // If we have a password hash, verify against it
if let Some(hash) = expected_hash { if let Some(hash) = expected_hash {
return pwhash::unix::verify(provided_password, hash); return pwhash::unix::verify(provided_password, hash);
} }
// Otherwise, do direct comparison // Otherwise, do direct comparison
let expected_credentials = format!("keep:{}", expected_password); let expected_credentials = format!("keep:{}", expected_password);
return decoded_str == expected_credentials; return decoded_str == expected_credentials;
@@ -667,16 +677,20 @@ fn check_basic_auth(auth_str: &str, expected_password: &str, expected_hash: &Opt
/// // Proceed /// // Proceed
/// } /// }
/// ``` /// ```
pub fn check_auth(headers: &HeaderMap, password: &Option<String>, password_hash: &Option<String>) -> bool { pub fn check_auth(
headers: &HeaderMap,
password: &Option<String>,
password_hash: &Option<String>,
) -> bool {
// If neither password nor hash is set, no authentication required // If neither password nor hash is set, no authentication required
if password.is_none() && password_hash.is_none() { if password.is_none() && password_hash.is_none() {
return true; return true;
} }
if let Some(auth_header) = headers.get("authorization") { if let Some(auth_header) = headers.get("authorization") {
if let Ok(auth_str) = auth_header.to_str() { if let Ok(auth_str) = auth_header.to_str() {
return check_bearer_auth(auth_str, password.as_deref().unwrap_or(""), password_hash) || return check_bearer_auth(auth_str, password.as_deref().unwrap_or(""), password_hash)
check_basic_auth(auth_str, password.as_deref().unwrap_or(""), password_hash); || check_basic_auth(auth_str, password.as_deref().unwrap_or(""), password_hash);
} }
} }
false false
@@ -707,28 +721,38 @@ pub async fn logging_middleware(
) -> Response { ) -> Response {
let method = request.method().clone(); let method = request.method().clone();
let uri = request.uri().clone(); let uri = request.uri().clone();
// Log the Accept header - extract before moving the request // Log the Accept header - extract before moving the request
let accept_header = request.headers() let accept_header = request
.headers()
.get("accept") .get("accept")
.and_then(|v| v.to_str().ok()) .and_then(|v| v.to_str().ok())
.unwrap_or("-") .unwrap_or("-")
.to_string(); .to_string();
let start = Instant::now(); let start = Instant::now();
let response = next.run(request).await; let response = next.run(request).await;
let duration = start.elapsed(); let duration = start.elapsed();
// Try to get response body size from content-length header, or default to 0 // Try to get response body size from content-length header, or default to 0
let response_content_length = response.headers() let response_content_length = response
.headers()
.get("content-length") .get("content-length")
.and_then(|v| v.to_str().ok()) .and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok()) .and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0); .unwrap_or(0);
info!("{} {} {} {} {} bytes - {:?} - Accept: {}", info!(
addr, method, uri, response.status(), response_content_length, duration, accept_header); "{} {} {} {} {} bytes - {:?} - Accept: {}",
addr,
method,
uri,
response.status(),
response_content_length,
duration,
accept_header
);
response response
} }
@@ -756,14 +780,21 @@ pub async fn logging_middleware(
pub fn create_auth_middleware( pub fn create_auth_middleware(
password: Option<String>, password: Option<String>,
password_hash: Option<String>, password_hash: Option<String>,
) -> impl Fn(ConnectInfo<SocketAddr>, Request, Next) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Response, StatusCode>> + Send>> + Clone + Send { ) -> impl Fn(
ConnectInfo<SocketAddr>,
Request,
Next,
)
-> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Response, StatusCode>> + Send>>
+ Clone
+ Send {
move |ConnectInfo(addr): ConnectInfo<SocketAddr>, request: Request, next: Next| { move |ConnectInfo(addr): ConnectInfo<SocketAddr>, request: Request, next: Next| {
let password = password.clone(); let password = password.clone();
let password_hash = password_hash.clone(); let password_hash = password_hash.clone();
Box::pin(async move { Box::pin(async move {
let headers = request.headers().clone(); let headers = request.headers().clone();
let uri = request.uri().clone(); let uri = request.uri().clone();
if !check_auth(&headers, &password, &password_hash) { if !check_auth(&headers, &password, &password_hash) {
warn!("Unauthorized request to {} from {}", uri, addr); warn!("Unauthorized request to {} from {}", uri, addr);
// Add WWW-Authenticate header to trigger basic auth in browsers // Add WWW-Authenticate header to trigger basic auth in browsers
@@ -771,11 +802,13 @@ pub fn create_auth_middleware(
*response.status_mut() = StatusCode::UNAUTHORIZED; *response.status_mut() = StatusCode::UNAUTHORIZED;
response.headers_mut().insert( response.headers_mut().insert(
"www-authenticate", "www-authenticate",
"Basic realm=\"Keep Server\", charset=\"UTF-8\"".parse().unwrap(), "Basic realm=\"Keep Server\", charset=\"UTF-8\""
.parse()
.unwrap(),
); );
return Ok(response); return Ok(response);
} }
let response = next.run(request).await; let response = next.run(request).await;
Ok(response) Ok(response)
}) })

108
src/modes/server/mcp/mod.rs
View File

@@ -7,17 +7,12 @@ pub use server::KeepMcpServer;
/// ///
/// Provides handlers for JSON-RPC style requests to interact with Keep's storage /// Provides handlers for JSON-RPC style requests to interact with Keep's storage
/// via the API. /// via the API.
use axum::{ use axum::{Json, extract::State, http::StatusCode, response::IntoResponse};
extract::State,
http::StatusCode,
response::IntoResponse,
Json,
};
use serde::Deserialize; use serde::Deserialize;
use serde_json::Value; use serde_json::Value;
use crate::modes::server::common::AppState;
use crate::modes::server::common::ApiResponse; use crate::modes::server::common::ApiResponse;
use crate::modes::server::common::AppState;
/// Request structure for MCP JSON-RPC calls. /// Request structure for MCP JSON-RPC calls.
/// ///
@@ -31,57 +26,58 @@ pub struct McpRequest {
pub params: Option<Value>, pub params: Option<Value>,
} }
/// Handles an MCP request via the Axum framework. /// Handles an MCP request via the Axum framework.
/// ///
/// Parses the JSON request, delegates to `KeepMcpServer`, and returns an API response. /// Parses the JSON request, delegates to `KeepMcpServer`, and returns an API response.
/// Attempts to parse the result as JSON; falls back to string if invalid. /// Attempts to parse the result as JSON; falls back to string if invalid.
/// ///
/// # Arguments /// # Arguments
/// ///
/// * `State(state)` - The application state. /// * `State(state)` - The application state.
/// * `Json(request)` - The deserialized MCP request. /// * `Json(request)` - The deserialized MCP request.
/// ///
/// # Returns /// # Returns
/// ///
/// An `IntoResponse` with status code and JSON API response. /// An `IntoResponse` with status code and JSON API response.
/// ///
/// # Errors /// # Errors
/// ///
/// Returns 400 Bad Request on handler errors. /// Returns 400 Bad Request on handler errors.
pub async fn handle_mcp_request( pub async fn handle_mcp_request(
State(state): State<AppState>, State(state): State<AppState>,
Json(request): Json<McpRequest>, Json(request): Json<McpRequest>,
) -> impl IntoResponse { ) -> impl IntoResponse {
let mcp_server = KeepMcpServer::new(state); let mcp_server = KeepMcpServer::new(state);
match mcp_server.handle_request(&request.method, request.params).await { match mcp_server
Ok(result) => { .handle_request(&request.method, request.params)
match serde_json::from_str(&result) { .await
Ok(parsed_result) => { {
let response = ApiResponse { Ok(result) => match serde_json::from_str(&result) {
success: true, Ok(parsed_result) => {
data: Some(parsed_result),
error: None,
};
(StatusCode::OK, Json(response))
}
Err(_) => {
let response = ApiResponse {
success: true,
data: Some(serde_json::Value::String(result)),
error: None,
};
(StatusCode::OK, Json(response))
}
}
}
Err(e) => {
let response = ApiResponse { let response = ApiResponse {
success: false, success: true,
data: None, data: Some(parsed_result),
error: Some(e.to_string()), error: None,
}; };
(StatusCode::BAD_REQUEST, Json(response)) (StatusCode::OK, Json(response))
} }
Err(_) => {
let response = ApiResponse {
success: true,
data: Some(serde_json::Value::String(result)),
error: None,
};
(StatusCode::OK, Json(response))
}
},
Err(e) => {
let response = ApiResponse {
success: false,
data: None,
error: Some(e.to_string()),
};
(StatusCode::BAD_REQUEST, Json(response))
} }
} }
}

63
src/modes/server/mcp/server.rs
View File

@@ -1,8 +1,8 @@
use log::debug; use log::debug;
use serde_json::Value; use serde_json::Value;
use crate::modes::server::common::AppState;
use super::tools::{KeepTools, ToolError}; use super::tools::{KeepTools, ToolError};
use crate::modes::server::common::AppState;
/// Server handler for MCP (Model Context Protocol) requests. /// Server handler for MCP (Model Context Protocol) requests.
/// ///
@@ -36,34 +36,41 @@ impl KeepMcpServer {
Self { state } Self { state }
} }
/// Handles an MCP request by routing to the appropriate tool. /// Handles an MCP request by routing to the appropriate tool.
/// ///
/// Supports methods like "save_item", "get_item", "list_items". Logs the request and delegates to KeepTools. /// Supports methods like "save_item", "get_item", "list_items". Logs the request and delegates to KeepTools.
/// ///
/// # Arguments /// # Arguments
/// ///
/// * `method` - The MCP method name (string). /// * `method` - The MCP method name (string).
/// * `params` - Optional JSON parameters as serde_json::Value. /// * `params` - Optional JSON parameters as serde_json::Value.
/// ///
/// # Returns /// # Returns
/// ///
/// `Ok(String)` with JSON-serialized response on success, or `Err(ToolError)` on failure. /// `Ok(String)` with JSON-serialized response on success, or `Err(ToolError)` on failure.
/// ///
/// # Errors /// # Errors
/// ///
/// * ToolError::UnknownTool if method unsupported. /// * ToolError::UnknownTool if method unsupported.
/// * Propagates tool-specific errors (e.g., invalid args, DB failures). /// * Propagates tool-specific errors (e.g., invalid args, DB failures).
/// ///
/// # Examples /// # Examples
/// ///
/// ``` /// ```
/// let result = server.handle_request("save_item", Some(params)).await?; /// let result = server.handle_request("save_item", Some(params)).await?;
/// ``` /// ```
pub async fn handle_request(&self, method: &str, params: Option<Value>) -> Result<String, ToolError> { pub async fn handle_request(
debug!("MCP: Handling request '{}' with params: {:?}", method, params); &self,
method: &str,
params: Option<Value>,
) -> Result<String, ToolError> {
debug!(
"MCP: Handling request '{}' with params: {:?}",
method, params
);
let tools = KeepTools::new(self.state.clone()); let tools = KeepTools::new(self.state.clone());
match method { match method {
"save_item" => tools.save_item(params).await, "save_item" => tools.save_item(params).await,
"get_item" => tools.get_item(params).await, "get_item" => tools.get_item(params).await,

159
src/modes/server/mcp/tools.rs
View File

@@ -1,7 +1,7 @@
use anyhow::{Result, anyhow}; use anyhow::{Result, anyhow};
use log::debug;
use serde_json::Value; use serde_json::Value;
use std::collections::HashMap; use std::collections::HashMap;
use log::{debug};
use crate::modes::server::common::AppState; use crate::modes::server::common::AppState;
use crate::services::async_item_service::AsyncItemService; use crate::services::async_item_service::AsyncItemService;
@@ -35,7 +35,8 @@ impl KeepTools {
} }
pub async fn save_item(&self, args: Option<Value>) -> Result<String, ToolError> { pub async fn save_item(&self, args: Option<Value>) -> Result<String, ToolError> {
let args = args.ok_or_else(|| ToolError::InvalidArguments("Missing arguments".to_string()))?; let args =
args.ok_or_else(|| ToolError::InvalidArguments("Missing arguments".to_string()))?;
let content = args let content = args
.get("content") .get("content")
@@ -70,11 +71,11 @@ impl KeepTools {
); );
let service = AsyncItemService::new( let service = AsyncItemService::new(
self.state.data_dir.clone(), self.state.data_dir.clone(),
self.state.db.clone(), self.state.db.clone(),
self.state.item_service.clone(), self.state.item_service.clone(),
self.state.cmd.clone(), self.state.cmd.clone(),
self.state.settings.clone() self.state.settings.clone(),
); );
let item_with_meta = service let item_with_meta = service
.save_item_from_mcp(content.as_bytes().to_vec(), tags, metadata) .save_item_from_mcp(content.as_bytes().to_vec(), tags, metadata)
@@ -90,28 +91,39 @@ impl KeepTools {
} }
pub async fn get_item(&self, args: Option<Value>) -> Result<String, ToolError> { pub async fn get_item(&self, args: Option<Value>) -> Result<String, ToolError> {
let args = args.ok_or_else(|| ToolError::InvalidArguments("Missing arguments".to_string()))?; let args =
args.ok_or_else(|| ToolError::InvalidArguments("Missing arguments".to_string()))?;
let item_id = args.get("id")
.and_then(|v| v.as_i64()) let item_id = args.get("id").and_then(|v| v.as_i64()).ok_or_else(|| {
.ok_or_else(|| ToolError::InvalidArguments("Missing or invalid 'id' field".to_string()))?; ToolError::InvalidArguments("Missing or invalid 'id' field".to_string())
})?;
let service = AsyncItemService::new( let service = AsyncItemService::new(
self.state.data_dir.clone(), self.state.data_dir.clone(),
self.state.db.clone(), self.state.db.clone(),
self.state.item_service.clone(), self.state.item_service.clone(),
self.state.cmd.clone(), self.state.cmd.clone(),
self.state.settings.clone() self.state.settings.clone(),
); );
let item_with_content = match service.get_item_content(item_id).await { let item_with_content = match service.get_item_content(item_id).await {
Ok(iwc) => iwc, Ok(iwc) => iwc,
Err(CoreError::ItemNotFound(_)) => return Err(ToolError::InvalidArguments(format!("Item {} not found", item_id))), Err(CoreError::ItemNotFound(_)) => {
return Err(ToolError::InvalidArguments(format!(
"Item {} not found",
item_id
)));
}
Err(e) => return Err(ToolError::Other(anyhow::Error::from(e))), Err(e) => return Err(ToolError::Other(anyhow::Error::from(e))),
}; };
let content = String::from_utf8_lossy(&item_with_content.content).to_string(); let content = String::from_utf8_lossy(&item_with_content.content).to_string();
let tags: Vec<String> = item_with_content.item_with_meta.tags.iter().map(|t| t.name.clone()).collect(); let tags: Vec<String> = item_with_content
.item_with_meta
.tags
.iter()
.map(|t| t.name.clone())
.collect();
let metadata = item_with_content.item_with_meta.meta_as_map(); let metadata = item_with_content.item_with_meta.meta_as_map();
let item = item_with_content.item_with_meta.item; let item = item_with_content.item_with_meta.item;
@@ -124,7 +136,7 @@ impl KeepTools {
"tags": tags, "tags": tags,
"metadata": metadata, "metadata": metadata,
}); });
Ok(serde_json::to_string_pretty(&response)?) Ok(serde_json::to_string_pretty(&response)?)
} }
@@ -133,28 +145,45 @@ impl KeepTools {
.as_ref() .as_ref()
.and_then(|v| v.get("tags")) .and_then(|v| v.get("tags"))
.and_then(|v| v.as_array()) .and_then(|v| v.as_array())
.map(|arr| arr.iter().filter_map(|v| v.as_str().map(|s| s.to_string())).collect()) .map(|arr| {
arr.iter()
.filter_map(|v| v.as_str().map(|s| s.to_string()))
.collect()
})
.unwrap_or_default(); .unwrap_or_default();
let service = AsyncItemService::new( let service = AsyncItemService::new(
self.state.data_dir.clone(), self.state.data_dir.clone(),
self.state.db.clone(), self.state.db.clone(),
self.state.item_service.clone(), self.state.item_service.clone(),
self.state.cmd.clone(), self.state.cmd.clone(),
self.state.settings.clone() self.state.settings.clone(),
); );
let item_with_meta = match service.find_item(vec![], tags, HashMap::new()).await { let item_with_meta = match service.find_item(vec![], tags, HashMap::new()).await {
Ok(iwm) => iwm, Ok(iwm) => iwm,
Err(CoreError::ItemNotFoundGeneric) => return Err(ToolError::InvalidArguments("No items found".to_string())), Err(CoreError::ItemNotFoundGeneric) => {
return Err(ToolError::InvalidArguments("No items found".to_string()));
}
Err(e) => return Err(ToolError::Other(anyhow::Error::from(e))), Err(e) => return Err(ToolError::Other(anyhow::Error::from(e))),
}; };
let item_id = item_with_meta.item.id.ok_or_else(|| anyhow!("Item missing ID after find"))?; let item_id = item_with_meta
let item_with_content = service.get_item_content(item_id).await.map_err(|e| ToolError::Other(anyhow::Error::from(e)))?; .item
.id
.ok_or_else(|| anyhow!("Item missing ID after find"))?;
let item_with_content = service
.get_item_content(item_id)
.await
.map_err(|e| ToolError::Other(anyhow::Error::from(e)))?;
let content = String::from_utf8_lossy(&item_with_content.content).to_string(); let content = String::from_utf8_lossy(&item_with_content.content).to_string();
let tags: Vec<String> = item_with_content.item_with_meta.tags.iter().map(|t| t.name.clone()).collect(); let tags: Vec<String> = item_with_content
.item_with_meta
.tags
.iter()
.map(|t| t.name.clone())
.collect();
let metadata = item_with_content.item_with_meta.meta_as_map(); let metadata = item_with_content.item_with_meta.meta_as_map();
let item = item_with_content.item_with_meta.item; let item = item_with_content.item_with_meta.item;
@@ -167,7 +196,7 @@ impl KeepTools {
"tags": tags, "tags": tags,
"metadata": metadata, "metadata": metadata,
}); });
Ok(serde_json::to_string_pretty(&response)?) Ok(serde_json::to_string_pretty(&response)?)
} }
@@ -176,40 +205,52 @@ impl KeepTools {
let tags: Vec<String> = args_ref let tags: Vec<String> = args_ref
.and_then(|v| v.get("tags")) .and_then(|v| v.get("tags"))
.and_then(|v| v.as_array()) .and_then(|v| v.as_array())
.map(|arr| arr.iter().filter_map(|v| v.as_str().map(|s| s.to_string())).collect()) .map(|arr| {
arr.iter()
.filter_map(|v| v.as_str().map(|s| s.to_string()))
.collect()
})
.unwrap_or_default(); .unwrap_or_default();
let limit = args_ref let limit = args_ref
.and_then(|v| v.get("limit")) .and_then(|v| v.get("limit"))
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.unwrap_or(10) as usize; .unwrap_or(10) as usize;
let offset = args_ref let offset = args_ref
.and_then(|v| v.get("offset")) .and_then(|v| v.get("offset"))
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.unwrap_or(0) as usize; .unwrap_or(0) as usize;
let service = AsyncItemService::new( let service = AsyncItemService::new(
self.state.data_dir.clone(), self.state.data_dir.clone(),
self.state.db.clone(), self.state.db.clone(),
self.state.item_service.clone(), self.state.item_service.clone(),
self.state.cmd.clone(), self.state.cmd.clone(),
self.state.settings.clone() self.state.settings.clone(),
); );
let mut items_with_meta = service.list_items(tags, HashMap::new()).await.map_err(|e| ToolError::Other(anyhow::Error::from(e)))?; let mut items_with_meta = service
.list_items(tags, HashMap::new())
.await
.map_err(|e| ToolError::Other(anyhow::Error::from(e)))?;
// Sort by timestamp (newest first) and apply pagination // Sort by timestamp (newest first) and apply pagination
items_with_meta.sort_by(|a, b| b.item.ts.cmp(&a.item.ts)); items_with_meta.sort_by(|a, b| b.item.ts.cmp(&a.item.ts));
let items_with_meta: Vec<_> = items_with_meta.into_iter().skip(offset).take(limit).collect(); let items_with_meta: Vec<_> = items_with_meta
.into_iter()
.skip(offset)
.take(limit)
.collect();
let items_info: Vec<_> = items_with_meta let items_info: Vec<_> = items_with_meta
.into_iter() .into_iter()
.map(|item_with_meta| { .map(|item_with_meta| {
let item_tags: Vec<String> = item_with_meta.tags.iter().map(|t| t.name.clone()).collect(); let item_tags: Vec<String> =
item_with_meta.tags.iter().map(|t| t.name.clone()).collect();
let item_meta = item_with_meta.meta_as_map(); let item_meta = item_with_meta.meta_as_map();
let item = item_with_meta.item; let item = item_with_meta.item;
let item_id = item.id.unwrap_or(0); let item_id = item.id.unwrap_or(0);
serde_json::json!({ serde_json::json!({
"id": item_id, "id": item_id,
"timestamp": item.ts.to_rfc3339(), "timestamp": item.ts.to_rfc3339(),
@@ -220,14 +261,14 @@ impl KeepTools {
}) })
}) })
.collect(); .collect();
let response = serde_json::json!({ let response = serde_json::json!({
"items": items_info, "items": items_info,
"count": items_info.len(), "count": items_info.len(),
"offset": offset, "offset": offset,
"limit": limit "limit": limit
}); });
Ok(serde_json::to_string_pretty(&response)?) Ok(serde_json::to_string_pretty(&response)?)
} }
@@ -236,38 +277,48 @@ impl KeepTools {
.as_ref() .as_ref()
.and_then(|v| v.get("tags")) .and_then(|v| v.get("tags"))
.and_then(|v| v.as_array()) .and_then(|v| v.as_array())
.map(|arr| arr.iter().filter_map(|v| v.as_str().map(|s| s.to_string())).collect()) .map(|arr| {
arr.iter()
.filter_map(|v| v.as_str().map(|s| s.to_string()))
.collect()
})
.unwrap_or_default(); .unwrap_or_default();
let metadata: HashMap<String, String> = args let metadata: HashMap<String, String> = args
.as_ref() .as_ref()
.and_then(|v| v.get("metadata")) .and_then(|v| v.get("metadata"))
.and_then(|v| v.as_object()) .and_then(|v| v.as_object())
.map(|obj| obj.iter().filter_map(|(k, v)| { .map(|obj| {
v.as_str().map(|s| (k.clone(), s.to_string())) obj.iter()
}).collect()) .filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_string())))
.collect()
})
.unwrap_or_default(); .unwrap_or_default();
let service = AsyncItemService::new( let service = AsyncItemService::new(
self.state.data_dir.clone(), self.state.data_dir.clone(),
self.state.db.clone(), self.state.db.clone(),
self.state.item_service.clone(), self.state.item_service.clone(),
self.state.cmd.clone(), self.state.cmd.clone(),
self.state.settings.clone() self.state.settings.clone(),
); );
let mut items_with_meta = service.list_items(tags.clone(), metadata.clone()).await.map_err(|e| ToolError::Other(anyhow::Error::from(e)))?; let mut items_with_meta = service
.list_items(tags.clone(), metadata.clone())
.await
.map_err(|e| ToolError::Other(anyhow::Error::from(e)))?;
// Sort by timestamp (newest first) // Sort by timestamp (newest first)
items_with_meta.sort_by(|a, b| b.item.ts.cmp(&a.item.ts)); items_with_meta.sort_by(|a, b| b.item.ts.cmp(&a.item.ts));
let items_info: Vec<_> = items_with_meta let items_info: Vec<_> = items_with_meta
.into_iter() .into_iter()
.map(|item_with_meta| { .map(|item_with_meta| {
let item_tags: Vec<String> = item_with_meta.tags.iter().map(|t| t.name.clone()).collect(); let item_tags: Vec<String> =
item_with_meta.tags.iter().map(|t| t.name.clone()).collect();
let item_meta = item_with_meta.meta_as_map(); let item_meta = item_with_meta.meta_as_map();
let item = item_with_meta.item; let item = item_with_meta.item;
let item_id = item.id.unwrap_or(0); let item_id = item.id.unwrap_or(0);
serde_json::json!({ serde_json::json!({
"id": item_id, "id": item_id,
"timestamp": item.ts.to_rfc3339(), "timestamp": item.ts.to_rfc3339(),
@@ -278,7 +329,7 @@ impl KeepTools {
}) })
}) })
.collect(); .collect();
let response = serde_json::json!({ let response = serde_json::json!({
"items": items_info, "items": items_info,
"count": items_info.len(), "count": items_info.len(),
@@ -287,7 +338,7 @@ impl KeepTools {
"metadata": metadata "metadata": metadata
} }
}); });
Ok(serde_json::to_string_pretty(&response)?) Ok(serde_json::to_string_pretty(&response)?)
} }
} }

77
src/modes/server/mod.rs
View File

@@ -1,27 +1,24 @@
use crate::config;
use crate::services::item_service::ItemService;
use anyhow::Result; use anyhow::Result;
use axum::{ use axum::{Router, routing::post};
Router,
routing::post,
};
use clap::Command; use clap::Command;
use log::{debug, info}; use log::{debug, info};
use std::net::SocketAddr; use std::net::SocketAddr;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use tower_http::cors::CorsLayer;
use tower::ServiceBuilder; use tower::ServiceBuilder;
use tower_http::cors::CorsLayer;
use tower_http::trace::TraceLayer; use tower_http::trace::TraceLayer;
use crate::config;
use crate::services::item_service::ItemService;
pub mod common;
mod api; mod api;
mod pages; pub mod common;
#[cfg(feature = "mcp")] #[cfg(feature = "mcp")]
mod mcp; mod mcp;
mod pages;
pub use common::{AppState, logging_middleware, create_auth_middleware}; pub use common::{AppState, create_auth_middleware, logging_middleware};
pub fn mode_server( pub fn mode_server(
cmd: &mut Command, cmd: &mut Command,
@@ -33,11 +30,14 @@ pub fn mode_server(
let server_address = if let Some(addr) = &settings.server_address() { let server_address = if let Some(addr) = &settings.server_address() {
addr.clone() addr.clone()
} else if let Some(server_config) = &settings.server { } else if let Some(server_config) = &settings.server {
server_config.address.clone().unwrap_or_else(|| "127.0.0.1".to_string()) server_config
.address
.clone()
.unwrap_or_else(|| "127.0.0.1".to_string())
} else { } else {
"127.0.0.1".to_string() "127.0.0.1".to_string()
}; };
// Get server port from args or config with default // Get server port from args or config with default
let server_port = if let Some(port) = settings.server_port() { let server_port = if let Some(port) = settings.server_port() {
port port
@@ -46,24 +46,31 @@ pub fn mode_server(
} else { } else {
21080 21080
}; };
let server_config = common::ServerConfig { let server_config = common::ServerConfig {
address: server_address, address: server_address,
port: Some(server_port), port: Some(server_port),
password: settings.server_password(), password: settings.server_password(),
password_hash: settings.server_password_hash(), password_hash: settings.server_password_hash(),
}; };
// Create ItemService once // Create ItemService once
let item_service = ItemService::new(data_path.clone()); let item_service = ItemService::new(data_path.clone());
// We need to move the connection into the async runtime // We need to move the connection into the async runtime
let rt = tokio::runtime::Runtime::new()?; let rt = tokio::runtime::Runtime::new()?;
// Take ownership of the connection and move it into the async runtime // Take ownership of the connection and move it into the async runtime
let owned_conn = std::mem::replace(conn, rusqlite::Connection::open_in_memory()?); let owned_conn = std::mem::replace(conn, rusqlite::Connection::open_in_memory()?);
let cmd = cmd.clone(); let cmd = cmd.clone();
let settings = settings.clone(); let settings = settings.clone();
rt.block_on(run_server(server_config, owned_conn, data_path, item_service, cmd, settings)) rt.block_on(run_server(
server_config,
owned_conn,
data_path,
item_service,
cmd,
settings,
))
} }
async fn run_server( async fn run_server(
@@ -80,12 +87,12 @@ async fn run_server(
} else { } else {
format!("{}:21080", config.address) format!("{}:21080", config.address)
}; };
debug!("SERVER: Starting REST HTTP server on {}", bind_address); debug!("SERVER: Starting REST HTTP server on {}", bind_address);
// Use the existing database connection // Use the existing database connection
let db_conn = Arc::new(Mutex::new(conn)); let db_conn = Arc::new(Mutex::new(conn));
let state = AppState { let state = AppState {
db: db_conn, db: db_conn,
data_dir: data_dir.clone(), data_dir: data_dir.clone(),
@@ -93,24 +100,25 @@ async fn run_server(
cmd: Arc::new(Mutex::new(Command::new("keep"))), cmd: Arc::new(Mutex::new(Command::new("keep"))),
settings: Arc::new(settings.clone()), settings: Arc::new(settings.clone()),
}; };
#[cfg(feature = "mcp")] #[cfg(feature = "mcp")]
let mcp_router = Router::new() let mcp_router = Router::new()
.route("/mcp", post(mcp::handle_mcp_request)) .route("/mcp", post(mcp::handle_mcp_request))
.with_state(state.clone()); .with_state(state.clone());
let mut protected_router = Router::new() let mut protected_router = Router::new()
.merge(api::add_routes(Router::new())) .merge(api::add_routes(Router::new()))
.merge(pages::add_routes(Router::new())); .merge(pages::add_routes(Router::new()));
#[cfg(feature = "mcp")] #[cfg(feature = "mcp")]
{ {
protected_router = protected_router.merge(mcp_router); protected_router = protected_router.merge(mcp_router);
} }
let protected_router = protected_router let protected_router = protected_router.layer(axum::middleware::from_fn(
.layer(axum::middleware::from_fn(create_auth_middleware(config.password.clone(), config.password_hash.clone()))); create_auth_middleware(config.password.clone(), config.password_hash.clone()),
));
// Create the app with documentation routes open and others protected // Create the app with documentation routes open and others protected
let app = Router::new() let app = Router::new()
// Add documentation routes without authentication // Add documentation routes without authentication
@@ -124,18 +132,19 @@ async fn run_server(
.layer( .layer(
ServiceBuilder::new() ServiceBuilder::new()
.layer(TraceLayer::new_for_http()) .layer(TraceLayer::new_for_http())
.layer(CorsLayer::permissive()) .layer(CorsLayer::permissive()),
); );
let addr: SocketAddr = bind_address.parse()?; let addr: SocketAddr = bind_address.parse()?;
info!("SERVER: HTTP server listening on {}", addr); info!("SERVER: HTTP server listening on {}", addr);
let listener = tokio::net::TcpListener::bind(addr).await?; let listener = tokio::net::TcpListener::bind(addr).await?;
axum::serve( axum::serve(
listener, listener,
app.into_make_service_with_connect_info::<SocketAddr>() app.into_make_service_with_connect_info::<SocketAddr>(),
).await?; )
.await?;
Ok(()) Ok(())
} }

156
src/modes/server/pages.rs
View File

@@ -1,3 +1,4 @@
use crate::config::ColumnConfig;
use crate::db; use crate::db;
use crate::modes::server::AppState; use crate::modes::server::AppState;
use anyhow::Result; use anyhow::Result;
@@ -8,7 +9,6 @@ use axum::{
use log::debug; use log::debug;
use rusqlite::Connection; use rusqlite::Connection;
use serde::Deserialize; use serde::Deserialize;
use crate::config::ColumnConfig;
use std::collections::HashMap; use std::collections::HashMap;
#[derive(Deserialize)] #[derive(Deserialize)]
@@ -72,8 +72,7 @@ fn default_count() -> usize {
/// let app = pages::add_routes(axum::Router::new()); /// let app = pages::add_routes(axum::Router::new());
/// ``` /// ```
pub fn add_routes(app: axum::Router<AppState>) -> axum::Router<AppState> { pub fn add_routes(app: axum::Router<AppState>) -> axum::Router<AppState> {
app app.route("/", axum::routing::get(list_items))
.route("/", axum::routing::get(list_items))
.route("/item/{item_id}", axum::routing::get(show_item)) .route("/item/{item_id}", axum::routing::get(show_item))
.route("/style.css", axum::routing::get(style_css)) .route("/style.css", axum::routing::get(style_css))
} }
@@ -84,9 +83,9 @@ async fn list_items(
) -> Result<Response, Html<String>> { ) -> Result<Response, Html<String>> {
let conn = state.db.lock().await; let conn = state.db.lock().await;
let settings = &state.settings; let settings = &state.settings;
let result = build_item_list(&conn, &params, &settings.list_format); let result = build_item_list(&conn, &params, &settings.list_format);
match result { match result {
Ok(html) => { Ok(html) => {
// Build response with explicit Content-Length // Build response with explicit Content-Length
@@ -96,23 +95,28 @@ async fn list_items(
.body(axum::body::Body::from(html)) .body(axum::body::Body::from(html))
.map_err(|_| Html("<html><body>Internal Server Error</body></html>".to_string()))?; .map_err(|_| Html("<html><body>Internal Server Error</body></html>".to_string()))?;
Ok(response) Ok(response)
}, }
Err(e) => Err(Html(format!("<html><body>Error: {}</body></html>", e))), Err(e) => Err(Html(format!("<html><body>Error: {}</body></html>", e))),
} }
} }
fn build_item_list(conn: &Connection, params: &ListQueryParams, columns: &[ColumnConfig]) -> Result<String> { fn build_item_list(
let tags: Vec<String> = params.tags conn: &Connection,
params: &ListQueryParams,
columns: &[ColumnConfig],
) -> Result<String> {
let tags: Vec<String> = params
.tags
.as_ref() .as_ref()
.map(|t| t.split(',').map(|s| s.trim().to_string()).collect()) .map(|t| t.split(',').map(|s| s.trim().to_string()).collect())
.unwrap_or_default(); .unwrap_or_default();
let items = if tags.is_empty() { let items = if tags.is_empty() {
db::query_all_items(conn)? db::query_all_items(conn)?
} else { } else {
db::query_tagged_items(conn, &tags)? db::query_tagged_items(conn, &tags)?
}; };
// Sort items // Sort items
let mut sorted_items = items; let mut sorted_items = items;
if params.sort == "newest" { if params.sort == "newest" {
@@ -120,7 +124,7 @@ fn build_item_list(conn: &Connection, params: &ListQueryParams, columns: &[Colum
} else { } else {
sorted_items.sort_by(|a, b| a.id.cmp(&b.id)); sorted_items.sort_by(|a, b| a.id.cmp(&b.id));
} }
// Apply pagination // Apply pagination
let start = params.start; let start = params.start;
let end = std::cmp::min(start + params.count, sorted_items.len()); let end = std::cmp::min(start + params.count, sorted_items.len());
@@ -129,29 +133,29 @@ fn build_item_list(conn: &Connection, params: &ListQueryParams, columns: &[Colum
} else { } else {
vec![] vec![]
}; };
// Get tags and meta for all items in the page // Get tags and meta for all items in the page
let item_ids: Vec<i64> = page_items.iter().filter_map(|item| item.id).collect(); let item_ids: Vec<i64> = page_items.iter().filter_map(|item| item.id).collect();
let tags_map = db::get_tags_for_items(conn, &item_ids)?; let tags_map = db::get_tags_for_items(conn, &item_ids)?;
let meta_map = db::get_meta_for_items(conn, &item_ids)?; let meta_map = db::get_meta_for_items(conn, &item_ids)?;
// Debug: print number of tags per item // Debug: print number of tags per item
for item_id in &item_ids { for item_id in &item_ids {
if let Some(tags) = tags_map.get(item_id) { if let Some(tags) = tags_map.get(item_id) {
debug!("Item {} has {} tags: {:?}", item_id, tags.len(), tags); debug!("Item {} has {} tags: {:?}", item_id, tags.len(), tags);
} }
} }
let mut html = String::new(); let mut html = String::new();
html.push_str("<html><head><title>Keep - Items</title>"); html.push_str("<html><head><title>Keep - Items</title>");
html.push_str("<link rel=\"stylesheet\" href=\"/style.css\">"); html.push_str("<link rel=\"stylesheet\" href=\"/style.css\">");
html.push_str("</head><body>"); html.push_str("</head><body>");
html.push_str("<h1>Items</h1>"); html.push_str("<h1>Items</h1>");
html.push_str("<p><a href=\"/swagger\">API Documentation</a></p>"); html.push_str("<p><a href=\"/swagger\">API Documentation</a></p>");
// Add recent tags section using the items we already have // Add recent tags section using the items we already have
html.push_str("<h2>Recent Tags</h2>"); html.push_str("<h2>Recent Tags</h2>");
// Collect all tags from all items, keeping track of their timestamps // Collect all tags from all items, keeping track of their timestamps
let mut all_tags_with_time: Vec<(String, chrono::DateTime<chrono::Utc>)> = Vec::new(); let mut all_tags_with_time: Vec<(String, chrono::DateTime<chrono::Utc>)> = Vec::new();
for item in &sorted_items { for item in &sorted_items {
@@ -163,10 +167,10 @@ fn build_item_list(conn: &Connection, params: &ListQueryParams, columns: &[Colum
} }
} }
} }
// Sort by timestamp descending (most recent first) // Sort by timestamp descending (most recent first)
all_tags_with_time.sort_by(|a, b| b.1.cmp(&a.1)); all_tags_with_time.sort_by(|a, b| b.1.cmp(&a.1));
// Get unique tags in order of most recent appearance // Get unique tags in order of most recent appearance
let mut seen = std::collections::HashSet::new(); let mut seen = std::collections::HashSet::new();
let mut recent_tags = Vec::new(); let mut recent_tags = Vec::new();
@@ -179,20 +183,23 @@ fn build_item_list(conn: &Connection, params: &ListQueryParams, columns: &[Colum
} }
} }
} }
if recent_tags.is_empty() { if recent_tags.is_empty() {
html.push_str("<p>No tags found</p>"); html.push_str("<p>No tags found</p>");
} else { } else {
html.push_str("<p>"); html.push_str("<p>");
for tag in recent_tags { for tag in recent_tags {
html.push_str(&format!("<a href=\"/?tags={}\" style=\"margin-right: 8px;\">{}</a>", tag, tag)); html.push_str(&format!(
"<a href=\"/?tags={}\" style=\"margin-right: 8px;\">{}</a>",
tag, tag
));
} }
html.push_str("</p>"); html.push_str("</p>");
} }
// Start table // Start table
html.push_str("<table>"); html.push_str("<table>");
// Table headers // Table headers
html.push_str("<tr>"); html.push_str("<tr>");
for column in columns { for column in columns {
@@ -200,19 +207,21 @@ fn build_item_list(conn: &Connection, params: &ListQueryParams, columns: &[Colum
} }
html.push_str("<th>Actions</th>"); html.push_str("<th>Actions</th>");
html.push_str("</tr>"); html.push_str("</tr>");
// Table rows // Table rows
for item in page_items { for item in page_items {
let item_id = item.id.unwrap_or(0); let item_id = item.id.unwrap_or(0);
let tags = tags_map.get(&item_id).cloned().unwrap_or_default(); let tags = tags_map.get(&item_id).cloned().unwrap_or_default();
let meta: HashMap<String, String> = meta_map.get(&item_id) let meta: HashMap<String, String> = meta_map
.get(&item_id)
.map(|metas| { .map(|metas| {
metas.iter() metas
.iter()
.map(|(name, value)| (name.clone(), value.clone())) .map(|(name, value)| (name.clone(), value.clone()))
.collect() .collect()
}) })
.unwrap_or_default(); .unwrap_or_default();
html.push_str("<tr>"); html.push_str("<tr>");
for column in columns { for column in columns {
let value = match column.name.as_str() { let value = match column.name.as_str() {
@@ -220,16 +229,17 @@ fn build_item_list(conn: &Connection, params: &ListQueryParams, columns: &[Colum
let id_value = item.id.map(|id| id.to_string()).unwrap_or_default(); let id_value = item.id.map(|id| id.to_string()).unwrap_or_default();
// Make the ID a link to the item details page // Make the ID a link to the item details page
format!("<a href=\"/item/{}\">{}</a>", item_id, id_value) format!("<a href=\"/item/{}\">{}</a>", item_id, id_value)
}, }
"time" => item.ts.format("%Y-%m-%d %H:%M:%S").to_string(), "time" => item.ts.format("%Y-%m-%d %H:%M:%S").to_string(),
"size" => item.size.map(|s| s.to_string()).unwrap_or_default(), "size" => item.size.map(|s| s.to_string()).unwrap_or_default(),
"tags" => { "tags" => {
// Make sure we're using all tags for the item // Make sure we're using all tags for the item
let tag_links: Vec<String> = tags.iter() let tag_links: Vec<String> = tags
.iter()
.map(|t| format!("<a href=\"/?tags={}\">{}</a>", t.name, t.name)) .map(|t| format!("<a href=\"/?tags={}\">{}</a>", t.name, t.name))
.collect(); .collect();
tag_links.join(", ") tag_links.join(", ")
}, }
_ => { _ => {
if column.name.starts_with("meta:") { if column.name.starts_with("meta:") {
let meta_key = &column.name[5..]; let meta_key = &column.name[5..];
@@ -239,7 +249,7 @@ fn build_item_list(conn: &Connection, params: &ListQueryParams, columns: &[Colum
} }
} }
}; };
// Apply max_len if specified, but skip for tags column to avoid truncating HTML // Apply max_len if specified, but skip for tags column to avoid truncating HTML
let display_value = if column.name == "tags" { let display_value = if column.name == "tags" {
value value
@@ -257,36 +267,41 @@ fn build_item_list(conn: &Connection, params: &ListQueryParams, columns: &[Colum
} else { } else {
value value
}; };
// Apply alignment // Apply alignment
let align_style = match column.align { let align_style = match column.align {
crate::config::ColumnAlignment::Left => "text-align: left;", crate::config::ColumnAlignment::Left => "text-align: left;",
crate::config::ColumnAlignment::Right => "text-align: right;", crate::config::ColumnAlignment::Right => "text-align: right;",
crate::config::ColumnAlignment::Center => "text-align: center;", crate::config::ColumnAlignment::Center => "text-align: center;",
}; };
html.push_str(&format!("<td style=\"{}\">{}</td>", align_style, display_value)); html.push_str(&format!(
"<td style=\"{}\">{}</td>",
align_style, display_value
));
} }
// Actions column // Actions column
html.push_str(&format!( html.push_str(&format!(
"<td><a href=\"/item/{}\">View</a> | <a href=\"/api/item/{}/content\">Download</a></td>", "<td><a href=\"/item/{}\">View</a> | <a href=\"/api/item/{}/content\">Download</a></td>",
item_id, item_id item_id, item_id
)); ));
html.push_str("</tr>"); html.push_str("</tr>");
} }
html.push_str("</table>"); html.push_str("</table>");
// Add pagination info // Add pagination info
html.push_str(&format!("<p>Showing {} to {} of {} items</p>", html.push_str(&format!(
start + 1, "<p>Showing {} to {} of {} items</p>",
std::cmp::min(end, sorted_items.len()), start + 1,
sorted_items.len())); std::cmp::min(end, sorted_items.len()),
sorted_items.len()
));
html.push_str("</body></html>"); html.push_str("</body></html>");
Ok(html) Ok(html)
} }
@@ -344,9 +359,9 @@ async fn show_item(
Path(id): Path<i64>, Path(id): Path<i64>,
) -> Result<Response, Html<String>> { ) -> Result<Response, Html<String>> {
let conn = state.db.lock().await; let conn = state.db.lock().await;
let result = build_item_details(&conn, id); let result = build_item_details(&conn, id);
match result { match result {
Ok(html) => { Ok(html) => {
// Build response with explicit Content-Length // Build response with explicit Content-Length
@@ -356,7 +371,7 @@ async fn show_item(
.body(axum::body::Body::from(html)) .body(axum::body::Body::from(html))
.map_err(|_| Html("<html><body>Internal Server Error</body></html>".to_string()))?; .map_err(|_| Html("<html><body>Internal Server Error</body></html>".to_string()))?;
Ok(response) Ok(response)
}, }
Err(e) => Err(Html(format!("<html><body>Error: {}</body></html>", e))), Err(e) => Err(Html(format!("<html><body>Error: {}</body></html>", e))),
} }
} }
@@ -366,51 +381,70 @@ fn build_item_details(conn: &Connection, id: i64) -> Result<String> {
Some(item) => item, Some(item) => item,
None => return Err(anyhow::anyhow!("Item not found")), None => return Err(anyhow::anyhow!("Item not found")),
}; };
let tags = db::get_item_tags(conn, &item)?; let tags = db::get_item_tags(conn, &item)?;
let metas = db::get_item_meta(conn, &item)?; let metas = db::get_item_meta(conn, &item)?;
let mut html = String::new(); let mut html = String::new();
html.push_str(&format!("<html><head><title>Keep - Item #{}</title>", id)); html.push_str(&format!("<html><head><title>Keep - Item #{}</title>", id));
html.push_str("<link rel=\"stylesheet\" href=\"/style.css\">"); html.push_str("<link rel=\"stylesheet\" href=\"/style.css\">");
html.push_str("</head><body>"); html.push_str("</head><body>");
html.push_str(&format!("<h1>Item #{}</h1>", id)); html.push_str(&format!("<h1>Item #{}</h1>", id));
// Single table for all details // Single table for all details
html.push_str("<table>"); html.push_str("<table>");
html.push_str(&format!("<tr><th>ID</th><td>{}</td></tr>", item.id.unwrap_or(0))); html.push_str(&format!(
html.push_str(&format!("<tr><th>Timestamp</th><td>{}</td></tr>", item.ts.format("%Y-%m-%d %H:%M:%S"))); "<tr><th>ID</th><td>{}</td></tr>",
html.push_str(&format!("<tr><th>Size</th><td>{}</td></tr>", item.size.unwrap_or(0))); item.id.unwrap_or(0)
html.push_str(&format!("<tr><th>Compression</th><td>{}</td></tr>", item.compression)); ));
html.push_str(&format!(
"<tr><th>Timestamp</th><td>{}</td></tr>",
item.ts.format("%Y-%m-%d %H:%M:%S")
));
html.push_str(&format!(
"<tr><th>Size</th><td>{}</td></tr>",
item.size.unwrap_or(0)
));
html.push_str(&format!(
"<tr><th>Compression</th><td>{}</td></tr>",
item.compression
));
// Tags row // Tags row
html.push_str("<tr><th>Tags</th><td>"); html.push_str("<tr><th>Tags</th><td>");
if tags.is_empty() { if tags.is_empty() {
html.push_str("No tags"); html.push_str("No tags");
} else { } else {
let tag_links: Vec<String> = tags.iter() let tag_links: Vec<String> = tags
.iter()
.map(|t| format!("<a href=\"/?tags={}\">{}</a>", t.name, t.name)) .map(|t| format!("<a href=\"/?tags={}\">{}</a>", t.name, t.name))
.collect(); .collect();
html.push_str(&tag_links.join(", ")); html.push_str(&tag_links.join(", "));
} }
html.push_str("</td></tr>"); html.push_str("</td></tr>");
// Metadata rows // Metadata rows
if metas.is_empty() { if metas.is_empty() {
html.push_str("<tr><th>Metadata</th><td>No metadata</td></tr>"); html.push_str("<tr><th>Metadata</th><td>No metadata</td></tr>");
} else { } else {
for meta in metas { for meta in metas {
html.push_str(&format!("<tr><th>{}</th><td>{}</td></tr>", meta.name, meta.value)); html.push_str(&format!(
"<tr><th>{}</th><td>{}</td></tr>",
meta.name, meta.value
));
} }
} }
html.push_str("</table>"); html.push_str("</table>");
// Links // Links
html.push_str("<h2>Actions</h2>"); html.push_str("<h2>Actions</h2>");
html.push_str(&format!("<p><a href=\"/api/item/{}/content\">Download Content</a></p>", id)); html.push_str(&format!(
"<p><a href=\"/api/item/{}/content\">Download Content</a></p>",
id
));
html.push_str("<p><a href=\"/\">Back to list</a></p>"); html.push_str("<p><a href=\"/\">Back to list</a></p>");
html.push_str("</body></html>"); html.push_str("</body></html>");
Ok(html) Ok(html)
} }

94
src/modes/status.rs
View File

@@ -1,14 +1,14 @@
use clap::*; use clap::*;
use log::debug;
use std::path::PathBuf; use std::path::PathBuf;
use std::str::FromStr; use std::str::FromStr;
use log::debug;
use crate::modes::common::OutputFormat;
use crate::config;
use crate::common::status::StatusInfo; use crate::common::status::StatusInfo;
use crate::config;
use crate::modes::common::OutputFormat;
use comfy_table::{Attribute, Cell, Table};
use serde_json; use serde_json;
use serde_yaml; use serde_yaml;
use comfy_table::{Table, Cell, Attribute};
use crate::common::status::PathInfo; use crate::common::status::PathInfo;
use crate::meta_plugin::MetaPluginType; use crate::meta_plugin::MetaPluginType;
@@ -28,7 +28,6 @@ fn build_path_table(path_info: &PathInfo) -> Table {
path_table path_table
} }
fn build_config_table(settings: &config::Settings) -> Table { fn build_config_table(settings: &config::Settings) -> Table {
let mut config_table = crate::modes::common::create_table(true); let mut config_table = crate::modes::common::create_table(true);
@@ -49,7 +48,7 @@ fn build_config_table(settings: &config::Settings) -> Table {
if let Some(compression) = settings.compression() { if let Some(compression) = settings.compression() {
config_table.add_row(vec!["Compression", &compression]); config_table.add_row(vec!["Compression", &compression]);
} }
config_table config_table
} }
@@ -70,61 +69,55 @@ fn build_meta_plugins_configured_table(status_info: &StatusInfo) -> Option<Table
Cell::new("Options").add_attribute(Attribute::Bold), Cell::new("Options").add_attribute(Attribute::Bold),
Cell::new("Outputs").add_attribute(Attribute::Bold), Cell::new("Outputs").add_attribute(Attribute::Bold),
]); ]);
for plugin_config in sorted_meta_plugins { for plugin_config in sorted_meta_plugins {
// Create the plugin to get its default options // Create the plugin to get its default options
let meta_plugin_type = match MetaPluginType::from_str(&plugin_config.name) { let meta_plugin_type = match MetaPluginType::from_str(&plugin_config.name) {
Ok(plugin_type) => plugin_type, Ok(plugin_type) => plugin_type,
Err(_) => continue, Err(_) => continue,
}; };
// First, create a default plugin to get its default options // First, create a default plugin to get its default options
let default_plugin = get_meta_plugin( let default_plugin = get_meta_plugin(meta_plugin_type.clone(), None, None);
meta_plugin_type.clone(),
None,
None,
);
// Start with the default options // Start with the default options
let mut effective_options = default_plugin.options().clone(); let mut effective_options = default_plugin.options().clone();
// Merge with the configured options // Merge with the configured options
for (key, value) in &plugin_config.options { for (key, value) in &plugin_config.options {
effective_options.insert(key.clone(), value.clone()); effective_options.insert(key.clone(), value.clone());
} }
// Convert outputs from HashMap<String, String> to HashMap<String, serde_yaml::Value> // Convert outputs from HashMap<String, String> to HashMap<String, serde_yaml::Value>
let outputs_converted: std::collections::HashMap<String, serde_yaml::Value> = plugin_config.outputs let outputs_converted: std::collections::HashMap<String, serde_yaml::Value> = plugin_config
.outputs
.iter() .iter()
.map(|(k, v)| (k.clone(), serde_yaml::Value::String(v.clone()))) .map(|(k, v)| (k.clone(), serde_yaml::Value::String(v.clone())))
.collect(); .collect();
// Create the actual plugin with merged options - the constructor will handle setting up outputs // Create the actual plugin with merged options - the constructor will handle setting up outputs
let actual_plugin = get_meta_plugin( let actual_plugin = get_meta_plugin(
meta_plugin_type.clone(), meta_plugin_type.clone(),
Some(effective_options.clone()), Some(effective_options.clone()),
Some(outputs_converted), Some(outputs_converted),
); );
// Get the default plugin to see its default options // Get the default plugin to see its default options
let default_plugin = get_meta_plugin( let default_plugin = get_meta_plugin(meta_plugin_type.clone(), None, None);
meta_plugin_type.clone(),
None,
None,
);
// Start with the default options // Start with the default options
let mut all_options = default_plugin.options().clone(); let mut all_options = default_plugin.options().clone();
// Merge with the configured options // Merge with the configured options
for (key, value) in &effective_options { for (key, value) in &effective_options {
all_options.insert(key.clone(), value.clone()); all_options.insert(key.clone(), value.clone());
} }
// Sort options by key and convert to a YAML string // Sort options by key and convert to a YAML string
let mut sorted_options: Vec<_> = all_options.iter().collect(); let mut sorted_options: Vec<_> = all_options.iter().collect();
sorted_options.sort_by(|a, b| a.0.cmp(b.0)); sorted_options.sort_by(|a, b| a.0.cmp(b.0));
let sorted_options_map: std::collections::BTreeMap<_, _> = sorted_options.into_iter().collect(); let sorted_options_map: std::collections::BTreeMap<_, _> =
sorted_options.into_iter().collect();
let options_str = if sorted_options_map.is_empty() { let options_str = if sorted_options_map.is_empty() {
"{}".to_string() "{}".to_string()
} else { } else {
@@ -133,7 +126,7 @@ fn build_meta_plugins_configured_table(status_info: &StatusInfo) -> Option<Table
.trim() .trim()
.to_string() .to_string()
}; };
// Show only non-null outputs from the plugin // Show only non-null outputs from the plugin
// Collect and sort outputs by their string representation // Collect and sort outputs by their string representation
let mut enabled_output_pairs = Vec::new(); let mut enabled_output_pairs = Vec::new();
@@ -142,7 +135,7 @@ fn build_meta_plugins_configured_table(status_info: &StatusInfo) -> Option<Table
if value.is_null() { if value.is_null() {
continue; continue;
} }
// Convert serde_yaml::Value to a string representation // Convert serde_yaml::Value to a string representation
let value_str = match value { let value_str = match value {
serde_yaml::Value::String(s) => s.clone(), serde_yaml::Value::String(s) => s.clone(),
@@ -167,31 +160,27 @@ fn build_meta_plugins_configured_table(status_info: &StatusInfo) -> Option<Table
enabled_output_pairs.push((key.clone(), format!("{}->{}", key, value_str))); enabled_output_pairs.push((key.clone(), format!("{}->{}", key, value_str)));
} }
} }
// Sort outputs by their display value (second element of the tuple) // Sort outputs by their display value (second element of the tuple)
enabled_output_pairs.sort_by(|a, b| a.1.cmp(&b.1)); enabled_output_pairs.sort_by(|a, b| a.1.cmp(&b.1));
// Join each output on a new line // Join each output on a new line
let outputs_str = if enabled_output_pairs.is_empty() { let outputs_str = if enabled_output_pairs.is_empty() {
"{}".to_string() "{}".to_string()
} else { } else {
enabled_output_pairs.into_iter() enabled_output_pairs
.into_iter()
.map(|(_, display)| display) .map(|(_, display)| display)
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join("\n") .join("\n")
}; };
table.add_row(vec![ table.add_row(vec![plugin_config.name.clone(), options_str, outputs_str]);
plugin_config.name.clone(),
options_str,
outputs_str,
]);
} }
Some(table) Some(table)
} }
pub fn mode_status( pub fn mode_status(
cmd: &mut Command, cmd: &mut Command,
settings: &config::Settings, settings: &config::Settings,
@@ -199,7 +188,7 @@ pub fn mode_status(
db_path: PathBuf, db_path: PathBuf,
) -> Result<(), anyhow::Error> { ) -> Result<(), anyhow::Error> {
debug!("STATUS: Starting mode_status function"); debug!("STATUS: Starting mode_status function");
let status_service = crate::services::status_service::StatusService::new(); let status_service = crate::services::status_service::StatusService::new();
let output_format = crate::modes::common::settings_output_format(settings); let output_format = crate::modes::common::settings_output_format(settings);
debug!("STATUS: About to generate status info"); debug!("STATUS: About to generate status info");
@@ -210,18 +199,27 @@ pub fn mode_status(
OutputFormat::Table => { OutputFormat::Table => {
println!("CONFIG:"); println!("CONFIG:");
let config_table = build_config_table(settings); let config_table = build_config_table(settings);
println!("{}", crate::modes::common::trim_lines_end(&config_table.trim_fmt())); println!(
"{}",
crate::modes::common::trim_lines_end(&config_table.trim_fmt())
);
println!(); println!();
println!("PATHS:"); println!("PATHS:");
let path_table = build_path_table(&status_info.paths); let path_table = build_path_table(&status_info.paths);
println!("{}", crate::modes::common::trim_lines_end(&path_table.trim_fmt())); println!(
"{}",
crate::modes::common::trim_lines_end(&path_table.trim_fmt())
);
println!(); println!();
// Always try to print META PLUGINS CONFIGURED section using status_info // Always try to print META PLUGINS CONFIGURED section using status_info
if let Some(meta_plugins_table) = build_meta_plugins_configured_table(&status_info) { if let Some(meta_plugins_table) = build_meta_plugins_configured_table(&status_info) {
println!("META PLUGINS CONFIGURED:"); println!("META PLUGINS CONFIGURED:");
println!("{}", crate::modes::common::trim_lines_end(&meta_plugins_table.trim_fmt())); println!(
"{}",
crate::modes::common::trim_lines_end(&meta_plugins_table.trim_fmt())
);
println!(); println!();
} else { } else {
println!("META PLUGINS CONFIGURED:"); println!("META PLUGINS CONFIGURED:");
@@ -229,12 +227,12 @@ pub fn mode_status(
println!(); println!();
} }
Ok(()) Ok(())
}, }
OutputFormat::Json => { OutputFormat::Json => {
// Create a subset for status info that includes everything // Create a subset for status info that includes everything
println!("{}", serde_json::to_string_pretty(&status_info)?); println!("{}", serde_json::to_string_pretty(&status_info)?);
Ok(()) Ok(())
}, }
OutputFormat::Yaml => { OutputFormat::Yaml => {
println!("{}", serde_yaml::to_string(&status_info)?); println!("{}", serde_yaml::to_string(&status_info)?);
Ok(()) Ok(())

71
src/modes/status_plugins.rs
View File

@@ -1,7 +1,7 @@
use clap::*; use clap::*;
use log::debug;
use std::path::PathBuf; use std::path::PathBuf;
use std::str::FromStr; use std::str::FromStr;
use log::debug;
/// Helper function to convert serde_json::Value to serde_yaml::Value. /// Helper function to convert serde_json::Value to serde_yaml::Value.
/// ///
@@ -49,17 +49,18 @@ fn convert_json_to_yaml_value(value: &serde_json::Value) -> serde_yaml::Value {
} }
} }
use crate::modes::common::OutputFormat;
use crate::config; use crate::config;
use crate::modes::common::OutputFormat;
use comfy_table::{Attribute, Cell, Table};
use serde_json; use serde_json;
use serde_yaml; use serde_yaml;
use comfy_table::{Table, Cell, Attribute};
use crate::common::status::{CompressionInfo, MetaPluginInfo};
use crate::meta_plugin::{MetaPluginType, get_meta_plugin}; use crate::meta_plugin::{MetaPluginType, get_meta_plugin};
use crate::common::status::{MetaPluginInfo, CompressionInfo};
fn build_meta_plugin_table(
fn build_meta_plugin_table(meta_plugin_info: &std::collections::HashMap<String, MetaPluginInfo>) -> Table { meta_plugin_info: &std::collections::HashMap<String, MetaPluginInfo>,
) -> Table {
// Builds a formatted table displaying meta plugin information. // Builds a formatted table displaying meta plugin information.
// //
// Sorts plugins by name and displays options as YAML and outputs as a list. // Sorts plugins by name and displays options as YAML and outputs as a list.
@@ -91,11 +92,7 @@ fn build_meta_plugin_table(meta_plugin_info: &std::collections::HashMap<String,
}; };
// Create a default plugin to get its default options // Create a default plugin to get its default options
let default_plugin = get_meta_plugin( let default_plugin = get_meta_plugin(meta_plugin_type.clone(), None, None);
meta_plugin_type.clone(),
None,
None,
);
// Get and sort options // Get and sort options
let mut options: Vec<_> = default_plugin.options().iter().collect(); let mut options: Vec<_> = default_plugin.options().iter().collect();
@@ -121,11 +118,7 @@ fn build_meta_plugin_table(meta_plugin_info: &std::collections::HashMap<String,
output_keys.join("\n") output_keys.join("\n")
}; };
meta_plugin_table.add_row(vec![ meta_plugin_table.add_row(vec![info.meta_name.clone(), options_str, outputs_display]);
info.meta_name.clone(),
options_str,
outputs_display,
]);
} }
meta_plugin_table meta_plugin_table
@@ -172,7 +165,7 @@ fn build_compression_table(compression_info: &Vec<CompressionInfo>) -> Table {
compression_table compression_table
} }
fn build_filter_plugin_table(filter_plugins: &Vec<crate::common::status::FilterPluginInfo>) -> Table { fn build_filter_plugin_table(filter_plugins: &[crate::common::status::FilterPluginInfo]) -> Table {
// Builds a formatted table displaying filter plugin information. // Builds a formatted table displaying filter plugin information.
// //
// Sorts plugins by name and formats options as YAML sequence. // Sorts plugins by name and formats options as YAML sequence.
@@ -244,10 +237,7 @@ fn build_filter_plugin_table(filter_plugins: &Vec<crate::common::status::FilterP
serde_yaml::Value::Mapping(yaml_mapping) serde_yaml::Value::Mapping(yaml_mapping)
} }
}; };
opt_map.insert( opt_map.insert(serde_yaml::Value::String("default".to_string()), yaml_value);
serde_yaml::Value::String("default".to_string()),
yaml_value,
);
} else { } else {
opt_map.insert( opt_map.insert(
serde_yaml::Value::String("default".to_string()), serde_yaml::Value::String("default".to_string()),
@@ -275,11 +265,7 @@ fn build_filter_plugin_table(filter_plugins: &Vec<crate::common::status::FilterP
// If no filter plugins are available, add a row indicating that // If no filter plugins are available, add a row indicating that
if filter_plugins.is_empty() { if filter_plugins.is_empty() {
filter_plugin_table.add_row(vec![ filter_plugin_table.add_row(vec!["No filter plugins available", "{}", ""]);
"No filter plugins available",
"{}",
"",
]);
} }
filter_plugin_table filter_plugin_table
@@ -306,7 +292,7 @@ pub fn mode_status_plugins(
// //
// `Ok(())` on success, or anyhow::Error. // `Ok(())` on success, or anyhow::Error.
debug!("STATUS_PLUGINS: Starting mode_status_plugins function"); debug!("STATUS_PLUGINS: Starting mode_status_plugins function");
let status_service = crate::services::status_service::StatusService::new(); let status_service = crate::services::status_service::StatusService::new();
let output_format = crate::modes::common::settings_output_format(settings); let output_format = crate::modes::common::settings_output_format(settings);
debug!("STATUS_PLUGINS: About to generate status info"); debug!("STATUS_PLUGINS: About to generate status info");
@@ -317,20 +303,29 @@ pub fn mode_status_plugins(
OutputFormat::Table => { OutputFormat::Table => {
println!("META PLUGINS:"); println!("META PLUGINS:");
let meta_table = build_meta_plugin_table(&status_info.meta_plugins); let meta_table = build_meta_plugin_table(&status_info.meta_plugins);
println!("{}", crate::modes::common::trim_lines_end(&meta_table.trim_fmt())); println!(
"{}",
crate::modes::common::trim_lines_end(&meta_table.trim_fmt())
);
println!(); println!();
println!("COMPRESSION PLUGINS:"); println!("COMPRESSION PLUGINS:");
let compression_table = build_compression_table(&status_info.compression); let compression_table = build_compression_table(&status_info.compression);
println!("{}", crate::modes::common::trim_lines_end(&compression_table.trim_fmt())); println!(
"{}",
crate::modes::common::trim_lines_end(&compression_table.trim_fmt())
);
println!(); println!();
println!("FILTER PLUGINS:"); println!("FILTER PLUGINS:");
let filter_table = build_filter_plugin_table(&status_info.filter_plugins); let filter_table = build_filter_plugin_table(&status_info.filter_plugins);
println!("{}", crate::modes::common::trim_lines_end(&filter_table.trim_fmt())); println!(
"{}",
crate::modes::common::trim_lines_end(&filter_table.trim_fmt())
);
println!(); println!();
Ok(()) Ok(())
}, }
OutputFormat::Json => { OutputFormat::Json => {
// Create a subset for plugins only using status_info // Create a subset for plugins only using status_info
let plugins_info = serde_json::json!({ let plugins_info = serde_json::json!({
@@ -340,18 +335,18 @@ pub fn mode_status_plugins(
}); });
println!("{}", serde_json::to_string_pretty(&plugins_info)?); println!("{}", serde_json::to_string_pretty(&plugins_info)?);
Ok(()) Ok(())
}, }
OutputFormat::Yaml => { OutputFormat::Yaml => {
// Create a proper structure for plugins info using status_info // Create a proper structure for plugins info using status_info
use serde_yaml::Mapping; use serde_yaml::Mapping;
let mut plugins_mapping = Mapping::new(); let mut plugins_mapping = Mapping::new();
// Add available plugins // Add available plugins
plugins_mapping.insert( plugins_mapping.insert(
serde_yaml::Value::String("meta_plugins_available".to_string()), serde_yaml::Value::String("meta_plugins_available".to_string()),
serde_yaml::to_value(&status_info.meta_plugins)?, serde_yaml::to_value(&status_info.meta_plugins)?,
); );
// Add configured plugins if they exist // Add configured plugins if they exist
if let Some(configured_plugins) = &status_info.configured_meta_plugins { if let Some(configured_plugins) = &status_info.configured_meta_plugins {
plugins_mapping.insert( plugins_mapping.insert(
@@ -359,13 +354,13 @@ pub fn mode_status_plugins(
serde_yaml::to_value(configured_plugins)?, serde_yaml::to_value(configured_plugins)?,
); );
} }
// Add filter plugins // Add filter plugins
plugins_mapping.insert( plugins_mapping.insert(
serde_yaml::Value::String("filter_plugins".to_string()), serde_yaml::Value::String("filter_plugins".to_string()),
serde_yaml::to_value(&status_info.filter_plugins)?, serde_yaml::to_value(&status_info.filter_plugins)?,
); );
println!("{}", serde_yaml::to_string(&plugins_mapping)?); println!("{}", serde_yaml::to_string(&plugins_mapping)?);
Ok(()) Ok(())
} }

105
src/services/async_item_service.rs
View File

@@ -44,15 +44,15 @@ impl AsyncItemService {
/// ///
/// A new `AsyncItemService`. /// A new `AsyncItemService`.
pub fn new( pub fn new(
data_dir: PathBuf, data_dir: PathBuf,
db: Arc<Mutex<Connection>>, db: Arc<Mutex<Connection>>,
item_service: Arc<ItemService>, item_service: Arc<ItemService>,
cmd: Arc<Mutex<Command>>, cmd: Arc<Mutex<Command>>,
settings: Arc<Settings>, settings: Arc<Settings>,
) -> Self { ) -> Self {
Self { Self {
data_dir, data_dir,
db, db,
item_service, item_service,
cmd, cmd,
settings, settings,
@@ -82,7 +82,7 @@ impl AsyncItemService {
{ {
let db = self.db.clone(); let db = self.db.clone();
let item_service = self.item_service.clone(); let item_service = self.item_service.clone();
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
let conn = db.blocking_lock(); let conn = db.blocking_lock();
f(&conn, &item_service) f(&conn, &item_service)
@@ -92,11 +92,13 @@ impl AsyncItemService {
} }
pub async fn get_item(&self, id: i64) -> Result<ItemWithMeta, CoreError> { pub async fn get_item(&self, id: i64) -> Result<ItemWithMeta, CoreError> {
self.execute_blocking(move |conn, item_service| item_service.get_item(conn, id)).await self.execute_blocking(move |conn, item_service| item_service.get_item(conn, id))
.await
} }
pub async fn get_item_content(&self, id: i64) -> Result<ItemWithContent, CoreError> { pub async fn get_item_content(&self, id: i64) -> Result<ItemWithContent, CoreError> {
self.execute_blocking(move |conn, item_service| item_service.get_item_content(conn, id)).await self.execute_blocking(move |conn, item_service| item_service.get_item_content(conn, id))
.await
} }
pub async fn get_item_content_info( pub async fn get_item_content_info(
@@ -104,7 +106,10 @@ impl AsyncItemService {
id: i64, id: i64,
filter: Option<String>, filter: Option<String>,
) -> Result<(Vec<u8>, String, bool), CoreError> { ) -> Result<(Vec<u8>, String, bool), CoreError> {
self.execute_blocking(move |conn, item_service| item_service.get_item_content_info(conn, id, filter)).await self.execute_blocking(move |conn, item_service| {
item_service.get_item_content_info(conn, id, filter)
})
.await
} }
pub async fn stream_item_content_by_id( pub async fn stream_item_content_by_id(
@@ -113,11 +118,25 @@ impl AsyncItemService {
allow_binary: bool, allow_binary: bool,
offset: u64, offset: u64,
length: u64, length: u64,
) -> Result<(std::pin::Pin<Box<dyn tokio_stream::Stream<Item = Result<tokio_util::bytes::Bytes, std::io::Error>> + Send>>, String), CoreError> { ) -> Result<
let content = self.execute_blocking(move |conn, item_service| { (
let item_with_content = item_service.get_item_content(conn, item_id)?; std::pin::Pin<
Ok::<_, CoreError>(item_with_content.content) Box<
}).await?; dyn tokio_stream::Stream<
Item = Result<tokio_util::bytes::Bytes, std::io::Error>,
> + Send,
>,
>,
String,
),
CoreError,
> {
let content = self
.execute_blocking(move |conn, item_service| {
let item_with_content = item_service.get_item_content(conn, item_id)?;
Ok::<_, CoreError>(item_with_content.content)
})
.await?;
// Clone content for use in the binary check closure // Clone content for use in the binary check closure
let content_clone = content.clone(); let content_clone = content.clone();
@@ -150,7 +169,9 @@ impl AsyncItemService {
// Check if content is binary when allow_binary is false // Check if content is binary when allow_binary is false
if !allow_binary && is_binary { if !allow_binary && is_binary {
return Err(CoreError::InvalidInput("Binary content not allowed".to_string())); return Err(CoreError::InvalidInput(
"Binary content not allowed".to_string(),
));
} }
// Create a stream that reads only the requested portion // Create a stream that reads only the requested portion
@@ -165,7 +186,8 @@ impl AsyncItemService {
}; };
let stream = if start < content_len { let stream = if start < content_len {
let chunk = tokio_util::bytes::Bytes::from(content[start as usize..end as usize].to_vec()); let chunk =
tokio_util::bytes::Bytes::from(content[start as usize..end as usize].to_vec());
Box::pin(tokio_stream::iter(vec![Ok(chunk)])) Box::pin(tokio_stream::iter(vec![Ok(chunk)]))
} else { } else {
Box::pin(tokio_stream::iter(vec![])) Box::pin(tokio_stream::iter(vec![]))
@@ -182,7 +204,19 @@ impl AsyncItemService {
offset: u64, offset: u64,
length: u64, length: u64,
filter: Option<String>, filter: Option<String>,
) -> Result<(std::pin::Pin<Box<dyn tokio_stream::Stream<Item = Result<tokio_util::bytes::Bytes, std::io::Error>> + Send>>, String), CoreError> { ) -> Result<
(
std::pin::Pin<
Box<
dyn tokio_stream::Stream<
Item = Result<tokio_util::bytes::Bytes, std::io::Error>,
> + Send,
>,
>,
String,
),
CoreError,
> {
// Use provided metadata to determine MIME type and binary status // Use provided metadata to determine MIME type and binary status
let mime_type = metadata let mime_type = metadata
.get("mime_type") .get("mime_type")
@@ -195,15 +229,14 @@ impl AsyncItemService {
text_val == "false" text_val == "false"
} else { } else {
// Get binary status using streaming approach // Get binary status using streaming approach
let (_, _, is_binary) = self.get_item_content_info_streaming( let (_, _, is_binary) = self.get_item_content_info_streaming(item_id, None).await?;
item_id,
None
).await?;
is_binary is_binary
}; };
if is_binary { if is_binary {
return Err(CoreError::InvalidInput("Binary content not allowed".to_string())); return Err(CoreError::InvalidInput(
"Binary content not allowed".to_string(),
));
} }
} }
@@ -215,11 +248,9 @@ impl AsyncItemService {
let filter = filter.clone(); let filter = filter.clone();
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
let conn = db.blocking_lock(); let conn = db.blocking_lock();
item_service.get_item_content_info_streaming( item_service
&conn, .get_item_content_info_streaming(&conn, item_id, filter)
item_id, .map(|(reader, _, _)| reader)
filter
).map(|(reader, _, _)| reader)
}) })
.await .await
.map_err(|e| CoreError::Other(anyhow::anyhow!("Blocking task failed: {}", e)))? .map_err(|e| CoreError::Other(anyhow::anyhow!("Blocking task failed: {}", e)))?
@@ -302,7 +333,10 @@ impl AsyncItemService {
item_id: i64, item_id: i64,
filter: Option<String>, filter: Option<String>,
) -> Result<(Box<dyn Read + Send>, String, bool), CoreError> { ) -> Result<(Box<dyn Read + Send>, String, bool), CoreError> {
self.execute_blocking(move |conn, item_service| item_service.get_item_content_info_streaming(conn, item_id, filter)).await self.execute_blocking(move |conn, item_service| {
item_service.get_item_content_info_streaming(conn, item_id, filter)
})
.await
} }
pub async fn find_item( pub async fn find_item(
@@ -314,7 +348,10 @@ impl AsyncItemService {
let ids_clone = ids.clone(); let ids_clone = ids.clone();
let tags_clone = tags.clone(); let tags_clone = tags.clone();
let meta_clone = meta.clone(); let meta_clone = meta.clone();
self.execute_blocking(move |conn, item_service| item_service.find_item(conn, &ids_clone, &tags_clone, &meta_clone)).await self.execute_blocking(move |conn, item_service| {
item_service.find_item(conn, &ids_clone, &tags_clone, &meta_clone)
})
.await
} }
pub async fn list_items( pub async fn list_items(
@@ -324,13 +361,16 @@ impl AsyncItemService {
) -> Result<Vec<ItemWithMeta>, CoreError> { ) -> Result<Vec<ItemWithMeta>, CoreError> {
let tags_clone = tags.clone(); let tags_clone = tags.clone();
let meta_clone = meta.clone(); let meta_clone = meta.clone();
self.execute_blocking(move |conn, item_service| item_service.list_items(conn, &tags_clone, &meta_clone)).await self.execute_blocking(move |conn, item_service| {
item_service.list_items(conn, &tags_clone, &meta_clone)
})
.await
} }
pub async fn delete_item(&self, id: i64) -> Result<(), CoreError> { pub async fn delete_item(&self, id: i64) -> Result<(), CoreError> {
let db = self.db.clone(); let db = self.db.clone();
let item_service = self.item_service.clone(); let item_service = self.item_service.clone();
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
let mut conn = db.blocking_lock(); let mut conn = db.blocking_lock();
item_service.delete_item(&mut conn, id) item_service.delete_item(&mut conn, id)
@@ -354,7 +394,8 @@ impl AsyncItemService {
let mut conn = db.blocking_lock(); let mut conn = db.blocking_lock();
let mut cmd = cmd.blocking_lock(); let mut cmd = cmd.blocking_lock();
let settings = settings.as_ref(); let settings = settings.as_ref();
item_service.save_item_from_mcp(&content, &tags, &metadata, &mut cmd, settings, &mut conn) item_service
.save_item_from_mcp(&content, &tags, &metadata, &mut cmd, settings, &mut conn)
}) })
.await .await
.unwrap() .unwrap()

21
src/services/compression_service.rs
View File

@@ -1,9 +1,9 @@
use crate::compression_engine::{get_compression_engine, CompressionType}; use crate::compression_engine::{CompressionType, get_compression_engine};
use crate::services::error::CoreError; use crate::services::error::CoreError;
use anyhow::anyhow;
use std::io::Read; use std::io::Read;
use std::path::PathBuf; use std::path::PathBuf;
use std::str::FromStr; use std::str::FromStr;
use anyhow::anyhow;
pub struct CompressionService; pub struct CompressionService;
@@ -28,7 +28,6 @@ pub struct CompressionService;
/// let service = CompressionService::new(); /// let service = CompressionService::new();
/// let content = service.get_item_content(path, "gzip")?; /// let content = service.get_item_content(path, "gzip")?;
/// ``` /// ```
impl CompressionService { impl CompressionService {
/// Creates a new CompressionService instance. /// Creates a new CompressionService instance.
/// ///
@@ -72,14 +71,19 @@ impl CompressionService {
/// let content = service.get_item_content(item_path, "lz4")?; /// let content = service.get_item_content(item_path, "lz4")?;
/// assert_eq!(content.len(), expected_size); /// assert_eq!(content.len(), expected_size);
/// ``` /// ```
pub fn get_item_content(&self, item_path: PathBuf, compression: &str) -> Result<Vec<u8>, CoreError> { pub fn get_item_content(
&self,
item_path: PathBuf,
compression: &str,
) -> Result<Vec<u8>, CoreError> {
let compression_type = CompressionType::from_str(compression) let compression_type = CompressionType::from_str(compression)
.map_err(|e| CoreError::Compression(e.to_string()))?; .map_err(|e| CoreError::Compression(e.to_string()))?;
let engine = get_compression_engine(compression_type) let engine = get_compression_engine(compression_type)
.map_err(|e| CoreError::Other(anyhow!(e.to_string())))?; .map_err(|e| CoreError::Other(anyhow!(e.to_string())))?;
let mut reader = engine.open(item_path.clone()) let mut reader = engine.open(item_path.clone()).map_err(|e| {
.map_err(|e| CoreError::Other(anyhow!("Failed to open item file {:?}: {}", item_path, e)))?; CoreError::Other(anyhow!("Failed to open item file {:?}: {}", item_path, e))
})?;
let mut content = Vec::new(); let mut content = Vec::new();
reader.read_to_end(&mut content)?; reader.read_to_end(&mut content)?;
Ok(content) Ok(content)
@@ -122,8 +126,9 @@ impl CompressionService {
let engine = get_compression_engine(compression_type) let engine = get_compression_engine(compression_type)
.map_err(|e| CoreError::Other(anyhow!(e.to_string())))?; .map_err(|e| CoreError::Other(anyhow!(e.to_string())))?;
let reader = engine.open(item_path.clone()) let reader = engine.open(item_path.clone()).map_err(|e| {
.map_err(|e| CoreError::Other(anyhow!("Failed to open item file {:?}: {}", item_path, e)))?; CoreError::Other(anyhow!("Failed to open item file {:?}: {}", item_path, e))
})?;
// Since we can't guarantee the reader implements Send, we need to wrap it // Since we can't guarantee the reader implements Send, we need to wrap it
// We'll read the content into a buffer and return a Cursor which is Send // We'll read the content into a buffer and return a Cursor which is Send
// This is not ideal for large files, but it ensures Send is implemented // This is not ideal for large files, but it ensures Send is implemented

35
src/services/filter_service.rs
View File

@@ -1,9 +1,11 @@
use crate::filter_plugin::{FilterChain, parse_filter_string}; use crate::filter_plugin::{FilterChain, parse_filter_string};
use std::collections::HashMap;
use std::io::{Result, Read, Write};
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use std::collections::HashMap;
use std::io::{Read, Result, Write};
use std::sync::Mutex; use std::sync::Mutex;
type FilterConstructor = fn() -> Box<dyn crate::filter_plugin::FilterPlugin>;
/// Service for managing filter chains and plugin registration. /// Service for managing filter chains and plugin registration.
/// ///
/// The `FilterService` provides functionality to parse filter strings, create filter chains, /// The `FilterService` provides functionality to parse filter strings, create filter chains,
@@ -20,6 +22,12 @@ use std::sync::Mutex;
/// ``` /// ```
pub struct FilterService; pub struct FilterService;
impl Default for FilterService {
fn default() -> Self {
Self::new()
}
}
impl FilterService { impl FilterService {
/// Creates a new `FilterService` instance. /// Creates a new `FilterService` instance.
/// ///
@@ -96,10 +104,10 @@ impl FilterService {
/// service.filter_data(&mut chain, &mut reader, &mut writer)?; /// service.filter_data(&mut chain, &mut reader, &mut writer)?;
/// ``` /// ```
pub fn filter_data<R: Read, W: Write>( pub fn filter_data<R: Read, W: Write>(
&self, &self,
chain: &mut Option<FilterChain>, chain: &mut Option<FilterChain>,
reader: &mut R, reader: &mut R,
writer: &mut W writer: &mut W,
) -> Result<()> { ) -> Result<()> {
if let Some(chain) = chain { if let Some(chain) = chain {
chain.filter(reader, writer) chain.filter(reader, writer)
@@ -139,13 +147,13 @@ impl FilterService {
let mut chain = self.create_filter_chain(filter_str)?; let mut chain = self.create_filter_chain(filter_str)?;
let mut reader = std::io::Cursor::new(data); let mut reader = std::io::Cursor::new(data);
let mut writer = Vec::new(); let mut writer = Vec::new();
if let Some(ref mut chain) = chain { if let Some(ref mut chain) = chain {
chain.filter(&mut reader, &mut writer)?; chain.filter(&mut reader, &mut writer)?;
} else { } else {
std::io::copy(&mut reader, &mut writer)?; std::io::copy(&mut reader, &mut writer)?;
} }
Ok(writer) Ok(writer)
} }
} }
@@ -158,7 +166,7 @@ impl FilterService {
/// # Panics /// # Panics
/// ///
/// Lock acquisition failures (rare) cause panics in accessors. /// Lock acquisition failures (rare) cause panics in accessors.
static FILTER_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<String, fn() -> Box<dyn crate::filter_plugin::FilterPlugin>>>> = static FILTER_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<String, FilterConstructor>>> =
Lazy::new(|| Mutex::new(HashMap::new())); Lazy::new(|| Mutex::new(HashMap::new()));
/// Registers a filter plugin in the global registry. /// Registers a filter plugin in the global registry.
@@ -180,8 +188,11 @@ static FILTER_PLUGIN_REGISTRY: Lazy<Mutex<HashMap<String, fn() -> Box<dyn crate:
/// ```rust /// ```rust
/// register_filter_plugin("custom_filter", || Box::new(CustomFilter::default())); /// register_filter_plugin("custom_filter", || Box::new(CustomFilter::default()));
/// ``` /// ```
pub fn register_filter_plugin(name: &str, constructor: fn() -> Box<dyn crate::filter_plugin::FilterPlugin>) { pub fn register_filter_plugin(name: &str, constructor: FilterConstructor) {
FILTER_PLUGIN_REGISTRY.lock().unwrap().insert(name.to_string(), constructor); FILTER_PLUGIN_REGISTRY
.lock()
.unwrap()
.insert(name.to_string(), constructor);
} }
/// Retrieves a snapshot of all registered filter plugins. /// Retrieves a snapshot of all registered filter plugins.
@@ -202,6 +213,6 @@ pub fn register_filter_plugin(name: &str, constructor: fn() -> Box<dyn crate::fi
/// let plugins = get_available_filter_plugins(); /// let plugins = get_available_filter_plugins();
/// assert!(plugins.contains_key("head_bytes")); /// assert!(plugins.contains_key("head_bytes"));
/// ``` /// ```
pub fn get_available_filter_plugins() -> HashMap<String, fn() -> Box<dyn crate::filter_plugin::FilterPlugin>> { pub fn get_available_filter_plugins() -> HashMap<String, FilterConstructor> {
FILTER_PLUGIN_REGISTRY.lock().unwrap().clone() FILTER_PLUGIN_REGISTRY.lock().unwrap().clone()
} }

234
src/services/item_service.rs
View File

@@ -1,14 +1,14 @@
use crate::common::PIPESIZE; use crate::common::PIPESIZE;
use crate::compression_engine::{CompressionType, get_compression_engine};
use crate::config::Settings; use crate::config::Settings;
use crate::db::{self, Meta};
use crate::filter_plugin;
use crate::modes::common::settings_compression_type;
use crate::services::compression_service::CompressionService; use crate::services::compression_service::CompressionService;
use crate::services::error::CoreError; use crate::services::error::CoreError;
use crate::services::filter_service::FilterService; use crate::services::filter_service::FilterService;
use crate::services::meta_service::MetaService; use crate::services::meta_service::MetaService;
use crate::services::types::{ItemWithContent, ItemWithMeta}; use crate::services::types::{ItemWithContent, ItemWithMeta};
use crate::db::{self, Meta};
use crate::compression_engine::{get_compression_engine, CompressionType};
use crate::modes::common::settings_compression_type;
use crate::filter_plugin;
use clap::Command; use clap::Command;
use log::debug; use log::debug;
use rusqlite::Connection; use rusqlite::Connection;
@@ -53,7 +53,10 @@ impl ItemService {
/// let service = ItemService::new(PathBuf::from("/data")); /// let service = ItemService::new(PathBuf::from("/data"));
/// ``` /// ```
pub fn new(data_path: PathBuf) -> Self { pub fn new(data_path: PathBuf) -> Self {
debug!("ITEM_SERVICE: Creating new ItemService with data_path: {:?}", data_path); debug!(
"ITEM_SERVICE: Creating new ItemService with data_path: {:?}",
data_path
);
Self { Self {
data_path, data_path,
compression_service: CompressionService::new(), compression_service: CompressionService::new(),
@@ -93,7 +96,11 @@ impl ItemService {
let tags = db::get_item_tags(conn, &item)?; let tags = db::get_item_tags(conn, &item)?;
debug!("ITEM_SERVICE: Found {} tags for item {}", tags.len(), id); debug!("ITEM_SERVICE: Found {} tags for item {}", tags.len(), id);
let meta = db::get_item_meta(conn, &item)?; let meta = db::get_item_meta(conn, &item)?;
debug!("ITEM_SERVICE: Found {} meta entries for item {}", meta.len(), id); debug!(
"ITEM_SERVICE: Found {} meta entries for item {}",
meta.len(),
id
);
Ok(ItemWithMeta { item, tags, meta }) Ok(ItemWithMeta { item, tags, meta })
} }
@@ -121,13 +128,23 @@ impl ItemService {
/// let item_with_content = item_service.get_item_content(&conn, 1)?; /// let item_with_content = item_service.get_item_content(&conn, 1)?;
/// assert!(!item_with_content.content.is_empty()); /// assert!(!item_with_content.content.is_empty());
/// ``` /// ```
pub fn get_item_content(&self, conn: &Connection, id: i64) -> Result<ItemWithContent, CoreError> { pub fn get_item_content(
&self,
conn: &Connection,
id: i64,
) -> Result<ItemWithContent, CoreError> {
debug!("ITEM_SERVICE: Getting item content for id: {}", id); debug!("ITEM_SERVICE: Getting item content for id: {}", id);
let item_with_meta = self.get_item(conn, id)?; let item_with_meta = self.get_item(conn, id)?;
let item_id = item_with_meta.item.id.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?; let item_id = item_with_meta
.item
.id
.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?;
if item_id <= 0 { if item_id <= 0 {
return Err(CoreError::InvalidInput(format!("Invalid item ID: {}", item_id))); return Err(CoreError::InvalidInput(format!(
"Invalid item ID: {}",
item_id
)));
} }
let mut item_path = self.data_path.clone(); let mut item_path = self.data_path.clone();
@@ -137,7 +154,11 @@ impl ItemService {
let content = self let content = self
.compression_service .compression_service
.get_item_content(item_path, &item_with_meta.item.compression)?; .get_item_content(item_path, &item_with_meta.item.compression)?;
debug!("ITEM_SERVICE: Read {} bytes of content for item {}", content.len(), id); debug!(
"ITEM_SERVICE: Read {} bytes of content for item {}",
content.len(),
id
);
Ok(ItemWithContent { Ok(ItemWithContent {
item_with_meta, item_with_meta,
@@ -176,14 +197,13 @@ impl ItemService {
filter: Option<String>, filter: Option<String>,
) -> Result<(Vec<u8>, String, bool), CoreError> { ) -> Result<(Vec<u8>, String, bool), CoreError> {
// Use streaming approach to handle all filtering options consistently // Use streaming approach to handle all filtering options consistently
let (mut reader, mime_type, is_binary) = self.get_item_content_info_streaming( let (mut reader, mime_type, is_binary) =
conn, id, filter self.get_item_content_info_streaming(conn, id, filter)?;
)?;
// Read all the filtered content into a buffer // Read all the filtered content into a buffer
let mut content = Vec::new(); let mut content = Vec::new();
reader.read_to_end(&mut content)?; reader.read_to_end(&mut content)?;
Ok((content, mime_type, is_binary)) Ok((content, mime_type, is_binary))
} }
@@ -222,13 +242,14 @@ impl ItemService {
} }
// Read only the first 8192 bytes for binary detection // Read only the first 8192 bytes for binary detection
let mut sample_reader = self.compression_service.stream_item_content( let mut sample_reader = self
item_path, .compression_service
compression .stream_item_content(item_path, compression)?;
)?;
let mut sample_buffer = vec![0; 8192]; let mut sample_buffer = vec![0; 8192];
let bytes_read = sample_reader.read(&mut sample_buffer)?; let bytes_read = sample_reader.read(&mut sample_buffer)?;
Ok(crate::common::is_binary::is_binary(&sample_buffer[..bytes_read])) Ok(crate::common::is_binary::is_binary(
&sample_buffer[..bytes_read],
))
} }
/// Retrieves a streaming reader for item content with optional filtering. /// Retrieves a streaming reader for item content with optional filtering.
@@ -263,12 +284,15 @@ impl ItemService {
) -> Result<(Box<dyn Read + Send>, String, bool), CoreError> { ) -> Result<(Box<dyn Read + Send>, String, bool), CoreError> {
// Convert filter string to FilterChain if provided // Convert filter string to FilterChain if provided
let filter_chain = if let Some(filter_str) = filter { let filter_chain = if let Some(filter_str) = filter {
self.filter_service.create_filter_chain(Some(&filter_str)) self.filter_service
.map_err(|e| CoreError::InvalidInput(format!("Failed to create filter chain: {}", e)))? .create_filter_chain(Some(&filter_str))
.map_err(|e| {
CoreError::InvalidInput(format!("Failed to create filter chain: {}", e))
})?
} else { } else {
None None
}; };
self.get_item_content_info_streaming_with_chain(conn, id, filter_chain.as_ref()) self.get_item_content_info_streaming_with_chain(conn, id, filter_chain.as_ref())
} }
@@ -302,19 +326,24 @@ impl ItemService {
filter_chain: Option<&filter_plugin::FilterChain>, filter_chain: Option<&filter_plugin::FilterChain>,
) -> Result<(Box<dyn Read + Send>, String, bool), CoreError> { ) -> Result<(Box<dyn Read + Send>, String, bool), CoreError> {
let item_with_meta = self.get_item(conn, id)?; let item_with_meta = self.get_item(conn, id)?;
let item_id = item_with_meta.item.id.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?; let item_id = item_with_meta
.item
.id
.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?;
if item_id <= 0 { if item_id <= 0 {
return Err(CoreError::InvalidInput(format!("Invalid item ID: {}", item_id))); return Err(CoreError::InvalidInput(format!(
"Invalid item ID: {}",
item_id
)));
} }
let mut item_path = self.data_path.clone(); let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string()); item_path.push(item_id.to_string());
let reader = self.compression_service.stream_item_content( let reader = self
item_path.clone(), .compression_service
&item_with_meta.item.compression .stream_item_content(item_path.clone(), &item_with_meta.item.compression)?;
)?;
// Wrap the reader with filtering // Wrap the reader with filtering
let filtered_reader = Box::new(FilteringReader::new(reader, filter_chain.cloned())); let filtered_reader = Box::new(FilteringReader::new(reader, filter_chain.cloned()));
@@ -326,11 +355,8 @@ impl ItemService {
.unwrap_or_else(|| "application/octet-stream".to_string()); .unwrap_or_else(|| "application/octet-stream".to_string());
// Check if content is binary // Check if content is binary
let is_binary = self.is_content_binary( let is_binary =
item_path, self.is_content_binary(item_path, &item_with_meta.item.compression, &metadata)?;
&item_with_meta.item.compression,
&metadata
)?;
Ok((filtered_reader, mime_type, is_binary)) Ok((filtered_reader, mime_type, is_binary))
} }
@@ -360,17 +386,26 @@ impl ItemService {
/// ``` /// ```
/// let item = item_service.find_item(&conn, vec![1], &vec![], &HashMap::new())?; /// let item = item_service.find_item(&conn, vec![1], &vec![], &HashMap::new())?;
/// ``` /// ```
pub fn find_item(&self, conn: &Connection, ids: &[i64], tags: &[String], meta: &HashMap<String, String>) -> Result<ItemWithMeta, CoreError> { pub fn find_item(
debug!("ITEM_SERVICE: Finding item with ids: {:?}, tags: {:?}, meta: {:?}", ids, tags, meta); &self,
conn: &Connection,
ids: &[i64],
tags: &[String],
meta: &HashMap<String, String>,
) -> Result<ItemWithMeta, CoreError> {
debug!(
"ITEM_SERVICE: Finding item with ids: {:?}, tags: {:?}, meta: {:?}",
ids, tags, meta
);
let item_maybe = match (ids.is_empty(), tags.is_empty() && meta.is_empty()) { let item_maybe = match (ids.is_empty(), tags.is_empty() && meta.is_empty()) {
(false, _) => { (false, _) => {
debug!("ITEM_SERVICE: Finding by ID: {}", ids[0]); debug!("ITEM_SERVICE: Finding by ID: {}", ids[0]);
db::get_item(conn, ids[0])? db::get_item(conn, ids[0])?
}, }
(true, true) => { (true, true) => {
debug!("ITEM_SERVICE: Finding last item"); debug!("ITEM_SERVICE: Finding last item");
db::get_item_last(conn)? db::get_item_last(conn)?
}, }
(true, false) => { (true, false) => {
debug!("ITEM_SERVICE: Finding by tags/meta"); debug!("ITEM_SERVICE: Finding by tags/meta");
db::get_item_matching(conn, &tags.to_vec(), meta)? db::get_item_matching(conn, &tags.to_vec(), meta)?
@@ -381,11 +416,21 @@ impl ItemService {
debug!("ITEM_SERVICE: Found matching item: {:?}", item); debug!("ITEM_SERVICE: Found matching item: {:?}", item);
// Get tags and meta directly instead of calling get_item which makes redundant queries // Get tags and meta directly instead of calling get_item which makes redundant queries
let item_id = item.id.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?; let item_id = item
.id
.ok_or_else(|| CoreError::InvalidInput("Item missing ID".to_string()))?;
let tags = db::get_item_tags(conn, &item)?; let tags = db::get_item_tags(conn, &item)?;
debug!("ITEM_SERVICE: Found {} tags for item {}", tags.len(), item_id); debug!(
"ITEM_SERVICE: Found {} tags for item {}",
tags.len(),
item_id
);
let meta = db::get_item_meta(conn, &item)?; let meta = db::get_item_meta(conn, &item)?;
debug!("ITEM_SERVICE: Found {} meta entries for item {}", meta.len(), item_id); debug!(
"ITEM_SERVICE: Found {} meta entries for item {}",
meta.len(),
item_id
);
Ok(ItemWithMeta { item, tags, meta }) Ok(ItemWithMeta { item, tags, meta })
} }
@@ -413,8 +458,16 @@ impl ItemService {
/// ``` /// ```
/// let items = item_service.list_items(&conn, &vec!["work"], &HashMap::new())?; /// let items = item_service.list_items(&conn, &vec!["work"], &HashMap::new())?;
/// ``` /// ```
pub fn list_items(&self, conn: &Connection, tags: &[String], meta: &HashMap<String, String>) -> Result<Vec<ItemWithMeta>, CoreError> { pub fn list_items(
debug!("ITEM_SERVICE: Listing items with tags: {:?}, meta: {:?}", tags, meta); &self,
conn: &Connection,
tags: &[String],
meta: &HashMap<String, String>,
) -> Result<Vec<ItemWithMeta>, CoreError> {
debug!(
"ITEM_SERVICE: Listing items with tags: {:?}, meta: {:?}",
tags, meta
);
let items = db::get_items_matching(conn, &tags.to_vec(), meta)?; let items = db::get_items_matching(conn, &tags.to_vec(), meta)?;
debug!("ITEM_SERVICE: Found {} matching items", items.len()); debug!("ITEM_SERVICE: Found {} matching items", items.len());
@@ -424,7 +477,10 @@ impl ItemService {
return Ok(Vec::new()); return Ok(Vec::new());
} }
debug!("ITEM_SERVICE: Getting tags and meta for {} items", item_ids.len()); debug!(
"ITEM_SERVICE: Getting tags and meta for {} items",
item_ids.len()
);
let tags_map = db::get_tags_for_items(conn, &item_ids)?; let tags_map = db::get_tags_for_items(conn, &item_ids)?;
let meta_map_db = db::get_meta_for_items(conn, &item_ids)?; let meta_map_db = db::get_meta_for_items(conn, &item_ids)?;
@@ -433,12 +489,22 @@ impl ItemService {
let item_id = item.id.unwrap(); let item_id = item.id.unwrap();
let tags = tags_map.get(&item_id).cloned().unwrap_or_default(); let tags = tags_map.get(&item_id).cloned().unwrap_or_default();
let meta_hm = meta_map_db.get(&item_id).cloned().unwrap_or_default(); let meta_hm = meta_map_db.get(&item_id).cloned().unwrap_or_default();
let meta = meta_hm.into_iter().map(|(name, value)| Meta { id: item_id, name, value }).collect(); let meta = meta_hm
.into_iter()
.map(|(name, value)| Meta {
id: item_id,
name,
value,
})
.collect();
result.push(ItemWithMeta { item, tags, meta }); result.push(ItemWithMeta { item, tags, meta });
} }
debug!("ITEM_SERVICE: Returning {} items with full metadata", result.len()); debug!(
"ITEM_SERVICE: Returning {} items with full metadata",
result.len()
);
Ok(result) Ok(result)
} }
@@ -478,7 +544,13 @@ impl ItemService {
debug!("ITEM_SERVICE: Deleting file at path: {:?}", item_path); debug!("ITEM_SERVICE: Deleting file at path: {:?}", item_path);
db::delete_item(conn, item)?; db::delete_item(conn, item)?;
fs::remove_file(&item_path).or_else(|e| if e.kind() == std::io::ErrorKind::NotFound { Ok(()) } else { Err(e) })?; fs::remove_file(&item_path).or_else(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
Ok(())
} else {
Err(e)
}
})?;
debug!("ITEM_SERVICE: Successfully deleted item {}", id); debug!("ITEM_SERVICE: Successfully deleted item {}", id);
Ok(()) Ok(())
@@ -522,12 +594,15 @@ impl ItemService {
) -> Result<i64, CoreError> { ) -> Result<i64, CoreError> {
debug!("ITEM_SERVICE: Starting save_item with tags: {:?}", tags); debug!("ITEM_SERVICE: Starting save_item with tags: {:?}", tags);
if tags.is_empty() { if tags.is_empty() {
tags.push("none".to_string()); tags.push("none".to_string());
debug!("ITEM_SERVICE: No tags provided, using default 'none' tag"); debug!("ITEM_SERVICE: No tags provided, using default 'none' tag");
} }
let compression_type = settings_compression_type(cmd, settings); let compression_type = settings_compression_type(cmd, settings);
debug!("ITEM_SERVICE: Using compression type: {:?}", compression_type); debug!(
"ITEM_SERVICE: Using compression type: {:?}",
compression_type
);
let compression_engine = get_compression_engine(compression_type.clone())?; let compression_engine = get_compression_engine(compression_type.clone())?;
let item_id; let item_id;
@@ -539,9 +614,12 @@ impl ItemService {
db::set_item_tags(conn, item.clone(), tags)?; db::set_item_tags(conn, item.clone(), tags)?;
debug!("ITEM_SERVICE: Set tags for item {}", item_id); debug!("ITEM_SERVICE: Set tags for item {}", item_id);
let item_meta = self.meta_service.collect_initial_meta(); let item_meta = self.meta_service.collect_initial_meta();
debug!("ITEM_SERVICE: Collected {} initial meta entries", item_meta.len()); debug!(
"ITEM_SERVICE: Collected {} initial meta entries",
item_meta.len()
);
for (k, v) in item_meta.iter() { for (k, v) in item_meta.iter() {
db::add_meta(conn, item_id, k, v)?; db::add_meta(conn, item_id, k, v)?;
} }
} }
@@ -571,7 +649,8 @@ impl ItemService {
let mut plugins = self.meta_service.get_plugins(cmd, settings); let mut plugins = self.meta_service.get_plugins(cmd, settings);
debug!("ITEM_SERVICE: Got {} meta plugins", plugins.len()); debug!("ITEM_SERVICE: Got {} meta plugins", plugins.len());
self.meta_service.initialize_plugins(&mut plugins, conn, item_id); self.meta_service
.initialize_plugins(&mut plugins, conn, item_id);
let mut item_path = self.data_path.clone(); let mut item_path = self.data_path.clone();
item_path.push(item_id.to_string()); item_path.push(item_id.to_string());
@@ -585,11 +664,14 @@ impl ItemService {
debug!("ITEM_SERVICE: Starting to read and process input data"); debug!("ITEM_SERVICE: Starting to read and process input data");
loop { loop {
let n = input.read(&mut buffer)?; let n = input.read(&mut buffer)?;
if n == 0 { break; } if n == 0 {
break;
}
total_bytes += n as i64; total_bytes += n as i64;
item_out.write_all(&buffer[..n])?; item_out.write_all(&buffer[..n])?;
self.meta_service.process_chunk(&mut plugins, &buffer[..n], conn, item_id); self.meta_service
.process_chunk(&mut plugins, &buffer[..n], conn, item_id);
} }
debug!("ITEM_SERVICE: Processed {} bytes total", total_bytes); debug!("ITEM_SERVICE: Processed {} bytes total", total_bytes);
@@ -597,7 +679,8 @@ impl ItemService {
drop(item_out); drop(item_out);
debug!("ITEM_SERVICE: Finalizing meta plugins"); debug!("ITEM_SERVICE: Finalizing meta plugins");
self.meta_service.finalize_plugins(&mut plugins, conn, item_id); self.meta_service
.finalize_plugins(&mut plugins, conn, item_id);
item.size = Some(total_bytes); item.size = Some(total_bytes);
db::update_item(conn, item.clone())?; db::update_item(conn, item.clone())?;
@@ -646,8 +729,12 @@ impl ItemService {
settings: &Settings, settings: &Settings,
conn: &mut Connection, conn: &mut Connection,
) -> Result<ItemWithMeta, CoreError> { ) -> Result<ItemWithMeta, CoreError> {
debug!("ITEM_SERVICE: Starting save_item_from_mcp with {} bytes, {} tags, {} metadata entries", debug!(
content.len(), tags.len(), metadata.len()); "ITEM_SERVICE: Starting save_item_from_mcp with {} bytes, {} tags, {} metadata entries",
content.len(),
tags.len(),
metadata.len()
);
let compression_type = CompressionType::LZ4; let compression_type = CompressionType::LZ4;
let compression_engine = get_compression_engine(compression_type.clone())?; let compression_engine = get_compression_engine(compression_type.clone())?;
@@ -669,7 +756,10 @@ impl ItemService {
for (key, value) in metadata { for (key, value) in metadata {
db::add_meta(conn, item_id, key, value)?; db::add_meta(conn, item_id, key, value)?;
} }
debug!("ITEM_SERVICE: Added {} custom metadata entries to MCP item", metadata.len()); debug!(
"ITEM_SERVICE: Added {} custom metadata entries to MCP item",
metadata.len()
);
} }
let mut item_path = self.data_path.clone(); let mut item_path = self.data_path.clone();
@@ -681,11 +771,17 @@ impl ItemService {
drop(writer); drop(writer);
let mut plugins = self.meta_service.get_plugins(cmd, settings); let mut plugins = self.meta_service.get_plugins(cmd, settings);
debug!("ITEM_SERVICE: Got {} configured meta plugins for MCP item", plugins.len()); debug!(
"ITEM_SERVICE: Got {} configured meta plugins for MCP item",
self.meta_service.initialize_plugins(&mut plugins, conn, item_id); plugins.len()
self.meta_service.process_chunk(&mut plugins, content, conn, item_id); );
self.meta_service.finalize_plugins(&mut plugins, conn, item_id);
self.meta_service
.initialize_plugins(&mut plugins, conn, item_id);
self.meta_service
.process_chunk(&mut plugins, content, conn, item_id);
self.meta_service
.finalize_plugins(&mut plugins, conn, item_id);
debug!("ITEM_SERVICE: Processed MCP item through configured meta plugins"); debug!("ITEM_SERVICE: Processed MCP item through configured meta plugins");
item.size = Some(content.len() as i64); item.size = Some(content.len() as i64);
@@ -713,7 +809,6 @@ impl ItemService {
pub fn get_data_path(&self) -> &PathBuf { pub fn get_data_path(&self) -> &PathBuf {
&self.data_path &self.data_path
} }
} }
/// A reader that applies a filter chain to the data as it's read. /// A reader that applies a filter chain to the data as it's read.
@@ -754,8 +849,8 @@ impl<R: Read> FilteringReader<R> {
/// let filtered = FilteringReader::new(reader, Some(filter_chain)); /// let filtered = FilteringReader::new(reader, Some(filter_chain));
/// ``` /// ```
pub fn new(reader: R, filter_chain: Option<filter_plugin::FilterChain>) -> Self { pub fn new(reader: R, filter_chain: Option<filter_plugin::FilterChain>) -> Self {
Self { Self {
reader, reader,
filter_chain, filter_chain,
buffer: Vec::new(), buffer: Vec::new(),
buffer_pos: 0, buffer_pos: 0,
@@ -793,7 +888,8 @@ impl<R: Read> Read for FilteringReader<R> {
// If we have data in our buffer, serve that first // If we have data in our buffer, serve that first
if self.buffer_pos < self.buffer.len() { if self.buffer_pos < self.buffer.len() {
let bytes_to_copy = std::cmp::min(buf.len(), self.buffer.len() - self.buffer_pos); let bytes_to_copy = std::cmp::min(buf.len(), self.buffer.len() - self.buffer_pos);
buf[..bytes_to_copy].copy_from_slice(&self.buffer[self.buffer_pos..self.buffer_pos + bytes_to_copy]); buf[..bytes_to_copy]
.copy_from_slice(&self.buffer[self.buffer_pos..self.buffer_pos + bytes_to_copy]);
self.buffer_pos += bytes_to_copy; self.buffer_pos += bytes_to_copy;
return Ok(bytes_to_copy); return Ok(bytes_to_copy);
} }
@@ -816,7 +912,7 @@ impl<R: Read> Read for FilteringReader<R> {
let mut input_cursor = std::io::Cursor::new(&temp_buf[..bytes_read]); let mut input_cursor = std::io::Cursor::new(&temp_buf[..bytes_read]);
// Write filtered output to our buffer // Write filtered output to our buffer
chain.filter(&mut input_cursor, &mut self.buffer)?; chain.filter(&mut input_cursor, &mut self.buffer)?;
if !self.buffer.is_empty() { if !self.buffer.is_empty() {
let bytes_to_copy = std::cmp::min(buf.len(), self.buffer.len()); let bytes_to_copy = std::cmp::min(buf.len(), self.buffer.len());
buf[..bytes_to_copy].copy_from_slice(&self.buffer[..bytes_to_copy]); buf[..bytes_to_copy].copy_from_slice(&self.buffer[..bytes_to_copy]);

63
src/services/meta_service.rs
View File

@@ -16,33 +16,37 @@ impl MetaService {
pub fn get_plugins(&self, cmd: &mut Command, settings: &Settings) -> Vec<Box<dyn MetaPlugin>> { pub fn get_plugins(&self, cmd: &mut Command, settings: &Settings) -> Vec<Box<dyn MetaPlugin>> {
debug!("META_SERVICE: get_plugins called"); debug!("META_SERVICE: get_plugins called");
let meta_plugin_types: Vec<MetaPluginType> = settings_meta_plugin_types(cmd, settings); let meta_plugin_types: Vec<MetaPluginType> = settings_meta_plugin_types(cmd, settings);
debug!("META_SERVICE: Meta plugin types from settings: {:?}", meta_plugin_types); debug!(
"META_SERVICE: Meta plugin types from settings: {:?}",
meta_plugin_types
);
// Create plugins with their configuration // Create plugins with their configuration
let meta_plugins: Vec<Box<dyn MetaPlugin>> = meta_plugin_types let meta_plugins: Vec<Box<dyn MetaPlugin>> = meta_plugin_types
.iter() .iter()
.map(|meta_plugin_type| { .map(|meta_plugin_type| {
debug!("META_SERVICE: Creating plugin: {:?}", meta_plugin_type); debug!("META_SERVICE: Creating plugin: {:?}", meta_plugin_type);
// Get the plugin name using strum's Display implementation // Get the plugin name using strum's Display implementation
let plugin_name = meta_plugin_type.to_string(); let plugin_name = meta_plugin_type.to_string();
// Get options and outputs from settings // Get options and outputs from settings
let (options, outputs) = if let Some(meta_plugin_configs) = &settings.meta_plugins { let (options, outputs) = if let Some(meta_plugin_configs) = &settings.meta_plugins {
if let Some(config) = meta_plugin_configs.iter().find(|c| c.name == plugin_name) { if let Some(config) = meta_plugin_configs.iter().find(|c| c.name == plugin_name)
{
// Convert options and outputs to the appropriate types // Convert options and outputs to the appropriate types
let options: std::collections::HashMap<String, serde_yaml::Value> = config let options: std::collections::HashMap<String, serde_yaml::Value> = config
.options .options
.iter() .iter()
.map(|(k, v)| (k.clone(), v.clone())) .map(|(k, v)| (k.clone(), v.clone()))
.collect(); .collect();
let outputs: std::collections::HashMap<String, serde_yaml::Value> = config let outputs: std::collections::HashMap<String, serde_yaml::Value> = config
.outputs .outputs
.iter() .iter()
.map(|(k, v)| (k.clone(), serde_yaml::Value::String(v.clone()))) .map(|(k, v)| (k.clone(), serde_yaml::Value::String(v.clone())))
.collect(); .collect();
(Some(options), Some(outputs)) (Some(options), Some(outputs))
} else { } else {
(None, None) (None, None)
@@ -50,7 +54,7 @@ impl MetaService {
} else { } else {
(None, None) (None, None)
}; };
crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone(), options, outputs) crate::meta_plugin::get_meta_plugin(meta_plugin_type.clone(), options, outputs)
}) })
.collect(); .collect();
@@ -65,7 +69,8 @@ impl MetaService {
item_id: i64, item_id: i64,
) { ) {
// Check for duplicate output names before initializing plugins // Check for duplicate output names before initializing plugins
let mut output_names: std::collections::HashMap<String, Vec<String>> = std::collections::HashMap::new(); let mut output_names: std::collections::HashMap<String, Vec<String>> =
std::collections::HashMap::new();
for plugin in plugins.iter() { for plugin in plugins.iter() {
let plugin_name = plugin.meta_type().to_string(); let plugin_name = plugin.meta_type().to_string();
@@ -80,8 +85,9 @@ impl MetaService {
// Only track outputs that will actually be written // Only track outputs that will actually be written
if !matches!(output_config, serde_yaml::Value::Bool(false)) { if !matches!(output_config, serde_yaml::Value::Bool(false)) {
output_names.entry(output_name) output_names
.or_insert_with(Vec::new) .entry(output_name)
.or_default()
.push(plugin_name.clone()); .push(plugin_name.clone());
} }
} }
@@ -90,15 +96,17 @@ impl MetaService {
// Print warnings for duplicate output names // Print warnings for duplicate output names
for (output_name, plugin_names) in &output_names { for (output_name, plugin_names) in &output_names {
if plugin_names.len() > 1 { if plugin_names.len() > 1 {
log::warn!("META_SERVICE: Output name '{}' is provided by multiple plugins: {}", log::warn!(
"META_SERVICE: Output name '{}' is provided by multiple plugins: {}",
output_name, output_name,
plugin_names.join(", ")); plugin_names.join(", ")
);
} }
} }
for meta_plugin in plugins.iter_mut() { for meta_plugin in plugins.iter_mut() {
let response = meta_plugin.initialize(); let response = meta_plugin.initialize();
self.process_plugin_response(conn, item_id, meta_plugin, response); self.process_plugin_response(conn, item_id, &mut **meta_plugin, response);
} }
} }
@@ -114,10 +122,10 @@ impl MetaService {
if meta_plugin.is_finalized() { if meta_plugin.is_finalized() {
continue; continue;
} }
let response = meta_plugin.update(chunk); let response = meta_plugin.update(chunk);
self.process_plugin_response(conn, item_id, meta_plugin, response.clone()); self.process_plugin_response(conn, item_id, &mut **meta_plugin, response.clone());
// Set finalized flag if response indicates finalization // Set finalized flag if response indicates finalization
if response.is_finalized { if response.is_finalized {
meta_plugin.set_finalized(true); meta_plugin.set_finalized(true);
@@ -125,16 +133,21 @@ impl MetaService {
} }
} }
pub fn finalize_plugins(&self, plugins: &mut [Box<dyn MetaPlugin>], conn: &Connection, item_id: i64) { pub fn finalize_plugins(
&self,
plugins: &mut [Box<dyn MetaPlugin>],
conn: &Connection,
item_id: i64,
) {
for meta_plugin in plugins.iter_mut() { for meta_plugin in plugins.iter_mut() {
// Skip plugins that are already finalized // Skip plugins that are already finalized
if meta_plugin.is_finalized() { if meta_plugin.is_finalized() {
continue; continue;
} }
let response = meta_plugin.finalize(); let response = meta_plugin.finalize();
self.process_plugin_response(conn, item_id, meta_plugin, response.clone()); self.process_plugin_response(conn, item_id, &mut **meta_plugin, response.clone());
// Set finalized flag if response indicates finalization // Set finalized flag if response indicates finalization
if response.is_finalized { if response.is_finalized {
meta_plugin.set_finalized(true); meta_plugin.set_finalized(true);
@@ -161,7 +174,7 @@ impl MetaService {
&self, &self,
conn: &Connection, conn: &Connection,
item_id: i64, item_id: i64,
_plugin: &Box<dyn MetaPlugin>, _plugin: &mut dyn MetaPlugin,
response: crate::meta_plugin::MetaPluginResponse, response: crate::meta_plugin::MetaPluginResponse,
) { ) {
for meta_data in response.metadata { for meta_data in response.metadata {
@@ -196,10 +209,10 @@ impl MetaService {
pub fn collect_initial_meta(&self) -> HashMap<String, String> { pub fn collect_initial_meta(&self) -> HashMap<String, String> {
let mut item_meta: HashMap<String, String> = crate::modes::common::get_meta_from_env(); let mut item_meta: HashMap<String, String> = crate::modes::common::get_meta_from_env();
if let Ok(hostname) = gethostname::gethostname().into_string() { if let Ok(hostname) = gethostname::gethostname().into_string()
if !item_meta.contains_key("hostname") { && !item_meta.contains_key("hostname")
item_meta.insert("hostname".to_string(), hostname); {
} item_meta.insert("hostname".to_string(), hostname);
} }
item_meta item_meta
} }

26
src/services/status_service.rs
View File

@@ -1,7 +1,7 @@
use crate::common::status::{generate_status_info, StatusInfo}; use crate::common::status::{StatusInfo, generate_status_info};
use crate::compression_engine::CompressionType;
use crate::config::Settings; use crate::config::Settings;
use crate::meta_plugin::MetaPluginType; use crate::meta_plugin::MetaPluginType;
use crate::compression_engine::CompressionType;
use crate::services::filter_service::get_available_filter_plugins; use crate::services::filter_service::get_available_filter_plugins;
use clap::Command; use clap::Command;
use std::path::PathBuf; use std::path::PathBuf;
@@ -75,8 +75,9 @@ impl StatusService {
db_path: PathBuf, db_path: PathBuf,
) -> StatusInfo { ) -> StatusInfo {
// Get meta plugins directly from config // Get meta plugins directly from config
let meta_plugin_types: Vec<MetaPluginType> = crate::modes::common::settings_meta_plugin_types(cmd, settings); let meta_plugin_types: Vec<MetaPluginType> =
crate::modes::common::settings_meta_plugin_types(cmd, settings);
// Determine which compression type would be enabled for a save operation // Determine which compression type would be enabled for a save operation
let enabled_compression_type = if let Some(compression_name) = &settings.compression() { let enabled_compression_type = if let Some(compression_name) = &settings.compression() {
CompressionType::from_str(compression_name).ok() CompressionType::from_str(compression_name).ok()
@@ -84,18 +85,23 @@ impl StatusService {
Some(crate::compression_engine::default_compression_type()) Some(crate::compression_engine::default_compression_type())
}; };
let mut status_info = generate_status_info(data_path, db_path, &meta_plugin_types, enabled_compression_type); let mut status_info = generate_status_info(
data_path,
db_path,
&meta_plugin_types,
enabled_compression_type,
);
// Add detailed filter plugins information // Add detailed filter plugins information
let filter_plugins_map = get_available_filter_plugins(); let filter_plugins_map = get_available_filter_plugins();
let mut filter_plugins_info = Vec::new(); let mut filter_plugins_info = Vec::new();
for (name, creator) in filter_plugins_map { for (name, creator) in filter_plugins_map {
let plugin = creator(); let plugin = creator();
let options = plugin.options(); let options = plugin.options();
// For now, use a default description // For now, use a default description
let description = "Filter plugin".to_string(); let description = "Filter plugin".to_string();
filter_plugins_info.push(crate::common::status::FilterPluginInfo { filter_plugins_info.push(crate::common::status::FilterPluginInfo {
name, name,
options, options,
@@ -103,10 +109,10 @@ impl StatusService {
}); });
} }
status_info.filter_plugins = filter_plugins_info; status_info.filter_plugins = filter_plugins_info;
// Add configured meta plugins information // Add configured meta plugins information
status_info.configured_meta_plugins = settings.meta_plugins.clone(); status_info.configured_meta_plugins = settings.meta_plugins.clone();
status_info status_info
} }
} }

6
src/services/types.rs
View File

@@ -34,7 +34,11 @@ impl ItemWithMeta {
/// assert_eq!(meta_map.get("hostname"), Some(&"example.com".to_string())); /// assert_eq!(meta_map.get("hostname"), Some(&"example.com".to_string()));
/// ``` /// ```
pub fn meta_as_map(&self) -> HashMap<String, String> { pub fn meta_as_map(&self) -> HashMap<String, String> {
self.meta.iter().cloned().map(|m| (m.name, m.value)).collect() self.meta
.iter()
.cloned()
.map(|m| (m.name, m.value))
.collect()
} }
} }

8
src/tests/common/is_binary_tests.rs
View File

@@ -6,7 +6,7 @@ mod tests {
fn test_is_binary_text() { fn test_is_binary_text() {
let text_data = b"Hello, World! This is plain text.\nWith newlines and spaces."; let text_data = b"Hello, World! This is plain text.\nWith newlines and spaces.";
let result = is_binary(text_data); let result = is_binary(text_data);
// Text data should not be detected as binary // Text data should not be detected as binary
assert!(!result); assert!(!result);
} }
@@ -15,7 +15,7 @@ mod tests {
fn test_is_binary_binary() { fn test_is_binary_binary() {
let binary_data = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09"; let binary_data = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09";
let result = is_binary(binary_data); let result = is_binary(binary_data);
// Binary data should be detected as binary // Binary data should be detected as binary
assert!(result); assert!(result);
} }
@@ -24,7 +24,7 @@ mod tests {
fn test_is_binary_png_signature() { fn test_is_binary_png_signature() {
let png_data = b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"; let png_data = b"\x89\x50\x4E\x47\x0D\x0A\x1A\x0A";
let result = is_binary(png_data); let result = is_binary(png_data);
// PNG signature should be detected as binary // PNG signature should be detected as binary
assert!(result); assert!(result);
} }
@@ -33,7 +33,7 @@ mod tests {
fn test_is_binary_empty() { fn test_is_binary_empty() {
let empty_data = b""; let empty_data = b"";
let result = is_binary(empty_data); let result = is_binary(empty_data);
// Empty data should not be detected as binary // Empty data should not be detected as binary
assert!(!result); assert!(!result);
} }

2
src/tests/common/status_tests.rs
View File

@@ -2,7 +2,7 @@
mod tests { mod tests {
// TODO: Add tests for common status functionality once implemented // TODO: Add tests for common status functionality once implemented
// This would test functions related to status checking in the common module // This would test functions related to status checking in the common module
#[test] #[test]
fn test_status_placeholder() { fn test_status_placeholder() {
// Placeholder test - to be implemented when status functionality is added // Placeholder test - to be implemented when status functionality is added

31
src/tests/common/test_helpers.rs
View File

@@ -1,11 +1,11 @@
//! Common test utilities and helper functions to reduce duplication in tests //! Common test utilities and helper functions to reduce duplication in tests
use tempfile::TempDir; use crate::db;
use rusqlite::Connection;
use std::fs::File; use std::fs::File;
use std::io::Write; use std::io::Write;
use std::path::PathBuf; use std::path::PathBuf;
use rusqlite::Connection; use tempfile::TempDir;
use crate::db;
/// Create a temporary directory for testing /// Create a temporary directory for testing
pub fn create_temp_dir() -> TempDir { pub fn create_temp_dir() -> TempDir {
@@ -37,10 +37,10 @@ pub fn test_temp_dir_setup() {
pub fn test_file_creation(dir: &TempDir, filename: &str, content: &str) -> PathBuf { pub fn test_file_creation(dir: &TempDir, filename: &str, content: &str) -> PathBuf {
let file_path = create_temp_file_with_content(dir, filename, content); let file_path = create_temp_file_with_content(dir, filename, content);
assert!(file_path.exists()); assert!(file_path.exists());
let metadata = std::fs::metadata(&file_path).expect("Failed to get file metadata"); let metadata = std::fs::metadata(&file_path).expect("Failed to get file metadata");
assert!(metadata.len() > 0); assert!(metadata.len() > 0);
file_path file_path
} }
@@ -64,21 +64,26 @@ pub fn create_test_item(conn: &Connection) -> i64 {
} }
/// Test compression and decompression with an engine /// Test compression and decompression with an engine
pub fn test_compression_engine(engine: &dyn crate::compression_engine::CompressionEngine, test_data: &[u8]) { pub fn test_compression_engine(
engine: &dyn crate::compression_engine::CompressionEngine,
test_data: &[u8],
) {
let dir = create_temp_dir(); let dir = create_temp_dir();
let file_path = dir.path().join("test_compression.dat"); let file_path = dir.path().join("test_compression.dat");
// Test compression // Test compression
{ {
let mut writer = engine.create(file_path.clone()).expect("Failed to create writer"); let mut writer = engine
.create(file_path.clone())
.expect("Failed to create writer");
writer.write_all(test_data).expect("Failed to write data"); writer.write_all(test_data).expect("Failed to write data");
} }
// Test decompression // Test decompression
let mut reader = engine.open(file_path).expect("Failed to open reader"); let mut reader = engine.open(file_path).expect("Failed to open reader");
let mut decompressed = Vec::new(); let mut decompressed = Vec::new();
std::io::copy(&mut reader, &mut decompressed).expect("Failed to read data"); std::io::copy(&mut reader, &mut decompressed).expect("Failed to read data");
assert_eq!(test_data, decompressed.as_slice()); assert_eq!(test_data, decompressed.as_slice());
} }
@@ -95,5 +100,9 @@ pub fn assert_file_exists(file_path: &PathBuf) {
/// Assert that a file does not exist /// Assert that a file does not exist
pub fn assert_file_not_exists(file_path: &PathBuf) { pub fn assert_file_not_exists(file_path: &PathBuf) {
assert!(!file_path.exists(), "File {:?} should not exist but it does", file_path); assert!(
!file_path.exists(),
"File {:?} should not exist but it does",
file_path
);
} }

2
src/tests/compression/gzip_tests.rs
View File

@@ -1,7 +1,7 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::compression_engine::gzip::CompressionEngineGZip;
use crate::compression_engine::CompressionEngine; use crate::compression_engine::CompressionEngine;
use crate::compression_engine::gzip::CompressionEngineGZip;
use crate::tests::common::test_helpers::test_compression_engine; use crate::tests::common::test_helpers::test_compression_engine;
#[test] #[test]

6
src/tests/compression_engine/program_tests.rs
View File

@@ -1,7 +1,7 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::compression_engine::program::CompressionEngineProgram;
use crate::compression_engine::CompressionEngine; use crate::compression_engine::CompressionEngine;
use crate::compression_engine::program::CompressionEngineProgram;
#[test] #[test]
fn test_compression_engine_program_creation() { fn test_compression_engine_program_creation() {
@@ -11,7 +11,7 @@ mod tests {
decompress: vec!["-d".to_string(), "-c".to_string()], decompress: vec!["-d".to_string(), "-c".to_string()],
supported: true, supported: true,
}; };
// If the program exists, it should be supported // If the program exists, it should be supported
let _ = engine.is_supported(); let _ = engine.is_supported();
} }
@@ -24,7 +24,7 @@ mod tests {
decompress: vec![], decompress: vec![],
supported: false, supported: false,
}; };
// Explicitly unsupported engine should report as such // Explicitly unsupported engine should report as such
assert!(!engine.is_supported()); assert!(!engine.is_supported());
} }

30
src/tests/compression_types/conversion_tests.rs
View File

@@ -12,13 +12,31 @@ mod tests {
#[test] #[test]
fn test_compression_type_from_str() { fn test_compression_type_from_str() {
assert_eq!(CompressionType::from_str("lz4").unwrap(), CompressionType::LZ4); assert_eq!(
assert_eq!(CompressionType::from_str("gzip").unwrap(), CompressionType::GZip); CompressionType::from_str("lz4").unwrap(),
assert_eq!(CompressionType::from_str("none").unwrap(), CompressionType::None); CompressionType::LZ4
);
assert_eq!(
CompressionType::from_str("gzip").unwrap(),
CompressionType::GZip
);
assert_eq!(
CompressionType::from_str("none").unwrap(),
CompressionType::None
);
// Test case insensitivity // Test case insensitivity
assert_eq!(CompressionType::from_str("LZ4").unwrap(), CompressionType::LZ4); assert_eq!(
assert_eq!(CompressionType::from_str("GZIP").unwrap(), CompressionType::GZip); CompressionType::from_str("LZ4").unwrap(),
assert_eq!(CompressionType::from_str("NONE").unwrap(), CompressionType::None); CompressionType::LZ4
);
assert_eq!(
CompressionType::from_str("GZIP").unwrap(),
CompressionType::GZip
);
assert_eq!(
CompressionType::from_str("NONE").unwrap(),
CompressionType::None
);
} }
#[test] #[test]

19
src/tests/compression_types/factory_tests.rs
View File

@@ -5,19 +5,16 @@ mod tests {
#[test] #[test]
fn test_compression_engine_factory() { fn test_compression_engine_factory() {
// Test getting different compression engines // Test getting different compression engines
let lz4_engine = compression_engine::get_compression_engine( let lz4_engine = compression_engine::get_compression_engine(CompressionType::LZ4)
CompressionType::LZ4 .expect("Failed to get LZ4 engine");
).expect("Failed to get LZ4 engine");
assert!(lz4_engine.is_supported()); assert!(lz4_engine.is_supported());
let gzip_engine = compression_engine::get_compression_engine( let gzip_engine = compression_engine::get_compression_engine(CompressionType::GZip)
CompressionType::GZip .expect("Failed to get GZip engine");
).expect("Failed to get GZip engine");
assert!(gzip_engine.is_supported()); assert!(gzip_engine.is_supported());
let none_engine = compression_engine::get_compression_engine( let none_engine = compression_engine::get_compression_engine(CompressionType::None)
CompressionType::None .expect("Failed to get None engine");
).expect("Failed to get None engine");
assert!(none_engine.is_supported()); assert!(none_engine.is_supported());
} }

2
src/tests/compression_types/mod.rs
View File

@@ -1,2 +1,2 @@
pub mod factory_tests;
pub mod conversion_tests; pub mod conversion_tests;
pub mod factory_tests;

10
src/tests/db/item_tests.rs
View File

@@ -1,16 +1,16 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::tests::common::test_helpers::create_temp_db;
use crate::db; use crate::db;
use crate::tests::common::test_helpers::create_temp_db;
#[test] #[test]
fn test_database_connection() { fn test_database_connection() {
// Create a temporary database // Create a temporary database
let (_temp_dir, _conn, db_path) = create_temp_db(); let (_temp_dir, _conn, db_path) = create_temp_db();
// Try to open the database // Try to open the database
let result = db::open(db_path); let result = db::open(db_path);
// Should succeed in creating a new database // Should succeed in creating a new database
assert!(result.is_ok()); assert!(result.is_ok());
} }
@@ -19,11 +19,11 @@ mod tests {
fn test_database_item_queries() { fn test_database_item_queries() {
// Create a temporary database // Create a temporary database
let (_temp_dir, conn, _db_path) = create_temp_db(); let (_temp_dir, conn, _db_path) = create_temp_db();
// Try to query all items (should be empty in new DB) // Try to query all items (should be empty in new DB)
let items = db::query_all_items(&conn); let items = db::query_all_items(&conn);
assert!(items.is_ok()); assert!(items.is_ok());
// Should start with no items // Should start with no items
assert_eq!(items.unwrap().len(), 0); assert_eq!(items.unwrap().len(), 0);
} }

12
src/tests/db/meta_tests.rs
View File

@@ -1,28 +1,28 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::tests::common::test_helpers::{create_temp_db, create_test_item};
use crate::db; use crate::db;
use crate::db::Meta; use crate::db::Meta;
use crate::tests::common::test_helpers::{create_temp_db, create_test_item};
#[test] #[test]
fn test_database_meta_operations() { fn test_database_meta_operations() {
// Create a temporary database // Create a temporary database
let (_temp_dir, conn, _db_path) = create_temp_db(); let (_temp_dir, conn, _db_path) = create_temp_db();
// First insert an item to have a valid ID // First insert an item to have a valid ID
let item_id = create_test_item(&conn); let item_id = create_test_item(&conn);
// Create a test meta with the valid item ID // Create a test meta with the valid item ID
let meta = Meta { let meta = Meta {
id: item_id, id: item_id,
name: "test_key".to_string(), name: "test_key".to_string(),
value: "test_value".to_string(), value: "test_value".to_string(),
}; };
// Try to insert meta // Try to insert meta
let insert_result = db::query_upsert_meta(&conn, meta.clone()); let insert_result = db::query_upsert_meta(&conn, meta.clone());
assert!(insert_result.is_ok()); assert!(insert_result.is_ok());
// Try to get meta for non-existent item // Try to get meta for non-existent item
let item = crate::db::Item { let item = crate::db::Item {
id: Some(999), // Non-existent item id: Some(999), // Non-existent item
@@ -30,7 +30,7 @@ mod tests {
size: Some(0), size: Some(0),
compression: crate::compression_engine::CompressionType::None.to_string(), compression: crate::compression_engine::CompressionType::None.to_string(),
}; };
let metas = db::get_item_meta(&conn, &item); let metas = db::get_item_meta(&conn, &item);
assert!(metas.is_ok()); assert!(metas.is_ok());
assert_eq!(metas.unwrap().len(), 0); assert_eq!(metas.unwrap().len(), 0);

4
src/tests/db/mod.rs
View File

@@ -3,6 +3,6 @@
#[cfg(test)] #[cfg(test)]
pub mod item_tests; pub mod item_tests;
#[cfg(test)] #[cfg(test)]
pub mod tag_tests;
#[cfg(test)]
pub mod meta_tests; pub mod meta_tests;
#[cfg(test)]
pub mod tag_tests;

12
src/tests/db/tag_tests.rs
View File

@@ -1,23 +1,23 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::tests::common::test_helpers::{create_temp_db, create_test_item};
use crate::db; use crate::db;
use crate::db::Tag; use crate::db::Tag;
use crate::tests::common::test_helpers::{create_temp_db, create_test_item};
#[test] #[test]
fn test_database_tag_operations() { fn test_database_tag_operations() {
// Create a temporary database // Create a temporary database
let (_temp_dir, conn, _db_path) = create_temp_db(); let (_temp_dir, conn, _db_path) = create_temp_db();
// First insert an item to have a valid ID // First insert an item to have a valid ID
let item_id = create_test_item(&conn); let item_id = create_test_item(&conn);
// Create a test tag with the valid item ID // Create a test tag with the valid item ID
let tag = Tag { let tag = Tag {
id: item_id, id: item_id,
name: "test_tag".to_string(), name: "test_tag".to_string(),
}; };
// Try to insert tag // Try to insert tag
let insert_result = db::insert_tag(&conn, tag.clone()); let insert_result = db::insert_tag(&conn, tag.clone());
assert!(insert_result.is_ok()); assert!(insert_result.is_ok());
@@ -27,7 +27,7 @@ mod tests {
fn test_database_item_tag_operations() { fn test_database_item_tag_operations() {
// Create a temporary database // Create a temporary database
let (_temp_dir, conn, _db_path) = create_temp_db(); let (_temp_dir, conn, _db_path) = create_temp_db();
// Try to delete tags for non-existent item // Try to delete tags for non-existent item
let item = crate::db::Item { let item = crate::db::Item {
id: Some(999), // Non-existent item id: Some(999), // Non-existent item
@@ -35,7 +35,7 @@ mod tests {
size: Some(0), size: Some(0),
compression: crate::compression_engine::CompressionType::None.to_string(), compression: crate::compression_engine::CompressionType::None.to_string(),
}; };
let delete_result = db::delete_item_tags(&conn, item); let delete_result = db::delete_item_tags(&conn, item);
assert!(delete_result.is_ok()); assert!(delete_result.is_ok());
} }

16
src/tests/meta_plugin/digest_tests.rs
View File

@@ -1,20 +1,20 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::meta_plugin::digest::*;
use crate::meta_plugin::MetaPlugin; use crate::meta_plugin::MetaPlugin;
use crate::meta_plugin::digest::*;
use std::io::Write; use std::io::Write;
#[test] #[test]
fn test_digest_sha256_meta_plugin() { fn test_digest_sha256_meta_plugin() {
let mut plugin = DigestSha256MetaPlugin::new(); let mut plugin = DigestSha256MetaPlugin::new();
assert_eq!(plugin.meta_name(), "digest_sha256"); assert_eq!(plugin.meta_name(), "digest_sha256");
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work // Creating a writer should work
let writer_result = plugin.create(); let writer_result = plugin.create();
assert!(writer_result.is_ok()); assert!(writer_result.is_ok());
// Writing some data // Writing some data
let mut writer = writer_result.unwrap(); let mut writer = writer_result.unwrap();
let write_result = writer.write(b"test data"); let write_result = writer.write(b"test data");
@@ -24,10 +24,10 @@ mod tests {
#[test] #[test]
fn test_read_time_meta_plugin() { fn test_read_time_meta_plugin() {
let mut plugin = ReadTimeMetaPlugin::new(); let mut plugin = ReadTimeMetaPlugin::new();
assert_eq!(plugin.meta_name(), "read_time"); assert_eq!(plugin.meta_name(), "read_time");
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work // Creating a writer should work
let writer_result = plugin.create(); let writer_result = plugin.create();
assert!(writer_result.is_ok()); assert!(writer_result.is_ok());
@@ -36,10 +36,10 @@ mod tests {
#[test] #[test]
fn test_read_rate_meta_plugin() { fn test_read_rate_meta_plugin() {
let mut plugin = ReadRateMetaPlugin::new(); let mut plugin = ReadRateMetaPlugin::new();
assert_eq!(plugin.meta_name(), "read_rate"); assert_eq!(plugin.meta_name(), "read_rate");
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work // Creating a writer should work
let writer_result = plugin.create(); let writer_result = plugin.create();
assert!(writer_result.is_ok()); assert!(writer_result.is_ok());

4
src/tests/meta_plugin/mod.rs
View File

@@ -1,8 +1,8 @@
// Meta plugin tests module // Meta plugin tests module
#[cfg(test)]
pub mod system_tests;
#[cfg(test)] #[cfg(test)]
pub mod digest_tests; pub mod digest_tests;
#[cfg(test)] #[cfg(test)]
pub mod program_tests; pub mod program_tests;
#[cfg(test)]
pub mod system_tests;

23
src/tests/meta_plugin/program_tests.rs
View File

@@ -1,17 +1,13 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::meta_plugin::program::MetaPluginProgram;
use crate::meta_plugin::MetaPlugin; use crate::meta_plugin::MetaPlugin;
use crate::meta_plugin::program::MetaPluginProgram;
#[test] #[test]
fn test_meta_plugin_program_creation() { fn test_meta_plugin_program_creation() {
let mut plugin = MetaPluginProgram::new( let mut plugin =
"echo", MetaPluginProgram::new("echo", vec!["test"], "test_plugin".to_string(), false);
vec!["test"],
"test_plugin".to_string(),
false,
);
assert_eq!(plugin.meta_name(), "test_plugin"); assert_eq!(plugin.meta_name(), "test_plugin");
// If echo is available, it should be supported // If echo is available, it should be supported
// We don't assert on is_supported() as it depends on system availability // We don't assert on is_supported() as it depends on system availability
@@ -19,13 +15,8 @@ mod tests {
#[test] #[test]
fn test_meta_plugin_program_create_writer() { fn test_meta_plugin_program_create_writer() {
let plugin = MetaPluginProgram::new( let plugin = MetaPluginProgram::new("cat", vec![], "cat_plugin".to_string(), false);
"cat",
vec![],
"cat_plugin".to_string(),
false,
);
// Creating a writer should work for valid programs // Creating a writer should work for valid programs
let result = plugin.create(); let result = plugin.create();
// We don't assert success as it depends on system availability // We don't assert success as it depends on system availability
@@ -41,7 +32,7 @@ mod tests {
"bad_plugin".to_string(), "bad_plugin".to_string(),
false, false,
); );
// An unsupported plugin should report as such // An unsupported plugin should report as such
// Note: This might still be supported if the program exists // Note: This might still be supported if the program exists
let _ = plugin.is_supported(); let _ = plugin.is_supported();

20
src/tests/meta_plugin/system_tests.rs
View File

@@ -1,19 +1,19 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::meta_plugin::system::*;
use crate::meta_plugin::MetaPlugin; use crate::meta_plugin::MetaPlugin;
use crate::meta_plugin::system::*;
#[test] #[test]
fn test_cwd_meta_plugin() { fn test_cwd_meta_plugin() {
let mut plugin = CwdMetaPlugin::new(); let mut plugin = CwdMetaPlugin::new();
assert_eq!(plugin.meta_name(), "cwd"); assert_eq!(plugin.meta_name(), "cwd");
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work // Creating a writer should work
let writer_result = plugin.create(); let writer_result = plugin.create();
assert!(writer_result.is_ok()); assert!(writer_result.is_ok());
// Finalize should return current working directory // Finalize should return current working directory
let result = plugin.finalize(); let result = plugin.finalize();
assert!(result.is_ok()); assert!(result.is_ok());
@@ -22,10 +22,10 @@ mod tests {
#[test] #[test]
fn test_binary_meta_plugin() { fn test_binary_meta_plugin() {
let mut plugin = BinaryMetaPlugin::new(); let mut plugin = BinaryMetaPlugin::new();
assert_eq!(plugin.meta_name(), "binary"); assert_eq!(plugin.meta_name(), "binary");
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work // Creating a writer should work
let writer_result = plugin.create(); let writer_result = plugin.create();
assert!(writer_result.is_ok()); assert!(writer_result.is_ok());
@@ -34,10 +34,10 @@ mod tests {
#[test] #[test]
fn test_uid_meta_plugin() { fn test_uid_meta_plugin() {
let mut plugin = UidMetaPlugin::new(); let mut plugin = UidMetaPlugin::new();
assert_eq!(plugin.meta_name(), "uid"); assert_eq!(plugin.meta_name(), "uid");
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work // Creating a writer should work
let writer_result = plugin.create(); let writer_result = plugin.create();
assert!(writer_result.is_ok()); assert!(writer_result.is_ok());
@@ -46,10 +46,10 @@ mod tests {
#[test] #[test]
fn test_user_meta_plugin() { fn test_user_meta_plugin() {
let mut plugin = UserMetaPlugin::new(); let mut plugin = UserMetaPlugin::new();
assert_eq!(plugin.meta_name(), "user"); assert_eq!(plugin.meta_name(), "user");
assert!(plugin.is_internal()); assert!(plugin.is_internal());
// Creating a writer should work // Creating a writer should work
let writer_result = plugin.create(); let writer_result = plugin.create();
assert!(writer_result.is_ok()); assert!(writer_result.is_ok());

6
src/tests/mod.rs
View File

@@ -1,8 +1,8 @@
pub mod common;
pub mod compression; pub mod compression;
pub mod compression_types;
pub mod compression_engine; pub mod compression_engine;
pub mod compression_types;
pub mod db;
pub mod meta_plugin; pub mod meta_plugin;
pub mod modes; pub mod modes;
pub mod server; pub mod server;
pub mod common;
pub mod db;

6
src/tests/modes/delete_tests.rs
View File

@@ -1,6 +1,8 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::tests::common::test_helpers::{create_temp_dir, create_empty_temp_file, assert_file_exists}; use crate::tests::common::test_helpers::{
assert_file_exists, create_empty_temp_file, create_temp_dir,
};
#[test] #[test]
fn test_delete_mode_setup() { fn test_delete_mode_setup() {
@@ -13,7 +15,7 @@ mod tests {
// Create a temporary directory for testing // Create a temporary directory for testing
let temp_dir = create_temp_dir(); let temp_dir = create_temp_dir();
let test_file = create_empty_temp_file(&temp_dir, "test_delete.txt"); let test_file = create_empty_temp_file(&temp_dir, "test_delete.txt");
// Verify file exists before deletion test // Verify file exists before deletion test
assert_file_exists(&test_file); assert_file_exists(&test_file);
} }

6
src/tests/modes/get_tests.rs
View File

@@ -1,6 +1,8 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::tests::common::test_helpers::{create_temp_dir, test_temp_dir_setup, assert_file_not_exists}; use crate::tests::common::test_helpers::{
assert_file_not_exists, create_temp_dir, test_temp_dir_setup,
};
#[test] #[test]
fn test_get_mode_basic_setup() { fn test_get_mode_basic_setup() {
@@ -12,7 +14,7 @@ mod tests {
// Create a temporary directory for testing // Create a temporary directory for testing
let temp_dir = create_temp_dir(); let temp_dir = create_temp_dir();
let test_file = temp_dir.path().join("test_get.txt"); let test_file = temp_dir.path().join("test_get.txt");
// Test path creation // Test path creation
assert_file_not_exists(&test_file); assert_file_not_exists(&test_file);
} }

8
src/tests/modes/info_tests.rs
View File

@@ -1,6 +1,8 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::tests::common::test_helpers::{create_temp_dir, test_file_creation, assert_file_not_exists, get_file_size}; use crate::tests::common::test_helpers::{
assert_file_not_exists, create_temp_dir, get_file_size, test_file_creation,
};
use std::path::PathBuf; use std::path::PathBuf;
#[test] #[test]
@@ -9,7 +11,7 @@ mod tests {
let dir = create_temp_dir(); let dir = create_temp_dir();
let content = "This is a test file for info mode\nWith multiple lines\n"; let content = "This is a test file for info mode\nWith multiple lines\n";
let file_path = test_file_creation(&dir, "info_test.txt", content); let file_path = test_file_creation(&dir, "info_test.txt", content);
// Additional verification specific to info mode // Additional verification specific to info mode
assert!(get_file_size(&file_path) > 0); assert!(get_file_size(&file_path) > 0);
} }
@@ -18,7 +20,7 @@ mod tests {
fn test_info_mode_nonexistent_file() { fn test_info_mode_nonexistent_file() {
// Create a path to a file that doesn't exist // Create a path to a file that doesn't exist
let nonexistent_path = PathBuf::from("/nonexistent/file/path.txt"); let nonexistent_path = PathBuf::from("/nonexistent/file/path.txt");
// Verify the file doesn't exist // Verify the file doesn't exist
assert_file_not_exists(&nonexistent_path); assert_file_not_exists(&nonexistent_path);
} }

4
src/tests/modes/list_tests.rs
View File

@@ -11,12 +11,12 @@ mod tests {
fn test_list_mode_directory_operations() { fn test_list_mode_directory_operations() {
// Create a temporary directory for testing // Create a temporary directory for testing
let temp_dir = create_temp_dir(); let temp_dir = create_temp_dir();
// Test reading directory contents (should be empty) // Test reading directory contents (should be empty)
let entries: Vec<_> = std::fs::read_dir(temp_dir.path()) let entries: Vec<_> = std::fs::read_dir(temp_dir.path())
.expect("Failed to read directory") .expect("Failed to read directory")
.collect(); .collect();
assert_eq!(entries.len(), 0); assert_eq!(entries.len(), 0);
} }
} }

14
src/tests/modes/mod.rs
View File

@@ -1,18 +1,18 @@
// Modes tests module // Modes tests module
#[cfg(test)] #[cfg(test)]
pub mod save_tests; pub mod delete_tests;
#[cfg(test)]
pub mod diff_tests;
#[cfg(test)] #[cfg(test)]
pub mod get_tests; pub mod get_tests;
#[cfg(test)] #[cfg(test)]
pub mod info_tests;
#[cfg(test)]
pub mod list_tests; pub mod list_tests;
#[cfg(test)] #[cfg(test)]
pub mod delete_tests; pub mod save_tests;
#[cfg(test)]
pub mod update_tests;
#[cfg(test)]
pub mod info_tests;
#[cfg(test)] #[cfg(test)]
pub mod status_tests; pub mod status_tests;
#[cfg(test)] #[cfg(test)]
pub mod diff_tests; pub mod update_tests;

14
src/tests/modes/save_tests.rs
View File

@@ -1,16 +1,20 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::tests::common::test_helpers::{create_temp_dir, create_temp_file_with_content, create_empty_temp_file, assert_file_exists, get_file_size}; use crate::tests::common::test_helpers::{
assert_file_exists, create_empty_temp_file, create_temp_dir, create_temp_file_with_content,
get_file_size,
};
#[test] #[test]
fn test_save_mode_basic_functionality() { fn test_save_mode_basic_functionality() {
// Create a temporary directory for testing // Create a temporary directory for testing
let dir = create_temp_dir(); let dir = create_temp_dir();
let file_path = create_temp_file_with_content(&dir, "test_input.txt", "test content for save mode\n"); let file_path =
create_temp_file_with_content(&dir, "test_input.txt", "test content for save mode\n");
// Verify file was created // Verify file was created
assert_file_exists(&file_path); assert_file_exists(&file_path);
// Note: Actual save mode testing would require integration with the keep database // Note: Actual save mode testing would require integration with the keep database
// and compression engines, which is complex for unit tests // and compression engines, which is complex for unit tests
} }
@@ -20,7 +24,7 @@ mod tests {
// Create a temporary directory for testing // Create a temporary directory for testing
let dir = create_temp_dir(); let dir = create_temp_dir();
let file_path = create_empty_temp_file(&dir, "empty_test.txt"); let file_path = create_empty_temp_file(&dir, "empty_test.txt");
// Verify empty file was created // Verify empty file was created
assert_file_exists(&file_path); assert_file_exists(&file_path);
assert_eq!(get_file_size(&file_path), 0); assert_eq!(get_file_size(&file_path), 0);

46
src/tests/server/auth_tests.rs
View File

@@ -1,13 +1,13 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use axum::http::{HeaderMap, HeaderValue};
use crate::modes::server::common::check_auth; use crate::modes::server::common::check_auth;
use axum::http::{HeaderMap, HeaderValue};
#[test] #[test]
fn test_auth_with_no_password_required() { fn test_auth_with_no_password_required() {
let headers = HeaderMap::new(); let headers = HeaderMap::new();
let password = None; let password = None;
// When no password is required, auth should pass // When no password is required, auth should pass
assert!(check_auth(&headers, &password)); assert!(check_auth(&headers, &password));
} }
@@ -15,10 +15,13 @@ mod tests {
#[test] #[test]
fn test_auth_with_bearer_token() { fn test_auth_with_bearer_token() {
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
headers.insert("authorization", HeaderValue::from_static("Bearer secret123")); headers.insert(
"authorization",
HeaderValue::from_static("Bearer secret123"),
);
let password = Some("secret123".to_string()); let password = Some("secret123".to_string());
// Valid bearer token should pass // Valid bearer token should pass
assert!(check_auth(&headers, &password)); assert!(check_auth(&headers, &password));
} }
@@ -26,10 +29,13 @@ mod tests {
#[test] #[test]
fn test_auth_with_invalid_bearer_token() { fn test_auth_with_invalid_bearer_token() {
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
headers.insert("authorization", HeaderValue::from_static("Bearer wrongtoken")); headers.insert(
"authorization",
HeaderValue::from_static("Bearer wrongtoken"),
);
let password = Some("secret123".to_string()); let password = Some("secret123".to_string());
// Invalid bearer token should fail // Invalid bearer token should fail
assert!(!check_auth(&headers, &password)); assert!(!check_auth(&headers, &password));
} }
@@ -38,10 +44,13 @@ mod tests {
fn test_auth_with_basic_auth() { fn test_auth_with_basic_auth() {
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
// Basic auth for "keep:secret123" base64 encoded // Basic auth for "keep:secret123" base64 encoded
headers.insert("authorization", HeaderValue::from_static("Basic a2VlcDpzZWNyZXQxMjM=")); headers.insert(
"authorization",
HeaderValue::from_static("Basic a2VlcDpzZWNyZXQxMjM="),
);
let password = Some("secret123".to_string()); let password = Some("secret123".to_string());
// Valid basic auth should pass // Valid basic auth should pass
assert!(check_auth(&headers, &password)); assert!(check_auth(&headers, &password));
} }
@@ -50,10 +59,13 @@ mod tests {
fn test_auth_with_invalid_basic_auth() { fn test_auth_with_invalid_basic_auth() {
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
// Basic auth for "keep:wrongpass" base64 encoded // Basic auth for "keep:wrongpass" base64 encoded
headers.insert("authorization", HeaderValue::from_static("Basic a2VlcDp3cm9uZ3Bhc3M=")); headers.insert(
"authorization",
HeaderValue::from_static("Basic a2VlcDp3cm9uZ3Bhc3M="),
);
let password = Some("secret123".to_string()); let password = Some("secret123".to_string());
// Invalid basic auth should fail // Invalid basic auth should fail
assert!(!check_auth(&headers, &password)); assert!(!check_auth(&headers, &password));
} }
@@ -62,7 +74,7 @@ mod tests {
fn test_auth_with_missing_auth_header() { fn test_auth_with_missing_auth_header() {
let headers = HeaderMap::new(); let headers = HeaderMap::new();
let password = Some("secret123".to_string()); let password = Some("secret123".to_string());
// Missing auth header should fail when password is required // Missing auth header should fail when password is required
assert!(!check_auth(&headers, &password)); assert!(!check_auth(&headers, &password));
} }
@@ -71,9 +83,9 @@ mod tests {
fn test_auth_with_malformed_auth_header() { fn test_auth_with_malformed_auth_header() {
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
headers.insert("authorization", HeaderValue::from_static("Invalid header")); headers.insert("authorization", HeaderValue::from_static("Invalid header"));
let password = Some("secret123".to_string()); let password = Some("secret123".to_string());
// Malformed auth header should fail // Malformed auth header should fail
assert!(!check_auth(&headers, &password)); assert!(!check_auth(&headers, &password));
} }