asp/keep
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add compression and digest support with database schema updates

Browse Source
This commit is contained in:
Andrew Phillips
2025-05-14 09:45:51 -03:00
committed by Andrew Phillips (aider)
parent 9b61a37036
commit bbdfe19836
19 changed files with 181 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@@ -2,7 +2,7 @@
name = "keep"
version = "0.1.0"
edition = "2024"
description = "Keep and manage temporary files with automatic metadata generation"
description = "Keep and manage temporary files with automatic compression and metadata generation"
readme = "README.md"
categories = ["command-line-utilities"]

9
src/compression_engine.rs
View File

@@ -1,4 +1,4 @@
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use std::io;
use std::io::{Read, Write};
use std::path::PathBuf;
@@ -128,7 +128,9 @@ lazy_static! {
};
}
pub fn get_compression_engine(compression_type: CompressionType) -> Result<Box<dyn CompressionEngine>> {
pub fn get_compression_engine(
compression_type: CompressionType,
) -> Result<Box<dyn CompressionEngine>> {
match compression_type {
CompressionType::LZ4 => Ok(Box::new(CompressionEngineLZ4::new())),
CompressionType::GZip => Ok(Box::new(CompressionEngineGZip::new())),
@@ -144,7 +146,8 @@ pub fn get_compression_engine(compression_type: CompressionType) -> Result<Box<d
pub fn default_compression_type() -> CompressionType {
let mut default = CompressionType::None;
for compression_type in CompressionType::iter() {
let compression_engine = get_compression_engine(compression_type.clone()).expect("Missing engine");
let compression_engine =
get_compression_engine(compression_type.clone()).expect("Missing engine");
if compression_engine.is_supported() {
default = compression_type;
break;

2
src/compression_engine/gzip.rs
View File

@@ -5,9 +5,9 @@ use std::io;
use std::io::{Read, Write};
use std::path::PathBuf;
use flate2::Compression;
use flate2::read::GzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
use crate::compression_engine::CompressionEngine;

2
src/compression_engine/program.rs
View File

@@ -1,4 +1,4 @@
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use log::*;
use std::env;
use std::fs;

18
src/db.rs
View File

@@ -3,7 +3,7 @@ use chrono::prelude::*;
use lazy_static::lazy_static;
use log::*;
use rusqlite::{Connection, OpenFlags};
use rusqlite_migration::{Migrations, M};
use rusqlite_migration::{M, Migrations};
use std::collections::HashMap;
use std::path::PathBuf;
use std::rc::Rc;
@@ -32,12 +32,8 @@ lazy_static! {
FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY(id, name));"
),
M::up(
"ALTER TABLE items ADD COLUMN digest_type TEXT NOT NULL DEFAULT 'none';"
),
M::up(
"ALTER TABLE items ADD COLUMN digest_value TEXT NULL;"
),
M::up("ALTER TABLE items ADD COLUMN digest_type TEXT NOT NULL DEFAULT 'none';"),
M::up("ALTER TABLE items ADD COLUMN digest_value TEXT NULL;"),
]);
}
@@ -97,7 +93,13 @@ pub fn update_item(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Updating item: {:?}", item);
conn.execute(
"UPDATE items SET size=?2, compression=?3, digest_type=?4, digest_value=?5 WHERE id=?1",
(item.id, item.size, item.compression, item.digest_type, item.digest_value),
(
item.id,
item.size,
item.compression,
item.digest_type,
item.digest_value,
),
)?;
Ok(())
}

1
src/digest_engine.rs
View File

@@ -64,7 +64,6 @@ lazy_static! {
};
}
pub fn get_digest_engine(digest_type: DigestType) -> Box<dyn DigestEngine> {
match digest_type {
DigestType::Sha256 => Box::new(DigestEngineSha256::new()),

2
src/digest_engine/none.rs
View File

@@ -1,6 +1,6 @@
use crate::digest_engine::DigestEngine;
use anyhow::Result;
use std::io::{self, Write};
use crate::digest_engine::DigestEngine;
#[derive(Debug, Eq, PartialEq, Clone, Default)]
pub struct DigestEngineNone {}

9
src/digest_engine/program.rs
View File

@@ -1,10 +1,10 @@
use anyhow::{anyhow, Result, Context};
use crate::digest_engine::ProgramWriter;
use anyhow::{Context, Result, anyhow};
use log::*;
use std::env;
use std::fs;
use std::io;
use std::io::Write;
use crate::digest_engine::ProgramWriter;
use std::os::unix::fs::PermissionsExt;
use std::process::{Command, Stdio};
@@ -41,10 +41,7 @@ impl DigestEngine for DigestEngineProgram {
let program = self.program.clone();
let args = self.args.clone();
debug!(
"DIGEST: Executing command: {:?} {:?}",
program, args
);
debug!("DIGEST: Executing command: {:?} {:?}", program, args);
let mut process = Command::new(program.clone())
.args(args.clone())

2
src/digest_engine/sha2.rs
View File

@@ -1,7 +1,7 @@
use anyhow::Result;
use std::io::Write;
use sha2::{Digest, Sha256};
use std::io;
use std::io::Write;
use crate::digest_engine::DigestEngine;

6
src/main.rs
View File

@@ -1,6 +1,6 @@
use std::path::PathBuf;
use anyhow::{anyhow, Context, Error, Result};
use anyhow::{Context, Error, Result, anyhow};
use clap::*;
use log::*;
mod modes;
@@ -102,7 +102,9 @@ struct ModeArgs {
#[derive(Parser, Debug)]
struct ItemArgs {
#[arg(help_heading("Item Options"), short, long, conflicts_with_all(["get", "delete", "status"]))]
#[arg(help("Set metadata for the item using the format KEY=[VALUE], the metadata will be removed if VALUE is not provided"))]
#[arg(help(
"Set metadata for the item using the format KEY=[VALUE], the metadata will be removed if VALUE is not provided"
))]
meta: Vec<KeyValue>,
#[arg(help_heading("Item Options"), long, env("KEEP_DIGEST"))]

2
src/modes/common.rs
View File

@@ -1,4 +1,4 @@
use humansize::{FormatSizeOptions, BINARY};
use humansize::{BINARY, FormatSizeOptions};
use log::debug;
use prettytable::format::TableFormat;
use regex::Regex;

4
src/modes/delete.rs
View File

@@ -1,10 +1,10 @@
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use std::fs;
use std::path::PathBuf;
use crate::db;
use clap::error::ErrorKind;
use clap::Command;
use clap::error::ErrorKind;
use log::{debug, warn};
use rusqlite::Connection;

8
src/modes/diff.rs
View File

@@ -3,11 +3,11 @@ use libc::c_int;
use std::path::PathBuf;
use std::str::FromStr;
use anyhow::{anyhow, Result};
use anyhow::{Result, anyhow};
use clap::Command;
use nix::Error as NixError;
use nix::fcntl::FdFlag;
use nix::unistd::{close, pipe};
use nix::Error as NixError;
use std::io::Read;
use std::os::fd::FromRawFd;
use std::process::Stdio;
@@ -133,8 +133,8 @@ pub fn mode_diff(
) {
use std::io::BufWriter;
let mut buffered_pipe_writer = BufWriter::new(pipe_writer_raw);
let engine = get_compression_engine(compression_type)
.expect("Unable to get compression engine");
let engine =
get_compression_engine(compression_type).expect("Unable to get compression engine");
log::debug!("THREAD: Sending item to diff");
engine
.copy(item_path, &mut buffered_pipe_writer)

12
src/modes/info.rs
View File

@@ -1,12 +1,12 @@
use crate::db::Item;
use crate::modes::common::format_size;
use anyhow::anyhow;
use clap::error::ErrorKind;
use clap::Command;
use clap::error::ErrorKind;
use std::path::PathBuf;
use std::str::FromStr;
use crate::db::Item;
use crate::compression_engine::{get_compression_engine, CompressionType};
use crate::compression_engine::{CompressionType, get_compression_engine};
use crate::db::{get_item, get_item_last, get_item_matching};
use crate::modes::common::get_format_box_chars_no_border_line_separator;
use chrono::prelude::*;
@@ -123,8 +123,8 @@ fn show_item(
// Corrected logic for file_magic_cell:
// compression_type_val is already the successfully parsed CompressionType.
// The .expect() here will panic if get_compression_engine returns an Err.
let compression_engine =
get_compression_engine(compression_type_val.clone()).expect("Unable to get compression engine");
let compression_engine = get_compression_engine(compression_type_val.clone())
.expect("Unable to get compression engine");
let magic_result = compression_engine.magic(item_path_buf.clone()); // Use cloned item_path_buf
let file_magic_cell = match magic_result {
@@ -162,7 +162,7 @@ fn show_item(
Cell::new("Digest Value").with_style(Attr::Bold),
Cell::new(&dv_str),
]));
},
}
None => { /* Do nothing if None, as per original logic */ }
}

14
src/modes/list.rs
View File

@@ -141,25 +141,19 @@ pub fn mode_list(
},
ColumnType::Compression => {
Cell::new(&string_column(item.compression.to_string(), column_width))
},
}
ColumnType::DigestType => {
Cell::new(&string_column(item.digest_type.to_string(), column_width))
},
ColumnType::DigestValue => {
match item.digest_value {
}
ColumnType::DigestValue => match item.digest_value {
Some(ref value) => Cell::new(&string_column(value.to_string(), column_width)),
None => Cell::new("Missing")
.with_style(Attr::ForegroundColor(color::RED))
.with_style(Attr::Bold),
}
},
ColumnType::FileSize => match item_path.metadata() {
Ok(metadata) => Cell::new_align(
&size_column(
metadata.len(),
args.options.human_readable,
column_width,
),
&size_column(metadata.len(), args.options.human_readable, column_width),
Alignment::RIGHT,
),
Err(_) => Cell::new_align("Missing", Alignment::RIGHT)

14
src/modes/save.rs
View File

@@ -1,18 +1,18 @@
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use is_terminal::IsTerminal;
use std::collections::HashMap;
use std::io::{self, Read};
use std::str::FromStr;
use clap::error::ErrorKind;
use clap::Command;
use clap::error::ErrorKind;
use log::debug;
use std::path::PathBuf;
use rusqlite::Connection;
use std::path::PathBuf;
use crate::compression_engine::{CompressionType, get_compression_engine};
use crate::db::{self};
use crate::digest_engine::{get_digest_engine, DigestType};
use crate::compression_engine::{get_compression_engine, CompressionType};
use crate::digest_engine::{DigestType, get_digest_engine};
use crate::modes::common::get_meta_from_env;
use chrono::Utc;
@@ -133,8 +133,8 @@ pub fn mode_save(
let mut stdout = io::stdout().lock();
let mut buffer = [0; libc::BUFSIZ as usize];
let compression_engine = get_compression_engine(compression_type.clone())
.expect("Unable to get compression engine");
let compression_engine =
get_compression_engine(compression_type.clone()).expect("Unable to get compression engine");
let mut item_out: Box<dyn Write> =
compression_engine
.create(item_path.clone())

20
src/modes/status.rs
View File

@@ -4,9 +4,9 @@ use std::path::PathBuf;
use strum::IntoEnumIterator;
use crate::compression_engine;
use crate::compression_engine::program::CompressionEngineProgram;
use crate::compression_engine::CompressionType;
use crate::compression_engine::COMPRESSION_PROGRAMS;
use crate::compression_engine::CompressionType;
use crate::compression_engine::program::CompressionEngineProgram;
use crate::FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR;
use crate::FORMAT_NO_BORDER_LINE_SEPARATOR;
@@ -109,9 +109,9 @@ fn build_compression_table() -> Table {
fn build_digest_table() -> Table {
use crate::digest_engine;
use crate::digest_engine::program::DigestEngineProgram;
use crate::digest_engine::DigestType;
use crate::digest_engine::DIGEST_PROGRAMS;
use crate::digest_engine::DigestType;
use crate::digest_engine::program::DigestEngineProgram;
let mut digest_table = Table::new();
if std::io::stdout().is_terminal() {
@@ -130,8 +130,7 @@ fn build_digest_table() -> Table {
let default_type = digest_engine::default_digest_type();
for digest_type in DigestType::iter() {
let digest_program: DigestEngineProgram =
match &DIGEST_PROGRAMS[digest_type.clone()] {
let digest_program: DigestEngineProgram = match &DIGEST_PROGRAMS[digest_type.clone()] {
Some(digest_program) => digest_program.clone(),
None => DigestEngineProgram {
program: "".to_string(),
@@ -172,15 +171,12 @@ pub fn mode_status(
db_path: PathBuf,
) -> Result<(), anyhow::Error> {
println!("PATHS:");
build_path_table(data_path, db_path)
.printstd();
build_path_table(data_path, db_path).printstd();
println!();
println!("COMPRESSION:");
build_compression_table()
.printstd();
build_compression_table().printstd();
println!();
println!("DIGEST:");
build_digest_table()
.printstd();
build_digest_table().printstd();
Ok(())
}

11
src/modes/update.rs
View File

@@ -4,10 +4,10 @@ use std::str::FromStr;
use crate::compression_engine::{CompressionType, get_compression_engine};
use crate::db;
use clap::error::ErrorKind;
use clap::Command;
use log::{debug, info};
use crate::digest_engine;
use clap::Command;
use clap::error::ErrorKind;
use log::{debug, info};
use rusqlite::Connection;
pub fn mode_update(
@@ -46,8 +46,8 @@ pub fn mode_update(
};
let compression_type = CompressionType::from_str(&item.compression)?;
let compression_engine = get_compression_engine(compression_type)
.expect("Unable to get compression engine");
let compression_engine =
get_compression_engine(compression_type).expect("Unable to get compression engine");
if item.size.is_none() {
info!("Updating unknown stream size");
@@ -66,7 +66,6 @@ pub fn mode_update(
}
}
if item.digest_value.is_none() {
let digest_type = digest_engine::DigestType::from_str(&item.digest_type)?;

104
src/tests.rs
View File

@@ -118,7 +118,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_A), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_A);
assert!(
output_str.contains(INPUT_A),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_A
);
let cmd = format!("{} -g 1", keep_cmd);
let output = run_sh(cmd.as_str());
@@ -129,7 +135,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_A), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_A);
assert!(
output_str.contains(INPUT_A),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_A
);
let cmd = format!("{} 1", keep_cmd);
let output = run_sh(cmd.as_str());
@@ -140,7 +152,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_A), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_A);
assert!(
output_str.contains(INPUT_A),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_A
);
let cmd = format!("{} --get", keep_cmd);
let output = run_sh(cmd.as_str());
@@ -151,7 +169,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_B), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_B);
assert!(
output_str.contains(INPUT_B),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_B
);
let cmd = format!("{} --get tag_a", keep_cmd);
let output = run_sh(cmd.as_str());
@@ -162,7 +186,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_A), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_A);
assert!(
output_str.contains(INPUT_A),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_A
);
let cmd = format!("{} --get tag_b", keep_cmd);
let output = run_sh(cmd.as_str());
@@ -173,7 +203,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_B), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_B);
assert!(
output_str.contains(INPUT_B),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_B
);
let cmd = format!("{} --get tag", keep_cmd);
let output = run_sh(cmd.as_str());
@@ -184,7 +220,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_B), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_B);
assert!(
output_str.contains(INPUT_B),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_B
);
});
}
@@ -292,9 +334,21 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_A), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_A);
assert!(
output_str.contains(INPUT_A),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_A
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_B), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_B);
assert!(
output_str.contains(INPUT_B),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_B
);
let cmd = format!("{} --diff tag_a tag_b", keep_cmd);
let output = run_sh(cmd.as_str());
@@ -373,7 +427,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_A), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_A);
assert!(
output_str.contains(INPUT_A),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_A
);
// Test with lz4 compression
let cmd = format!("echo {} | {} -c lz4 lz4", INPUT_A, keep_cmd);
@@ -396,7 +456,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_A), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_A);
assert!(
output_str.contains(INPUT_A),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_A
);
// Test with gzip compression
let cmd = format!("echo {} | {} -c gzip gzip", INPUT_A, keep_cmd);
@@ -419,7 +485,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_A), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_A);
assert!(
output_str.contains(INPUT_A),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_A
);
// Test with bzip2 compression
let cmd = format!("echo {} | {} -c bzip2 bzip2", INPUT_A, keep_cmd);
@@ -442,7 +514,13 @@ mod tests {
output.status
);
let output_str = String::from_utf8_lossy(&output.stdout).to_string();
assert!(output_str.contains(INPUT_A), "Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"", cmd, output_str, INPUT_A);
assert!(
output_str.contains(INPUT_A),
"Command output does not contain expected string. Command: {} Output: {} Expected: \"{}\"",
cmd,
output_str,
INPUT_A
);
});
}
}