From 49a77f9090036691021f6bf43e79a1849e7ebb6d Mon Sep 17 00:00:00 2001 From: Andrew Phillips Date: Thu, 31 Aug 2023 19:38:59 +0000 Subject: [PATCH] Most of basic functionality implemented --- Cargo.lock | 325 ++++++++++++++++++++++++- Cargo.toml | 16 +- src/compression.rs | 225 ++++++++++++----- src/db.rs | 374 +++++++++++++++++++++++++++-- src/main.rs | 584 ++++++++++++++++++++++++++++++++++++++++----- 5 files changed, 1382 insertions(+), 142 deletions(-) mode change 100644 => 100755 src/compression.rs diff --git a/Cargo.lock b/Cargo.lock index 5303ccb..fe190c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,6 +92,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "anyhow" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" + [[package]] name = "atty" version = "0.2.14" @@ -109,6 +115,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.3.3" @@ -143,6 +155,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "time", "wasm-bindgen", "winapi", ] @@ -200,6 +213,95 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "csv" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "enum-map" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9705d8de4776df900a4a0b2384f8b0ab42f775e93b083b42f8ce71bdc32a47e3" +dependencies = [ + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccb14d927583dd5c2eac0f2cf264fc4762aefe1ae14c47a8a20fc1939d3a5fc0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "errno" version = "0.3.1" @@ -233,6 +335,27 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "gethostname" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" +dependencies = [ + "libc", + "windows-targets", +] + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + [[package]] name = "hashbrown" version = "0.14.0" @@ -273,6 +396,15 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -298,15 +430,21 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.2", "rustix", "windows-sys", ] +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + [[package]] name = "js-sys" version = "0.3.64" @@ -320,13 +458,26 @@ dependencies = [ name = "keep-rust" version = "0.1.0" dependencies = [ + "anyhow", + "chrono", "clap", + "directories", + "enum-map", + "gethostname", + "humansize", + "is-terminal", "lazy_static", + "libc", "log", + "prettytable-rs", "regex", "rusqlite", "rusqlite_migration", + "signal-hook", "stderrlog", + "strum", + "strum_macros", + "term", ] [[package]] @@ -341,6 +492,12 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + [[package]] name = "libsqlite3-sys" version = "0.26.0" @@ -385,12 +542,32 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "pkg-config" version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "prettytable-rs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a" +dependencies = [ + "csv", + "encode_unicode", + "is-terminal", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "proc-macro2" version = "1.0.63" @@ -409,6 +586,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" version = "1.9.1" @@ -444,7 +641,8 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" dependencies = [ - "bitflags", + "bitflags 2.3.3", + "chrono", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -468,13 +666,50 @@ version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4" dependencies = [ - "bitflags", + "bitflags 2.3.3", "errno", "libc", "linux-raw-sys", "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.185" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" + +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + [[package]] name = "smallvec" version = "1.11.0" @@ -500,6 +735,28 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "syn" version = "2.0.23" @@ -511,6 +768,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "termcolor" version = "1.1.3" @@ -520,6 +788,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.1.7" @@ -530,12 +818,29 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + [[package]] name = "unicode-ident" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "utf8parse" version = "0.2.1" @@ -554,6 +859,18 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.87" diff --git a/Cargo.toml b/Cargo.toml index b36b6be..a597837 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,10 +6,24 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = "1.0.72" clap = { version = "4.3.10", features = ["derive", "env"] } +directories = "5.0.1" lazy_static = "1.4.0" +libc = "0.2.147" log = "0.4.19" regex = "1.9.1" -rusqlite = { version = "0.29.0", features = ["bundled"] } +rusqlite = { version = "0.29.0", features = ["bundled", "array", "chrono"] } rusqlite_migration = "1.0.2" stderrlog = "0.5.4" +strum_macros = "0.25" +strum = { version = "0.25", features = ["derive"] } +signal-hook = "0.3.17" +prettytable-rs = "0.10.0" +chrono = "0.4.26" +gethostname = "0.4.3" +humansize = "2.1.3" +enum-map = "2.6.1" +is-terminal = "0.4.9" +term = "0.7.0" + diff --git a/src/compression.rs b/src/compression.rs old mode 100644 new mode 100755 index 5c2474b..7fc9021 --- a/src/compression.rs +++ b/src/compression.rs @@ -1,86 +1,166 @@ -use std::fmt; -use std::os::unix::fs::PermissionsExt; +use anyhow::{Context, Result, anyhow}; +use strum::IntoEnumIterator; +use std::fs::File; +use std::io; +use std::io::Write; +use std::process::{Command,Stdio}; +use std::path::PathBuf; use std::env; use std::fs; +use std::os::unix::fs::PermissionsExt; use log::*; +use lazy_static::lazy_static; +extern crate enum_map; +use enum_map::enum_map; +use enum_map::{EnumMap,Enum}; -#[derive(Debug, Clone)] -pub struct CompressionType { - name: String, - binary: String, - compress: String, - decompress: String, + +#[derive(Debug, Eq, PartialEq, Clone, strum::EnumIter, strum::Display, strum::EnumString, Enum)] +#[strum(ascii_case_insensitive)] +pub enum CompressionType { + LZ4, + GZip, + BZip2, + XZ, + ZStd, + None } -impl fmt::Display for CompressionType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "[name='{}', binary='{}', compress='{}', decompress='{}']", self.name, self.binary, self.compress, self.decompress) + +#[derive(Debug, Eq, PartialEq, Clone)] +pub struct CompressionProgram { + pub program: String, + pub compress: Vec, + pub decompress: Vec, + pub supported: bool +} + + +lazy_static! { + static ref COMPRESSION_PROGRAMS: EnumMap> = enum_map! { + CompressionType::LZ4 => Some(CompressionProgram::new("lz4", vec!["-qcf"], vec!["-dcf"])), + CompressionType::GZip => Some(CompressionProgram::new("gzip", vec!["-qcf"], vec!["-dcf"])), + CompressionType::BZip2 => Some(CompressionProgram::new("bzip2", vec!["-qcf"], vec!["-dcf"])), + CompressionType::XZ => Some(CompressionProgram::new("xz", vec!["-qcf"], vec!["-dcf"])), + CompressionType::ZStd => Some(CompressionProgram::new("zstd", vec!["-qcf"], vec!["-dcf"])), + CompressionType::None => None + }; +} + +impl CompressionProgram { + pub fn new(program: &str, compress: Vec<&str>, decompress: Vec<&str>) -> CompressionProgram { + let program_path = get_program_path(program); + let supported = program_path.is_ok(); + + CompressionProgram { + program: program_path.unwrap_or(program.to_string()), + compress: compress.iter().map(|s| {s.to_string()}).collect(), + decompress: decompress.iter().map(|s| {s.to_string()}).collect(), + supported + } } } -pub fn add_compression_type(compression_types: &mut Vec, name: String, binary: String, compress: String, decompress: String) { - let path = is_program_in_path(binary); +pub trait CompressionEngine { + fn is_supported(&self) -> bool; + fn cat(&self, file_path: PathBuf) -> Result<()>; + fn create(&self, file_path: PathBuf) -> Result>; +} - if let Ok(path) = path { - compression_types.push( - CompressionType { - name, - binary: path, - compress, - decompress - }); + +impl CompressionEngine for CompressionProgram { + fn is_supported(&self) -> bool { + self.supported + } + + fn cat(&self, file_path: PathBuf) -> Result<()> { + debug!("COMPRESSION: Outputting {:?} to STDOUT using {:?}", file_path, *self); + let program = self.program.clone(); + let args = self.decompress.clone(); + + debug!("COMPRESSION: Executing command: {:?} {:?} writing to {:?}", program, args, file_path); + + let file = File::open(file_path).context("Unable to open file for reading")?; + + let mut process = Command::new(program.clone()) + .args(args.clone()) + .stdin(file) + .spawn() + .context(anyhow!("Unable to spawn child process: {:?} {:?}", program, args))?; + + let result = process.wait() + .context(anyhow!("Unable to wait for child process: {:?} {:?}", program, args))?; + + if result.success() { + Ok(()) + } else { + Err(anyhow!("Decompression program returned {}", result)) + } + } + + fn create(&self, file_path: PathBuf) -> Result> { + debug!("COMPRESSION: Writting to {:?} using {:?}", file_path, *self); + + let program = self.program.clone(); + let args = self.compress.clone(); + + debug!("COMPRESSION: Executing command: {:?} {:?} writing to {:?}", program, args, file_path); + + let file = File::create(file_path).context("Unable to open file for writing")?; + + let process = Command::new(program.clone()) + .args(args.clone()) + .stdin(Stdio::piped()) + .stdout(file) + .spawn() + .context(anyhow!("Problem spawning child process: {:?} {:?}", program, args))?; + + Ok(Box::new(process.stdin.unwrap())) } } -pub fn supported_compression_types() -> Vec { - let mut compression_types = Vec::new(); - - add_compression_type(&mut compression_types, "lz4".to_string(), "lz4".to_string(), "-qc".to_string() ,"-dc".to_string()); - add_compression_type(&mut compression_types, "gzip".to_string(), "gzip".to_string(), "-qc".to_string() ,"-dc".to_string()); - add_compression_type(&mut compression_types, "bzip2".to_string(), "bzip2".to_string(), "-qc".to_string() ,"-dc".to_string()); - add_compression_type(&mut compression_types, "xz".to_string(), "xz".to_string(), "-qc".to_string() ,"-dc".to_string()); - - compression_types.push( - CompressionType { - name: "none".to_string(), - binary: "".to_string(), - compress: "".to_string(), - decompress: "".to_string(), - }); - - return compression_types; +#[derive(Debug, Eq, PartialEq, Clone)] +pub struct CompressionEngineNone { } - -pub fn get_compression_default(compression_types: Vec) -> Result { - debug!("Compression type: default"); - - match compression_types.first() { - None => Err(String::from("Unable to find default compression type")), - Some(compression_type) => Ok(compression_type.clone()) +impl CompressionEngineNone { + pub fn new() -> CompressionEngineNone { + CompressionEngineNone {} } } -pub fn get_compression_named(compression_types: Vec, compression_name: String) -> Result { - debug!("Compression type: {}", compression_name); - match compression_types.iter().find(|&c| c.name == *compression_name) { - None => Err(format!("Unable to find compression type: {}", compression_name)), - Some(compression_type) => Ok(compression_type.clone()) +impl Default for CompressionEngineNone { + fn default() -> Self { + Self::new() } } -pub fn get_compression(compression_name: Option) -> Result { - let compression_types = supported_compression_types(); +impl CompressionEngine for CompressionEngineNone { + fn is_supported(&self) -> bool { + true + } - match compression_name { - None => get_compression_default(compression_types), - Some(compression_name) => get_compression_named(compression_types, compression_name), + fn cat(&self, file_path: PathBuf) -> Result<()> { + debug!("COMPRESSION: Outputting {:?} to STDOUT using {:?}", file_path, *self); + let mut stdout = io::stdout().lock(); + let mut file = File::open(file_path)?; + + io::copy(&mut file, &mut stdout)?; + stdout.flush()?; + + Ok(()) + } + + fn create(&self, file_path: PathBuf) -> Result> { + debug!("COMPRESSION: Writting to {:?} using {:?}", file_path, *self); + Ok(Box::new(File::create(file_path)?)) } } -fn is_program_in_path(program: String) -> Result { - debug!("Looking for executable: {}", program); + +fn get_program_path(program: &str) -> Result { + debug!("COMPRESSION: Looking for executable: {}", program); if let Ok(path) = env::var("PATH") { for p in path.split(':') { let p_str = format!("{}/{}", p, program); @@ -94,5 +174,34 @@ fn is_program_in_path(program: String) -> Result { } } } - Err(()) + Err(anyhow!("Unable to find binary {} in PATH", program)) +} + + +pub fn get_program(compression_type: CompressionType) -> Result { + match &COMPRESSION_PROGRAMS[compression_type.clone()] { + Some(compression_program) => Ok(compression_program.clone()), + None => Err(anyhow!("Compression type {} has no program", compression_type)) + } +} + + +pub fn get_engine(compression_type: CompressionType) -> Result> { + match compression_type { + CompressionType::None => Ok(Box::new(CompressionEngineNone::new())), + compression_type => Ok(Box::new(COMPRESSION_PROGRAMS[compression_type.clone()].clone().unwrap())) + } +} + + +pub fn default_type() -> CompressionType { + let mut default = CompressionType::None; + for compression_type in CompressionType::iter() { + let compression_engine = get_engine(compression_type.clone()).expect("Missing engine"); + if compression_engine.is_supported() { + default = compression_type; + break; + } + } + default } diff --git a/src/db.rs b/src/db.rs index 6f2bae5..53161b7 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,36 +1,372 @@ -use rusqlite::{params, Connection, Error}; +use std::path::PathBuf; +use std::collections::HashMap; +use std::rc::Rc; +use anyhow::{Context, Result, Error}; +use rusqlite::{Connection, OpenFlags}; use rusqlite_migration::{Migrations, M}; -use log::*; +use chrono::prelude::*; use lazy_static::lazy_static; +use log::*; lazy_static! { static ref MIGRATIONS: Migrations<'static> = Migrations::new(vec![ - M::up("CREATE TABLE keep( - id INTEGER AUTOINCREMENT NOT NULL, + M::up("CREATE TABLE items( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, ts TEXT NOT NULL, - compress TEXT NOT NULL, - hostname TEXT NOT NULL, - comment TEXT NOT NULL) - PRIMARY KEY(id);"), + size INTEGER NULL, + compression TEXT NOT NULL)"), M::up("CREATE TABLE tags ( id INTEGER NOT NULL, name TEXT NOT NULL, - FOREIGN KEY(id) REFERENCES keep(id) ON DELETE CASCADE, + FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE, + PRIMARY KEY(id, name));"), + M::up("CREATE TABLE metas ( + id INTEGER NOT NULL, + name TEXT NOT NULL, + value TEXT NOT NULL, + FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE, PRIMARY KEY(id, name));") ]); } -fn open(path: String) -> Result { - debug!("Opening DB {}", path); +#[derive(Debug, Clone)] +pub struct Item { + pub id: Option, + pub ts: DateTime, + pub size: Option, + pub compression: String +} - match Connection::open(path) { - Ok(mut conn) => match conn.pragma_update(None, "foreign_keys", "ON") { - Ok(()) => match MIGRATIONS.to_latest(&mut conn) { - Ok(()) => Ok(conn), - Err(e) => Err(format!("Error migrating sqlite schema: {}", e)) - }, - Err(e) => Err(format!("Error setting sqlite pragma: {}", e)) +#[derive(Debug, Clone)] +pub struct Tag { + pub id: i64, + pub name: String +} + +#[derive(Debug, Clone)] +pub struct Meta { + pub id: i64, + pub name: String, + pub value: String +} + + +pub fn open(path: PathBuf) -> Result { + debug!("DB: Opening file: {:?}", path); + let mut conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE) + .context("Problem opening file")?; + + conn.pragma_update(None, "foreign_keys", "ON") + .context("Problem enabling SQLite foreign_keys pragma")?; + + MIGRATIONS.to_latest(&mut conn) + .context("Problem performing database migrations")?; + + rusqlite::vtab::array::load_module(&conn) + .context("Problem enabling array module")?; + + Ok(conn) +} + + +pub fn insert_item(conn: &Connection, item: Item) -> Result { + debug!("DB: Inserting item: {:?}", item); + conn.execute( + "INSERT INTO items (ts, size, compression) VALUES (?1, ?2, ?3)", + (item.ts, item.size, item.compression))?; + Ok(conn.last_insert_rowid()) +} + + +pub fn update_item(conn: &Connection, item: Item) -> Result<()> { + debug!("DB: Updating item: {:?}", item); + conn.execute( + "UPDATE items SET size=?2, compression=?3 WHERE id=?1", + (item.id, item.size, item.compression))?; + Ok(()) +} + + +pub fn delete_item(conn: &Connection, item: Item) -> Result<()> { + debug!("DB: Deleting item: {:?}", item); + conn.execute("DELETE FROM items WHERE id=?1", [item.id])?; + Ok(()) +} + + +pub fn query_delete_meta(conn: &Connection, meta: Meta) -> Result<()> { + debug!("DB: Deleting meta: {:?}", meta); + conn.execute( + "DELETE FROM metas WHERE id=?1 AND name=?2", + (meta.id, meta.name))?; + Ok(()) +} + +pub fn query_upsert_meta(conn: &Connection, meta: Meta) -> Result<()> { + debug!("DB: Inserting meta: {:?}", meta); + conn.execute( + "INSERT INTO metas (id, name, value) VALUES (?1, ?2, ?3) + ON CONFLICT(id, name) DO UPDATE SET value=?3", + (meta.id, meta.name, meta.value))?; + Ok(()) +} + +pub fn store_meta(conn: &Connection, meta: Meta) -> Result<()> { + debug!("DB: Storing meta: {:?}", meta); + if meta.value.eq("") { + query_delete_meta(conn, meta)?; + } else { + query_upsert_meta(conn, meta)?; + } + Ok(()) +} + + +pub fn insert_tag(conn: &Connection, tag: Tag) -> Result<()> { + debug!("DB: Inserting tag: {:?}", tag); + conn.execute( + "INSERT INTO tags (id, name) VALUES (?1, ?2)", + (tag.id, tag.name))?; + Ok(()) +} + + +pub fn delete_item_tags(conn: &Connection, item: Item) -> Result<()> { + debug!("DB: Deleting all item tags: {:?}", item); + conn.execute( + "DELETE FROM tags WHERE id=?1", + [item.id])?; + Ok(()) +} + + +pub fn set_item_tags(conn: &Connection, item: Item, tags: &Vec) -> Result<()> { + debug!("DB: Setting tags for item: {:?} ?{:?}", item, tags); + delete_item_tags(conn, item.clone())?; + let item_id = item.id.unwrap(); + for tag_name in tags { + insert_tag(conn, + Tag { + id: item_id, + name: tag_name.to_string() + })?; + } + + Ok(()) +} + + +pub fn query_all_items(conn: &Connection) -> Result> { + debug!("DB: Querying all items"); + let mut statement = conn + .prepare("SELECT id, ts, size, compression FROM items ORDER BY id ASC") + .context("Problem preparing SQL statement")?; + let mut rows = statement.query([])?; + let mut items = Vec::new(); + + while let Some(row) = rows.next()? { + let item = Item { + id: row.get(0)?, + ts: row.get(1)?, + size: row.get(2)?, + compression: row.get(3)? + }; + items.push(item); + } + + Ok(items) +} + +pub fn query_tagged_items<'a>(conn: &'a Connection, tags: &'a Vec) -> Result> { + debug!("DB: Querying tagged items: {:?}", tags); + let mut statement = conn + .prepare_cached(" + SELECT items.id, + items.ts, + items.size, + items.compression, + count(tags_match.id) as tags_score + FROM items, + (SELECT tags.id FROM tags WHERE tags.name IN rarray(?1)) as tags_match + WHERE items.id = tags_match.id + GROUP BY items.id + HAVING tags_score = ?2 + ORDER BY items.id ASC") + .context("Problem preparing SQL statement")?; + + let tags_values: Vec = tags + .iter() + .map(|s| {rusqlite::types::Value::from(s.clone())}) + .collect(); + + let tags_ptr = Rc::new(tags_values); + let mut rows = statement.query((&tags_ptr, &tags.len()))?; + + let mut items = Vec::new(); + + while let Some(row) = rows.next()? { + let item = Item { + id: row.get(0)?, + ts: row.get(1)?, + size: row.get(2)?, + compression: row.get(3)? + }; + items.push(item); + } + + Ok(items) +} + + + +pub fn get_items(conn: &Connection) -> Result> { + debug!("DB: Getting all items"); + query_all_items(conn) +} + + +pub fn get_items_matching(conn: &Connection, tags: &Vec, meta: &HashMap) -> Result> { + debug!("DB: Getting items matching: tags={:?} meta={:?}", tags, meta); + + let items = match tags.is_empty() { + true => query_all_items(conn)?, + false => query_tagged_items(conn, tags)? + }; + + if meta.is_empty() { + debug!("DB: Not filtering on meta"); + Ok(items) + } else { + debug!("DB: Filtering on meta"); + let mut filtered_items: Vec = Vec::new(); + for item in items.iter() { + let mut item_ok = true; + let mut item_meta: HashMap = HashMap::new(); + for meta in get_item_meta(conn, item)? { + item_meta.insert(meta.name, meta.value); + } + + debug!("DB: Matching: {:?}: {:?}", item, item_meta); + + for (k, v) in meta.iter() { + match item_meta.get(k) { + Some(value) => item_ok = v.eq(value), + None => item_ok = false + } + + if item_ok { + break; + } + } + + if item_ok { + filtered_items.push(item.clone()); + } } - Err(e) => Err(format!("Error connecting to database: {}", e)) + Ok(filtered_items) + } + +} + + +pub fn get_item_matching(conn: &Connection, tags: &Vec, _meta: &HashMap) -> Result> { + debug!("DB: Get item matching tags: {:?}", tags); + let mut statement = conn + .prepare_cached(" + SELECT items.id, + items.ts, + items.size, + items.compression, + count(sel.id) as score + FROM items, + (SELECT tags.id FROM tags WHERE tags.name IN rarray(?1)) as sel + WHERE items.id = sel.id + GROUP BY items.id + HAVING score = ?2 + ORDER BY items.id DESC + LIMIT 1") + .context("Problem preparing SQL statement")?; + + let tags_values: Vec = tags + .iter() + .map(|s| {rusqlite::types::Value::from(s.clone()) }) + .collect(); + + let tags_ptr = Rc::new(tags_values); + + let mut rows = statement.query((&tags_ptr, &tags.len()))?; + + match rows.next()? { + Some(row) => Ok(Some(Item { + id: row.get(0)?, + ts: row.get(1)?, + size: row.get(2)?, + compression: row.get(3)? + })), + None => Ok(None) } } + + +pub fn get_item(conn: &Connection, item_id: i64) -> Result> { + debug!("DB: Getting item {:?}", item_id); + let mut statement = conn + .prepare_cached(" + SELECT id, ts, size, compression + FROM items + WHERE items.id = ?1") + .context("Problem preparing SQL statement")?; + + let mut rows = statement.query([item_id])?; + + match rows.next()? { + Some(row) => Ok(Some(Item { + id: row.get(0)?, + ts: row.get(1)?, + size: row.get(2)?, + compression: row.get(3)? + })), + None => Ok(None) + } +} + + +pub fn get_item_tags(conn: &Connection, item: &Item) -> Result> { + debug!("DB: Getting tags for item: {:?}", item); + let mut statement = conn + .prepare_cached("SELECT id, name FROM tags WHERE id=?1 ORDER BY name ASC") + .context("Problem preparing SQL statement")?; + let mut rows = statement.query([item.id])?; + + let mut tags = Vec::new(); + + while let Some(row) = rows.next()? { + tags.push(Tag { + id: row.get(0)?, + name: row.get(1)?, + }); + } + + Ok(tags) +} + + +pub fn get_item_meta(conn: &Connection, item: &Item) -> Result> { + debug!("DB: Getting item meta: {:?}", item); + let mut statement = conn + .prepare_cached("SELECT id, name, value FROM metas WHERE id=?1 ORDER BY name ASC") + .context("Problem preparing SQL statement")?; + let mut rows = statement.query([item.id])?; + + let mut metas = Vec::new(); + + while let Some(row) = rows.next()? { + metas.push(Meta { + id: row.get(0)?, + name: row.get(1)?, + value: row.get(2)? + }); + } + + Ok(metas) +} diff --git a/src/main.rs b/src/main.rs index 1d05954..9a6f859 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,69 @@ +use std::io; +use std::io::{Read, Write}; +use std::fs; use std::str::FromStr; use std::path::PathBuf; +use std::collections::{HashMap, HashSet}; + +use anyhow::{Context, Result, Error, anyhow}; +use rusqlite::Connection; +use gethostname::gethostname; +use strum::IntoEnumIterator; use clap::error::ErrorKind; use clap::*; use log::*; +extern crate directories; +use directories::ProjectDirs; + +extern crate prettytable; +use prettytable::{Table, Row, Cell, Attr}; +use prettytable::format; +use prettytable::format::{TableFormat, Alignment}; +use prettytable::format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR; +use prettytable::row; +use prettytable::color; + +use chrono::prelude::*; + +#[macro_use] +extern crate lazy_static; + +use crate::compression::CompressionType; pub mod compression; pub mod db; +use humansize::{format_size, BINARY}; + +use is_terminal::IsTerminal; + +extern crate term; + +const BUFSIZ: usize = 8192; + +lazy_static! { + static ref FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR: TableFormat = format::FormatBuilder::new() + .column_separator('│') + .borders('│') + .separators(&[format::LinePosition::Top], + format::LineSeparator::new('─', + '┬', + '┌', + '┐')) + .separators(&[format::LinePosition::Title], + format::LineSeparator::new('─', + '┼', + '├', + '┤')) + .separators(&[format::LinePosition::Bottom], + format::LineSeparator::new('─', + '┴', + '└', + '┘')) + .padding(1, 1) + .build(); +} + #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { @@ -17,60 +74,66 @@ struct Args { #[command(flatten)] options: OptionsArgs, - #[arg()] + #[arg(help("A list of either item IDs or tags"))] ids_or_tags: Vec } + #[derive(Parser, Debug)] struct ModeArgs { - #[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["get", "list", "update", "delete", "status"]))] + #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["get", "list", "update", "delete", "status"]))] + #[arg(help("Save an item using any tags or metadata provided"))] save: bool, - #[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "list", "update", "delete", "status"]))] + #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "list", "update", "delete", "status"]))] + #[arg(help("Get an item either by it's ID or by a combination of matching tags and metatdata"))] get: bool, - #[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "get", "update", "delete", "status"]))] + #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "update", "delete", "status"]))] + #[arg(help("List items, filtering on tags or metadata if given"))] list: bool, - #[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "get", "list", "delete", "status"]), requires("ids_or_tags"))] + #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "list", "delete", "status"]), requires("ids_or_tags"))] + #[arg(help("Update a specified item ID's tags and/or metadata"))] update: bool, - #[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "get", "list", "update", "status"]), requires("ids_or_tags"))] + #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "list", "update", "status"]), requires("ids_or_tags"))] + #[arg(help("Delete items either by ID or by matching tags"))] delete: bool, - #[arg(group("mode"), help_heading("Mode"), short('S'), long, conflicts_with_all(["save", "get", "list", "update", "delete"]))] + #[arg(group("mode"), help_heading("Mode Options"), short('S'), long, conflicts_with_all(["save", "get", "list", "update", "delete"]))] + #[arg(help("Show status of directories and supported compression algorithms"))] status: bool } + #[derive(Parser, Debug)] struct ItemArgs { - #[arg(help_heading("Item"), short, long, conflicts_with("get"), conflicts_with("list"))] - comment: Option, + #[arg(help_heading("Item Options"), short, long, conflicts_with_all(["get", "delete", "status"]))] + #[arg(help("Set metadata for the item using the format KEY=[VALUE], the metadata will be removed if VALUE is not provided"))] + meta: Option>, - #[arg(help_heading("Item"), short('C'), long, conflicts_with("get"), conflicts_with("list"), env("KEEP_COMPRESS"))] - compress: Option, + #[arg(help_heading("Item Options"), short('C'), long, conflicts_with("get"), conflicts_with("list"), env("KEEP_COMPRESSION"), )] + #[arg(help("Compression algorithm to use when saving items"))] + compression: Option, } + #[derive(Parser, Debug)] struct OptionsArgs { - #[arg(help_heading("Options"), long, env("KEEP_DIR"))] + #[arg(long, env("KEEP_DIR"))] + #[arg(help("Specify the directory to use for storage"))] dir: Option, - #[arg(help_heading("Options"), short, long)] - force: bool, - - #[arg(help_heading("Options"), short, long, action = clap::ArgAction::Count, conflicts_with("quiet"))] + #[arg(short, long, action = clap::ArgAction::Count, conflicts_with("quiet"))] + #[arg(help("Increase message verbosity, can be given more than once"))] verbose: u8, - #[arg(help_heading("Options"), short, long)] + #[arg(short, long)] + #[arg(help("Do show any messages"))] quiet: bool, } -#[derive(Debug,Clone)] -enum NumberOrString { - Number(u32), - Str(String), -} #[derive(Debug,PartialEq)] enum KeepModes { @@ -83,35 +146,60 @@ enum KeepModes { Status } +#[derive(Debug,Clone)] +struct KeyValue { + key: String, + value: String +} + +impl FromStr for KeyValue { + type Err = Error; + fn from_str(s: &str) -> Result { + match s.split_once('=') { + Some(kv) => Ok(KeyValue { + key: kv.0.to_string(), + value: kv.1.to_string() + }), + None => Err(anyhow!("Unable to parse key=value pair")) + } + } +} + + +#[derive(Debug,Clone)] +enum NumberOrString { + Number(i64), + Str(String), +} impl FromStr for NumberOrString { - type Err = &'static str; // The actual type doesn't matter since we never error, but it must implement `Display` - fn from_str(s: &str) -> Result - { - Ok (s.parse::() + type Err = Error; + fn from_str(s: &str) -> Result { + Ok (s.parse::() .map(NumberOrString::Number) .unwrap_or_else(|_| NumberOrString::Str (s.to_string()))) } } +fn main() -> Result<(), Error> { + let proj_dirs = ProjectDirs::from("gt0.ca", "Andrew Phillips", "Keep"); -fn main() { let mut cmd = Args::command(); - let args = Args::parse(); + let mut args = Args::parse(); stderrlog::new() .module(module_path!()) .quiet(args.options.quiet) .verbosity(usize::from(args.options.verbose + 2)) - .timestamp(stderrlog::Timestamp::Second) + //.timestamp(stderrlog::Timestamp::Second) .init() .unwrap(); - debug!("Start"); - + debug!("MAIN: Start"); + let ids = &mut Vec::new(); let tags = &mut Vec::new(); - + for v in args.ids_or_tags.iter() { match v.clone() { NumberOrString::Number(num) => ids.push(num), @@ -119,7 +207,11 @@ fn main() { } } + tags.sort(); + tags.dedup(); + let mut mode: KeepModes = KeepModes::Unknown; + if args.mode.save { mode = KeepModes::Save; } else if args.mode.get { @@ -133,7 +225,7 @@ fn main() { } else if args.mode.status { mode = KeepModes::Status; } - + if mode == KeepModes::Unknown { if ! ids.is_empty() { mode = KeepModes::Get; @@ -142,24 +234,53 @@ fn main() { } } - debug!("args: {:?}", args); - debug!("ids: {:?}", ids); - debug!("tags: {:?}", tags); - debug!("mode: {:?}", mode); - + debug!("MAIN: args: {:?}", args); + debug!("MAIN: ids: {:?}", ids); + debug!("MAIN: tags: {:?}", tags); + debug!("MAIN: mode: {:?}", mode); + + if args.options.dir.is_none() { + match proj_dirs { + Some(proj_dirs) => args.options.dir = Some(proj_dirs.data_dir().to_path_buf()), + None => return Err(anyhow!("Unable to determine data directory")) + } + } + + unsafe { + libc::umask(0o077); + } + + let data_path = args.options.dir.clone().unwrap(); + let mut db_path = data_path.clone(); + db_path.push("keep-1.db"); + + debug!("MAIN: Data directory: {:?}", data_path); + debug!("MAIN: DB file: {:?}", db_path); + + fs::create_dir_all(data_path.clone()) + .context("Problem creating data directory")?; + debug!("MAIN: Data directory created or already exists"); + + let mut conn = db::open(db_path.clone()) + .context("Problem opening database")?; + debug!("MAIN: DB opened successfully"); + match mode { - KeepModes::Save => mode_save(&mut cmd, args, ids, tags), - KeepModes::Get => mode_get(&mut cmd, args, ids, tags), - KeepModes::List => mode_list(&mut cmd, args, ids, tags), - KeepModes::Update => mode_update(&mut cmd, args, ids, tags), - KeepModes::Delete => mode_delete(&mut cmd, args, ids, tags), - KeepModes::Status => mode_status(&mut cmd, args), + KeepModes::Save => mode_save(&mut cmd, args, ids, tags, conn, data_path)?, + KeepModes::Get => mode_get(&mut cmd, args, ids, tags, &mut conn, data_path)?, + KeepModes::List => mode_list(&mut cmd, args, ids, tags, &mut conn, data_path)?, + KeepModes::Update => mode_update(&mut cmd, args, ids, tags, &mut conn)?, + KeepModes::Delete => mode_delete(&mut cmd, args, ids, tags, &mut conn, data_path)?, + KeepModes::Status => mode_status(&mut cmd, args, data_path, db_path)?, _ => todo!() } + + Ok(()) } -fn mode_save(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec) { + +fn mode_save(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec, conn: Connection, data_path: PathBuf) -> Result<()> { if ! ids.is_empty() { cmd.error(ErrorKind::InvalidValue, "ID given, you cannot supply IDs when using --save").exit(); } @@ -168,37 +289,294 @@ fn mode_save(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec = HashMap::new(); + + if let Ok(hostname) = gethostname().into_string() { + item_meta.insert("hostname".to_string(), hostname); + } + + if args.item.meta.is_some() { + for item in args.item.meta.unwrap().iter() { + let item = item.clone(); + item_meta.insert(item.key, item.value); + } + } + for kv in item_meta.iter() { + let meta = db::Meta { + id: item.id.unwrap(), + name: kv.0.to_string(), + value: kv.1.to_string() + }; + db::store_meta(&conn, meta)?; + } + + let mut item_path = data_path.clone(); + item_path.push(id.to_string()); + + let mut stdin = io::stdin().lock(); + let mut stdout = io::stdout().lock(); + let mut buffer = [0; BUFSIZ]; + + let compression_engine = compression::get_engine(compression_type.clone()).expect("Unable to get compression engine"); + let mut item_out: Box = compression_engine.create(item_path.clone()) + .context(anyhow!("Unable to write file {:?} using compression {:?}", item_path, compression_type))?; + + debug!("MAIN: Starting IO loop"); + loop { + let n = stdin.read(&mut buffer[..BUFSIZ])?; + + if n == 0 { + debug!("MAIN: EOF on STDIN"); + break; + } + + stdout.write_all(&buffer[..n])?; + item_out.write_all(&buffer[..n])?; + item.size = match item.size { + None => Some(n as i64), + Some(prev_n) => Some(prev_n + n as i64) + }; + } + debug!("MAIN: Ending IO loop"); + + stdout.flush()?; + item_out.flush()?; + + db::update_item(&conn, item.clone())?; + + Ok(()) } -fn mode_get(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec) { +fn mode_get(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec, conn: &mut Connection, data_path: PathBuf) -> Result<()> { if ids.is_empty() && tags.is_empty() { - cmd.error(ErrorKind::InvalidValue, "No ID or tags given, ou must supply one ID or atleast one tag when using --get").exit(); + cmd.error(ErrorKind::InvalidValue, "No ID or tags given, you must supply one ID or atleast one tag when using --get").exit(); } else if ! ids.is_empty() && ! tags.is_empty() { cmd.error(ErrorKind::InvalidValue, "Both ID and tags given, you must supply one ID or atleast one tag when using --get").exit(); } else if ids.len() > 1 { cmd.error(ErrorKind::InvalidValue, "More than one ID given, you must supply one ID or atleast one tag when using --get").exit(); } + + let mut meta: HashMap = HashMap::new(); + if args.item.meta.is_some() { + for item in args.item.meta.unwrap().iter() { + let item = item.clone(); + meta.insert(item.key, item.value); + } + } + + let item_maybe = match tags.is_empty() && meta.is_empty() { + true => match ids.iter().next() { + Some(item_id) => db::get_item(conn, *item_id)?, + None => None + }, + false => db::get_item_matching(conn, tags, &meta)? + }; + + if let Some(item) = item_maybe { + debug!("MAIN: Found item {:?}", item); + + let mut item_path = data_path.clone(); + item_path.push(item.id.unwrap().to_string()); + + let compression_type = CompressionType::from_str(&item.compression)?; + debug!("MAIN: Item has compression type {:?}", compression_type.clone()); + + let compression_engine = compression::get_engine(compression_type).expect("Unable to get compression engine"); + compression_engine.cat(item_path.clone()) + } else { + Err(anyhow!("Unable to find matching item in database")) + } } -fn mode_list(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec) { +fn mode_list(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &Vec, conn: &mut Connection, data_path: PathBuf) -> Result<()> { if ! ids.is_empty() { cmd.error(ErrorKind::InvalidValue, "ID given, you can only supply tags when using --list").exit(); } + + let mut meta: HashMap = HashMap::new(); + if args.item.meta.is_some() { + for item in args.item.meta.unwrap().iter() { + let item = item.clone(); + meta.insert(item.key, item.value); + } + } + + + let items = match tags.is_empty() && meta.is_empty() { + true => db::get_items(conn)?, + false => db::get_items_matching(conn, tags, &meta)? + }; + + debug!("MAIN: Items: {:?}", items); + + let mut tags_by_item: HashMap> = HashMap::new(); + let mut meta_by_item: HashMap> = HashMap::new(); + let mut meta_columns = HashSet::new(); + + for item in items.iter() { + let item_id = item.id.unwrap(); + + let item_tags: Vec = db::get_item_tags(conn, item)? + .into_iter() + .map(|x| {x.name}) + .collect(); + tags_by_item.insert(item_id, item_tags); + + let mut item_meta: HashMap = HashMap::new(); + + for meta in db::get_item_meta(conn, item)? { + meta_columns.insert(meta.name.clone()); + item_meta.insert(meta.name.clone(), meta.value); + } + meta_by_item.insert(item_id, item_meta); + }; + + let mut meta_columns_sorted = Vec::from_iter(meta_columns); + meta_columns_sorted.sort(); + + let mut table = Table::new(); + if std::io::stdout().is_terminal() { + table.set_format(*FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR); + } else { + table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR); + } + + let mut title_row = row!( + b->"ID", + b->"Time", + b->"Stream Size", + b->"Comp", + b->"File Size", + b->"Tags", + ); + + for name in &meta_columns_sorted { + title_row.add_cell(Cell::new(name).with_style(Attr::Bold)); + } + + table.set_titles(title_row); + + for item in items { + let item_id = item.id.unwrap(); + let mut item_path = data_path.clone(); + item_path.push(item.id.unwrap().to_string()); + + let id_cell = Cell::new_align(&item.id.unwrap_or(0).to_string(), Alignment::RIGHT); + let ts_cell = Cell::new(&item.ts.with_timezone(&Local).format("%F %T").to_string()); + let size_cell = match item.size { + Some(size) => Cell::new_align(format_size(size as u64, BINARY).as_str(), Alignment::RIGHT), + None => Cell::new_align("Missing", Alignment::RIGHT).with_style(Attr::ForegroundColor(color::RED)).with_style(Attr::Bold) + }; + let compression_cell = Cell::new(&item.compression); + let file_size_cell = match item_path.metadata() { + Ok(metadata) => Cell::new_align(format_size(metadata.len(), BINARY).as_str(), Alignment::RIGHT), + Err(_) => Cell::new_align("Missing", Alignment::RIGHT).with_style(Attr::ForegroundColor(color::RED)).with_style(Attr::Bold) + }; + + let item_tags = tags_by_item.get(&item_id).unwrap(); + let tags_cell = Cell::new(&item_tags.join(" ")); + + let mut table_row = Row::new(vec![id_cell,ts_cell,size_cell, compression_cell, file_size_cell, tags_cell]); + + let item_meta = meta_by_item.get(&item_id).unwrap(); + for name in &meta_columns_sorted { + match item_meta.get(name) { + Some(value) => table_row.add_cell(Cell::new(value)), + None => table_row.add_cell(Cell::new("")) + }; + } + + table.add_row(table_row); + } + + table.printstd(); + + Ok(()) } -fn mode_update(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec) { +fn mode_update(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec, conn: &mut Connection) -> Result<()> { if ids.is_empty() { cmd.error(ErrorKind::InvalidValue, "No ID given, you must supply one ID when using --update").exit(); } + + let item_id = ids.iter().next().expect("Unable to determine item id"); + let item_maybe = db::get_item(conn, *item_id)?; + + let item = item_maybe.expect("Unable to find item in database"); + debug!("MAIN: Found item {:?}", item); + + if ! tags.is_empty() { + debug!("MAIN: Updating item tags"); + db::set_item_tags(conn, item.clone(), tags)?; + } + + if args.item.meta.is_some() { + debug!("MAIN: Updating item meta"); + for kv in args.item.meta.unwrap().iter() { + let meta = db::Meta { + id: item.id.unwrap(), + name: kv.key.to_string(), + value: kv.value.to_string() + }; + db::store_meta(conn, meta)?; + } + } + + Ok(()) } -fn mode_delete(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec) { +fn mode_delete(cmd: &mut Command, _args: Args, ids: &mut Vec, tags: &mut Vec, conn: &mut Connection, data_path: PathBuf) -> Result<()> { if ids.is_empty() { cmd.error(ErrorKind::InvalidValue, "No ID given, you must supply one ID when using --delete").exit(); } else if ! tags.is_empty() { @@ -206,19 +584,105 @@ fn mode_delete(cmd: &mut Command, args: Args, ids: &mut Vec, tags: &mut Vec } else if ids.len() > 1 { cmd.error(ErrorKind::InvalidValue, "More than one ID given, you must supply one ID when using --delete").exit(); } + + let item_id = ids.iter().next().expect("Unable to determine item id"); + let item_maybe = db::get_item(conn, *item_id)?; + + let item = item_maybe.expect("Unable to find item in database"); + debug!("MAIN: Found item {:?}", item); + + db::delete_item(conn, item)?; + + let mut item_path = data_path.clone(); + item_path.push(item_id.to_string()); + + fs::remove_file(&item_path).context(anyhow!("Unable to remove item file {:?}", item_path))?; + + Ok(()) } -fn mode_status_show_compression() { - let compression_types = compression::supported_compression_types(); - - println!("compression_types:"); - for compression_type in compression_types.into_iter() { - println!(" {}", compression_type); +fn mode_status(_cmd: &mut Command, args: Args, data_path: PathBuf, db_path: PathBuf) -> Result<()> { + let mut path_table = Table::new(); + + if std::io::stdout().is_terminal() { + path_table.set_format(*FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR); + } else { + path_table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR); } -} + + path_table.set_titles(Row::new(vec![ + Cell::new("Type").with_style(Attr::Bold), + Cell::new("Path").with_style(Attr::Bold), + ])); + + path_table.add_row(Row::new(vec![ + Cell::new("Data"), + Cell::new(&data_path.into_os_string().into_string().expect("Unable to convert data path to string")) + ])); + + path_table.add_row(Row::new(vec![ + Cell::new("Database"), + Cell::new(&db_path.into_os_string().into_string().expect("Unable to convert DB path to string")) + ])); -fn mode_status(cmd: &mut Command, args: Args) { - mode_status_show_compression(); + let mut compression_table = Table::new(); + if std::io::stdout().is_terminal() { + compression_table.set_format(*FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR); + } else { + compression_table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR); + } + + compression_table.set_titles(row!( + b->"Type", + b->"Found", + b->"Default", + b->"Binary", + b->"Compress", + b->"Decompress")); + + + let default_type = match args.item.compression { + Some(compression_name) => CompressionType::from_str(&compression_name) + .context(anyhow!("Invalid compression type {}", compression_name))?, + None => compression::default_type() + }; + + for compression_type in CompressionType::iter() { + let compression_program = match compression_type { + CompressionType::None => compression::CompressionProgram { + program: "".to_string(), + compress: Vec::new(), + decompress: Vec::new(), + supported: true + }, + _ => compression::get_program(compression_type.clone())? + }; + + let is_default = compression_type == default_type; + + compression_table.add_row(Row::new(vec![ + Cell::new(&compression_type.to_string()), + match compression_program.supported { + true => Cell::new("Yes").with_style(Attr::ForegroundColor(color::GREEN)), + false => Cell::new("No").with_style(Attr::ForegroundColor(color::RED)) + }, + match is_default { + true => Cell::new("Yes").with_style(Attr::ForegroundColor(color::GREEN)), + false => Cell::new("No") + }, + Cell::new(&compression_program.program), + Cell::new(&compression_program.compress.join(" ")), + Cell::new(&compression_program.decompress.join(" ")), + ])); + } + + println!("PATHS:"); + path_table.printstd(); + println!(); + println!("COMPRESSION:"); + compression_table.printstd(); + + Ok(()) }