Most of basic functionality implemented

This commit is contained in:
Andrew Phillips
2023-08-31 19:38:59 +00:00
parent b9bf5a831e
commit 49a77f9090
5 changed files with 1382 additions and 142 deletions

325
Cargo.lock generated
View File

@@ -92,6 +92,12 @@ dependencies = [
"windows-sys", "windows-sys",
] ]
[[package]]
name = "anyhow"
version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
[[package]] [[package]]
name = "atty" name = "atty"
version = "0.2.14" version = "0.2.14"
@@ -109,6 +115,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "2.3.3" version = "2.3.3"
@@ -143,6 +155,7 @@ dependencies = [
"iana-time-zone", "iana-time-zone",
"js-sys", "js-sys",
"num-traits", "num-traits",
"time",
"wasm-bindgen", "wasm-bindgen",
"winapi", "winapi",
] ]
@@ -200,6 +213,95 @@ version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]]
name = "csv"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086"
dependencies = [
"csv-core",
"itoa",
"ryu",
"serde",
]
[[package]]
name = "csv-core"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
dependencies = [
"memchr",
]
[[package]]
name = "directories"
version = "5.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-next"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
dependencies = [
"cfg-if",
"dirs-sys-next",
]
[[package]]
name = "dirs-sys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys",
]
[[package]]
name = "dirs-sys-next"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "encode_unicode"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "enum-map"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9705d8de4776df900a4a0b2384f8b0ab42f775e93b083b42f8ce71bdc32a47e3"
dependencies = [
"enum-map-derive",
]
[[package]]
name = "enum-map-derive"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccb14d927583dd5c2eac0f2cf264fc4762aefe1ae14c47a8a20fc1939d3a5fc0"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "errno" name = "errno"
version = "0.3.1" version = "0.3.1"
@@ -233,6 +335,27 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "gethostname"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818"
dependencies = [
"libc",
"windows-targets",
]
[[package]]
name = "getrandom"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
dependencies = [
"cfg-if",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.0" version = "0.14.0"
@@ -273,6 +396,15 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
[[package]]
name = "humansize"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
dependencies = [
"libm",
]
[[package]] [[package]]
name = "iana-time-zone" name = "iana-time-zone"
version = "0.1.57" version = "0.1.57"
@@ -298,15 +430,21 @@ dependencies = [
[[package]] [[package]]
name = "is-terminal" name = "is-terminal"
version = "0.4.8" version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
dependencies = [ dependencies = [
"hermit-abi 0.3.2", "hermit-abi 0.3.2",
"rustix", "rustix",
"windows-sys", "windows-sys",
] ]
[[package]]
name = "itoa"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.64" version = "0.3.64"
@@ -320,13 +458,26 @@ dependencies = [
name = "keep-rust" name = "keep-rust"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow",
"chrono",
"clap", "clap",
"directories",
"enum-map",
"gethostname",
"humansize",
"is-terminal",
"lazy_static", "lazy_static",
"libc",
"log", "log",
"prettytable-rs",
"regex", "regex",
"rusqlite", "rusqlite",
"rusqlite_migration", "rusqlite_migration",
"signal-hook",
"stderrlog", "stderrlog",
"strum",
"strum_macros",
"term",
] ]
[[package]] [[package]]
@@ -341,6 +492,12 @@ version = "0.2.147"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
[[package]]
name = "libm"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
[[package]] [[package]]
name = "libsqlite3-sys" name = "libsqlite3-sys"
version = "0.26.0" version = "0.26.0"
@@ -385,12 +542,32 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "option-ext"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]] [[package]]
name = "pkg-config" name = "pkg-config"
version = "0.3.27" version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "prettytable-rs"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
dependencies = [
"csv",
"encode_unicode",
"is-terminal",
"lazy_static",
"term",
"unicode-width",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.63" version = "1.0.63"
@@ -409,6 +586,26 @@ dependencies = [
"proc-macro2", "proc-macro2",
] ]
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_users"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
"getrandom",
"redox_syscall",
"thiserror",
]
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.9.1" version = "1.9.1"
@@ -444,7 +641,8 @@ version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.3.3",
"chrono",
"fallible-iterator", "fallible-iterator",
"fallible-streaming-iterator", "fallible-streaming-iterator",
"hashlink", "hashlink",
@@ -468,13 +666,50 @@ version = "0.38.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4" checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4"
dependencies = [ dependencies = [
"bitflags", "bitflags 2.3.3",
"errno", "errno",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
"windows-sys", "windows-sys",
] ]
[[package]]
name = "rustversion"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
[[package]]
name = "ryu"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
[[package]]
name = "serde"
version = "1.0.185"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31"
[[package]]
name = "signal-hook"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
dependencies = [
"libc",
"signal-hook-registry",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.11.0" version = "1.11.0"
@@ -500,6 +735,28 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "strum"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
dependencies = [
"strum_macros",
]
[[package]]
name = "strum_macros"
version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059"
dependencies = [
"heck",
"proc-macro2",
"quote",
"rustversion",
"syn",
]
[[package]] [[package]]
name = "syn" name = "syn"
version = "2.0.23" version = "2.0.23"
@@ -511,6 +768,17 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "term"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
dependencies = [
"dirs-next",
"rustversion",
"winapi",
]
[[package]] [[package]]
name = "termcolor" name = "termcolor"
version = "1.1.3" version = "1.1.3"
@@ -520,6 +788,26 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "thiserror"
version = "1.0.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "thread_local" name = "thread_local"
version = "1.1.7" version = "1.1.7"
@@ -530,12 +818,29 @@ dependencies = [
"once_cell", "once_cell",
] ]
[[package]]
name = "time"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
dependencies = [
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
"winapi",
]
[[package]] [[package]]
name = "unicode-ident" name = "unicode-ident"
version = "1.0.10" version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73"
[[package]]
name = "unicode-width"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
[[package]] [[package]]
name = "utf8parse" name = "utf8parse"
version = "0.2.1" version = "0.2.1"
@@ -554,6 +859,18 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasi"
version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]] [[package]]
name = "wasm-bindgen" name = "wasm-bindgen"
version = "0.2.87" version = "0.2.87"

View File

@@ -6,10 +6,24 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
anyhow = "1.0.72"
clap = { version = "4.3.10", features = ["derive", "env"] } clap = { version = "4.3.10", features = ["derive", "env"] }
directories = "5.0.1"
lazy_static = "1.4.0" lazy_static = "1.4.0"
libc = "0.2.147"
log = "0.4.19" log = "0.4.19"
regex = "1.9.1" regex = "1.9.1"
rusqlite = { version = "0.29.0", features = ["bundled"] } rusqlite = { version = "0.29.0", features = ["bundled", "array", "chrono"] }
rusqlite_migration = "1.0.2" rusqlite_migration = "1.0.2"
stderrlog = "0.5.4" stderrlog = "0.5.4"
strum_macros = "0.25"
strum = { version = "0.25", features = ["derive"] }
signal-hook = "0.3.17"
prettytable-rs = "0.10.0"
chrono = "0.4.26"
gethostname = "0.4.3"
humansize = "2.1.3"
enum-map = "2.6.1"
is-terminal = "0.4.9"
term = "0.7.0"

229
src/compression.rs Normal file → Executable file
View File

@@ -1,86 +1,166 @@
use std::fmt; use anyhow::{Context, Result, anyhow};
use std::os::unix::fs::PermissionsExt; use strum::IntoEnumIterator;
use std::fs::File;
use std::io;
use std::io::Write;
use std::process::{Command,Stdio};
use std::path::PathBuf;
use std::env; use std::env;
use std::fs; use std::fs;
use std::os::unix::fs::PermissionsExt;
use log::*; use log::*;
use lazy_static::lazy_static;
extern crate enum_map;
use enum_map::enum_map;
use enum_map::{EnumMap,Enum};
#[derive(Debug, Clone)]
pub struct CompressionType { #[derive(Debug, Eq, PartialEq, Clone, strum::EnumIter, strum::Display, strum::EnumString, Enum)]
name: String, #[strum(ascii_case_insensitive)]
binary: String, pub enum CompressionType {
compress: String, LZ4,
decompress: String, GZip,
BZip2,
XZ,
ZStd,
None
} }
impl fmt::Display for CompressionType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { #[derive(Debug, Eq, PartialEq, Clone)]
write!(f, "[name='{}', binary='{}', compress='{}', decompress='{}']", self.name, self.binary, self.compress, self.decompress) pub struct CompressionProgram {
pub program: String,
pub compress: Vec<String>,
pub decompress: Vec<String>,
pub supported: bool
}
lazy_static! {
static ref COMPRESSION_PROGRAMS: EnumMap<CompressionType, Option<CompressionProgram>> = enum_map! {
CompressionType::LZ4 => Some(CompressionProgram::new("lz4", vec!["-qcf"], vec!["-dcf"])),
CompressionType::GZip => Some(CompressionProgram::new("gzip", vec!["-qcf"], vec!["-dcf"])),
CompressionType::BZip2 => Some(CompressionProgram::new("bzip2", vec!["-qcf"], vec!["-dcf"])),
CompressionType::XZ => Some(CompressionProgram::new("xz", vec!["-qcf"], vec!["-dcf"])),
CompressionType::ZStd => Some(CompressionProgram::new("zstd", vec!["-qcf"], vec!["-dcf"])),
CompressionType::None => None
};
}
impl CompressionProgram {
pub fn new(program: &str, compress: Vec<&str>, decompress: Vec<&str>) -> CompressionProgram {
let program_path = get_program_path(program);
let supported = program_path.is_ok();
CompressionProgram {
program: program_path.unwrap_or(program.to_string()),
compress: compress.iter().map(|s| {s.to_string()}).collect(),
decompress: decompress.iter().map(|s| {s.to_string()}).collect(),
supported
}
} }
} }
pub fn add_compression_type(compression_types: &mut Vec<CompressionType>, name: String, binary: String, compress: String, decompress: String) { pub trait CompressionEngine {
let path = is_program_in_path(binary); fn is_supported(&self) -> bool;
fn cat(&self, file_path: PathBuf) -> Result<()>;
if let Ok(path) = path { fn create(&self, file_path: PathBuf) -> Result<Box<dyn Write>>;
compression_types.push(
CompressionType {
name,
binary: path,
compress,
decompress
});
}
}
pub fn supported_compression_types() -> Vec<CompressionType> {
let mut compression_types = Vec::new();
add_compression_type(&mut compression_types, "lz4".to_string(), "lz4".to_string(), "-qc".to_string() ,"-dc".to_string());
add_compression_type(&mut compression_types, "gzip".to_string(), "gzip".to_string(), "-qc".to_string() ,"-dc".to_string());
add_compression_type(&mut compression_types, "bzip2".to_string(), "bzip2".to_string(), "-qc".to_string() ,"-dc".to_string());
add_compression_type(&mut compression_types, "xz".to_string(), "xz".to_string(), "-qc".to_string() ,"-dc".to_string());
compression_types.push(
CompressionType {
name: "none".to_string(),
binary: "".to_string(),
compress: "".to_string(),
decompress: "".to_string(),
});
return compression_types;
} }
pub fn get_compression_default(compression_types: Vec<CompressionType>) -> Result<CompressionType, String> { impl CompressionEngine for CompressionProgram {
debug!("Compression type: default"); fn is_supported(&self) -> bool {
self.supported
}
match compression_types.first() { fn cat(&self, file_path: PathBuf) -> Result<()> {
None => Err(String::from("Unable to find default compression type")), debug!("COMPRESSION: Outputting {:?} to STDOUT using {:?}", file_path, *self);
Some(compression_type) => Ok(compression_type.clone()) let program = self.program.clone();
let args = self.decompress.clone();
debug!("COMPRESSION: Executing command: {:?} {:?} writing to {:?}", program, args, file_path);
let file = File::open(file_path).context("Unable to open file for reading")?;
let mut process = Command::new(program.clone())
.args(args.clone())
.stdin(file)
.spawn()
.context(anyhow!("Unable to spawn child process: {:?} {:?}", program, args))?;
let result = process.wait()
.context(anyhow!("Unable to wait for child process: {:?} {:?}", program, args))?;
if result.success() {
Ok(())
} else {
Err(anyhow!("Decompression program returned {}", result))
} }
} }
pub fn get_compression_named(compression_types: Vec<CompressionType>, compression_name: String) -> Result<CompressionType, String> { fn create(&self, file_path: PathBuf) -> Result<Box<dyn Write>> {
debug!("Compression type: {}", compression_name); debug!("COMPRESSION: Writting to {:?} using {:?}", file_path, *self);
match compression_types.iter().find(|&c| c.name == *compression_name) {
None => Err(format!("Unable to find compression type: {}", compression_name)), let program = self.program.clone();
Some(compression_type) => Ok(compression_type.clone()) let args = self.compress.clone();
debug!("COMPRESSION: Executing command: {:?} {:?} writing to {:?}", program, args, file_path);
let file = File::create(file_path).context("Unable to open file for writing")?;
let process = Command::new(program.clone())
.args(args.clone())
.stdin(Stdio::piped())
.stdout(file)
.spawn()
.context(anyhow!("Problem spawning child process: {:?} {:?}", program, args))?;
Ok(Box::new(process.stdin.unwrap()))
} }
} }
pub fn get_compression(compression_name: Option<String>) -> Result<CompressionType, String> { #[derive(Debug, Eq, PartialEq, Clone)]
let compression_types = supported_compression_types(); pub struct CompressionEngineNone {
}
match compression_name { impl CompressionEngineNone {
None => get_compression_default(compression_types), pub fn new() -> CompressionEngineNone {
Some(compression_name) => get_compression_named(compression_types, compression_name), CompressionEngineNone {}
} }
} }
fn is_program_in_path(program: String) -> Result<String, ()> { impl Default for CompressionEngineNone {
debug!("Looking for executable: {}", program); fn default() -> Self {
Self::new()
}
}
impl CompressionEngine for CompressionEngineNone {
fn is_supported(&self) -> bool {
true
}
fn cat(&self, file_path: PathBuf) -> Result<()> {
debug!("COMPRESSION: Outputting {:?} to STDOUT using {:?}", file_path, *self);
let mut stdout = io::stdout().lock();
let mut file = File::open(file_path)?;
io::copy(&mut file, &mut stdout)?;
stdout.flush()?;
Ok(())
}
fn create(&self, file_path: PathBuf) -> Result<Box<dyn Write>> {
debug!("COMPRESSION: Writting to {:?} using {:?}", file_path, *self);
Ok(Box::new(File::create(file_path)?))
}
}
fn get_program_path(program: &str) -> Result<String> {
debug!("COMPRESSION: Looking for executable: {}", program);
if let Ok(path) = env::var("PATH") { if let Ok(path) = env::var("PATH") {
for p in path.split(':') { for p in path.split(':') {
let p_str = format!("{}/{}", p, program); let p_str = format!("{}/{}", p, program);
@@ -94,5 +174,34 @@ fn is_program_in_path(program: String) -> Result<String, ()> {
} }
} }
} }
Err(()) Err(anyhow!("Unable to find binary {} in PATH", program))
}
pub fn get_program(compression_type: CompressionType) -> Result<CompressionProgram> {
match &COMPRESSION_PROGRAMS[compression_type.clone()] {
Some(compression_program) => Ok(compression_program.clone()),
None => Err(anyhow!("Compression type {} has no program", compression_type))
}
}
pub fn get_engine(compression_type: CompressionType) -> Result<Box<dyn CompressionEngine>> {
match compression_type {
CompressionType::None => Ok(Box::new(CompressionEngineNone::new())),
compression_type => Ok(Box::new(COMPRESSION_PROGRAMS[compression_type.clone()].clone().unwrap()))
}
}
pub fn default_type() -> CompressionType {
let mut default = CompressionType::None;
for compression_type in CompressionType::iter() {
let compression_engine = get_engine(compression_type.clone()).expect("Missing engine");
if compression_engine.is_supported() {
default = compression_type;
break;
}
}
default
} }

374
src/db.rs
View File

@@ -1,36 +1,372 @@
use rusqlite::{params, Connection, Error}; use std::path::PathBuf;
use std::collections::HashMap;
use std::rc::Rc;
use anyhow::{Context, Result, Error};
use rusqlite::{Connection, OpenFlags};
use rusqlite_migration::{Migrations, M}; use rusqlite_migration::{Migrations, M};
use log::*; use chrono::prelude::*;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use log::*;
lazy_static! { lazy_static! {
static ref MIGRATIONS: Migrations<'static> = Migrations::new(vec![ static ref MIGRATIONS: Migrations<'static> = Migrations::new(vec![
M::up("CREATE TABLE keep( M::up("CREATE TABLE items(
id INTEGER AUTOINCREMENT NOT NULL, id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
ts TEXT NOT NULL, ts TEXT NOT NULL,
compress TEXT NOT NULL, size INTEGER NULL,
hostname TEXT NOT NULL, compression TEXT NOT NULL)"),
comment TEXT NOT NULL)
PRIMARY KEY(id);"),
M::up("CREATE TABLE tags ( M::up("CREATE TABLE tags (
id INTEGER NOT NULL, id INTEGER NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
FOREIGN KEY(id) REFERENCES keep(id) ON DELETE CASCADE, FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY(id, name));"),
M::up("CREATE TABLE metas (
id INTEGER NOT NULL,
name TEXT NOT NULL,
value TEXT NOT NULL,
FOREIGN KEY(id) REFERENCES items(id) ON DELETE CASCADE,
PRIMARY KEY(id, name));") PRIMARY KEY(id, name));")
]); ]);
} }
fn open(path: String) -> Result<Connection, String> { #[derive(Debug, Clone)]
debug!("Opening DB {}", path); pub struct Item {
pub id: Option<i64>,
pub ts: DateTime<Utc>,
pub size: Option<i64>,
pub compression: String
}
match Connection::open(path) { #[derive(Debug, Clone)]
Ok(mut conn) => match conn.pragma_update(None, "foreign_keys", "ON") { pub struct Tag {
Ok(()) => match MIGRATIONS.to_latest(&mut conn) { pub id: i64,
Ok(()) => Ok(conn), pub name: String
Err(e) => Err(format!("Error migrating sqlite schema: {}", e))
},
Err(e) => Err(format!("Error setting sqlite pragma: {}", e))
} }
Err(e) => Err(format!("Error connecting to database: {}", e))
#[derive(Debug, Clone)]
pub struct Meta {
pub id: i64,
pub name: String,
pub value: String
}
pub fn open(path: PathBuf) -> Result<Connection, Error> {
debug!("DB: Opening file: {:?}", path);
let mut conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE)
.context("Problem opening file")?;
conn.pragma_update(None, "foreign_keys", "ON")
.context("Problem enabling SQLite foreign_keys pragma")?;
MIGRATIONS.to_latest(&mut conn)
.context("Problem performing database migrations")?;
rusqlite::vtab::array::load_module(&conn)
.context("Problem enabling array module")?;
Ok(conn)
}
pub fn insert_item(conn: &Connection, item: Item) -> Result<i64> {
debug!("DB: Inserting item: {:?}", item);
conn.execute(
"INSERT INTO items (ts, size, compression) VALUES (?1, ?2, ?3)",
(item.ts, item.size, item.compression))?;
Ok(conn.last_insert_rowid())
}
pub fn update_item(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Updating item: {:?}", item);
conn.execute(
"UPDATE items SET size=?2, compression=?3 WHERE id=?1",
(item.id, item.size, item.compression))?;
Ok(())
}
pub fn delete_item(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Deleting item: {:?}", item);
conn.execute("DELETE FROM items WHERE id=?1", [item.id])?;
Ok(())
}
pub fn query_delete_meta(conn: &Connection, meta: Meta) -> Result<()> {
debug!("DB: Deleting meta: {:?}", meta);
conn.execute(
"DELETE FROM metas WHERE id=?1 AND name=?2",
(meta.id, meta.name))?;
Ok(())
}
pub fn query_upsert_meta(conn: &Connection, meta: Meta) -> Result<()> {
debug!("DB: Inserting meta: {:?}", meta);
conn.execute(
"INSERT INTO metas (id, name, value) VALUES (?1, ?2, ?3)
ON CONFLICT(id, name) DO UPDATE SET value=?3",
(meta.id, meta.name, meta.value))?;
Ok(())
}
pub fn store_meta(conn: &Connection, meta: Meta) -> Result<()> {
debug!("DB: Storing meta: {:?}", meta);
if meta.value.eq("") {
query_delete_meta(conn, meta)?;
} else {
query_upsert_meta(conn, meta)?;
}
Ok(())
}
pub fn insert_tag(conn: &Connection, tag: Tag) -> Result<()> {
debug!("DB: Inserting tag: {:?}", tag);
conn.execute(
"INSERT INTO tags (id, name) VALUES (?1, ?2)",
(tag.id, tag.name))?;
Ok(())
}
pub fn delete_item_tags(conn: &Connection, item: Item) -> Result<()> {
debug!("DB: Deleting all item tags: {:?}", item);
conn.execute(
"DELETE FROM tags WHERE id=?1",
[item.id])?;
Ok(())
}
pub fn set_item_tags(conn: &Connection, item: Item, tags: &Vec<String>) -> Result<()> {
debug!("DB: Setting tags for item: {:?} ?{:?}", item, tags);
delete_item_tags(conn, item.clone())?;
let item_id = item.id.unwrap();
for tag_name in tags {
insert_tag(conn,
Tag {
id: item_id,
name: tag_name.to_string()
})?;
}
Ok(())
}
pub fn query_all_items(conn: &Connection) -> Result<Vec<Item>> {
debug!("DB: Querying all items");
let mut statement = conn
.prepare("SELECT id, ts, size, compression FROM items ORDER BY id ASC")
.context("Problem preparing SQL statement")?;
let mut rows = statement.query([])?;
let mut items = Vec::new();
while let Some(row) = rows.next()? {
let item = Item {
id: row.get(0)?,
ts: row.get(1)?,
size: row.get(2)?,
compression: row.get(3)?
};
items.push(item);
}
Ok(items)
}
pub fn query_tagged_items<'a>(conn: &'a Connection, tags: &'a Vec<String>) -> Result<Vec<Item>> {
debug!("DB: Querying tagged items: {:?}", tags);
let mut statement = conn
.prepare_cached("
SELECT items.id,
items.ts,
items.size,
items.compression,
count(tags_match.id) as tags_score
FROM items,
(SELECT tags.id FROM tags WHERE tags.name IN rarray(?1)) as tags_match
WHERE items.id = tags_match.id
GROUP BY items.id
HAVING tags_score = ?2
ORDER BY items.id ASC")
.context("Problem preparing SQL statement")?;
let tags_values: Vec<rusqlite::types::Value> = tags
.iter()
.map(|s| {rusqlite::types::Value::from(s.clone())})
.collect();
let tags_ptr = Rc::new(tags_values);
let mut rows = statement.query((&tags_ptr, &tags.len()))?;
let mut items = Vec::new();
while let Some(row) = rows.next()? {
let item = Item {
id: row.get(0)?,
ts: row.get(1)?,
size: row.get(2)?,
compression: row.get(3)?
};
items.push(item);
}
Ok(items)
}
pub fn get_items(conn: &Connection) -> Result<Vec<Item>> {
debug!("DB: Getting all items");
query_all_items(conn)
}
pub fn get_items_matching(conn: &Connection, tags: &Vec<String>, meta: &HashMap<String,String>) -> Result<Vec<Item>> {
debug!("DB: Getting items matching: tags={:?} meta={:?}", tags, meta);
let items = match tags.is_empty() {
true => query_all_items(conn)?,
false => query_tagged_items(conn, tags)?
};
if meta.is_empty() {
debug!("DB: Not filtering on meta");
Ok(items)
} else {
debug!("DB: Filtering on meta");
let mut filtered_items: Vec<Item> = Vec::new();
for item in items.iter() {
let mut item_ok = true;
let mut item_meta: HashMap<String, String> = HashMap::new();
for meta in get_item_meta(conn, item)? {
item_meta.insert(meta.name, meta.value);
}
debug!("DB: Matching: {:?}: {:?}", item, item_meta);
for (k, v) in meta.iter() {
match item_meta.get(k) {
Some(value) => item_ok = v.eq(value),
None => item_ok = false
}
if item_ok {
break;
} }
} }
if item_ok {
filtered_items.push(item.clone());
}
}
Ok(filtered_items)
}
}
pub fn get_item_matching(conn: &Connection, tags: &Vec<String>, _meta: &HashMap<String, String>) -> Result<Option<Item>> {
debug!("DB: Get item matching tags: {:?}", tags);
let mut statement = conn
.prepare_cached("
SELECT items.id,
items.ts,
items.size,
items.compression,
count(sel.id) as score
FROM items,
(SELECT tags.id FROM tags WHERE tags.name IN rarray(?1)) as sel
WHERE items.id = sel.id
GROUP BY items.id
HAVING score = ?2
ORDER BY items.id DESC
LIMIT 1")
.context("Problem preparing SQL statement")?;
let tags_values: Vec<rusqlite::types::Value> = tags
.iter()
.map(|s| {rusqlite::types::Value::from(s.clone()) })
.collect();
let tags_ptr = Rc::new(tags_values);
let mut rows = statement.query((&tags_ptr, &tags.len()))?;
match rows.next()? {
Some(row) => Ok(Some(Item {
id: row.get(0)?,
ts: row.get(1)?,
size: row.get(2)?,
compression: row.get(3)?
})),
None => Ok(None)
}
}
pub fn get_item(conn: &Connection, item_id: i64) -> Result<Option<Item>> {
debug!("DB: Getting item {:?}", item_id);
let mut statement = conn
.prepare_cached("
SELECT id, ts, size, compression
FROM items
WHERE items.id = ?1")
.context("Problem preparing SQL statement")?;
let mut rows = statement.query([item_id])?;
match rows.next()? {
Some(row) => Ok(Some(Item {
id: row.get(0)?,
ts: row.get(1)?,
size: row.get(2)?,
compression: row.get(3)?
})),
None => Ok(None)
}
}
pub fn get_item_tags(conn: &Connection, item: &Item) -> Result<Vec<Tag>> {
debug!("DB: Getting tags for item: {:?}", item);
let mut statement = conn
.prepare_cached("SELECT id, name FROM tags WHERE id=?1 ORDER BY name ASC")
.context("Problem preparing SQL statement")?;
let mut rows = statement.query([item.id])?;
let mut tags = Vec::new();
while let Some(row) = rows.next()? {
tags.push(Tag {
id: row.get(0)?,
name: row.get(1)?,
});
}
Ok(tags)
}
pub fn get_item_meta(conn: &Connection, item: &Item) -> Result<Vec<Meta>> {
debug!("DB: Getting item meta: {:?}", item);
let mut statement = conn
.prepare_cached("SELECT id, name, value FROM metas WHERE id=?1 ORDER BY name ASC")
.context("Problem preparing SQL statement")?;
let mut rows = statement.query([item.id])?;
let mut metas = Vec::new();
while let Some(row) = rows.next()? {
metas.push(Meta {
id: row.get(0)?,
name: row.get(1)?,
value: row.get(2)?
});
}
Ok(metas)
}

View File

@@ -1,12 +1,69 @@
use std::io;
use std::io::{Read, Write};
use std::fs;
use std::str::FromStr; use std::str::FromStr;
use std::path::PathBuf; use std::path::PathBuf;
use std::collections::{HashMap, HashSet};
use anyhow::{Context, Result, Error, anyhow};
use rusqlite::Connection;
use gethostname::gethostname;
use strum::IntoEnumIterator;
use clap::error::ErrorKind; use clap::error::ErrorKind;
use clap::*; use clap::*;
use log::*; use log::*;
extern crate directories;
use directories::ProjectDirs;
extern crate prettytable;
use prettytable::{Table, Row, Cell, Attr};
use prettytable::format;
use prettytable::format::{TableFormat, Alignment};
use prettytable::format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR;
use prettytable::row;
use prettytable::color;
use chrono::prelude::*;
#[macro_use]
extern crate lazy_static;
use crate::compression::CompressionType;
pub mod compression; pub mod compression;
pub mod db; pub mod db;
use humansize::{format_size, BINARY};
use is_terminal::IsTerminal;
extern crate term;
const BUFSIZ: usize = 8192;
lazy_static! {
static ref FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR: TableFormat = format::FormatBuilder::new()
.column_separator('│')
.borders('│')
.separators(&[format::LinePosition::Top],
format::LineSeparator::new('─',
'┬',
'┌',
'┐'))
.separators(&[format::LinePosition::Title],
format::LineSeparator::new('─',
'┼',
'├',
'┤'))
.separators(&[format::LinePosition::Bottom],
format::LineSeparator::new('─',
'┴',
'└',
'┘'))
.padding(1, 1)
.build();
}
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)] #[command(author, version, about, long_about = None)]
struct Args { struct Args {
@@ -17,60 +74,66 @@ struct Args {
#[command(flatten)] #[command(flatten)]
options: OptionsArgs, options: OptionsArgs,
#[arg()] #[arg(help("A list of either item IDs or tags"))]
ids_or_tags: Vec<NumberOrString> ids_or_tags: Vec<NumberOrString>
} }
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
struct ModeArgs { struct ModeArgs {
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["get", "list", "update", "delete", "status"]))] #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["get", "list", "update", "delete", "status"]))]
#[arg(help("Save an item using any tags or metadata provided"))]
save: bool, save: bool,
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "list", "update", "delete", "status"]))] #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "list", "update", "delete", "status"]))]
#[arg(help("Get an item either by it's ID or by a combination of matching tags and metatdata"))]
get: bool, get: bool,
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "get", "update", "delete", "status"]))] #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "update", "delete", "status"]))]
#[arg(help("List items, filtering on tags or metadata if given"))]
list: bool, list: bool,
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "get", "list", "delete", "status"]), requires("ids_or_tags"))] #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "list", "delete", "status"]), requires("ids_or_tags"))]
#[arg(help("Update a specified item ID's tags and/or metadata"))]
update: bool, update: bool,
#[arg(group("mode"), help_heading("Mode"), short, long, conflicts_with_all(["save", "get", "list", "update", "status"]), requires("ids_or_tags"))] #[arg(group("mode"), help_heading("Mode Options"), short, long, conflicts_with_all(["save", "get", "list", "update", "status"]), requires("ids_or_tags"))]
#[arg(help("Delete items either by ID or by matching tags"))]
delete: bool, delete: bool,
#[arg(group("mode"), help_heading("Mode"), short('S'), long, conflicts_with_all(["save", "get", "list", "update", "delete"]))] #[arg(group("mode"), help_heading("Mode Options"), short('S'), long, conflicts_with_all(["save", "get", "list", "update", "delete"]))]
#[arg(help("Show status of directories and supported compression algorithms"))]
status: bool status: bool
} }
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
struct ItemArgs { struct ItemArgs {
#[arg(help_heading("Item"), short, long, conflicts_with("get"), conflicts_with("list"))] #[arg(help_heading("Item Options"), short, long, conflicts_with_all(["get", "delete", "status"]))]
comment: Option<String>, #[arg(help("Set metadata for the item using the format KEY=[VALUE], the metadata will be removed if VALUE is not provided"))]
meta: Option<Vec<KeyValue>>,
#[arg(help_heading("Item"), short('C'), long, conflicts_with("get"), conflicts_with("list"), env("KEEP_COMPRESS"))] #[arg(help_heading("Item Options"), short('C'), long, conflicts_with("get"), conflicts_with("list"), env("KEEP_COMPRESSION"), )]
compress: Option<String>, #[arg(help("Compression algorithm to use when saving items"))]
compression: Option<String>,
} }
#[derive(Parser, Debug)] #[derive(Parser, Debug)]
struct OptionsArgs { struct OptionsArgs {
#[arg(help_heading("Options"), long, env("KEEP_DIR"))] #[arg(long, env("KEEP_DIR"))]
#[arg(help("Specify the directory to use for storage"))]
dir: Option<PathBuf>, dir: Option<PathBuf>,
#[arg(help_heading("Options"), short, long)] #[arg(short, long, action = clap::ArgAction::Count, conflicts_with("quiet"))]
force: bool, #[arg(help("Increase message verbosity, can be given more than once"))]
#[arg(help_heading("Options"), short, long, action = clap::ArgAction::Count, conflicts_with("quiet"))]
verbose: u8, verbose: u8,
#[arg(help_heading("Options"), short, long)] #[arg(short, long)]
#[arg(help("Do show any messages"))]
quiet: bool, quiet: bool,
} }
#[derive(Debug,Clone)]
enum NumberOrString {
Number(u32),
Str(String),
}
#[derive(Debug,PartialEq)] #[derive(Debug,PartialEq)]
enum KeepModes { enum KeepModes {
@@ -83,31 +146,56 @@ enum KeepModes {
Status Status
} }
#[derive(Debug,Clone)]
struct KeyValue {
key: String,
value: String
}
impl FromStr for KeyValue {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Error> {
match s.split_once('=') {
Some(kv) => Ok(KeyValue {
key: kv.0.to_string(),
value: kv.1.to_string()
}),
None => Err(anyhow!("Unable to parse key=value pair"))
}
}
}
#[derive(Debug,Clone)]
enum NumberOrString {
Number(i64),
Str(String),
}
impl FromStr for NumberOrString { impl FromStr for NumberOrString {
type Err = &'static str; // The actual type doesn't matter since we never error, but it must implement `Display` type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> fn from_str(s: &str) -> Result<Self, Self::Err> {
{ Ok (s.parse::<i64>()
Ok (s.parse::<u32>()
.map(NumberOrString::Number) .map(NumberOrString::Number)
.unwrap_or_else(|_| NumberOrString::Str (s.to_string()))) .unwrap_or_else(|_| NumberOrString::Str (s.to_string())))
} }
} }
fn main() -> Result<(), Error> {
let proj_dirs = ProjectDirs::from("gt0.ca", "Andrew Phillips", "Keep");
fn main() {
let mut cmd = Args::command(); let mut cmd = Args::command();
let args = Args::parse(); let mut args = Args::parse();
stderrlog::new() stderrlog::new()
.module(module_path!()) .module(module_path!())
.quiet(args.options.quiet) .quiet(args.options.quiet)
.verbosity(usize::from(args.options.verbose + 2)) .verbosity(usize::from(args.options.verbose + 2))
.timestamp(stderrlog::Timestamp::Second) //.timestamp(stderrlog::Timestamp::Second)
.init() .init()
.unwrap(); .unwrap();
debug!("Start"); debug!("MAIN: Start");
let ids = &mut Vec::new(); let ids = &mut Vec::new();
let tags = &mut Vec::new(); let tags = &mut Vec::new();
@@ -119,7 +207,11 @@ fn main() {
} }
} }
tags.sort();
tags.dedup();
let mut mode: KeepModes = KeepModes::Unknown; let mut mode: KeepModes = KeepModes::Unknown;
if args.mode.save { if args.mode.save {
mode = KeepModes::Save; mode = KeepModes::Save;
} else if args.mode.get { } else if args.mode.get {
@@ -142,24 +234,53 @@ fn main() {
} }
} }
debug!("args: {:?}", args); debug!("MAIN: args: {:?}", args);
debug!("ids: {:?}", ids); debug!("MAIN: ids: {:?}", ids);
debug!("tags: {:?}", tags); debug!("MAIN: tags: {:?}", tags);
debug!("mode: {:?}", mode); debug!("MAIN: mode: {:?}", mode);
if args.options.dir.is_none() {
match proj_dirs {
Some(proj_dirs) => args.options.dir = Some(proj_dirs.data_dir().to_path_buf()),
None => return Err(anyhow!("Unable to determine data directory"))
}
}
unsafe {
libc::umask(0o077);
}
let data_path = args.options.dir.clone().unwrap();
let mut db_path = data_path.clone();
db_path.push("keep-1.db");
debug!("MAIN: Data directory: {:?}", data_path);
debug!("MAIN: DB file: {:?}", db_path);
fs::create_dir_all(data_path.clone())
.context("Problem creating data directory")?;
debug!("MAIN: Data directory created or already exists");
let mut conn = db::open(db_path.clone())
.context("Problem opening database")?;
debug!("MAIN: DB opened successfully");
match mode { match mode {
KeepModes::Save => mode_save(&mut cmd, args, ids, tags), KeepModes::Save => mode_save(&mut cmd, args, ids, tags, conn, data_path)?,
KeepModes::Get => mode_get(&mut cmd, args, ids, tags), KeepModes::Get => mode_get(&mut cmd, args, ids, tags, &mut conn, data_path)?,
KeepModes::List => mode_list(&mut cmd, args, ids, tags), KeepModes::List => mode_list(&mut cmd, args, ids, tags, &mut conn, data_path)?,
KeepModes::Update => mode_update(&mut cmd, args, ids, tags), KeepModes::Update => mode_update(&mut cmd, args, ids, tags, &mut conn)?,
KeepModes::Delete => mode_delete(&mut cmd, args, ids, tags), KeepModes::Delete => mode_delete(&mut cmd, args, ids, tags, &mut conn, data_path)?,
KeepModes::Status => mode_status(&mut cmd, args), KeepModes::Status => mode_status(&mut cmd, args, data_path, db_path)?,
_ => todo!() _ => todo!()
} }
Ok(())
} }
fn mode_save(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) {
fn mode_save(cmd: &mut Command, args: Args, ids: &mut Vec<i64>, tags: &mut Vec<String>, conn: Connection, data_path: PathBuf) -> Result<()> {
if ! ids.is_empty() { if ! ids.is_empty() {
cmd.error(ErrorKind::InvalidValue, "ID given, you cannot supply IDs when using --save").exit(); cmd.error(ErrorKind::InvalidValue, "ID given, you cannot supply IDs when using --save").exit();
} }
@@ -168,37 +289,294 @@ fn mode_save(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<S
tags.push("none".to_string()); tags.push("none".to_string());
} }
let compression_type = compression::get_compression(args.item.compress); let compression_name = args.item.compression.unwrap_or(compression::default_type().to_string());
let compression_type_opt = CompressionType::from_str(&compression_name);
if compression_type_opt.is_err() {
cmd.error(ErrorKind::InvalidValue, format!("Unknown compression type: {}", compression_name)).exit();
}
let compression_type = compression_type_opt.unwrap();
debug!("MAIN: Compression type: {}", compression_type);
let mut item = db::Item {
id: None,
ts: Utc::now(),
size: None,
compression: compression_type.to_string()
};
let id = db::insert_item(&conn, item.clone())?;
item.id = Some(id);
debug!("MAIN: Added item {:?}", item.clone());
if ! args.options.quiet {
if std::io::stderr().is_terminal() {
let mut t = term::stderr().unwrap();
t.reset().unwrap_or(());
t.attr(term::Attr::Bold).unwrap_or(());
write!(t, "KEEP:").unwrap_or(());
t.reset().unwrap_or(());
write!(t, " New item ").unwrap_or(());
t.attr(term::Attr::Bold).unwrap_or(());
write!(t, "{id}")?;
t.reset().unwrap_or(());
write!(t, " tags: ")?;
t.attr(term::Attr::Bold).unwrap_or(());
write!(t, "{}", tags.join(" "))?;
t.reset().unwrap_or(());
writeln!(t, "")?;
std::io::stderr().flush()?;
} else {
let mut t = std::io::stderr();
writeln!(t, "KEEP: New item: {} tags: {:?}", id, tags)?;
}
}
db::set_item_tags(&conn, item.clone(), tags)?;
let mut item_meta: HashMap<String, String> = HashMap::new();
if let Ok(hostname) = gethostname().into_string() {
item_meta.insert("hostname".to_string(), hostname);
}
if args.item.meta.is_some() {
for item in args.item.meta.unwrap().iter() {
let item = item.clone();
item_meta.insert(item.key, item.value);
}
}
for kv in item_meta.iter() {
let meta = db::Meta {
id: item.id.unwrap(),
name: kv.0.to_string(),
value: kv.1.to_string()
};
db::store_meta(&conn, meta)?;
}
let mut item_path = data_path.clone();
item_path.push(id.to_string());
let mut stdin = io::stdin().lock();
let mut stdout = io::stdout().lock();
let mut buffer = [0; BUFSIZ];
let compression_engine = compression::get_engine(compression_type.clone()).expect("Unable to get compression engine");
let mut item_out: Box<dyn Write> = compression_engine.create(item_path.clone())
.context(anyhow!("Unable to write file {:?} using compression {:?}", item_path, compression_type))?;
debug!("MAIN: Starting IO loop");
loop {
let n = stdin.read(&mut buffer[..BUFSIZ])?;
if n == 0 {
debug!("MAIN: EOF on STDIN");
break;
}
stdout.write_all(&buffer[..n])?;
item_out.write_all(&buffer[..n])?;
item.size = match item.size {
None => Some(n as i64),
Some(prev_n) => Some(prev_n + n as i64)
};
}
debug!("MAIN: Ending IO loop");
stdout.flush()?;
item_out.flush()?;
db::update_item(&conn, item.clone())?;
Ok(())
} }
fn mode_get(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) { fn mode_get(cmd: &mut Command, args: Args, ids: &mut Vec<i64>, tags: &mut Vec<String>, conn: &mut Connection, data_path: PathBuf) -> Result<()> {
if ids.is_empty() && tags.is_empty() { if ids.is_empty() && tags.is_empty() {
cmd.error(ErrorKind::InvalidValue, "No ID or tags given, ou must supply one ID or atleast one tag when using --get").exit(); cmd.error(ErrorKind::InvalidValue, "No ID or tags given, you must supply one ID or atleast one tag when using --get").exit();
} else if ! ids.is_empty() && ! tags.is_empty() { } else if ! ids.is_empty() && ! tags.is_empty() {
cmd.error(ErrorKind::InvalidValue, "Both ID and tags given, you must supply one ID or atleast one tag when using --get").exit(); cmd.error(ErrorKind::InvalidValue, "Both ID and tags given, you must supply one ID or atleast one tag when using --get").exit();
} else if ids.len() > 1 { } else if ids.len() > 1 {
cmd.error(ErrorKind::InvalidValue, "More than one ID given, you must supply one ID or atleast one tag when using --get").exit(); cmd.error(ErrorKind::InvalidValue, "More than one ID given, you must supply one ID or atleast one tag when using --get").exit();
} }
let mut meta: HashMap<String, String> = HashMap::new();
if args.item.meta.is_some() {
for item in args.item.meta.unwrap().iter() {
let item = item.clone();
meta.insert(item.key, item.value);
}
}
let item_maybe = match tags.is_empty() && meta.is_empty() {
true => match ids.iter().next() {
Some(item_id) => db::get_item(conn, *item_id)?,
None => None
},
false => db::get_item_matching(conn, tags, &meta)?
};
if let Some(item) = item_maybe {
debug!("MAIN: Found item {:?}", item);
let mut item_path = data_path.clone();
item_path.push(item.id.unwrap().to_string());
let compression_type = CompressionType::from_str(&item.compression)?;
debug!("MAIN: Item has compression type {:?}", compression_type.clone());
let compression_engine = compression::get_engine(compression_type).expect("Unable to get compression engine");
compression_engine.cat(item_path.clone())
} else {
Err(anyhow!("Unable to find matching item in database"))
}
} }
fn mode_list(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) { fn mode_list(cmd: &mut Command, args: Args, ids: &mut Vec<i64>, tags: &Vec<String>, conn: &mut Connection, data_path: PathBuf) -> Result<()> {
if ! ids.is_empty() { if ! ids.is_empty() {
cmd.error(ErrorKind::InvalidValue, "ID given, you can only supply tags when using --list").exit(); cmd.error(ErrorKind::InvalidValue, "ID given, you can only supply tags when using --list").exit();
} }
let mut meta: HashMap<String, String> = HashMap::new();
if args.item.meta.is_some() {
for item in args.item.meta.unwrap().iter() {
let item = item.clone();
meta.insert(item.key, item.value);
}
} }
fn mode_update(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) { let items = match tags.is_empty() && meta.is_empty() {
true => db::get_items(conn)?,
false => db::get_items_matching(conn, tags, &meta)?
};
debug!("MAIN: Items: {:?}", items);
let mut tags_by_item: HashMap<i64, Vec<String>> = HashMap::new();
let mut meta_by_item: HashMap<i64, HashMap<String, String>> = HashMap::new();
let mut meta_columns = HashSet::new();
for item in items.iter() {
let item_id = item.id.unwrap();
let item_tags: Vec<String> = db::get_item_tags(conn, item)?
.into_iter()
.map(|x| {x.name})
.collect();
tags_by_item.insert(item_id, item_tags);
let mut item_meta: HashMap<String, String> = HashMap::new();
for meta in db::get_item_meta(conn, item)? {
meta_columns.insert(meta.name.clone());
item_meta.insert(meta.name.clone(), meta.value);
}
meta_by_item.insert(item_id, item_meta);
};
let mut meta_columns_sorted = Vec::from_iter(meta_columns);
meta_columns_sorted.sort();
let mut table = Table::new();
if std::io::stdout().is_terminal() {
table.set_format(*FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR);
} else {
table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR);
}
let mut title_row = row!(
b->"ID",
b->"Time",
b->"Stream Size",
b->"Comp",
b->"File Size",
b->"Tags",
);
for name in &meta_columns_sorted {
title_row.add_cell(Cell::new(name).with_style(Attr::Bold));
}
table.set_titles(title_row);
for item in items {
let item_id = item.id.unwrap();
let mut item_path = data_path.clone();
item_path.push(item.id.unwrap().to_string());
let id_cell = Cell::new_align(&item.id.unwrap_or(0).to_string(), Alignment::RIGHT);
let ts_cell = Cell::new(&item.ts.with_timezone(&Local).format("%F %T").to_string());
let size_cell = match item.size {
Some(size) => Cell::new_align(format_size(size as u64, BINARY).as_str(), Alignment::RIGHT),
None => Cell::new_align("Missing", Alignment::RIGHT).with_style(Attr::ForegroundColor(color::RED)).with_style(Attr::Bold)
};
let compression_cell = Cell::new(&item.compression);
let file_size_cell = match item_path.metadata() {
Ok(metadata) => Cell::new_align(format_size(metadata.len(), BINARY).as_str(), Alignment::RIGHT),
Err(_) => Cell::new_align("Missing", Alignment::RIGHT).with_style(Attr::ForegroundColor(color::RED)).with_style(Attr::Bold)
};
let item_tags = tags_by_item.get(&item_id).unwrap();
let tags_cell = Cell::new(&item_tags.join(" "));
let mut table_row = Row::new(vec![id_cell,ts_cell,size_cell, compression_cell, file_size_cell, tags_cell]);
let item_meta = meta_by_item.get(&item_id).unwrap();
for name in &meta_columns_sorted {
match item_meta.get(name) {
Some(value) => table_row.add_cell(Cell::new(value)),
None => table_row.add_cell(Cell::new(""))
};
}
table.add_row(table_row);
}
table.printstd();
Ok(())
}
fn mode_update(cmd: &mut Command, args: Args, ids: &mut Vec<i64>, tags: &mut Vec<String>, conn: &mut Connection) -> Result<()> {
if ids.is_empty() { if ids.is_empty() {
cmd.error(ErrorKind::InvalidValue, "No ID given, you must supply one ID when using --update").exit(); cmd.error(ErrorKind::InvalidValue, "No ID given, you must supply one ID when using --update").exit();
} }
let item_id = ids.iter().next().expect("Unable to determine item id");
let item_maybe = db::get_item(conn, *item_id)?;
let item = item_maybe.expect("Unable to find item in database");
debug!("MAIN: Found item {:?}", item);
if ! tags.is_empty() {
debug!("MAIN: Updating item tags");
db::set_item_tags(conn, item.clone(), tags)?;
}
if args.item.meta.is_some() {
debug!("MAIN: Updating item meta");
for kv in args.item.meta.unwrap().iter() {
let meta = db::Meta {
id: item.id.unwrap(),
name: kv.key.to_string(),
value: kv.value.to_string()
};
db::store_meta(conn, meta)?;
}
}
Ok(())
} }
fn mode_delete(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec<String>) { fn mode_delete(cmd: &mut Command, _args: Args, ids: &mut Vec<i64>, tags: &mut Vec<String>, conn: &mut Connection, data_path: PathBuf) -> Result<()> {
if ids.is_empty() { if ids.is_empty() {
cmd.error(ErrorKind::InvalidValue, "No ID given, you must supply one ID when using --delete").exit(); cmd.error(ErrorKind::InvalidValue, "No ID given, you must supply one ID when using --delete").exit();
} else if ! tags.is_empty() { } else if ! tags.is_empty() {
@@ -206,19 +584,105 @@ fn mode_delete(cmd: &mut Command, args: Args, ids: &mut Vec<u32>, tags: &mut Vec
} else if ids.len() > 1 { } else if ids.len() > 1 {
cmd.error(ErrorKind::InvalidValue, "More than one ID given, you must supply one ID when using --delete").exit(); cmd.error(ErrorKind::InvalidValue, "More than one ID given, you must supply one ID when using --delete").exit();
} }
let item_id = ids.iter().next().expect("Unable to determine item id");
let item_maybe = db::get_item(conn, *item_id)?;
let item = item_maybe.expect("Unable to find item in database");
debug!("MAIN: Found item {:?}", item);
db::delete_item(conn, item)?;
let mut item_path = data_path.clone();
item_path.push(item_id.to_string());
fs::remove_file(&item_path).context(anyhow!("Unable to remove item file {:?}", item_path))?;
Ok(())
} }
fn mode_status_show_compression() { fn mode_status(_cmd: &mut Command, args: Args, data_path: PathBuf, db_path: PathBuf) -> Result<()> {
let compression_types = compression::supported_compression_types(); let mut path_table = Table::new();
println!("compression_types:"); if std::io::stdout().is_terminal() {
for compression_type in compression_types.into_iter() { path_table.set_format(*FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR);
println!(" {}", compression_type); } else {
} path_table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR);
} }
path_table.set_titles(Row::new(vec![
Cell::new("Type").with_style(Attr::Bold),
Cell::new("Path").with_style(Attr::Bold),
]));
fn mode_status(cmd: &mut Command, args: Args) { path_table.add_row(Row::new(vec![
mode_status_show_compression(); Cell::new("Data"),
Cell::new(&data_path.into_os_string().into_string().expect("Unable to convert data path to string"))
]));
path_table.add_row(Row::new(vec![
Cell::new("Database"),
Cell::new(&db_path.into_os_string().into_string().expect("Unable to convert DB path to string"))
]));
let mut compression_table = Table::new();
if std::io::stdout().is_terminal() {
compression_table.set_format(*FORMAT_BOX_CHARS_NO_BORDER_LINE_SEPARATOR);
} else {
compression_table.set_format(*FORMAT_NO_BORDER_LINE_SEPARATOR);
}
compression_table.set_titles(row!(
b->"Type",
b->"Found",
b->"Default",
b->"Binary",
b->"Compress",
b->"Decompress"));
let default_type = match args.item.compression {
Some(compression_name) => CompressionType::from_str(&compression_name)
.context(anyhow!("Invalid compression type {}", compression_name))?,
None => compression::default_type()
};
for compression_type in CompressionType::iter() {
let compression_program = match compression_type {
CompressionType::None => compression::CompressionProgram {
program: "".to_string(),
compress: Vec::new(),
decompress: Vec::new(),
supported: true
},
_ => compression::get_program(compression_type.clone())?
};
let is_default = compression_type == default_type;
compression_table.add_row(Row::new(vec![
Cell::new(&compression_type.to_string()),
match compression_program.supported {
true => Cell::new("Yes").with_style(Attr::ForegroundColor(color::GREEN)),
false => Cell::new("No").with_style(Attr::ForegroundColor(color::RED))
},
match is_default {
true => Cell::new("Yes").with_style(Attr::ForegroundColor(color::GREEN)),
false => Cell::new("No")
},
Cell::new(&compression_program.program),
Cell::new(&compression_program.compress.join(" ")),
Cell::new(&compression_program.decompress.join(" ")),
]));
}
println!("PATHS:");
path_table.printstd();
println!();
println!("COMPRESSION:");
compression_table.printstd();
Ok(())
} }